Refined build scripts

.gitlab-ci.yml

@@ -6,6 +6,8 @@ before_script:
 
 variables:
   SINGULARITY_CACHEDIR: /cache/singularity
+  SINGULARITY_TMPDIR: /cache/singularity-temp
+  OUTPUT_DIR: /cache/build-temp
 
 buid-default-notebooks:
   only:
@@ -13,22 +15,22 @@ buid-default-notebooks:
       - tags
   when: manual
   variables:
-    VARIANT_PATH: './docker-stacks'
+    SOURCE_DIR: './docker-stacks'
   script:
-    - ./build_variant.sh $VARIANT_PATH all-spark $CI_COMMIT_TAG $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH pyspark $CI_COMMIT_TAG $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH datascience $CI_COMMIT_TAG $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH minimal $CI_COMMIT_TAG $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH r $CI_COMMIT_TAG $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH scipy $CI_COMMIT_TAG $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH tensorflow $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $OUTPUT_DIR $SOURCE_DIR all-spark $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $OUTPUT_DIR $SOURCE_DIR datascience $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $OUTPUT_DIR $SOURCE_DIR minimal $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $OUTPUT_DIR $SOURCE_DIR r $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $OUTPUT_DIR $SOURCE_DIR scipy $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $OUTPUT_DIR $SOURCE_DIR tensorflow $CI_COMMIT_TAG $CI_COMMIT_TAG
 
 buid-gpu-jupyter:
+  # Use this with a tag of e.g.
+  #   - v1.4_cuda-11.0_ubuntu-20.04
+  #   - v1.4_cuda-11.2_ubuntu-20.04
   only:
     refs:
       - tags
   when: manual
-  variables:
-    VARIANT_PATH: './gpu-jupyter'
-  script:   # v1.4_cuda-11.0_ubuntu-20.04
-    - ./build_variant.sh $VARIANT_PATH gpu-jupyter misc $CI_COMMIT_TAG
+  script:   
+    - ./build_variant.sh output gpu-jupyter gpu-jupyter misc $CI_COMMIT_TAG only-vsc3

build_variant.sh

@@ -31,26 +31,24 @@ VARIANT_GROUP="$4"
 VARIANT_DOCKER_TAG="$5"
 
 PARAMETER="$6"
-if [ "$PARAMETER" == 'only-vsc3' ]; then
-    ONLY_VSC3=1
-    echo "Only copying to VSC3."
-fi
-
-shift 
-PARAMETER="$6"
-if [ "$PARAMETER" == 'dry-run' ]; then
-    DRY_RUN=1
-    echo "!!! Dry-run enabled !!!"
-fi
+while [ ! -z "$PARAMETER" ]; do
+    if [ "$PARAMETER" == 'only-vsc3' ]; then
+        ONLY_VSC3=1
+        echo "Only copying to VSC3."
+    elif [ "$PARAMETER" == 'dry-run' ]; then
+        DRY_RUN=1
+        echo "!!! Dry-run enabled !!!"
+    elif [ ! -z "$PARAMETER" ]; then
+        echo "Unknown parameter: $PARAMETER"
+        echo ""
+        usage
+        exit 2
+    fi
+    shift
+    PARAMETER="$6"
+done
 
-shift
-PARAMETER="$6"
-if [ ! -z "$PARAMETER" ]; then
-    echo "Unknown parameter: $PARAMETER"
-    echo ""
-    usage
-    exit 2
-fi
+mkdir -p $OUTPUT_PATH
 
 ./tools/generate_config.py $OUTPUT_PATH $SOURCE_PATH tools \
     $VARIANT_NAME $VARIANT_GROUP $VARIANT_DOCKER_TAG
@@ -61,14 +59,16 @@ VARIANT_META="$OUTPUT_PATH/${VARIANT_NAME}.meta"
 
 echo "Starting singularity build for $VARIANT_NAME: $VARIANT_IMAGE"
 if [ -z "$DRY_RUN" ]; then
-    singularity build $VARIANT_IMAGE $VARIANT_DEF
+    # singularity build $VARIANT_IMAGE $VARIANT_DEF
+    # build with debug and verbosity flags
+    singularity -v -d build $VARIANT_IMAGE $VARIANT_DEF
 else
     echo "singularity build $VARIANT_IMAGE $VARIANT_DEF"
 fi
 
 echo "Built files will be placed in subfolder: \"$VARIANT_GROUP\""
 
-if [ ! -z "$ONLY_VSC3" ]; then
+if [ -z "$ONLY_VSC3" ]; then
     echo "Copying built image to VSC-4 jupyterhub NFS share"
     if [ -z "$DRY_RUN" ]; then
         mkdir -p $VSC4_IMAGE_PATH/$VARIANT_GROUP/

docker-stacks/999_extend-r-path.sh

+#!/usr/bin/env bash
+
+# Additional hook for /usr/local/bin/before-notebook.d
+# to make r user package installs work in the singularity images
+
+VSC_R_PACKAGES=$HOME/.rpackages
+VSC_R_LIB_USER_FOLDER=$VSC_R_PACKAGES/user
+VSC_TRAINING_R_LIB_USER_FOLDER=$VSC_R_PACKAGES/training
+
+export R_LIBS_USER=$VSC_R_LIB_USER_FOLDER:$VSC_TRAINING_R_LIB_USER_FOLDER:$R_LIBS_USER
+
+mkdir -p $VSC_R_LIB_USER_FOLDER
+
+# create a (broken) link
+# note: will only work on training profiles where the training folder is mounted
+pushd $VSC_R_PACKAGES
+ln -s $HOME/../training/rpackages

docker-stacks/all-spark.json

@@ -8,5 +8,8 @@
     "exclude-caps": ["gpu"],
     "required-groups": [],
     "order": 6,
-    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
+    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}",
+    "additional-files": [
+        "./docker-stacks/999_extend-r-path.sh /usr/local/bin/before-notebook.d/999_extend-r-path.sh"
+    ]
 }

docker-stacks/pyspark.json

-{
-    "title": "PySpark Notebook ({{base.group}})",
-    "description": "The PySpark image is based on the 'SciPy' image and contains Apache Spark, Hadoop binaries and the pyarrow library in addition",
-    "link": "https://github.com/jupyter/docker-stacks",
-    "author": "VSC",
-    "contact": "service@vsc.ac.at",
-    "required-caps": [],
-    "exclude-caps": ["gpu"],
-    "required-groups": [],
-    "order": 5,
-    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
-}
\ No newline at end of file

docker-stacks/r.json

@@ -8,5 +8,8 @@
     "exclude-caps": ["gpu"],
     "required-groups": [],
     "order": 1,
-    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
+    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}",
+    "additional-files": [
+        "./docker-stacks/999_extend-r-path.sh /usr/local/bin/before-notebook.d/999_extend-r-path.sh"
+    ]    
 }
\ No newline at end of file

tools/meta-template.meta.j2

 {
-    "version": "{{base['docker-tag']}}",
-    "title": "{{var.title}}",
-    "description": "{{var.description}}",
-    "link": "{{var.link}}",
-    "author": "{{var.author}}",
-    "contact": "{{var.contact}}",
-    "required-caps": {{var['required-caps']}},
-    "exclude-caps": {{var['exclude-caps']}},
-    "required-groups": {{var['required-groups']}},
-    "order": {{var.order}}
+    "version": {{base['docker-tag']|tojson}},
+    "title": {{var.title|tojson}},
+    "description": {{var.description|tojson}},
+    "link": {{var.link|tojson}},
+    "author": {{var.author|tojson}},
+    "contact": {{var.contact|tojson}},
+    "required-caps": {{var['required-caps']|tojson}},
+    "exclude-caps": {{var['exclude-caps']|tojson}},
+    "required-groups": {{var['required-groups']|tojson}},
+    "order": {{var.order|tojson}}
 }
\ No newline at end of file

tools/singularity-template.def.j2

@@ -5,6 +5,11 @@ From: {{var['docker-image']}}
 ./tools/requirements.txt /home/jovyan/requirements.txt
 ./tools/install-packages.sh /home/jovyan/install-packages.sh
 ./tools/vsc-singleuser.sh /usr/local/bin/vsc-singleuser.sh
+{% if 'additional-files' in var -%}
+{%- for line in var['additional-files'] -%}
+{{line}}
+{%- endfor -%}
+{%- endif %}
 
 %environment
 export TINI_SUBREAPER=true