Reworked build scripts

.gitignore

 .DS_Store
-__pycache__
\ No newline at end of file
+__pycache__/**
+output/**
+venv
\ No newline at end of file

.gitlab-ci.yml

@@ -13,15 +13,15 @@ buid-default-notebooks:
       - tags
   when: manual
   variables:
-    VARIANT_PATH: './default'
+    VARIANT_PATH: './docker-stacks'
   script:
-    - ./build_variant.sh $VARIANT_PATH all-spark $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH pyspark $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH datascience $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH minimal $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH r $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH scipy $CI_COMMIT_TAG
-    - ./build_variant.sh $VARIANT_PATH tensorflow $CI_COMMIT_TAG
+    - ./build_variant.sh $VARIANT_PATH all-spark $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $VARIANT_PATH pyspark $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $VARIANT_PATH datascience $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $VARIANT_PATH minimal $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $VARIANT_PATH r $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $VARIANT_PATH scipy $CI_COMMIT_TAG $CI_COMMIT_TAG
+    - ./build_variant.sh $VARIANT_PATH tensorflow $CI_COMMIT_TAG $CI_COMMIT_TAG
 
 buid-gpu-jupyter:
   only:
@@ -30,5 +30,5 @@ buid-gpu-jupyter:
   when: manual
   variables:
     VARIANT_PATH: './gpu-jupyter'
-  script:
-    - ./build_variant.sh $VARIANT_PATH gpu-jupyter $CI_COMMIT_TAG
+  script:   # v1.4_cuda-11.0_ubuntu-20.04
+    - ./build_variant.sh $VARIANT_PATH gpu-jupyter misc $CI_COMMIT_TAG

build_variant.sh

@@ -4,84 +4,89 @@ set -e
 
 VSC3_IMAGE_PATH=/mnt/NFS/vsc3_src/USIF/jupyter-notebook
 VSC4_IMAGE_PATH=/mnt/NFS/vsc4_src/usif/jupyter-notebook
-DEF_FILE_TEMPLATE=./tools/singularity-template.def.j2
-META_FILE_TEMPLATE=./tools/meta-template.meta.j2
 
-if [ $# -ne 4 ]; then
-    echo "usage: $0 <path> <name> <group> <version>"
+function usage() {
+    echo "usage: $0 <output> <source> <name> <group> <docker-tag> [only-vsc3] [dry-run]"
     echo ""
-    echo "  path        path to the def, meta files and also where the sif file is built"
+    echo "  output      temp output path where the files will be built"
+    echo "  source      path to the json, def and meta source files"
     echo "  name        name of the input and output files e.g. all-spark.sif"
     echo "  group       output group directory"
-    echo "  version     docker source version tag"
+    echo "  docker-tag  docker image tag to use"
+    echo "  [only-vsc3|dry-run]"
+    echo "      only-vsc3   if this argument is set the output files will only be copied to VSC3 folders"
+    echo "      dry-run     only render the config files and exit"
     echo ""
+}
+
+if [[ $# -ne 5 ]] && [[ $# -ne 6 ]] && [[ $# -ne 7 ]]; then
+    usage
     exit 1
 fi
 
-VARIANT_PATH="$1"
-VARIANT_NAME="$2"
-VARIANT_GROUP="$3"
-VARIANT_VERSION="$4"
-
-echo "Building for $VARIANT_NAME"
+OUTPUT_PATH=$( readlink -f $1 )
+SOURCE_PATH=$( readlink -f $2 )
+VARIANT_NAME="$3"
+VARIANT_GROUP="$4"
+VARIANT_DOCKER_TAG="$5"
 
-BASE_ARGS="{ \"name\": \"$VARIANT_NAME\", \"group\": \"$VARIANT_GROUP\", \"version\": \"$VARIANT_VERSION\" }"
+PARAMETER="$6"
+if [ "$PARAMETER" == 'only-vsc3' ]; then
+    ONLY_VSC3=1
+    echo "Only copying to VSC3."
+fi
 
-VAR_FILE="$VARIANT_PATH/$VARIANT_NAME.json"
-if [ -f "$VAR_FILE" ]; then
-    VAR_ARGS=$( echo "$BASE_ARGS" | j2 --format=json $VAR_FILE )    
-else
-    VAR_ARGS="{}"
+shift 
+PARAMETER="$6"
+if [ "$PARAMETER" == 'dry-run' ]; then
+    DRY_RUN=1
+    echo "!!! Dry-run enabled !!!"
 fi
 
-INPUT_ARGS="{ \"base\": $BASE_ARGS, \"var\": $VAR_ARGS }"
-echo "Using input args:"
-echo "$INPUT_ARGS"
+shift
+PARAMETER="$6"
+if [ ! -z "$PARAMETER" ]; then
+    echo "Unknown parameter: $PARAMETER"
+    echo ""
+    usage
+    exit 2
+fi
 
-function _render() {
-    INPUT_FILE="$1"
-    OUTPUT_FILE="$2"
-    echo "$INPUT_ARGS" | j2 --format=json $INPUT_FILE -o $OUTPUT_FILE
-}
+./tools/generate_config.py $OUTPUT_PATH $SOURCE_PATH tools \
+    $VARIANT_NAME $VARIANT_GROUP $VARIANT_DOCKER_TAG
 
-function _print() {
-    TYPE="$1"
-    FILE="$2"
-    echo "Using $TYPE file: $FILE"
-    echo "-----"
-    cat $FILE
-    echo "-----"
-}
+VARIANT_IMAGE="$OUTPUT_PATH/${VARIANT_NAME}.sif"
+VARIANT_DEF="$OUTPUT_PATH/${VARIANT_NAME}.def"
+VARIANT_META="$OUTPUT_PATH/${VARIANT_NAME}.meta"
 
-if [ -f "$VARIANT_PATH/$VARIANT_NAME.def" ]; then
-    VARIANT_DEF="$VARIANT_PATH/$VARIANT_NAME.def"
+echo "Starting singularity build for $VARIANT_NAME: $VARIANT_IMAGE"
+if [ -z "$DRY_RUN" ]; then
+    singularity build $VARIANT_IMAGE $VARIANT_DEF
 else
-    VARIANT_DEF="$( mktemp -d )/$VARIANT_NAME.def"
-    _render $DEF_FILE_TEMPLATE $VARIANT_DEF
+    echo "singularity build $VARIANT_IMAGE $VARIANT_DEF"
 fi
-_print 'def' $VARIANT_DEF
 
-if [ -f "$VARIANT_PATH/$VARIANT_NAME.meta" ]; then
-    VARIANT_META="$VARIANT_PATH/$VARIANT_NAME.meta"
-else
-    VARIANT_META="$( mktemp -d )/$VARIANT_NAME.meta"
-    _render $META_FILE_TEMPLATE $VARIANT_META
+echo "Built files will be placed in subfolder: \"$VARIANT_GROUP\""
+
+if [ ! -z "$ONLY_VSC3" ]; then
+    echo "Copying built image to VSC-4 jupyterhub NFS share"
+    if [ -z "$DRY_RUN" ]; then
+        mkdir -p $VSC4_IMAGE_PATH/$VARIANT_GROUP/
+        cp -v $VARIANT_IMAGE $VARIANT_DEF $VARIANT_META $VSC4_IMAGE_PATH/$VARIANT_GROUP/
+    else
+        echo "mkdir -p $VSC4_IMAGE_PATH/$VARIANT_GROUP/"
+        echo "cp -v $VARIANT_IMAGE $VARIANT_DEF $VARIANT_META $VSC4_IMAGE_PATH/$VARIANT_GROUP/"
+    fi
 fi
-_print 'meta' $VARIANT_META
-
-VARIANT_IMAGE="$VARIANT_PATH/$VARIANT_NAME.sif"
-
-echo "Starting singularity build for $VARIANT_NAME: $VARIANT_IMAGE"
-# singularity build $VARIANT_IMAGE $VARIANT_DEF
-
-# # echo "Copying built image to VSC-3 jupyterhub NFS share"
-# mkdir -p $VSC3_IMAGE_PATH/$VARIANT_GROUP/
-# cp -v $VARIANT_IMAGE $VARIANT_DEF $VARIANT_META $VSC3_IMAGE_PATH/$VARIANT_GROUP/
 
-# # echo "Copying built image to VSC-4 jupyterhub NFS share"
-# mkdir -p /mnt/NFS/vsc4_src/usif/jupyter-notebook/$VARIANT_GROUP
-# mv -v $VARIANT_IMAGE $VARIANT_DEF $VSC3_IMAGE_PATH/$VARIANT_GROUP/
-# cp -v $VARIANT_META $VSC3_IMAGE_PATH/$VARIANT_GROUP/
+echo "Copying built image to VSC-3 jupyterhub NFS share"
+if [ -z "$DRY_RUN" ]; then
+    mkdir -p $VSC3_IMAGE_PATH/$VARIANT_GROUP/
+    mv -v $VARIANT_IMAGE $VARIANT_DEF $VARIANT_META $VSC3_IMAGE_PATH/$VARIANT_GROUP/
+else
+    echo "mkdir -p $VSC3_IMAGE_PATH/$VARIANT_GROUP/"
+    echo "mv -v $VARIANT_IMAGE $VARIANT_DEF $VARIANT_META $VSC3_IMAGE_PATH/$VARIANT_GROUP/"
+fi
 
 echo "Done $VARIANT_NAME."
-echo ""
\ No newline at end of file
+echo ""

default/datascience.def

-BootStrap: docker
-From: jupyter/datascience-notebook:hub-2.1.1
-
-%files
-requirements.txt /home/jovyan/requirements.txt
-install-packages.sh /home/jovyan/install-packages.sh
-
-%post
-/usr/bin/bash /home/jovyan/install-packages.sh
-rm -f /home/jovyan/requirements.txt
-rm -f /home/jovyan/install-packages.sh

default/minimal.def

-BootStrap: docker
-From: jupyter/minimal-notebook:hub-2.1.1
-
-%files
-requirements.txt /home/jovyan/requirements.txt
-install-packages.sh /home/jovyan/install-packages.sh
-
-%post
-/usr/bin/bash /home/jovyan/install-packages.sh
-rm -f /home/jovyan/requirements.txt
-rm -f /home/jovyan/install-packages.sh

default/pyspark.def

-BootStrap: docker
-From: jupyter/pyspark-notebook:hub-2.1.1
-
-%files
-requirements.txt /home/jovyan/requirements.txt
-install-packages.sh /home/jovyan/install-packages.sh
-
-%post
-/usr/bin/bash /home/jovyan/install-packages.sh
-rm -f /home/jovyan/requirements.txt
-rm -f /home/jovyan/install-packages.sh

default/r.def

-BootStrap: docker
-From: jupyter/r-notebook:hub-2.1.1
-
-%files
-requirements.txt /home/jovyan/requirements.txt
-install-packages.sh /home/jovyan/install-packages.sh
-
-%post
-/usr/bin/bash /home/jovyan/install-packages.sh
-rm -f /home/jovyan/requirements.txt
-rm -f /home/jovyan/install-packages.sh

default/scipy.def

-BootStrap: docker
-From: jupyter/scipy-notebook:hub-2.1.1
-
-%files
-requirements.txt /home/jovyan/requirements.txt
-install-packages.sh /home/jovyan/install-packages.sh
-
-%post
-/usr/bin/bash /home/jovyan/install-packages.sh
-rm -f /home/jovyan/requirements.txt
-rm -f /home/jovyan/install-packages.sh

default/tensorflow.def

-BootStrap: docker
-From: jupyter/tensorflow-notebook:hub-2.1.1
-
-%files
-requirements.txt /home/jovyan/requirements.txt
-install-packages.sh /home/jovyan/install-packages.sh
-
-%post
-/usr/bin/bash /home/jovyan/install-packages.sh
-rm -f /home/jovyan/requirements.txt
-rm -f /home/jovyan/install-packages.sh

default/all-spark.json → docker-stacks/all-spark.json

 {
-    "title": "All-Spark Notebook",
+    "title": "All-Spark Notebook ({{base.group}})",
     "description": "Based on the 'PySpark' image with the addition of IRKernel, rcurl, sparklyr, ggplot2 & spylon-kernel",
     "link": "https://github.com/jupyter/docker-stacks",
     "author": "VSC",
@@ -8,5 +8,5 @@
     "exclude-caps": ["gpu"],
     "required-groups": [],
     "order": 6,
-    "image": "jupyter/all-spark-notebook"
+    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
 }

default/datascience.meta → docker-stacks/datascience.json

 {
-    "title": "Datascience Notebook ({group})",
+    "title": "Datascience Notebook ({{base.group}})",
     "description": "The Datascience image contains everything from 'SciPy' & 'R' as well as an Julia environment.",
     "link": "https://github.com/jupyter/docker-stacks",
-    "version": "hub-2.1.1",
     "author": "VSC",
     "contact": "service@vsc.ac.at",
     "required-caps": [],
     "exclude-caps": ["gpu"],
     "required-groups": [],
-    "order": 4
-}
\ No newline at end of file
+    "order": 4,
+    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
+}

default/minimal.meta → docker-stacks/minimal.json

 {
-    "title": "Minimal Notebook ({group})",
+    "title": "Minimal Notebook ({base.group})",
     "description": "Minimal image contains everything from 'base' as well as TeX Live",
     "link": "https://github.com/jupyter/docker-stacks",
-    "version": "hub-2.1.1",
     "author": "VSC",
     "contact": "service@vsc.ac.at",
     "required-caps": [],
     "exclude-caps": ["gpu"],
     "required-groups": [],
-    "order": 0
+    "order": 0,
+    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
 }
\ No newline at end of file

default/pyspark.meta → docker-stacks/pyspark.json

 {
-    "title": "PySpark Notebook ({group})",
+    "title": "PySpark Notebook ({{base.group}})",
     "description": "The PySpark image is based on the 'SciPy' image and contains Apache Spark, Hadoop binaries and the pyarrow library in addition",
     "link": "https://github.com/jupyter/docker-stacks",
-    "version": "hub-2.1.1",
     "author": "VSC",
     "contact": "service@vsc.ac.at",
     "required-caps": [],
     "exclude-caps": ["gpu"],
     "required-groups": [],
-    "order": 5
+    "order": 5,
+    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
 }
\ No newline at end of file

default/r.meta → docker-stacks/r.json

 {
-    "title": "R Notebook ({group})",
+    "title": "R Notebook ({{base.group}})",
     "description": "R image contains everything from 'minimal' as well as the R interpreter and base environment.",
     "link": "https://github.com/jupyter/docker-stacks",
-    "version": "hub-2.1.1",
     "author": "VSC",
     "contact": "service@vsc.ac.at",
     "required-caps": [],
     "exclude-caps": ["gpu"],
     "required-groups": [],
-    "order": 1
+    "order": 1,
+    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
 }
\ No newline at end of file

default/scipy.meta → docker-stacks/scipy.json

 {
-    "title": "SciPy Notebook ({group})",
+    "title": "SciPy Notebook ({{base.group}})",
     "description": "The SciPy image contains everything from 'minimal' and popular packages from the scientific Python ecosystem.",
     "link": "https://github.com/jupyter/docker-stacks",
-    "version": "hub-2.1.1",
     "author": "VSC",
     "contact": "service@vsc.ac.at",
     "required-caps": [],
     "exclude-caps": ["gpu"],
     "required-groups": [],
-    "order": 2
+    "order": 2,
+    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
 }
\ No newline at end of file

default/tensorflow.meta → docker-stacks/tensorflow.json

 {
-    "title": "Tensorflow Notebook ({group})",
+    "title": "Tensorflow Notebook ({{base.group}})",
     "description": "The Tensorflow image contains everything from 'SciPy' as well as the tensorflow machine learning library.",
     "link": "https://github.com/jupyter/docker-stacks",
-    "version": "hub-2.1.1",
     "author": "VSC",
     "contact": "service@vsc.ac.at",
     "required-caps": [],
     "exclude-caps": ["gpu"],
     "required-groups": [],
-    "order": 3
+    "order": 3,
+    "docker-image": "jupyter/{{base.name}}-notebook:{{base['docker-tag']}}"
 }
\ No newline at end of file

gpu-jupyter/gpu-jupyter.def → gpu-jupyter/_old_gpu-jupyter.def.j2

 BootStrap: docker
-From: cschranz/gpu-jupyter:v1.4_cuda-11.0_ubuntu-20.04
+From: {{var['docker-image']}}
 
 %post
 /opt/conda/bin/pip3 install batchspawner==1.1.0

gpu-jupyter/gpu-jupyter.meta → gpu-jupyter/gpu-jupyter.json

@@ -2,9 +2,11 @@
     "title": "PyTorch/Tensorflow GPU enabled Notebook",
     "description": "This notebook uses the NVIDIA CUDA image as the base image and installs the jupyter toolstack on top of it to enable GPU calculations in the Jupyter notebooks",
     "link": "https://github.com/iot-salzburg/gpu-jupyter",
-    "version": "v1.4_cuda-11.0_ubuntu-20.04",
     "author": "Christoph Schranz",
     "contact": "",
     "required-caps": ["vsc3", "gpu"],
-    "exclude-caps": []
+    "exclude-caps": [],
+    "required-groups": [],    
+    "order": 0,
+    "docker-image": "cschranz/gpu-jupyter:{{base['docker-tag']}}"
 }
\ No newline at end of file

tools/generate_config.py

+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+from pprint import pprint
+import shutil
+from typing import Dict
+
+from jinja2 import Template
+
+
+def render_template(file_path: str, variables: Dict):
+    with open(file_path, 'r') as file:
+        file_content = file.read()
+        template = Template(file_content)
+        return template.render(**variables)
+
+
+def render_template_to_file(file_path: str, variables: Dict, output_file_path: str):
+    with open(output_file_path, 'w') as output_file:
+        output_file.write(render_template(file_path=file_path, variables=variables))
+
+
+def print_file(purpose: str, file_path: str):
+    print(f'Content of {purpose} file: {file_path}')
+    print('>-----')
+    with open(file_path, 'r') as file:
+        print(file.read())
+    print('<-----')
+
+
+def prepare_file(source_path: str, output_path: str, name: str, extension: str, template_file: str, input_args: Dict):
+    input_file = os.path.join(source_path, name + extension)
+    custom_template_file = os.path.join(source_path, name + extension + '.j2')
+    output_file = os.path.join(output_path, name + extension)
+    if os.path.exists(input_file):
+        print(f'Using existing {extension} file: {input_file}')
+        shutil.copy(src=input_file, dst=output_file)
+    elif os.path.exists(custom_template_file):
+        print(f'Rendering {extension} file: {input_file} using custom template {custom_template_file}')
+        render_template_to_file(custom_template_file, input_args, output_file)
+    else:
+        print(f'Rendering {extension} file: {input_file}')
+        render_template_to_file(template_file, input_args, output_file)
+    print_file(extension, output_file)    
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Render config files for singularity build and VSC JupyterHub.')
+    parser.add_argument('output', type=str, help='Output path')
+    parser.add_argument('source', type=str, help='Path to the source files')
+    parser.add_argument('template', type=str, help='Path to the template files')
+    parser.add_argument('name', type=str, help='Common name of the data source files (.json, .meta, .def) - will also be used for output .sif file name')
+    parser.add_argument('group', type=str, help='Output group name')
+    parser.add_argument('docker_tag', type=str, help='Docker iamge tag to use')
+    args = parser.parse_args()
+    
+    def_file_template = os.path.join(args.template, 'singularity-template.def.j2')
+    meta_file_template = os.path.join(args.template, 'meta-template.meta.j2')
+    
+    print(f'Generating config files using name: {args.name}')
+    
+    base_args = {
+        'name': args.name,
+        'group': args.group,
+        'docker-tag': args.docker_tag
+    }
+    
+    input_args = dict(base=base_args)
+    
+    # render input arguments if they are there
+    var_args = {}
+    var_file = os.path.join(args.source, args.name + '.json')
+    if os.path.exists(var_file):
+        print(f'Using var file: {var_file}')
+        var_args = json.loads(render_template(var_file, input_args))
+    else:
+        print(f'No var file found - skipping. path={var_file}')
+
+    input_args['var'] = var_args
+    
+    print('Using input args:')
+    print('>-----')
+    pprint(input_args)
+    print('<-----')
+
+    # DEF file
+    prepare_file(args.source, args.output, args.name, '.def', def_file_template, input_args)
+
+    # META file
+    prepare_file(args.source, args.output, args.name, '.meta', meta_file_template, input_args)
+    
+    print('Config file generation done.')    
+
+
+if __name__ == '__main__':
+    main()

tools/meta-template.meta.j2

 {
-    "title": "{{var.title}} ({{base.group}})",
+    "version": "{{base['docker-tag']}}",
+    "title": "{{var.title}}",
     "description": "{{var.description}}",
     "link": "{{var.link}}",
-    "version": "{{base.version}}",
     "author": "{{var.author}}",
     "contact": "{{var.contact}}",
     "required-caps": {{var['required-caps']}},
     "exclude-caps": {{var['exclude-caps']}},
     "required-groups": {{var['required-groups']}},
     "order": {{var.order}}
-}
+}
\ No newline at end of file