#2: add packages config to deps in config file and generate setup scr…

…ipts per environment

.github/workflows/build-and-test.yml

@@ -1,6 +1,6 @@
 name: Build and Test (Template)
 
-# This if a template to build and test a project in all available test environments on Github actions
+# Template workkflow to build and test a project in all available test environments on Github actions
 # WIP: This is currently a work in progress (#2: Docker image usage)
 
 on:

ci/build-docker-base

@@ -2,7 +2,6 @@
 
 import copy
 import os
-import uuid
 import sys
 
 from util import resolve_conf
@@ -56,66 +55,24 @@ class DockerBuilder:
 
         # Add compiler packages
         compiler = environment.get('compiler')
-        packages = []
-        if compiler is not None:
-            packages.append(compiler.get("cc"))
-            packages.append(compiler.get("cxx"))
-
-        # Add setup instructions
-        instructions = []
-        deps = environment.get("deps")
-        for dep_id in deps:
-            dep_args = deps.get(dep_id)
-            # Install deps with arguments
-            if dep_args is not None and len(dep_args) > 0:
-                # dep script multiple call
-                if isinstance(dep_args[0], list):
-                    for (i, dep_args_sub) in enumerate(dep_args):
-                        dep_args_sub = [ f"\"{a}\"" for a in dep_args_sub]
-                        instructions.append(f"./{dep_id}.sh {' '.join(dep_args_sub)}")
-                # dep script single call
-                else:
-                    dep_args = [ f"\"{a}\"" for a in dep_args]
-                    instructions.append(f"./{dep_id}.sh {' '.join(dep_args)}")
-            # No arguments
-            else:
-                instructions.append(f"./{dep_id}.sh")
-
-        setup_script = ""
-        with open(
-            os.path.dirname(__file__) + '/scripts/setup-deps-template.sh',
-            'r',
-            encoding="utf-8"
-        ) as file:
-            setup_script = file.read()
-        setup_script = setup_script.replace('%INSTRUCTIONS%', '\n'.join(instructions))
-
-        setup_filename = f"~setup-deps-{uuid.uuid4()}.sh"
-        setup_filepath = os.path.join(os.path.dirname(__file__), 'scripts', 'deps', setup_filename)
-
-        with open(setup_filepath, "w+", encoding="utf-8") as f:
-            f.write(setup_script)
-
         cmd = ("docker build"
+                # General
                 f" --tag {repo}:{tag}"
                 f" --build-arg ARCH={image.get('arch')}"
                 f" --build-arg BASE={image.get('base')}"
+                # Compiler
                 f" --build-arg CC={compiler.get('cc', '') if compiler is not None else ''}"
                 f" --build-arg CXX={compiler.get('cxx', '') if compiler is not None else ''}"
                 f" --build-arg GCOV={compiler.get('gcov', '') if compiler is not None else ''}"
-                 " --build-arg INSTALL_DEFAULT_PACKAGES=1"
-                f" --build-arg PACKAGES=\"{' '.join(packages)}\""
-                f" --build-arg SETUP_FILE=\'{setup_filename}\'"
+                f" --build-arg CI_ENVIRONMENT={image.get('environment')}"
+                #
                 f" --file {os.path.dirname(__file__)}/docker/base.dockerfile ."
                 " --progress=plain"
                 # " --no-cache"
             )
         print(cmd)
         os.system(cmd)
 
-        # remove temporary setup script
-        os.remove(setup_filepath)
-
         # TODO: option to push to Dockerhub
 
 DockerBuilder().build(sys.argv[1:])

ci/build-setup

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+
+import copy
+import os
+from typing import List
+
+from util import resolve_conf
+import yaml
+
+class SetupBuilder:
+    """Dockerfile generator class"""
+
+    def __instructions(self, dep_id, args: list) -> List[str]:
+        """ Generate shell instructions to setup a dependency"""
+
+        call_args = []
+        if args is not None and len(args) > 0:
+            if isinstance(args[0], list):
+                instructions = []
+                for (_, sub_args) in enumerate(args):
+                    instructions.extend(self.__instructions(dep_id, sub_args))
+                return instructions
+
+            call_args = [ f"\"{a}\"" for a in args]
+
+        return [ f"./deps/{dep_id}.sh {' '.join(call_args)}" ]
+
+    def build(self):
+        """Build environment setup scripts for each environments defined in config"""
+
+        raw_config: dict = {}
+        with open(os.path.dirname(__file__) + '/config.yaml', 'r', encoding="utf-8") as file:
+            raw_config = yaml.safe_load(file)
+        config = resolve_conf(copy.deepcopy(raw_config))
+
+        environments = config.get("environments")
+
+        for (env_id, environment) in environments.items():
+
+
+            # Add compiler packages
+            compiler = environment.get('compiler')
+            packages = copy.deepcopy(environment.get("deps", {}).get("packages", []))
+            if compiler is not None:
+                if environment.get("deps") is None:
+                    environment["deps"] = {}
+                if packages is None:
+                    environment["deps"]["packages"] = []
+
+                if compiler.get("cc") is not None and not compiler.get("cc") in packages:
+                    packages.append(compiler.get("cc"))
+                if compiler.get("cxx") is not None and not compiler.get("cxx") in packages:
+                    packages.append(compiler.get("cxx"))
+                environment["deps"]["packages"] = packages
+
+            # Add setup instructions
+            instructions = []
+            for (dep_id, args) in environment.get("deps").items():
+                instructions.extend(self.__instructions(dep_id, args))
+
+            setup_script = ""
+            with open(
+                os.path.dirname(__file__) + '/setup-template.sh',
+                'r',
+                encoding="utf-8"
+            ) as file:
+                setup_script = file.read()
+            setup_script = setup_script.replace('%INSTRUCTIONS%', '\n'.join(instructions))
+
+            setup_filename = f"setup-{env_id}.sh"
+            setup_filepath = os.path.join(os.path.dirname(__file__),
+                                          'shared', 'scripts', setup_filename)
+
+            with open(setup_filepath, "w+", encoding="utf-8") as f:
+                f.write(setup_script)
+
+SetupBuilder().build()

ci/build-workflow-matrix

@@ -3,7 +3,6 @@
 import copy
 import os
 import json
-import sys
 
 from util import resolve_conf
 import yaml
@@ -51,7 +50,7 @@ class MatrixGenerator:
             data = json.dumps({
                 "_comment": "This file has been generated. Please do not edit",
                 "matrix": matrix}, indent=2)
-            with open(os.path.dirname(__file__) + f"/shared/{runner_type}-matrix.json", 'w+',
+            with open(os.path.dirname(__file__) + f"/shared/workflows/{runner_type}-matrix.json", 'w+',
                         encoding="utf-8") as file:
                 file.write(data)
 

ci/config.yaml

@@ -14,6 +14,23 @@
 
 # Common parameters
 parameters:
+    # Package common dependencies
+    apt_default_packages:
+      - ca-certificates
+      - less
+      - curl
+      - jq
+      - git
+      - wget
+      - zlib1g
+      - zlib1g-dev
+      - ninja-build
+      - valgrind
+      - make-guile
+      - libomp5
+      - libunwind-dev
+      - ccache
+    brew_default_packages: []
     # Python
     common_pip_packages: "nanobind yaml setuptools"
     conda-py[3.8-3.12]:
@@ -24,7 +41,7 @@ parameters:
       - ["3.12", "@common_pip_packages"]
     # C/C++
     gcc-11: { type: gnu, cc: gcc-11, cxx: g++-11, gcov: gcov }  
-    gcc-14: { type: gnu, cc: gcc-14, cxx: g++-14, gcov: gcov }  
+    gcc-14: { type: gnu, cc: gcc-14, cxx: g++-14, gcov: gcov }
 
 # Environments (OS, commpiler, dependencies)
 environments:
@@ -34,29 +51,32 @@ environments:
     compiler: "@gcc-14"
     # list of optional dependencies to install (matching script at ci/scripts/deps/{dep}.sh {arg1} {arg2} ...)
     deps:
+      packages: "@apt_default_packages"
       cmake: ['3.23.4']
-      conda: ~
-      conda-python-env: "@conda-py[3.8-3.12]" 
+      # conda: ~
+      # conda-python-env: [ "@conda-py[3.8-3.12]" ]
       openmpi: ['v5.0', '5.0.4', '-j4']
-      zoltan: [-j4', '/trilinos-install']
-      mpich: ['4.0.2', '-j4']
-      vtk: [ '9.3.1' ]
+      # zoltan: [-j4', '/trilinos-install']
+      # mpich: ['4.0.2', '-j4']
+      # vtk: [ '9.3.1' ]
 
   ubuntu-22.04-gcc-11-vtk-9.2.2-py-3x-mpi:
     label: Ubuntu 22.04 / gcc-11 / Python[3.8-3.12] / MPI 5.0.4 / VTK 9.3.1
     compiler: "@gcc-11"
-    # list of optional dependencies to install (matching script at ci/scripts/deps/{dep}.sh {arg1} {arg2} ...)
+    # list dependencies to install (matching script at ci/scripts/deps/{dep}.sh {arg1} {arg2} ...)
     deps:
+      packages: "@apt_default_packages"
       cmake: ['3.23.4']
       conda: [ ]
-      conda-python-env: "conda-py[3.8-3.12]"
+      conda-python-env: [ "@conda-py[3.8-3.12]" ]
       vtk: [ '9.3.1' ]
       openmpi: ['v5.0', '5.0.4']
 
   macos-12-clang-14-vtk-9.3.1:
     label: MacOSX / clang / VTK 9.3.1
     compiler: { type: clang, cc: clang-14, cxx: clang-14++, gcov: llvm-gcov }
     deps:
+      packages: "@apt_default_packages"
       conda: [ ]
       vtk: [ '9.3.1' ]
 

ci/docker/base.dockerfile

@@ -3,36 +3,29 @@ ARG ARCH=amd64
 ARG BASE=ubuntu:22.04
 ARG CC=${CC:-""}
 ARG CXX=${CXX:-""}
+ARG CI_ENVIRONMENT=${CXX:-"ubuntu-develop"}
 
 FROM ${ARCH}/${BASE} as base
 
 SHELL ["/bin/bash", "-c"]
 
 # Copy setup scripts
-COPY ci/scripts/deps /opt/scripts/install
-RUN ls /opt/scripts/install
-RUN chmod u+x /opt/scripts/install/*.sh
-
-# Install packages (works)
-ARG PACKAGES=""
-ARG INSTALL_DEFAULT_PACKAGES=1
-RUN /opt/scripts/install/packages.sh "${PACKAGES}" ${INSTALL_DEFAULT_PACKAGES}
+ARG CI_ENVIRONMENT
+COPY ci/shared/scripts/setup-${CI_ENVIRONMENT}.sh /opt/scripts/setup.sh
+COPY ci/shared/scripts/deps /opt/scripts/deps
+RUN chmod u+x /opt/scripts/setup.sh /opt/scripts/deps/*.sh
 
 # Environment variables (conda path, python environments, compiler, cmake path, vtk)
 ENV PATH=/opt/cmake/bin:$PATH
-
 ENV CC=$CC
 ENV CXX=$CXX
 ENV GCOV=$GCOV
-
 ENV CONDA_INSTALL_DIR=/opt/conda
 ENV CONDA_AUTO_ACTIVATE_BASE=false
-
 ENV VTK_DIR=/opt/vtk/build
 
-# Run additional setup commands from setup file
-ARG SETUP_FILE=""
-RUN /opt/scripts/install/${SETUP_FILE}
+# Run setup script
+RUN /opt/scripts/setup.sh
 
 # Clean
 RUN rm -rf /opt/scripts

ci/scripts/setup-deps-template.sh → ci/setup-template.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# This file has been generated by a script. Do not edit.
+
 echo "Installing dependencies..."
 
 CURRENT_DIR="$(dirname -- "$(realpath -- "$0")")"

ci/shared/scripts/deps/packages-macos.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+PACKAGES=$@
+
+echo "PACKAGES=\"${PACKAGES}\""
+
+apt-get brew install -y -q && \
+    $PACKAGES

ci/shared/scripts/deps/packages-ubuntu.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+PACKAGES=$@
+
+echo "PACKAGES=\"${PACKAGES}\""
+
+apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+    $PACKAGES && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*

ci/scripts/deps/packages.sh → ci/shared/scripts/deps/packages.sh

@@ -3,20 +3,20 @@
 # Set up packages through the package manager
 
 DEPS_DIR="$(dirname -- "$(realpath -- "$0")")"
-PACKAGES="${1:-""}"
-INSTALL_DEFAULT_PACKAGES=${2:-"1"}
 
-echo "Set up dependencies..."
+PACKAGES=$@
+
+echo "Install system packages..."
 if [[ $(uname -a) == *"Darwin"* ]]; then
-    bash $DEPS_DIR/packages-macos.sh "$PACKAGES" $INSTALL_DEFAULT_PACKAGES
+    bash $DEPS_DIR/packages-macos.sh $PACKAGES
     exit 0
 elif [[ $(uname -a) == *"Linux"* ]]; then
     if [[ -f "/etc/lsb-release" ]]
     then
         . /etc/lsb-release
         if [[ $DISTRIB_ID == "Ubuntu" ]]
         then
-            bash $DEPS_DIR/packages-ubuntu.sh "$PACKAGES"
+            bash $DEPS_DIR/packages-ubuntu.sh $PACKAGES
             exit 0
         fi
     else

ci/shared/scripts/setup-macos-12-clang-14-vtk-9.3.1.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# This file has been generated by a script. Do not edit.
+
+echo "Installing dependencies..."
+
+CURRENT_DIR="$(dirname -- "$(realpath -- "$0")")"
+
+pushd $CURRENT_DIR
+
+./deps/packages.sh "ca-certificates" "less" "curl" "jq" "git" "wget" "zlib1g" "zlib1g-dev" "ninja-build" "valgrind" "make-guile" "libomp5" "libunwind-dev" "ccache" "clang-14" "clang-14++"
+./deps/conda.sh 
+./deps/vtk.sh "9.3.1"
+
+popd

ci/shared/scripts/setup-ubuntu-22.04-gcc-11-vtk-9.2.2-py-3x-mpi.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# This file has been generated by a script. Do not edit.
+
+echo "Installing dependencies..."
+
+CURRENT_DIR="$(dirname -- "$(realpath -- "$0")")"
+
+pushd $CURRENT_DIR
+
+./deps/packages.sh "ca-certificates" "less" "curl" "jq" "git" "wget" "zlib1g" "zlib1g-dev" "ninja-build" "valgrind" "make-guile" "libomp5" "libunwind-dev" "ccache" "gcc-11" "g++-11"
+./deps/cmake.sh "3.23.4"
+./deps/conda.sh 
+./deps/conda-python-env.sh "3.8" "nanobind yaml setuptools"
+./deps/conda-python-env.sh "3.9" "nanobind yaml setuptools"
+./deps/conda-python-env.sh "3.10" "nanobind yaml setuptools"
+./deps/conda-python-env.sh "3.11" "nanobind yaml setuptools"
+./deps/conda-python-env.sh "3.12" "nanobind yaml setuptools"
+./deps/vtk.sh "9.3.1"
+./deps/openmpi.sh "v5.0" "5.0.4"
+
+popd

ci/shared/scripts/setup-ubuntu-develop.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# This file has been generated by a script. Do not edit.
+
+echo "Installing dependencies..."
+
+CURRENT_DIR="$(dirname -- "$(realpath -- "$0")")"
+
+pushd $CURRENT_DIR
+
+./deps/packages.sh "ca-certificates" "less" "curl" "jq" "git" "wget" "zlib1g" "zlib1g-dev" "ninja-build" "valgrind" "make-guile" "libomp5" "libunwind-dev" "ccache" "gcc-14" "g++-14"
+./deps/cmake.sh "3.23.4"
+./deps/openmpi.sh "v5.0" "5.0.4" "-j4"
+
+popd