Merge pull request #3 from DARMA-tasking/2-implement-common-docker-co…

…ntainers

#2: implement common docker containers

.github/workflows/action-git-diff-check.yml

@@ -7,7 +7,7 @@ jobs:
     name: Run git check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}

.github/workflows/build-docker-image.yml

@@ -0,0 +1,82 @@
+name: Build & Push docker image
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches: "*"
+
+jobs:
+  get-matrix:
+    runs-on: ubuntu-latest
+    name: Get matrix
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get matrix
+        id: get-matrix
+        run: |
+          matrix=$(cat ci/shared/matrix/github.json | jq '.matrix' | jq -c '[ .[] | select( .image != null) ]')
+          echo "runner=$(echo $matrix)" >> $GITHUB_OUTPUT
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.runner }}
+
+  build-image:
+    name: Build ${{ matrix.runner.name }}
+    runs-on: ${{ matrix.runner.runs-on }}
+    needs: get-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: ${{ fromJson(needs.get-matrix.outputs.matrix ) }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - run: pip install pyyaml
+
+      - name: Build Docker image
+        run: |
+          python ci/build-docker-image.py ${{ matrix.runner.image }}
+          docker image inspect ${{ matrix.runner.image }}
+
+      - name: Test Docker image (VT build & test)
+        run: |
+          CMD='echo "CC=$CC" ; \
+          echo "CXX=$CXX" ; \
+          echo "FC=$FC" ; \
+          echo "CMPLR_ROOT=$CMPLR_ROOT" ; \
+          echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" ; \
+          echo "CPATH=$CPATH" ; \
+          echo "INFOPATH=$INFOPATH" ; \
+          echo "INTEL_LICENSE_FILE=$INTEL_LICENSE_FILE" ; \
+          echo "LIBRARY_PATH=$LIBRARY_PATH" ; \
+          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" ; \
+          echo "ONEAPI_ROOT=$ONEAPI_ROOT" ; \
+          echo "PATH=$PATH" ; \
+          echo "TBBROOT=$TBBROOT" ; \
+          mkdir -p "/opt/vt/src" "/opt/vt/build/vt" ; \
+            git clone https://github.com/DARMA-tasking/vt /opt/vt/src ; \
+            cd /opt/vt/src ; \
+            bash ci/build_cpp.sh /opt/vt/src /opt/vt/build ; \
+            bash ci/test_cpp.sh /opt/vt/src /opt/vt/build ; \
+            bash ci/build_vt_sample.sh /opt/vt/src /opt/vt/build ;
+            rm -rf "/opt/vt/src" "/opt/vt/build"'
+          echo "Running ${CMD}"
+          docker run \
+            --name test-container \
+            -e CI="1" \
+            -e CMAKE_CXX_STANDARD="17" \
+            -e CMAKE_BUILD_TYPE="Release" \
+            ${{ matrix.runner.image }} \
+            bash -c "$CMD"
+          exit $(docker container inspect --format '{{.State.ExitCode}}' test-container)
+
+      - name: Push Docker image to DockerHub Container Registry
+        if: ${{ success() && github.ref == 'refs/heads/master' }}
+        run: |
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_TOKEN }}
+          docker push ${{ matrix.runner.image }}

.github/workflows/check-commit-format.yml

@@ -7,7 +7,7 @@ jobs:
     name: Check commit message format
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}

.github/workflows/find-trailing-whitespace.yml

@@ -7,7 +7,7 @@ jobs:
     name: Find Trailing Whitespace
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: DARMA-tasking/find-trailing-whitespace@master
         with:
           exclude: "sketches;lib"

.gitignore

@@ -0,0 +1 @@
+__pycache__

ci/build-docker-image.py

@@ -0,0 +1,102 @@
+"""This script enable to build a docker image"""
+import copy
+import os
+import sys
+
+from util import resolve_conf
+import yaml
+
+class DockerBuilder:
+    """Dockerfile generator class"""
+
+    def build(self, args: list):
+        """Build an image using a given docker configuration fro the config file"""
+
+        raw_config: dict = {}
+        with open(os.path.dirname(__file__) + "/config.yaml", 'r', encoding="utf-8") as file:
+            raw_config = yaml.safe_load(file)
+
+        config = resolve_conf(copy.deepcopy(raw_config))
+        images = config.get("images")
+        setup = config.get("setup")
+
+        image_tag = None
+        if len(args) > 0:
+            image_tag = args[0]
+            if image_tag not in images.keys():
+                print(f"[error] Image not found {image_tag}.\n"
+                      f"Available images:{(os.linesep + '- ')}"
+                      f"{(os.linesep + '- ') . join(images.keys())}")
+                raise SystemExit(1)
+        else:
+            # Step 1: list platforms and their configurations
+            choices = {k: v for k, v in enumerate(images.keys())}
+            print("Choose image: ")
+            for i in choices:
+                image = images.get(choices[i])
+                setup_id = image.get("setup")
+                current_setup = setup.get(setup_id)
+                if current_setup is None:
+                    raise RuntimeError(f"Invalid setup {setup_id}")
+                lbl = current_setup.get("label", image.get("setup"))
+                print(
+                    f"\033[1m[{i}] {choices[i]}\033[0m\n"
+                    f"    \033[3;34m{lbl}\033[0m"
+                )
+            choice = input("> ")
+
+            image_tag = choices[int(choice)]
+
+        image = images.get(image_tag)
+        print("Selected image:")
+        print("---------------------------")
+        print(yaml.dump(image, default_flow_style=True))
+        print("---------------------------")
+
+        image_setup = setup.get(image.get("setup"))
+        dockerfile = image.get("dockerfile")
+        env = image_setup.get("env")
+
+        args = {
+            "ARCH": image.get("arch"),
+            "BASE": image.get("base"),
+            "SETUP_ID": image.get("setup")
+        }
+        # Env
+        supported_env_keys = [
+            # Compiler
+            "CC", "CXX", "FC",
+            # MPI
+            "MPICH_CC", "MPICH_CXX",
+            # Intel
+            "CMPLR_ROOT", "INTEL_LICENSE_FILE", "ONEAPI_ROOT", "TBBROOT",
+            "CMAKE_PREFIX_PATH", "CPATH", "INFOPATH", "LIBRARY_PATH", "LD_LIBRARY_PATH",
+            # Path
+            "PATH_PREFIX"
+        ]
+        for env_key in supported_env_keys:
+            args[env_key] = env.get(env_key, '')
+
+        invalid_keys = list(key for key in env if not key in supported_env_keys)
+        if len(invalid_keys) > 0:
+            raise RuntimeError(f"warning: env keys not supported: {invalid_keys}")
+
+        space = ' '
+
+        escaped_args = { k:f'"{v}"' for (k,v) in args.items()}
+
+        cmd = ("docker build . "
+                f" --tag {image_tag}"
+                f" --file {os.path.dirname(__file__)}/docker/{dockerfile}"
+                f" {space.join([f'--build-arg {k}={v}' for (k,v) in escaped_args.items()])}"
+                # f" --cache-from={image_tag}" # use previously built image as a cache layer.
+                " --no-cache"
+                " --progress=plain"
+            )
+        print(cmd)
+        status = os.system(cmd)
+        exit(status)
+
+        # ENHANCEMENT: option to push to Dockerhub
+
+DockerBuilder().build(sys.argv[1:])

ci/build-matrix.py

@@ -0,0 +1,86 @@
+"""This script generates CI matrix file(s)"""
+import copy
+import os
+import json
+import re
+
+from typing import Union
+from util import resolve_conf
+import yaml
+
+
+class MatrixBuilder:
+    """A class to generate matrix files for either Github Workflows or Azure Pipelines"""
+
+    def generate(self):
+        """Generate a matrix of runners and inner environments to be used by CI pipelines"""
+
+        raw_config: dict = {}
+        with open(os.path.dirname(__file__) + "/config.yaml", 'r', encoding="utf-8") as file:
+            raw_config = yaml.safe_load(file)
+        config = resolve_conf(copy.deepcopy(raw_config))
+
+        for runner_type in ["github", "azure"]:
+            # Configured runner type could be an array if the 2 platforms are targetted.
+            # Useful for testing on both github and Azure.
+            runners = [runner for runner in config.get("runners")
+                if (
+                    (runner.get("type") is str and runner.get("type") == runner_type) or
+                    (runner_type in runner.get("type")) # list
+                )]
+            matrix: Union[dict,list] = []
+            for runner in runners:
+                matrix_item = {
+                    "label": runner.get("label"),
+                    ("runs-on" if runner_type == "github" else "vmImage"): runner.get("runs-on")
+                }
+
+                # xor
+                assert( (runner.get("setup") is not None) != (runner.get("image") is not None))
+
+                if runner.get("setup") is not None:
+                    setup = config.get("setup").get(runner.get("setup"))
+
+                    if setup is None:
+                        raise RuntimeError(f"Setup not found {runner.get('setup')}")
+                    matrix_item["setup"] = runner.get("setup")
+                    matrix_item["name"] = runner.get("setup")
+
+                elif runner.get("image") is not None:
+                    image_name = (runner.get("image", {}).get("repository", "") + ":"
+                                + runner.get("image", {}).get("tag", ""))
+                    image = config.get("images").get(image_name)
+
+                    if image is None:
+                        raise RuntimeError(f"Image not found {runner.get('image')}")
+
+                    setup = config.get("setup").get(image.get("setup"))
+                    if setup is None:
+                        raise RuntimeError(f"Setup not found {runner.get('setup')}")
+
+                    matrix_item["image"] = image.get("repository") + ":" + image.get("tag")
+
+                    if matrix_item["label"] is None:
+                        matrix_item["label"] = image.get("label")
+
+                    matrix_item["name"] = image.get("tag")
+
+                if matrix_item["label"] is None:
+                    matrix_item["label"] = setup.get("label")
+
+                matrix.append(matrix_item)
+
+            if runner_type == "azure":
+                matrix = { re.sub("[^0-9a-zA-Z]+", '-', v.get("name")):v for v in matrix }
+
+            data = json.dumps({
+                "_comment": "This file has been generated. Please do not edit",
+                "matrix": matrix}, indent=2)
+            with open(
+                os.path.dirname(__file__) + f"/shared/matrix/{runner_type}.json",
+                "w+",
+                encoding="utf-8"
+            ) as file:
+                file.write(data)
+
+MatrixBuilder().generate()

ci/build-setup.py

@@ -0,0 +1,83 @@
+"""This script generates setup scripts"""
+import copy
+import os
+from typing import List, Union
+
+from util import resolve_conf
+import yaml
+
+class SetupBuilder:
+    """Setup files generator class"""
+
+    def __instructions(self, dep_id, args: Union[list, dict]) -> List[str]:
+        """ Generate shell instructions to setup a dependency"""
+
+        # basic command to run in setup script (bash)
+        if dep_id == "cmd":
+            assert isinstance(args, list)
+            return [ f"{' '.join(args)}" ]
+
+        call_args = []
+        env = []
+
+        # repeat instructions if args is an array of array
+        if args is not None and len(args) > 0:
+            if isinstance(args, list) and isinstance(args[0], list):
+                instructions = []
+                for (_, sub_args) in enumerate(args):
+                    instructions.extend(self.__instructions(dep_id, sub_args))
+                return instructions
+
+            env = []
+            if isinstance(args, dict):
+                call_args = [ f"\"{a}\"" for a in args.get("args", [])]
+                env = [ f"{k}=\"{v}\"" for k, v in args.get("env", {}).items()]
+            else:
+                call_args = [ f"\"{a}\"" for a in args]
+                env = []
+
+        cmd = f"./{dep_id}.sh"
+        if len(call_args) > 0:
+            cmd = f"{cmd} {' '.join(call_args)}"
+        if len(env) > 0:
+            cmd = f"{' '.join(env)} {cmd}"
+
+        return [ cmd ]
+
+    def build(self):
+        """Build setup scripts for each setup configuration defined in config"""
+
+        raw_config: dict = {}
+        with open(os.path.dirname(__file__) + "/config.yaml", 'r', encoding="utf-8") as file:
+            raw_config = yaml.safe_load(file)
+        config = resolve_conf(copy.deepcopy(raw_config))
+
+        setup = config.get("setup")
+        for (setup_id, setup_config) in setup.items():
+            # generate install instructions and install dependencies commands
+            instructions = []
+            downloads = []
+            for (dep_id, args) in setup_config.get("deps").items():
+                if dep_id != "cmd":
+                    downloads.append(f"wget $WF_DEPS_URL/{dep_id}.sh")
+                instructions.extend(self.__instructions(dep_id, args))
+
+            setup_script = ""
+            with open(
+                os.path.dirname(__file__) + "/setup-template.sh",
+                'r',
+                encoding="utf-8"
+            ) as file:
+                setup_script = file.read()
+            setup_script = setup_script.replace("%ENVIRONMENT_LABEL%", setup_config.get("label"))
+            setup_script = setup_script.replace("%DEPS_DOWNLOAD%", '\n    '.join(downloads))
+            setup_script = setup_script.replace("%DEPS_INSTALL%", '\n'.join(instructions))
+
+            setup_filename = f"setup-{setup_id}.sh"
+            setup_filepath = os.path.join(os.path.dirname(__file__),
+                                          "shared", "scripts", setup_filename)
+
+            with open(setup_filepath, "w+", encoding="utf-8") as f:
+                f.write(setup_script)
+
+SetupBuilder().build()