Build spacktainers on codebuild

.github/actions/build_container/action.yaml

@@ -0,0 +1,70 @@
+---
+name: Build Base Container
+description: Build runtime / builder depending on variables
+inputs:
+  AWS_ECR_URL:
+    description: Base URL for AWS ECR where our containers live
+    required: true
+  AWS_ECR_PATH:
+    description: path under the AWS ECR where this specific container lives
+    required: true
+  AWS_ACCESS_KEY_ID:
+    description: Access key ID for AWS
+    required: true
+  AWS_SECRET_ACCESS_KEY:
+    description: Secret access key for AWS
+    required: true
+  BUILD_PATH:
+    description: Under which directory in this repo the Dockerfile lives
+    required: true
+  BUILDAH_EXTRA_ARGS:
+    description: Extra args to pass to buildah
+    required: false
+    default: ''
+  DOCKERHUB_USER:
+    description: Username for Dockerhub authentication
+    required: true
+  DOCKERHUB_PASSWORD:
+    description: Password for Dockerhub authentication
+    required: true
+  GHCR_USER:
+    description: Username for GHCR authentication
+    required: true
+  GHCR_TOKEN:
+    description: Token for GHCR authentication
+    required: true
+  SPACK_DEPLOYMENT_KEY_PUB:
+    description: Public key for spack deployments
+    required: true
+  SPACK_DEPLOYMENT_KEY_PRIVATE:
+    description: Private key for spack deployments
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: create builder
+      shell: bash
+      run: |-
+        echo "Building container in ${{ inputs.BUILD_PATH }}"
+        set -x
+        apt-get update
+        apt-get install -y awscli buildah podman
+        export STORAGE_DRIVER=vfs  # allows to build inside containers without additional mounts
+        export BUILDAH_FORMAT=docker  # enables ONBUILD instructions which are not OCI compatible
+        export REGISTRY_IMAGE_TAG=latest  # for now
+        echo "${{ inputs.SPACK_DEPLOYMENT_KEY_PUB }}" > ${{ inputs.BUILD_PATH }}/key.pub
+        echo "${{ inputs.SPACK_DEPLOYMENT_KEY_PRIVATE }}" > ${{ inputs.BUILD_PATH }}/key
+        aws ecr get-login-password --region us-east-1 | buildah login --username AWS --password-stdin ${{ inputs.AWS_ECR_URL }}
+        buildah login --username ${{ inputs.DOCKERHUB_USER }} --password ${{ inputs.DOCKERHUB_PASSWORD }} docker.io
+        buildah login --username ${{ inputs.GHCR_USER }} --password ${{ inputs.GHCR_TOKEN }} ghcr.io
+        # This is written like that in case $BUILDAH_EXTRA_ARGS has args that require spaces,
+        # which is tricky with shell variable expansion.  Similar to Kaniko, see also:
+        # https://github.com/GoogleContainerTools/kaniko/issues/1803
+        export IFS=''
+        COMMAND="buildah bud --iidfile image_id ${{ inputs.BUILDAH_EXTRA_ARGS }} ${{ inputs.BUILD_PATH }}"
+        echo "${COMMAND}"
+        eval "${COMMAND}"
+        # Sometimes buildah push fails on the first attempt
+        buildah push $(<image_id) "docker://${{ inputs.AWS_ECR_URL }}${{ inputs.AWS_ECR_PATH }}:${REGISTRY_IMAGE_TAG}" || sleep 10; buildah push $(<image_id) "docker://${{ inputs.AWS_ECR_URL }}${{ inputs.AWS_ECR_PATH }}:${REGISTRY_IMAGE_TAG}"
+        # Also push to ghcr
+        buildah push $(<image_id) "docker://ghcr.io${{ inputs.GHCR_PATH }}:${REGISTRY_IMAGE_TAG}" || sleep 10; buildah push $(<image_id) "docker://ghcr.io${{ inputs.GHCR_PATH }}:${REGISTRY_IMAGE_TAG}"

.github/workflows/spacktainer.yaml

@@ -0,0 +1,121 @@
+---
+name: Build Spacktainers
+on: [push]
+jobs:
+  builder-container-job:
+    runs-on:
+      - codebuild-spacktainers-tf-${{ github.run_id }}-${{ github.run_attempt }}
+      - instance-size:small
+    steps:
+      - name: clone repo
+        uses: actions/checkout@v4
+      - name: create builder
+        uses: ./.github/actions/build_container
+        with:
+          AWS_ECR_URL: ${{ secrets.AWS_ECR_URL }}
+          AWS_ECR_PATH: /spacktainers/builder
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ECR_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }}
+          GHCR_USER: ${{ secrets.GHCR_USER }}
+          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+          GHCR_PATH: /bluebrain/spack-builder
+          BUILD_PATH: builder
+          BUILDAH_EXTRA_ARGS: --label org.opencontainers.image.revision="$GITHUB_SHA"
+            --label org.opencontainers.image.authors="$GITHUB_TRIGGERING_ACTOR" --label
+            org.opencontainers.image.url="https://github.com/${GITHUB_REPOSITORY}"
+            --label org.opencontainers.image.source="https://github.com/${GITHUB_REPOSITORY}"
+            --label ch.epfl.bbpgitlab.ci-pipeline-url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+            --label ch.epfl.bbpgitlab.ci-commit-branch="$GITHUB_REF_NAME" --build-arg
+            SPACK_BRANCH=develop
+         # ' --label org.opencontainers.image.created="$CI_JOB_STARTED_AT"'
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          SPACK_DEPLOYMENT_KEY_PUB: ${{ secrets.SPACK_DEPLOYMENT_KEY_PUB }}
+          SPACK_DEPLOYMENT_KEY_PRIVATE: ${{ secrets.SPACK_DEPLOYMENT_KEY_PRIVATE }}
+  runtime-container-job:
+    runs-on:
+      - codebuild-spacktainers-tf-${{ github.run_id }}-${{ github.run_attempt }}
+      - instance-size:small
+    steps:
+      - name: clone repo
+        uses: actions/checkout@v4
+      - name: create runtime
+        uses: ./.github/actions/build_container
+        with:
+          AWS_ECR_URL: ${{ secrets.AWS_ECR_URL }}
+          AWS_ECR_PATH: /spacktainers/runtime
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ECR_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }}
+          GHCR_USER: ${{ secrets.GHCR_USER }}
+          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+          GHCR_PATH: /bluebrain/spack-runtime
+          BUILD_PATH: runtime
+          BUILDAH_EXTRA_ARGS: --label org.opencontainers.image.revision="$GITHUB_SHA"
+            --label org.opencontainers.image.authors="$GITHUB_TRIGGERING_ACTOR" --label
+            org.opencontainers.image.url="https://github.com/${GITHUB_REPOSITORY}"
+            --label org.opencontainers.image.source="https://github.com/${GITHUB_REPOSITORY}"
+            --label ch.epfl.bbpgitlab.ci-pipeline-url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+            --label ch.epfl.bbpgitlab.ci-commit-branch="$GITHUB_REF_NAME" --build-arg
+            SPACK_BRANCH=develop
+         # ' --label org.opencontainers.image.created="$CI_JOB_STARTED_AT"'
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          SPACK_DEPLOYMENT_KEY_PUB: ${{ secrets.SPACK_DEPLOYMENT_KEY_PUB }}
+          SPACK_DEPLOYMENT_KEY_PRIVATE: ${{ secrets.SPACK_DEPLOYMENT_KEY_PRIVATE }}
+  spacktainer-build-job:
+    strategy:
+      matrix:
+        spacktainer:
+          - appositionizer
+          - brain-indexer
+          - brayns
+          - connectome-manipulator
+          - functionalizer
+          - multiscale-run
+          - neurodamus-hippocampus
+          - neurodamus-neocortex
+          - system-benchmarks
+    runs-on:
+      - codebuild-spacktainers-tf-${{ github.run_id }}-${{ github.run_attempt }}
+    continue-on-error: true
+    needs: [builder-container-job, runtime-container-job]
+    steps:
+      - name: clone repo
+        uses: actions/checkout@v4
+      - name: prepare to build container
+        env:
+          AWS_ECR_URL: ${{ secrets.AWS_ECR_URL }}
+        run: |-
+          cd container_definitions/amd64/${{ matrix.spacktainer }}
+          cat << EOF > Dockerfile
+          FROM ${AWS_ECR_URL}/spacktainers/builder:latest AS builder
+          FROM ${AWS_ECR_URL}/spacktainers/runtime:latest
+
+          # Triggers building the 'builder' image, otherwise it is optimized away
+          COPY --from=builder /etc/debian_version /etc/debian_version
+          EOF
+      - name: build ${{ matrix.spacktainer }}
+        uses: ./.github/actions/build_container
+        with:
+          AWS_ECR_URL: ${{ secrets.AWS_ECR_URL }}
+          AWS_ECR_PATH: /spacktainers/${{ matrix.spacktainer }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ECR_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }}
+          GHCR_USER: ${{ secrets.GHCR_USER }}
+          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+          GHCR_PATH: /bluebrain/spack-${{ matrix.spacktainer }}
+          BUILD_PATH: container_definitions/amd64/${{ matrix.spacktainer }}
+          BUILDAH_EXTRA_ARGS: --label org.opencontainers.image.revision="$GITHUB_SHA"
+            --label org.opencontainers.image.authors="$GITHUB_TRIGGERING_ACTOR" --label
+            org.opencontainers.image.url="https://github.com/${GITHUB_REPOSITORY}"
+            --label org.opencontainers.image.source="https://github.com/${GITHUB_REPOSITORY}"
+            --label ch.epfl.bbpgitlab.ci-pipeline-url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+            --label ch.epfl.bbpgitlab.ci-commit-branch="$GITHUB_REF_NAME" --build-arg
+            SPACK_BRANCH=develop --build-arg CACHE_BUCKET=${{ secrets.AWS_CACHE_BUCKET }}
+            --build-arg MIRROR_AUTH_ARG="\"--s3-access-key-id='${{ secrets.AWS_CACHE_ACCESS_KEY_ID }}
+            --s3-access-key-secret=${{ secrets.AWS_CACHE_SECRET_ACCESS_KEY }}'\""
+          # ' --label org.opencontainers.image.created="$CI_JOB_STARTED_AT"'
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          SPACK_DEPLOYMENT_KEY_PUB: ${{ secrets.SPACK_DEPLOYMENT_KEY_PUB }}
+          SPACK_DEPLOYMENT_KEY_PRIVATE: ${{ secrets.SPACK_DEPLOYMENT_KEY_PRIVATE }}

README.md

@@ -9,7 +9,7 @@ This repository aims to be the one-stop shop for all of our container needs.
 
 ## Defining containers
 
-The only files you should have to edit as an end-user are located in the `container_definitions` folder. There's a subfolder per architecture (currently supported: `amd64` and `arm64`) under which both `yaml` and `def` files can live.
+The only files you should have to edit as an end-user are located in the `container_definitions` folder. There's a subfolder per architecture (currently supported: `amd64` and `arm64`) under which both `yaml` (in subdirectories) and `def` files can live.
 * A YAML file file defines a Spack container - in it you can define the Spack specs as you would in a Spack environment. If you have specific requirements for dependencies, you can add `spack: packages: ...` keys to define those, again, as in a Spack environment.
 * A def file defines a singularity container that will be built from an existing container on docker-hub. nexus-storage is already defined for amd64 as an example.
 
@@ -45,38 +45,15 @@ spacktainer:
 
 # Developer documentation
 
-## Components
-
-* Spacktainerizer: the base image which contains our spack fork
-* Singularitah: arm64 container with singularity and s3cmd installation for sif manipulation on arm nodes
-* Spack-cacher: builds spack packages and puts them in a build cache
-* Spackitor: cleans the build cache: anything that is too old or no longer used gets removed
-* Spackah: builds the actual containers
-
 ## Build Order
 
 1. base containers
    * Build runtime / builder
-   * Build singularitah
-2. packages
-   * Build cache
-3. containers
+2. application containers
    * Build containers
+     * Every package build will be pushed to the cash directly after build
    * Publish containers
 
-## Pipeline logic
-
-While the pipeline is organised in stages, jobs jump the queue wherever they can to optimise build times. As such, we'll ignore the stages here and look at the actual execution order:
-* `generate base pipeline`: the "entrypoint" that will generate the necessary jobs to:
-    * build the builder, runtime and singularitah containers if necessary. These containers will be built only for the architectures needed for the final containers. These jobs will be generated only for the containers that need to be built.
-    * run `spack ci generate` and process its output. This is needed because Gitlab imposes a fairly tight restriction on how large a YAML file can be and Spack can easily surpass that. To work around this, we take the output YAML and split it into multiple pipelines along the generated stages.
-    * Clean the build cache buckets
-* `base containers and pipeline generation`: will run the pipeline that was generated in the first step
-* `gather child artifacts`: will collect the yaml generated in the `base containers and pipeline generation` child pipeline. This is needed because Gitlab doesn't allow triggering artifacts from a child pipeline
-* `populate buildcache for amd64`: run the jobs that `spack ci generate` produced in order to populate the buildcache
-* `build spacktainers for amd64`: this workflow was also generated in the `base containers and pipeline generation` child pipeline and will build the actual containers, if necessary.
-
-
 ## CI/CD Variables
 
 * `AWS_CACHE_ACCESS_KEY_ID` / `AWS_CACHE_SECRET_ACCESS_KEY`: AWS keypair for accessing the cache bucket hosted by Amazon
@@ -87,12 +64,6 @@ While the pipeline is organised in stages, jobs jump the queue wherever they can
 * `DOCKERHUB_USER` / `DOCKERHUB_PASSWORD`: credentials for docker hub
 * `GITLAB_API_TOKEN`: private (!) gitlab token with API_READ access (CI_JOB_TOKEN does not have enough permissions). Change this once I'm gone
 
-## Base containers
-
-* [Singularitah](bbpgitlab.epfl.ch:5050/hpc/spacktainers/singularitah)
-* [Builder](bbpgitlab.epfl.ch:5050/hpc/spacktainers/builder)
-* [Runner](bbpgitlab.epfl.ch:5050/hpc/spacktainers/runtime)
-
 ## Repository layout
 
 There are a few python projects in this repository:
@@ -129,6 +100,114 @@ The main entrypoints can be found, unsurprisingly, in the `__main__.py` file. Th
 
 `utils.py` contains utility functions for reading/writing yaml, getting the multiarch job for a container, ...
 
+## Pulling images with Sarus or Podman
+
+Make sure you have your AWS credentials set up.  Then identify the image you want to run.
+In the following, `spacktainers/neurodamus-neocortex` is going to be used.  Identify the
+URL of the registry:
+```
+❯ aws ecr describe-repositories --repository-names spacktainers/neurodamus-neocortex
+{
+    "repositories": [
+        {
+            "repositoryArn": "arn:aws:ecr:us-east-1:130659266700:repository/spacktainers/neurodamus-neocortex",
+            "registryId": "130659266700",
+            "repositoryName": "spacktainers/neurodamus-neocortex",
+            "repositoryUri": "130659266700.dkr.ecr.us-east-1.amazonaws.com/spacktainers/neurodamus-neocortex",
+            "createdAt": "2024-11-20T17:32:11.169000+01:00",
+            "imageTagMutability": "MUTABLE",
+            "imageScanningConfiguration": {
+                "scanOnPush": false
+            },
+            "encryptionConfiguration": {
+                "encryptionType": "AES256"
+            }
+        }
+    ]
+}
+
+```
+Note the `repositoryUri` key. This will be used to log in with either Podman or Sarus.
+
+Get a login token from AWS:
+```
+❯ aws ecr get-login-password
+[secret]
+```
+
+### Pulling with Podman
+
+Log into the registry, using `AWS` as the username:
+```
+❯ aws ecr get-login-password|podman login -u AWS --password-stdin 130659266700.dkr.ecr.us-east-1.amazonaws.com
+```
+Then pull the full `repositoryUri`:
+```
+❯ podman pull 130659266700.dkr.ecr.us-east-1.amazonaws.com/spacktainers/neurodamus-neocortex
+```
+
+### Pulling with Sarus
+
+Everything in Sarus goes into one command:
+```
+❯ sarus pull --login -u AWS 130659266700.dkr.ecr.us-east-1.amazonaws.com/spacktainers/neurodamus-neocortex
+```
+
+## Reproducing GitHub Action builds containerized
+
+See above instructions under [pulling containers](#user-content-pulling-with-podman) to
+login and pull the `spacktainers/builder` container.
+Then launch the container and install something, i.e., with:
+```
+❯ podman run -it 130659266700.dkr.ecr.us-east-1.amazonaws.com/spacktainers/builder
+root@43dec0527c62:/# (cd /opt/spack-repos/ && git pull)
+Already up to date.
+root@43dec0527c62:/# spack install zlib
+[...]
+```
+Environments may be recreated as present under
+[`container_definitions/`][(./container_definitions).
+
+## Reproducing GitHub Action builds locally
+
+Prerequisites needed to try the container building locally:
+
+1. The upstream Spack commit we are using in the
+   [`builder/Dockerfile`](builder/Dockerfile), in the argument `SPACK_BRANCH` (may be
+   overwritten by the CI).  Referred to as `${SPACK_BRANCH}` here.
+2. Access to the S3 bucket that holds the binary cache, denoted by the `CACHE_BUCKET`
+   argument in the same file.  Referred to as `${CACHE_BUCKET}` here.
+
+Set up upstream Spack, and source it:
+```
+❯ gh repo clone spack/spack
+❯ cd spack
+❯ git fetch --depth=1 origin ${SPACK_BRANCH}
+❯ git reset --hard FETCH_HEAD
+❯ . ./share/spack/setup-env.sh
+❯ cd ..
+```
+Then clone our own Spack fork and add the repositories:
+```
+❯ gh repo clone BlueBrain/spack spack-blue
+❯ spack repo add --scope=site spack-blue/bluebrain/repo-patches
+❯ spack repo add --scope=site spack-blue/bluebrain/repo-bluebrain
+```
+Configure the mirror and set the generic architecture:
+```
+❯ spack mirror add --scope=site build_s3 ${CACHE_BUCKET}
+❯ spack config --scope=site add packages:all:require:target=x86_64_v3
+```
+Now the basic Spack installation should be ready to use and pull from the build cache.
+
+Then one may pick a container specification and create environments from it, i.e.:
+```
+❯ spack env create brindex spacktainers/container_definitions/amd64/py-brain-indexer/spack.yaml
+❯ spack env activate brindex
+❯ spack concretize -f
+❯ spack install
+```
+
 # Acknowledgment
 
 The development of this software was supported by funding to the Blue Brain Project,