split images

.github/workflows/main.yml

@@ -8,8 +8,12 @@ on:
     branches:
       - main
 
+
+# https://github.com/docker/build-push-action/issues/461
+# https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311
+
 jobs:
-  build:
+  build_old:
     runs-on: ubuntu-latest
     timeout-minutes:  30
     strategy:
@@ -36,3 +40,69 @@ jobs:
           push: ${{ github.event_name == 'push' && github.repository == 'DIRACGrid/container-images' && github.ref_name == 'main' }}
           tags: ghcr.io/diracgrid/diracx/${{ matrix.image-name }}:latest
           platforms: linux/amd64,linux/arm64
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes:  30
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build base
+        id: build_base
+        uses: docker/build-push-action@v5
+        with:
+          context: base
+          outputs: type=oci,dest=output-base.tar
+          tags: ghcr.io/diracgrid/diracx/base:latest
+          platforms: linux/amd64,linux/arm64
+      - name: Extract base
+        run: |
+          mkdir output-base
+          tar -C output-base -xf output-base.tar
+
+      - name: Build services-base
+        uses: docker/build-push-action@v5
+        with:
+          context: services-base
+          build-contexts: |
+            ghcr.io/diracgrid/diracx/base=oci-layout://output-base@${{steps.build_base.outputs.digest}}
+          outputs: type=oci,dest=output-services-base.tar
+          tags: ghcr.io/diracgrid/diracx/services-base:latest
+          platforms: linux/amd64,linux/arm64
+      - name: Extract services-base
+        run: |
+          mkdir output-services-base
+          tar -C output-services-base -xf output-services-base.tar
+
+      - name: Build client-base
+        uses: docker/build-push-action@v5
+        with:
+          context: client-base
+          build-contexts: |
+            ghcr.io/diracgrid/diracx/base=oci-layout://output-base@${{steps.build_base.outputs.digest}}
+          outputs: type=oci,dest=output-client-base.tar
+          tags: ghcr.io/diracgrid/diracx/client-base:latest
+          platforms: linux/amd64,linux/arm64
+      - name: Extract client-base
+        run: |
+          mkdir output-client-base
+          tar -C output-client-base -xf output-client-base.tar
+
+      - name: Push images
+        if: ${{ github.event_name != 'pull_grequest' && github.repository == 'DIRACGrid/container-images' && github.ref_name == 'main' }}
+        run: |
+          for image_name in base services-base client-base; do
+            oras cp --from-oci-layout "$PWD/output-${image_name}:latest" "ghcr.io/diracgrid/diracx/${image_name}:latest"
+          done

base/Dockerfile

@@ -0,0 +1,22 @@
+FROM mambaorg/micromamba:latest
+
+# Copying in ENTRYPOINT script and environment specification
+COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml dirac_dependencies.yml /tmp/
+COPY --chown=$MAMBA_USER:$MAMBA_USER entrypoint.sh /
+RUN chmod 755 /entrypoint.sh
+
+RUN micromamba install --freeze-installed --yes --file /tmp/environment.yml  --name=base && \
+    micromamba install --freeze-installed --yes --file /tmp/dirac_dependencies.yml --name=base && \
+    micromamba clean --all --yes --force-pkgs-dirs && \
+    rm -rf /tmp/environment.yml /tmp/dirac_dependencies.yml
+
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
+
+# In many clusters the container is ran as a random uid for security reasons.
+# If we mark the conda directory as group 0 and give it group write permissions
+# then we're still able to manage the environment from inside the container.
+USER 0
+RUN chown -R $MAMBA_USER:0 /opt/conda && chmod -R g=u /opt/conda
+USER $MAMBA_USER
+
+ENTRYPOINT [ "/entrypoint.sh" ]

base/dirac_dependencies.yml

@@ -0,0 +1,29 @@
+# This yaml file contains the list of DIRAC dependencies that must be satisfied
+# even if we won't need it. For example, gfal2 needs to be installed in order to
+# be able to install DIRAC, even in the services.
+# These dependencies are separated from the main environment.yml in the hope that
+# we can one day remove entirely this file
+
+name: diracx
+channels:
+  - diracgrid
+  - conda-forge
+  - nodefaults
+dependencies:
+  - aiobotocore
+  - boto3
+  - botocore
+  - db12
+  - diraccfg
+  - dominate
+  - fts3
+  - importlib-metadata
+  - m2crypto >=0.38.0
+  - pexpect
+  - prompt-toolkit
+  - psutil
+  - pyasn1-modules
+  - pyparsing
+  - python-gfal2
+  - pytz
+  - rucio-clients

base/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+eval "$(micromamba shell hook --shell=posix)"
+micromamba activate base
+exec "$@"

base/environment.yml

@@ -0,0 +1,19 @@
+name: diracx
+channels:
+  - diracgrid
+  - conda-forge
+  - nodefaults
+dependencies:
+  - cachetools
+  # Needed because coverage runs inside the image
+  # when doing the integration tests
+  - coverage
+  - git
+  - gitpython
+  - httpx
+  - pip
+  - pydantic =1.10.10
+  - python =3.11
+  - pyyaml
+  - requests
+  - urllib3 <2

client-base/Dockerfile

@@ -0,0 +1,18 @@
+FROM ghcr.io/diracgrid/diracx/base
+
+
+# Copying in ENTRYPOINT script and environment specification
+COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/
+
+RUN micromamba install --freeze-installed --yes --file /tmp/environment.yml  --name=base && \
+    micromamba clean --all --yes --force-pkgs-dirs && \
+    rm -rf /tmp/environment.yml 
+
+
+# In many clusters the container is ran as a random uid for security reasons.
+# If we mark the conda directory as group 0 and give it group write permissions
+# then we're still able to manage the environment from inside the container.
+USER 0
+RUN chown -R $MAMBA_USER:0 /opt/conda && chmod -R g=u /opt/conda
+USER $MAMBA_USER
+

client-base/environment.yml

@@ -0,0 +1,13 @@
+name: diracx
+channels:
+  - diracgrid
+  - conda-forge
+  - nodefaults
+dependencies:
+  - aiohttp
+  - azure-core
+  - cachetools
+  - python-dotenv
+  - python-multipart
+  - rich
+  - typer

services-base/Dockerfile

@@ -0,0 +1,18 @@
+FROM ghcr.io/diracgrid/diracx/base
+
+
+# Copying in ENTRYPOINT script and environment specification
+COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/
+
+RUN micromamba install --freeze-installed --yes --file /tmp/environment.yml  --name=base && \
+    micromamba clean --all --yes --force-pkgs-dirs && \
+    rm -rf /tmp/environment.yml 
+
+
+# In many clusters the container is ran as a random uid for security reasons.
+# If we mark the conda directory as group 0 and give it group write permissions
+# then we're still able to manage the environment from inside the container.
+USER 0
+RUN chown -R $MAMBA_USER:0 /opt/conda && chmod -R g=u /opt/conda
+USER $MAMBA_USER
+

services-base/environment.yml

@@ -0,0 +1,22 @@
+name: diracx
+channels:
+  - diracgrid
+  - conda-forge
+  - nodefaults
+dependencies:
+  - authlib
+  - aiomysql
+  - aiosqlite
+  - email-validator
+  - fastapi
+  - isodate
+  - opensearch-py
+  - pyjwt
+  - coverage
+  - python-dotenv
+  - python-jose
+  - python-multipart
+  - sqlalchemy
+  - uvicorn
+  - aiobotocore
+  - botocore