Merge pull request #143 from FNNDSC/pixi

Switch to pixi package manager

.dockerignore

@@ -10,3 +10,7 @@ Dockerfile
 LICENSE
 CHRIS_REMOTE_FS
 swift_storage
+
+dist/
+.pixi/
+

.gitattributes

@@ -0,0 +1,2 @@
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML linguist-generated=true

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ on:
 jobs:
   test-pfcon:
     name: tests (pfcon)
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - name: Start Docker Swarm
@@ -28,7 +28,7 @@ jobs:
           docker swarm leave --force
   test-cube:
     name: tests (CUBE)
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - run: docker build -t localhost/fnndsc/pfcon .
@@ -38,26 +38,40 @@ jobs:
 
   build:
     needs: [test-pfcon, test-cube]
-    if: github.event_name == 'push' || github.event_name == 'release'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
+    - name: Install pixi
+      id: install-pixi
+      if: startsWith(github.ref, 'refs/tags/v')
+      uses: prefix-dev/[email protected]
       with:
-        fetch-depth: "0"
+        pixi-version: v0.28.2
+    - name: Set version
+      id: set-version
+      if: steps.install-pixi.outcome == 'success'
+      run:
+        ref_name='${{ github.ref_name }}'
+        version_number="${ref_name:1}"
+        pixi project version set "$version_number"
+        echo "LABEL org.opencontainers.image.version=\"$version_number\"" >> Dockerfile
+        echo "version=$version_number" >> "$GITHUB_OUTPUT"
     - name: Get build tags
       id: info
       shell: python
       run: |
         import os
         import itertools
 
-        short_sha = os.getenv('GITHUB_SHA', 'unknown')[:7]
+        commit_sha = '${{ github.sha }}'
+        short_sha = commit_sha[:7]
         git_refs = []
-        if os.getenv('GITHUB_REF', '').startswith('refs/tags/v'):
-            version_from_tag = os.getenv('GITHUB_REF')[11:]
-            git_refs.append(version_from_tag.replace('+', '.'))
+        version_number = '${{ steps.set-version.outputs.version }}'
+        if version_number:
+            sanitized_version_number = version_number.replace('+', '.')
+            git_refs.append(sanitized_version_number)
         registries = ['docker.io', 'ghcr.io']
-        repo = os.environ['GITHUB_REPOSITORY'].lower()
+        repo = '${{ github.repository }}'.lower()
         tags = ['latest'] + git_refs
         names = ','.join(''.join(c) for c in itertools.product(
             (r + '/' for r in registries),
@@ -69,27 +83,29 @@ jobs:
     - uses: docker/setup-qemu-action@v3
     - uses: docker/setup-buildx-action@v3
     - name: Login to DockerHub
+      id: login-dockerhub
+      if: github.event_name == 'push'
       uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_PASSWORD }}
     - name: Login to GitHub Container Registry
+      id: login-ghcr
+      if: github.event_name == 'push'
       uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Build and push
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
-        build-args: |
-          ENVIRONMENT=production
-          BUILD_VERSION=${{ github.ref_name }}
-        push: true
+        build-args: ENVIRONMENT=prod
+        push: ${{ steps.login-dockerhub.outcome == 'success' && steps.login-ghcr.outcome == 'success' }}
         context: .
         file: ./Dockerfile
         tags: "${{ steps.info.outputs.tags }}"
-        platforms: linux/amd64,linux/ppc64le,linux/arm64
+        platforms: linux/amd64,linux/arm64
         cache-from: type=gha
         cache-to: type=gha,mode=max

.gitignore

@@ -17,3 +17,7 @@
 dc.out
 CHRIS_REMOTE_FS/
 swift_storage/
+
+# pixi environments
+.pixi
+*.egg-info

Dockerfile

@@ -1,55 +1,41 @@
-#
-# Docker file for pfcon image
-#
-# Build production image:
-#
-#   docker build -t <name> .
-#
-# For example if building a local production image:
-#
-#   docker build -t local/pfcon .
-#
-# Build development image:
-#
-#   docker build --build-arg ENVIRONMENT=local -t <name>:<tag> .
-#
-# For example if building a local development image:
-#
-#   docker build --build-arg ENVIRONMENT=local -t local/pfcon:dev .
-#
-# In the case of a proxy (located at say proxy.tch.harvard.edu:3128), do:
-#
-#    export PROXY="http://proxy.tch.harvard.edu:3128"
-#
-# then add to any of the previous build commands:
-#
-#    --build-arg http_proxy=${PROXY}
-#
-# For example if building a local development image:
-#
-# docker build --build-arg http_proxy=${PROXY} --build-arg ENVIRONMENT=local -t local/pfcon:dev .
-#
-
-FROM python:3.10.5-bullseye
+FROM ghcr.io/prefix-dev/pixi:0.27.1 AS install
 
+COPY . /app
 WORKDIR /app
 
-COPY ./requirements ./requirements
-ARG ENVIRONMENT=production
-RUN pip install --no-cache-dir -r /app/requirements/$ENVIRONMENT.txt
+RUN --mount=type=cache,target=/root/.cache/rattler/cache,sharing=locked pixi install
 
-COPY . .
-ARG BUILD_VERSION=unknown
-RUN if [ "$ENVIRONMENT" = "local" ]; then pip install -e .; else pip install .; fi
+# development stage
+FROM install AS dev
 
+CMD ["pixi", "run", "python", "-m", "pfcon"]
 EXPOSE 5005
+
+# production build stage
+FROM install AS build
+
+RUN pixi run build
+
+ARG ENVIRONMENT=prod
+RUN printf '#!/bin/sh\n%s\nexec "$@"' "$(pixi shell-hook -e ${ENVIRONMENT})" > /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+# must be the last command of this stage, or else pixi will overwrite the installed package.
+RUN pixi run postinstall-production
+
+# production minimal image
+FROM docker.io/library/debian:bookworm-slim
+
+COPY --from=build /app/.pixi/envs/${ENVIRONMENT} /app/.pixi/envs/${ENVIRONMENT}
+COPY --from=build /entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT [ "/entrypoint.sh" ]
 CMD ["gunicorn", "--bind", "0.0.0.0:5005", "--workers", "8", "--timeout", "3600", "pfcon.wsgi:application"]
 
+EXPOSE 5005
+
 LABEL org.opencontainers.image.authors="FNNDSC <[email protected]>" \
       org.opencontainers.image.title="pfcon" \
       org.opencontainers.image.description="ChRIS compute resource controller" \
       org.opencontainers.image.url="https://chrisproject.org/" \
       org.opencontainers.image.source="https://github.com/FNNDSC/pfcon" \
-      org.opencontainers.image.version=$BUILD_VERSION \
-      org.opencontainers.image.revision=$BUILD_VERSION \
       org.opencontainers.image.licenses="MIT"

make.sh

@@ -8,7 +8,7 @@
 #
 #   make.sh                     [-h] [-i] [-s] [-N] [-U]   \
 #                               [-O <swarm|kubernetes>]    \
-#                               [-F <swift|filesystem|fslink|zipfile>]    \
+#                               [-F <swift|filesystem|zipfile>]    \
 #                               [-S <storeBase>]           \
 #                               [local|fnndsc[:dev]]
 #
@@ -32,11 +32,6 @@
 #
 #       unmake.sh -N -F filesystem; sudo rm -fr CHRIS_REMOTE_FS; rm -fr CHRIS_REMOTE_FS; make.sh -N -F filesystem
 #
-#   Run full pfcon instantiation operating in-network on Swarm using mounted filesystem
-#   with ChRIS links storage:
-#
-#       unmake.sh -N -F fslink; sudo rm -fr CHRIS_REMOTE_FS; rm -fr CHRIS_REMOTE_FS; make.sh -N -F fslink
-#
 #   Skip the intro:
 #
 #       unmake.sh ; sudo rm -fr CHRIS_REMOTE_FS; rm -fr CHRIS_REMOTE_FS; make.sh -s
@@ -71,9 +66,9 @@
 #       Optional set pfcon to operate in-network mode (using a swift storage instead of
 #       a zip file).
 #
-#   -F <swift|filesystem|fslink|zipfile>
+#   -F <swift|filesystem|zipfile>
 #
-#       Explicitly set the storage environment. This option must be swift, fslink or filesystem
+#       Explicitly set the storage environment. This option must be swift or filesystem
 #       for pfcon operating in-network mode. For pfcon operating in out-of-network mode
 #       it must be set to zipfile (default).
 #
@@ -106,11 +101,11 @@ source ./cparse.sh
 
 declare -i STEP=0
 ORCHESTRATOR=swarm
-STORAGE_ENV=zipfile
+STORAGE=zipfile
 HERE=$(pwd)
 
 print_usage () {
-    echo "Usage: ./make.sh [-h] [-i] [-s] [-N] [-F <swift|filesystem|fslink|zipfile>] [-U] [-O <swarm|kubernetes>] [-S <storeBase>] [local|fnndsc[:dev]]"
+    echo "Usage: ./make.sh [-h] [-i] [-s] [-N] [-F <swift|filesystem|zipfile>] [-U] [-O <swarm|kubernetes>] [-S <storeBase>] [local|fnndsc[:dev]]"
     exit 1
 }
 
@@ -124,8 +119,8 @@ while getopts ":hsiNUF:O:S:" opt; do
           ;;
         N) b_pfconInNetwork=1
           ;;
-        F) STORAGE_ENV=$OPTARG
-           if ! [[ "$STORAGE_ENV" =~ ^(swift|filesystem|fslink|zipfile)$ ]]; then
+        F) STORAGE=$OPTARG
+           if ! [[ "$STORAGE" =~ ^(swift|filesystem|zipfile)$ ]]; then
               echo "Invalid value for option -- F"
               print_usage
            fi
@@ -180,16 +175,15 @@ title -d 1 "Setting global exports..."
     fi
     if (( b_pfconInNetwork )) ; then
         echo -e "PFCON_INNETWORK=True"                             | ./boxes.sh
-        if [[ $STORAGE_ENV == 'zipfile' ]]; then
-            echo -e "Need to pass '-F <swift|filesystem|fslink|>' when PFCON_INNETWORK=True"  | ./boxes.sh
+        if [[ $STORAGE == 'zipfile' ]]; then
+            echo -e "Need to pass '-F <swift|filesystem>' when PFCON_INNETWORK=True"  | ./boxes.sh
             exit 1
         fi
     else
         echo -e "PFCON_INNETWORK=False"                            | ./boxes.sh
     fi
     echo -e "ORCHESTRATOR=$ORCHESTRATOR"                           | ./boxes.sh
-    echo -e "exporting STORAGE_ENV=$STORAGE_ENV"                   | ./boxes.sh
-    export STORAGE_ENV=$STORAGE_ENV
+    echo -e "STORAGE=$STORAGE"                                     | ./boxes.sh
     echo -e "exporting STOREBASE=$STOREBASE "                      | ./boxes.sh
     export STOREBASE=$STOREBASE
     export SOURCEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
@@ -216,9 +210,9 @@ windowBottom
 title -d 1 "Building :dev"
     cd $HERE
     if (( b_pfconInNetwork )) ; then
-        if [[ $STORAGE_ENV == 'swift' ]]; then
+        if [[ $STORAGE == 'swift' ]]; then
             CMD="docker compose -f swarm/docker-compose_dev_innetwork.yml build"
-        else
+        elif [[ $STORAGE == 'filesystem' ]]; then
             CMD="docker compose -f swarm/docker-compose_dev_innetwork_fs.yml build"
         fi
     else
@@ -265,10 +259,10 @@ windowBottom
 title -d 1 "Starting pfcon containerized dev environment on $ORCHESTRATOR"
     if [[ $ORCHESTRATOR == swarm ]]; then
         if (( b_pfconInNetwork )) ; then
-            if [[ $STORAGE_ENV == 'swift' ]]; then
+            if [[ $STORAGE == 'swift' ]]; then
                 echo "docker stack deploy -c swarm/docker-compose_dev_innetwork.yml pfcon_dev_stack" | ./boxes.sh ${LightCyan}
                 docker stack deploy -c swarm/docker-compose_dev_innetwork.yml pfcon_dev_stack
-            else
+            elif [[ $STORAGE == 'filesystem' ]]; then
                 echo "docker stack deploy -c swarm/docker-compose_dev_innetwork_fs.yml pfcon_dev_stack" | ./boxes.sh ${LightCyan}
                 docker stack deploy -c swarm/docker-compose_dev_innetwork_fs.yml pfcon_dev_stack
             fi
@@ -312,21 +306,19 @@ if (( ! b_skipUnitTests )) ; then
     sleep 5
     if [[ $ORCHESTRATOR == swarm ]]; then
         if (( b_pfconInNetwork )) ; then
-            if [[ $STORAGE_ENV == 'swift' ]]; then
-                docker exec $pfcon_dev pytest tests/test_resources_innetwork.py --color=yes
-            elif [[ $STORAGE_ENV == 'filesystem' ]]; then
-                docker exec $pfcon_dev pytest tests/test_resources_innetwork_fs.py --color=yes
-            elif [[ $STORAGE_ENV == 'fslink' ]]; then
-                docker exec $pfcon_dev pytest tests/test_resources_innetwork_fslink.py --color=yes
+            if [[ $STORAGE == 'swift' ]]; then
+                docker exec $pfcon_dev pixi run pytest tests/test_resources_innetwork.py --color=yes
+            elif [[ $STORAGE == 'filesystem' ]]; then
+                docker exec $pfcon_dev pixi run pytest tests/test_resources_innetwork_fs.py --color=yes
             fi
         else
             docker exec $pfcon_dev pytest tests/test_resources.py --color=yes
         fi
     elif [[ $ORCHESTRATOR == kubernetes ]]; then
         if (( b_pfconInNetwork )) ; then
-            kubectl exec $pfcon_dev -- pytest tests/test_resources_innetwork.py --color=yes
+            kubectl exec $pfcon_dev -- pixi run pytest tests/test_resources_innetwork.py --color=yes
         else
-            kubectl exec $pfcon_dev -- pytest tests/test_resources.py --color=yes
+            kubectl exec $pfcon_dev -- pixi run pytest tests/test_resources.py --color=yes
         fi
     fi
     status=$?