Build and host algorithm images on GitHub (base images) #2
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build images and test algorithms | |
| on: | |
| pull_request: | |
| branches: main | |
| push: | |
| branches: main | |
| defaults: | |
| run: | |
| shell: 'bash -Eeuo pipefail -xl {0}' | |
| jobs: | |
| init: | |
| name: Generate Jobs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| base_matrix: ${{ steps.generate-jobs.outputs.base_matrix }} | |
| intermediate_matrix: ${{ steps.generate-jobs.outputs.intermediate_matrix }} | |
| algo_matrix: ${{ steps.generate-jobs.outputs.algo_matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: generate-jobs | |
| name: Generate Jobs for modified base images or algorithms | |
| run: | | |
| base_matrix="$(./.ci/generate-build-matrix.sh 0-base-images)" | |
| echo "base_matrix=$base_matrix" >> "$GITHUB_OUTPUT" | |
| jq . <<<"$base_matrix" # print generated json for debugging | |
| intermediate_matrix="$(./.ci/generate-build-matrix.sh 1-intermediate-images)" | |
| echo "intermediate_matrix=$intermediate_matrix" >> "$GITHUB_OUTPUT" | |
| jq . <<<"$intermediate_matrix" # print generated json for debugging | |
| algo_matrix="$(./.ci/generate-build-matrix.sh)" | |
| echo "algo_matrix=$algo_matrix" >> "$GITHUB_OUTPUT" | |
| jq . <<<"$algo_matrix" # print generated json for debugging | |
| build-base-images: | |
| name: Build base images | |
| runs-on: ubuntu-latest | |
| needs: init | |
| permissions: | |
| contents: read | |
| packages: write | |
| if: fromJson(needs.init.outputs.base_matrix).algorithm_name[0] != null | |
| strategy: | |
| max-parallel: 3 | |
| matrix: ${{ fromJson(needs.init.outputs.base_matrix) }} | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Compute (and check) version | |
| id: version | |
| run: | | |
| version="$(./.ci/get-image-version.sh 0-base-images/${{ matrix.algorithm_name }})" | |
| echo "version=$version" >> "$GITHUB_OUTPUT" | |
| - name: Compute Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ghcr.io/timeeval/${{ matrix.algorithm_name }} | |
| tags: | | |
| type=raw,value=latest | |
| type=raw,value=${{ steps.version.outputs.version }} | |
| labels: | | |
| org.opencontainers.image.title=TimeEval algorithm ${{ matrix.algorithm_name }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to registry | |
| # if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build algorithm image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: "./0-base-images/${{ matrix.algorithm_name }}" | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| pull: true | |
| load: true | |
| push: true | |
| # push: ${{ github.event_name != 'pull_request' }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| build-intermediate-images: | |
| name: Build intermediate images | |
| runs-on: ubuntu-latest | |
| needs: init | |
| permissions: | |
| contents: read | |
| packages: write | |
| if: fromJson(needs.init.outputs.intermediate_matrix).algorithm_name[0] != null | |
| strategy: | |
| max-parallel: 3 | |
| matrix: ${{ fromJson(needs.init.outputs.intermediate_matrix) }} | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Compute (and check) version | |
| id: version | |
| run: | | |
| version="$(./.ci/get-image-version.sh 1-intermediate-images/${{ matrix.algorithm_name }})" | |
| echo "version=$version" >> "$GITHUB_OUTPUT" | |
| - name: Compute Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ghcr.io/timeeval/${{ matrix.algorithm_name }} | |
| tags: | | |
| type=raw,value=latest | |
| type=raw,value=${{ steps.version.outputs.version }} | |
| labels: | | |
| org.opencontainers.image.title=TimeEval algorithm ${{ matrix.algorithm_name }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build algorithm image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: "./1-intermediate-images/${{ matrix.algorithm_name }}" | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| pull: true | |
| load: true | |
| push: ${{ github.event_name != 'pull_request' }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| test-algorithms: | |
| name: Build and test algorithm images | |
| runs-on: ubuntu-latest | |
| needs: | |
| - init | |
| permissions: | |
| contents: read | |
| packages: write | |
| if: fromJson(needs.init.outputs.algo_matrix).algorithm_name[0] != null | |
| strategy: | |
| max-parallel: 3 | |
| matrix: ${{ fromJson(needs.init.outputs.algo_matrix) }} | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Validate manifest | |
| run: python validate_manifest.py --path ${{ matrix.algorithm_name }}/manifest.json | |
| - name: Determine correct test dataset | |
| run: | | |
| dataset_name="$(python .ci/get_dataset_name.py ${{ matrix.algorithm_name }})" | |
| echo "${{ matrix.algorithm_name }}_dataset_name=$dataset_name" >> "$GITHUB_OUTPUT" | |
| - name: Compute (and check) version | |
| id: version | |
| run: | | |
| version="$(./.ci/get-image-version.sh ${{ matrix.algorithm_name }})" | |
| echo "version=$version" >> "$GITHUB_OUTPUT" | |
| - name: Compute Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ghcr.io/timeeval/${{ matrix.algorithm_name }} | |
| tags: | | |
| type=raw,value=latest | |
| type=raw,value=${{ steps.version.outputs.version }} | |
| labels: | | |
| org.opencontainers.image.title=TimeEval algorithm ${{ matrix.algorithm_name }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build algorithm image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: "./${{ matrix.algorithm_name }}" | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| pull: true | |
| load: true | |
| push: false | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Prepare result folder | |
| run: mkdir results && chmod -R 777 results | |
| - name: Check manifest available in container | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: "${{ matrix.algorithm_name }}:${{ github.sha }}" | |
| options: -e LOCAL_UID=1000 -e LOCAL_GID=1000 | |
| run: manifest | |
| - name: Test training call | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: "${{ matrix.algorithm_name }}:${{ github.sha }}" | |
| options: -e LOCAL_UID=1000 -e LOCAL_GID=1000 -v "${${{ matrix.algorithm_name }}_dataset_name}:/data/dataset.csv:ro" -v "$(pwd)/results:/results:rw" | |
| run: | | |
| execute-algorithm '{ | |
| "dataInput": "/data/dataset.csv", "dataOutput": "/results/scores.csv", | |
| "modelInput": "/results/model.pkl", "modelOutput": "/results/model.pkl", | |
| "executionType": "train", | |
| "customParameters": {"epochs": 1} | |
| }' | |
| - name: Test execution call | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: "${{ matrix.algorithm_name }}:${{ github.sha }}" | |
| options: -e LOCAL_UID=1000 -e LOCAL_GID=1000 -v "${${{ matrix.algorithm_name }}_dataset_name}:/data/dataset.csv:ro" -v "$(pwd)/results:/results:rw" | |
| run: | | |
| execute-algorithm '{ | |
| "dataInput": "/data/dataset.csv", "dataOutput": "/results/scores.csv", | |
| "modelInput": "/results/model.pkl", "modelOutput": "/results/model.pkl", | |
| "executionType": "execute", | |
| "customParameters": {"epochs": 1} | |
| }' | |
| - name: Validate output | |
| run: | | |
| ls -alh results/* | |
| python .ci/check_output.py "${{ matrix.algorithm_name }}" | |
| # deployment for non-PRs and if all checks are successfull: | |
| - name: Login to registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push algorithm image | |
| if: github.event_name != 'pull_request' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: "./${{ matrix.algorithm_name }}" | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| push: true | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max |