Build and host algorithm images on GitHub (base images) #11
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build images and test algorithms | |
| on: | |
| pull_request: | |
| branches: main | |
| push: | |
| branches: main | |
| defaults: | |
| run: | |
| shell: 'bash -Eeuo pipefail -xl {0}' | |
| jobs: | |
| init: | |
| name: Generate Jobs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| base_matrix: ${{ steps.generate-jobs.outputs.base_matrix }} | |
| intermediate_matrix: ${{ steps.generate-jobs.outputs.intermediate_matrix }} | |
| algo_matrix: ${{ steps.generate-jobs.outputs.algo_matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: generate-jobs | |
| name: Generate Jobs for modified base images or algorithms | |
| run: | | |
| base_matrix="$(./.ci/generate-build-matrix.sh 0-base-images)" | |
| echo "base_matrix=$base_matrix" >> "$GITHUB_OUTPUT" | |
| jq . <<<"$base_matrix" # print generated json for debugging | |
| intermediate_matrix='{"algorithm_name": []}' | |
| echo "intermediate_matrix=$intermediate_matrix" >> "$GITHUB_OUTPUT" | |
| jq . <<<"$intermediate_matrix" # print generated json for debugging | |
| algo_matrix="$(./.ci/generate-build-matrix.sh)" | |
| echo "algo_matrix=$algo_matrix" >> "$GITHUB_OUTPUT" | |
| jq . <<<"$algo_matrix" # print generated json for debugging | |
| build-base-images: | |
| name: Build base images | |
| runs-on: ubuntu-latest | |
| needs: init | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| max-parallel: 3 | |
| matrix: ${{ fromJson(needs.init.outputs.base_matrix) }} | |
| # algorithm_name: ["python3-base", "java-base"] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Compute Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ${{ matrix.algorithm_name }} | |
| ghcr.io/hpi-information-systems/${{ matrix.algorithm_name }} | |
| tags: | | |
| type=ref,event=branch | |
| type=sha | |
| type=raw,value=latest | |
| # type=raw,${{ version }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build algorithm image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: "./0-base-images/${{ matrix.algorithm_name }}" | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| pull: true | |
| load: true | |
| # push: ${{ github.event_name != 'pull_request' }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| build-intermediate-images: | |
| name: Build intermediate images | |
| runs-on: ubuntu-latest | |
| needs: | |
| - init | |
| - build-base-images | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| max-parallel: 3 | |
| matrix: ${{ fromJson(needs.init.outputs.intermediate_matrix) }} | |
| # algorithm_name: ["tsmp"] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Compute Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ${{ matrix.algorithm_name }} | |
| ghcr.io/hpi-information-systems/${{ matrix.algorithm_name }} | |
| tags: | | |
| type=ref,event=branch | |
| type=sha | |
| type=raw,value=latest | |
| # type=raw,${{ version }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build algorithm image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: "./1-intermediate-images/${{ matrix.algorithm_name }}" | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| pull: true | |
| load: true | |
| # push: ${{ github.event_name != 'pull_request' }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| test-algorithms: | |
| name: Test algorithms | |
| runs-on: ubuntu-latest | |
| needs: | |
| - init | |
| - build-base-images | |
| - build-intermediate-images | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| max-parallel: 3 | |
| matrix: ${{ fromJson(needs.init.outputs.algo_matrix) }} | |
| # algorithm_name: ["subsequence_lof", "lof"] | |
| # or with multiple variables: | |
| # include: | |
| # - algorithm_name: "subsequence_lof" | |
| # image_name: "ghcr.io/hpi-information-systems/pyod/subsequence_lof" | |
| # - algorithm_name: "lof" | |
| # image_name: "ghcr.io/hpi-information-systems/pyod/lof" | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Validate manifest | |
| run: python validate_manifest.py --path ${{ matrix.algorithm_name }}/manifest.json | |
| - name: Determine correct test dataset | |
| run: | | |
| dataset_name="$(python .ci/get_dataset_name.py ${{ matrix.algorithm_name }} > dataset_name.txt)" | |
| echo "${{ matrix.algorithm_name }}_dataset_name=$dataset_name" >> "$GITHUB_OUTPUT" | |
| - name: Compute Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ${{ matrix.algorithm_name }} | |
| ghcr.io/hpi-information-systems/${{ matrix.algorithm_name }} | |
| tags: | | |
| type=ref,event=branch | |
| type=sha | |
| # type=raw,${{ version }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build algorithm image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: "./${{ matrix.algorithm_name }}" | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| pull: true | |
| load: true | |
| push: false # TODO: push when tests are successfull! | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Prepare result folder | |
| run: mkdir results && chmod -R 777 results | |
| - name: Test training call | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: "${{ matrix.algorithm_name }}:${{ github.sha }}" | |
| options: -e LOCAL_UID=1000 -e LOCAL_GID=1000 -v "${${{ matrix.algorithm_name }}_dataset_name}:/data/dataset.csv:ro" -v "$(pwd)/results:/results:rw" | |
| run: | | |
| execute-algorithm '{ | |
| "dataInput": "/data/dataset.csv", "dataOutput": "/results/scores.csv", | |
| "modelInput": "/results/model.pkl", "modelOutput": "/results/model.pkl", | |
| "executionType": "train", | |
| "customParameters": {"epochs": 1} | |
| }' | |
| - name: Test execution call | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: "${{ matrix.algorithm_name }}:${{ github.sha }}" | |
| options: -e LOCAL_UID=1000 -e LOCAL_GID=1000 -v "${${{ matrix.algorithm_name }}_dataset_name}:/data/dataset.csv:ro" -v "$(pwd)/results:/results:rw" | |
| run: | | |
| execute-algorithm '{ | |
| "dataInput": "/data/dataset.csv", "dataOutput": "/results/scores.csv", | |
| "modelInput": "/results/model.pkl", "modelOutput": "/results/model.pkl", | |
| "executionType": "execute", | |
| "customParameters": {"epochs": 1} | |
| }' | |
| - name: Validate output | |
| run: | | |
| ls -alh results/* | |
| python .ci/check_output.py "${{ matrix.algorithm_name }}" | |
| # deployment for non-PRs and if all checks are successfull: | |
| # - name: Login to registry | |
| # uses: docker/login-action@v4 | |
| # if: github.event_name != 'pull_request' | |
| # with: | |
| # registry: ghcr.io | |
| # username: ${{ github.actor }} | |
| # password: ${{ secrets.GITHUB_TOKEN }} | |
| # - name: Build and push algorithm image | |
| # uses: docker/build-push-action@v5 | |
| # if: github.event_name != 'pull_request' | |
| # with: | |
| # context: "./${{ matrix.algorithm_name }}" | |
| # tags: | | |
| # ghcr.io/hpi-information-systems/"${{ matrix.algorithm_name }}:latest" | |
| # ghcr.io/hpi-information-systems/"${{ matrix.algorithm_name }}:0.0.0" | |
| # labels: ${{ steps.meta.outputs.labels }} | |
| # pull: true | |
| # load: true | |
| # push: true # TODO: push when tests are successfull! | |
| # cache-from: type=gha | |
| # cache-to: type=gha,mode=max | |
| # GITHUB_EVENT_NAME=pull_request | |
| # GITHUB_BASE_REF=PR target branch (probably main) | |
| # GITHUB_HEAD_REF=PR source branch | |
| # GITHUB_REF=refs/pull/<pr_number>/merge | |
| # GITHUB_REF_TYPE=tag or branch | |
| # RUNNER_ARCH=X86, X64, ARM, or ARM64 | |
| # RUNNER_OD=Linux, Windows, or macOS |