Build and host algorithm images on GitHub (base images) #6
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Pull Request | |
| on: | |
| pull_request: | |
| branches: main | |
| defaults: | |
| run: | |
| shell: 'bash -Eeuo pipefail -xl {0}' | |
| jobs: | |
| init: | |
| name: Generate Jobs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.generate-jobs.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: generate-jobs | |
| name: Generate Jobs for modified algorithms | |
| run: | | |
| matrix="$(./.ci/generate-build-matrix.sh)" | |
| echo "matrix=$matrix" >> "$GITHUB_OUTPUT" | |
| jq . <<<"$matrix" # print generated json for debugging | |
| validate: | |
| name: Test algorithms | |
| runs-on: ubuntu-latest | |
| needs: init | |
| permissions: | |
| contents: read | |
| # packages: write # just for pushing the image to the registry | |
| strategy: | |
| max-parallel: 3 | |
| matrix: ${{ fromJson(needs.init.outputs.matrix) }} | |
| # algorithm_name: ["subsequence_lof", "lof"] | |
| # or with multiple variables: | |
| # include: | |
| # - algorithm_name: "subsequence_lof" | |
| # image_name: "ghcr.io/HPI-Information-Systems/pyod/subsequence_lof" | |
| # - algorithm_name: "lof" | |
| # image_name: "ghcr.io/HPI-Information-Systems/pyod/lof" | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Validate manifest | |
| run: python validate_manifest.py --path ${{ matrix.algorithm_name }}/manifest.json | |
| - name: Determine correct test dataset | |
| run: | | |
| dataset_name="$(python .ci/get_dataset_name.py ${{ matrix.algorithm_name }} > dataset_name.txt)" | |
| echo "${{ matrix.algorithm_name }}_dataset_name=$dataset_name" >> "$GITHUB_OUTPUT" | |
| - name: Prepare build | |
| run: mkdir results && chmod -R 777 results | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build algorithm image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: "./${{ matrix.algorithm_name }}" | |
| tags: "${{ matrix.algorithm_name }}:${{ github.sha }}" | |
| pull: true | |
| push: false | |
| load: true | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Test training call | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: "${{ matrix.algorithm_name }}:${{ github.sha }}" | |
| options: -e LOCAL_UID=1000 -e LOCAL_GID=1000 -v "${${{ matrix.algorithm_name }}_dataset_name}:/data/dataset.csv:ro" -v "$(pwd)/results:/results:rw" | |
| run: | | |
| execute-algorithm '{ | |
| "dataInput": "/data/dataset.csv", "dataOutput": "/results/scores.csv", | |
| "modelInput": "/results/model.pkl", "modelOutput": "/results/model.pkl", | |
| "executionType": "train", | |
| "customParameters": {"epochs": 1} | |
| }' | |
| - name: Test execution call | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: "${{ matrix.algorithm_name }}:${{ github.sha }}" | |
| options: -e LOCAL_UID=1000 -e LOCAL_GID=1000 -v "${${{ matrix.algorithm_name }}_dataset_name}:/data/dataset.csv:ro" -v "$(pwd)/results:/results:rw" | |
| run: | | |
| execute-algorithm '{ | |
| "dataInput": "/data/dataset.csv", "dataOutput": "/results/scores.csv", | |
| "modelInput": "/results/model.pkl", "modelOutput": "/results/model.pkl", | |
| "executionType": "execute", | |
| "customParameters": {"epochs": 1} | |
| }' | |
| - name: Validate output | |
| run: | | |
| ls -alh results/* | |
| python .ci/check_output.py "${{ matrix.algorithm_name }}" | |
| # no deployment for PRs | |
| # - name: Login to registry | |
| # uses: docker/login-action@v4 | |
| # with: | |
| # registry: ghcr.io | |
| # username: ${{ github.actor }} | |
| # password: ${{ secrets.GITHUB_TOKEN }} | |
| # GITHUB_EVENT_NAME=pull_request | |
| # GITHUB_BASE_REF=PR target branch (probably main) | |
| # GITHUB_HEAD_REF=PR source branch | |
| # GITHUB_REF=refs/pull/<pr_number>/merge | |
| # GITHUB_REF_TYPE=tag or branch | |
| # RUNNER_ARCH=X86, X64, ARM, or ARM64 | |
| # RUNNER_OD=Linux, Windows, or macOS |