Build #760
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name}}-build | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| push_containers: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: push the containers to the registry | |
| target_tag: | |
| type: string | |
| required: false | |
| default: main_build | |
| description: Version tag of existing containers to use. Is `main_build` by default. | |
| deploy_to_viash_hub: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: Also build packages and docker images for viash-hub.com and push them. | |
| push: | |
| branches: [ 'main' ] | |
| jobs: | |
| # phase 1 | |
| build_and_deploy_target_folder: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| component_matrix: ${{ steps.set_matrix.outputs.matrix }} | |
| cache_key: ${{ steps.cache.outputs.cache_key }} | |
| steps: | |
| - name: Keep symlinks as-is | |
| run: | | |
| git config --global core.symlinks true | |
| - name: Branch to checkout (use existing target branch if it exists) | |
| id: get_checkout_branch | |
| run: | | |
| if ! git ls-remote --heads --exit-code https://github.com/openpipelines-bio/openpipeline.git ${{ github.ref_name }}_build > /dev/null; then | |
| echo "Remote branch does not exist, fetching current branch and building on top of it" | |
| echo "checkout_branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Remote branch exists, checking out existing branch" | |
| echo "checkout_branch=${{ github.ref_name }}_build" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.get_checkout_branch.outputs.checkout_branch }} | |
| fetch-depth: 0 | |
| - name: Fetch changes from main | |
| run: | | |
| git fetch origin ${{github.ref_name}} | |
| git checkout -f --no-overlay origin/${{github.ref_name}} -- '.' | |
| - uses: viash-io/viash-actions/setup@v4 | |
| - name: Remove target folder from .gitignore | |
| run: | | |
| # allow publishing the target folder | |
| sed -i '/^\/target\/$/d' .gitignore | |
| - uses: viash-io/viash-actions/ns-build@v4 | |
| with: | |
| config_mod: | | |
| .functionality.version := '${{ github.ref_name }}_build' | |
| .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }}' | |
| parallel: true | |
| query: ^(?!workflows) | |
| - uses: viash-io/viash-actions/ns-build@v4 | |
| with: | |
| config_mod: .functionality.version := '${{ github.ref_name }}_build' | |
| parallel: true | |
| query: ^workflows | |
| - name: Build nextflow schemas | |
| uses: viash-io/viash-actions/pro/build-nextflow-schemas@v4 | |
| with: | |
| components: src | |
| workflows: src | |
| viash_pro_token: ${{ secrets.GTHB_PAT }} | |
| tools_version: 'main_build' | |
| - name: Build parameter files | |
| uses: viash-io/viash-actions/pro/build-nextflow-params@v4 | |
| with: | |
| workflows: src | |
| components: src | |
| viash_pro_token: ${{ secrets.GTHB_PAT }} | |
| tools_version: 'main_build' | |
| - name: Deploy to target branch | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| create_branch: true | |
| commit_message: "deploy: ${{github.sha}}" | |
| skip_dirty_check: true | |
| branch: ${{ github.ref_name }}_build | |
| - id: ns_list | |
| uses: viash-io/viash-actions/ns-list@v4 | |
| with: | |
| platform: docker | |
| src: src | |
| format: json | |
| query_namespace: ^(?!workflows) | |
| - id: set_matrix | |
| run: | | |
| echo "matrix=$(jq -c '[ .[] | | |
| { | |
| "name": (.functionality.namespace + "/" + .functionality.name), | |
| "config": .info.config, | |
| "dir": .info.config | capture("^(?<dir>.*\/)").dir | |
| } | |
| ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT | |
| - uses: viash-io/viash-actions/ns-build@v4 | |
| if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }} | |
| with: | |
| config_mod: | | |
| .functionality.version := '${{ github.ref_name }}_build' | |
| .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }}' | |
| .platforms[.type == 'docker'].target_organization := 'openpipelines-bio/openpipeline' | |
| .platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050' | |
| .platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/openpipelines-bio/openpipeline' | |
| parallel: true | |
| query: ^(?!workflows) | |
| - uses: viash-io/viash-actions/ns-build@v4 | |
| if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }} | |
| with: | |
| config_mod: | | |
| .functionality.version := '${{ github.ref_name }}_build' | |
| parallel: true | |
| query: ^workflows | |
| - name: Push source branch to Viash-hub | |
| if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }} | |
| run: | | |
| git fetch ${{ github.ref_name }} | |
| git push -f https://x-access-token:${{ secrets.VIASHHUB_PAT }}@viash-hub.com/openpipelines-bio/openpipeline.git -u ${{ github.ref_name }} ${{ github.ref_name }} | |
| - name: Set origin to viash-hub | |
| if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }} | |
| # This is needed because git-auto-commit-action uses origin by default | |
| run: git remote set-url origin https://x-access-token:${{ secrets.VIASHHUB_PAT }}@viash-hub.com/openpipelines-bio/openpipeline.git | |
| - name: Deploy to target branch | |
| if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }} | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| create_branch: true | |
| commit_message: "deploy: ${{github.sha}}" | |
| skip_dirty_check: true | |
| branch: ${{ github.ref_name }}_build | |
| # phase 2 | |
| build_and_deploy_docker_containers: | |
| needs: build_and_deploy_target_folder | |
| if: ${{github.event_name == 'push' || github.event.inputs.push_containers == 'true' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| component: ${{ fromJson(needs.list.outputs.component_matrix) }} | |
| steps: | |
| # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' | |
| - uses: data-intuitive/reclaim-the-bytes@v2 | |
| - uses: actions/checkout@v4 | |
| - uses: viash-io/viash-actions/setup@v4 | |
| - name: Build container | |
| uses: viash-io/viash-actions/ns-build@v4 | |
| with: | |
| config_mod: | | |
| .functionality.version := '${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }}' | |
| .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }}' | |
| platform: docker | |
| src: ${{ matrix.component.dir }} | |
| setup: build | |
| - name: Login to container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GTHB_USER }} | |
| password: ${{ secrets.GTHB_PAT }} | |
| - name: Push container | |
| uses: viash-io/viash-actions/ns-build@v4 | |
| with: | |
| config_mod: .functionality.version := '${{ github.ref_name }}_build' | |
| platform: docker | |
| src: ${{ matrix.component.dir }} | |
| setup: push | |
| - name: Login to Viash-Hub container registry | |
| if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }} | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: viash-hub.com:5050 | |
| username: ${{ secrets.VIASHHUB_USER }} | |
| password: ${{ secrets.VIASHHUB_PAT }} | |
| - name: Update Docker settings | |
| if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }} | |
| run: | | |
| sudo sed -i 's/ }/, \"max-concurrent-uploads\": 2 }/' /etc/docker/daemon.json | |
| sudo systemctl restart docker | |
| - name: "Re-tag containers for viash-hub" | |
| if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }} | |
| run: | | |
| viash ns exec --apply_platform -p docker \ | |
| 'docker tag ghcr.io/openpipelines-bio/{namespace}_{functionality-name}:${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }} \ | |
| viash-hub.com:5050/openpipelines-bio/openpipeline/{namespace}_{functionality-name}:${{ github.ref_name }}_build' | |
| - name: Push container to Viash-Hub | |
| if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }} | |
| uses: viash-io/viash-actions/ns-build@v4 | |
| with: | |
| config_mod: | | |
| .functionality.version := '${{ github.ref_name }}_build' | |
| .platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050' | |
| .platforms[.type == 'docker'].target_organization := 'openpipelines-bio/openpipeline' | |
| .platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/openpipelines-bio/openpipeline' | |
| platform: docker | |
| src: ${{ matrix.component.dir }} | |
| setup: push |