main-build.yml

name: Build
concurrency: 
  group: ${{ github.workflow }}-${{ inputs.deploy_branch && inputs.deploy_branch || format('{0}_build', github.ref_name) }}
  cancel-in-progress: true

on:
  workflow_dispatch:
    inputs:
      push_containers:
        type: boolean
        required: false
        default: false
        description: Build docker images and push them to the registry
      version:
        type: string
        required: false
        description: |
          Version to tag the build components with (e.i functionality.version).
          Defaults to name of the branch that triggered the workflow, suffixed by "_build".
      target_tag:
        type: string
        required: false
        description: |
          Version tag of containers to use. Defaults to name of the branch that triggered the workflow, 
          suffixed by "_build". Can be used in combination with 'push_containers' (by unchecking it)
          to re-use existing docker images or set the tag for new builds.

  # when used as a subworkflow
  workflow_call:
    inputs:
      push_containers:
        type: boolean
        required: false
        default: false
        description: push the containers to the registry
      version:
        type: string
        required: false
        description: |
          Version to tag the build components with (e.i functionality.version).
          Defaults to name of the branch that triggered the workflow, suffixed by "_build".
      target_tag:
        type: string
        required: false
        description: |
          Version tag of containers to use. Defaults to name of the branch that triggered the workflow, 
          suffixed by "_build". Can be used in combination with 'push_containers' (by setting it to 'false')
          to re-use existing docker images or set the tag for new builds.
      deploy_branch:
        type: string
        required: false
        description: |
          Branch to deploy the build to. Defaults to name of the branch 
          that triggered the workflow, suffixed by "_build".
      is_workflow_call:
          description: |
            Flag to destinguish between a 'workflow_call' and 'workflow_dispatch'.
            github.event_name can not be used for this because it refers to the event of
            the top-level workflow. See https://github.com/actions/runner/issues/3146
          type: boolean
          required: false
          default: true
    outputs:
      component_matrix:
        description: "A JSON object that can be used to populate a github actions matrix for component jobs."
        value: ${{ jobs.build_and_deploy_target_folder.outputs.component_matrix }}
      workflow_matrix:
        description: "A JSON object that can be used to populate a github actions matrix for workflow jobs."
        value: ${{ jobs.build_and_deploy_target_folder.outputs.workflow_matrix}}
    secrets:
      GTHB_USER:
        required: true 
      GTHB_PAT:
        required: true
  push:
    branches: [ 'main' ]


jobs:
  # phase 1
  build_and_deploy_target_folder:
    name: "Build and push target folder"
    runs-on: ubuntu-latest

    outputs:
      component_matrix: ${{ steps.set_matrix.outputs.components }}
      workflow_matrix: ${{ steps.set_matrix.outputs.workflows }}

    env:
      DEPLOY_BRANCH: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}

    steps:
    - name: Do not allow manual trigger if the branch is main
      # github.event_name is the event of the top-level workflow, so use dummy 'is_workflow_call' input
      if: ${{ github.event_name != 'push' && (!inputs.is_workflow_call) && github.ref == 'refs/heads/main' }}
      run: |
        echo "This workflow should not be triggered with workflow_dispatch from the main branch"
        exit 1
    - name: Check input arguments
      run: |
        input_version="${{ inputs.version || format('{0}_build', github.ref_name) }}"
        target_tag="${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}"
        should_push="${{inputs.push_containers }}"
        if [ "$input_version" != "$target_tag" ] && [ "$should_push" == "true" ]; then
          echo "When trying to push new docker images, the tag for the components must be equal to the target_tag for the docker images."
          exit 1
        fi

    - name: Keep symlinks as-is
      run: |
        git config --global core.symlinks true

    - name: Branch to checkout (use existing target branch if it exists)
      id: get_checkout_branch
      run: |
        if ! git ls-remote --heads --exit-code https://github.com/openpipelines-bio/openpipeline.git "$DEPLOY_BRANCH" > /dev/null; then
          echo "Remote branch does not exist, fetching current branch and building on top of it"
          echo "checkout_branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
        else
          echo "Remote branch exists, checking out existing branch"
          echo "checkout_branch=$DEPLOY_BRANCH" >> "$GITHUB_OUTPUT"
        fi
  
    - uses: actions/checkout@v4
      with:
        ref: ${{ steps.get_checkout_branch.outputs.checkout_branch }}
        fetch-depth: 0

    - name: Fetch changes from ${{github.ref_name}} 
      run: |
        git fetch origin ${{github.ref_name}} 
        git checkout -f --no-overlay origin/${{github.ref_name}} -- '.'
    
    - uses: viash-io/viash-actions/setup@v6

    - name: Remove target folder from .gitignore
      run: |
        # allow publishing the target folder
        sed -i '/^\/target\/$/d' .gitignore

    - uses: viash-io/viash-actions/ns-build@v6
      with:
        config_mod: |
          .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
          .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}'
        parallel: true
        query: ^(?!workflows|test_workflows)
    
    - uses: viash-io/viash-actions/ns-build@v6
      with:
        config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
        parallel: true
        query: ^workflows
  
    - name: Build nextflow schemas
      uses: viash-io/viash-actions/pro/build-nextflow-schemas@v6
      with:
        target_dir: target/nextflow
        viash_pro_token: ${{ secrets.GTHB_PAT }}
        tools_version: 'main_build'
        enable_dataset_input: true
   
    - name: Build parameter files
      uses: viash-io/viash-actions/pro/build-nextflow-params@v6
      with:
        target_dir: target/nextflow
        viash_pro_token: ${{ secrets.GTHB_PAT }}
        tools_version: 'main_build'

    - name: Deploy to target branch
      uses: stefanzweifel/git-auto-commit-action@v5
      with:
        create_branch: true
        commit_message: "deploy: ${{github.sha}}"
        skip_dirty_check: true
        branch:  ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}
    - name: "List components"
      id: ns_list
      uses: viash-io/viash-actions/ns-list@v6
      with:
        engine: docker
        src: src
        format: json
        query_namespace: ^(?!workflows)|(workflows/test_workflows)

    - name: "List workflows"
      id: ns_list_workflows
      uses: viash-io/viash-actions/ns-list@v6
      with:
        src: src
        format: json
        query_namespace: ^workflows/(?!test_workflows)

    - name: "Parse JSON output from 'viash ns list' as input for matrix."
      id: set_matrix
      run: |
        echo "components=$(jq -c '[ .[] | 
          { 
            "name": .name,
            "namespace_separator": .engines | map(select(.type == "docker"))[0].namespace_separator,
            "namespace": .namespace,
            "config": .build_info.config,
            "dir": .build_info.config | capture("^(?<dir>.*\/)").dir
          }
        ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT

        echo "workflows=$(jq -c '[ .[] | . as $config | (.test_resources // [])[] | select(.type == "nextflow_script", .entrypoint) |
        { 
          "name": $config.name,
          "namespace": $config.namespace,
          "main_script": (($config.build_info.config | capture("^(?<dir>.*\/)").dir) + "/" + .path),
          "entry": .entrypoint,
          "config": $config.build_info.config
        }
        ] | unique' ${{ steps.ns_list_workflows.outputs.output_file }} )" >> $GITHUB_OUTPUT

  # phase 2
  build_and_deploy_docker_containers:
    name: "Build and Deploy Docker Images"
    needs: build_and_deploy_target_folder
    if: ${{github.event_name == 'push' || inputs.push_containers }}

    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        component: ${{ fromJson(needs.build_and_deploy_target_folder.outputs.component_matrix) }}

    steps:
    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
    - uses: data-intuitive/reclaim-the-bytes@v2

    - uses: actions/checkout@v4

    - uses: viash-io/viash-actions/setup@v6

    - name: Build container
      uses: viash-io/viash-actions/ns-build@v6
      with:
        config_mod: |
          .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
          .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}'
        engine: docker
        src: ${{ matrix.component.dir }}
        setup: build
        
    - name: Login to container registry
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ secrets.GTHB_USER }}
        password: ${{ secrets.GTHB_PAT }}

    - name: Push container
      uses: viash-io/viash-actions/ns-build@v6
      with:
        config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
        engine: docker
        src: ${{ matrix.component.dir }}
        setup: push