diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c7ca24b7..bca78889 100755 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,6 +1,22 @@ name: nf-scil checks on: workflow_call: + inputs: + nextflow_version: + description: "Nextflow version to use" + required: false + type: string + default: "24.04.4" + nf_core_version: + description: "nf-core version to use" + required: false + type: string + default: "2.14.*" + nf_test_version: + description: "nf-test version to use" + required: false + type: string + default: "0.9.0-rc1" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -11,6 +27,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + cache: 'pip' - uses: pre-commit/action@v3.0.1 # FIXME Flip this off once we get to less than a couple hundred. Adding # this so it will only run against changed files. It'll make it much @@ -21,13 +39,11 @@ jobs: prettier: runs-on: ubuntu-latest steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Install NodeJS - uses: actions/setup-node@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: "20" + cache: 'npm' - name: Install Prettier run: npm install -g prettier@3.1.0 @@ -39,10 +55,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 with: node-version: "20" + cache: 'npm' - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -111,6 +127,8 @@ jobs: with: type: "module" component: ${{ matrix.module }} + nextflow_version: ${{ inputs.nextflow_version }} + nf_core_version: ${{ inputs.nf_core_version }} secrets: inherit lint-subworkflows: @@ -127,6 +145,8 @@ jobs: with: type: "subworkflow" component: ${{ matrix.subworkflow }} + nextflow_version: ${{ inputs.nextflow_version }} + nf_core_version: ${{ inputs.nf_core_version }} secrets: inherit nf-test: @@ -149,6 +169,8 @@ jobs: profile: ${{ matrix.profile }} paths: ${{ matrix.path }} runner: ${{ matrix.runner }} + nextflow_version: ${{ inputs.nextflow_version }} + nf_test_version: ${{ inputs.nf_test_version }} secrets: inherit confirm-pass: diff --git a/.github/workflows/lint_module.yml b/.github/workflows/lint_module.yml index 225840a1..43479814 100755 --- a/.github/workflows/lint_module.yml +++ b/.github/workflows/lint_module.yml @@ -14,6 +14,16 @@ on: options: - module - subworkflow + nextflow_version: + description: "Nextflow version to use" + required: false + type: string + default: "24.04.4" + nf_core_version: + description: "nf-core version to use" + required: false + type: string + default: "2.14.*" workflow_call: inputs: component: @@ -24,6 +34,16 @@ on: description: "Type of component" required: true type: string + nextflow_version: + description: "Nextflow version to use" + required: false + type: string + default: "24.04.4" + nf_core_version: + description: "nf-core version to use" + required: false + type: string + default: "2.14.*" run-name: Lint ${{ inputs.component }} jobs: @@ -33,36 +53,31 @@ jobs: steps: - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: "3.11" - - - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip - ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - - - name: Install pip - run: python -m pip install --upgrade pip - + cache: 'pip' - uses: actions/setup-java@v4 with: distribution: "temurin" java-version: "17" + cache: 'gradle' + - uses: nf-core/setup-nextflow@v2 + with: + version: ${{ inputs.nextflow_version }} - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 + - name: Upgrade pip + run: python -m pip install --upgrade pip - name: Install nf-core tools run: | - python -m pip install --upgrade --force-reinstall nf-core==2.14.* + python -m pip install --upgrade --force-reinstall nf-core==${{ inputs.nf_core_version }} - name: Lint ${{ inputs.type }} ${{ inputs.component }} env: GIT_CLONE_PROTECTION_ACTIVE: false - run: nf-core --verbose ${{ inputs.type }}s --git-remote https://github.com/scilus/nf-scil.git lint ${{ inputs.component }} + run: | + nf-core --verbose ${{ inputs.type }}s \ + --git-remote https://github.com/${{ github.repository }}.git \ + --branch ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} + lint ${{ inputs.component }} diff --git a/.github/workflows/nf-test_module.yml b/.github/workflows/nf-test_module.yml index 0faf0841..3d5f5ed3 100644 --- a/.github/workflows/nf-test_module.yml +++ b/.github/workflows/nf-test_module.yml @@ -15,9 +15,19 @@ on: - docker - singularity runner: - description: "Runner label to use for the test run" - default: 'ubuntu-latest' - type: string + description: "Runner label to use for the test run" + default: 'ubuntu-latest' + type: string + nextflow_version: + description: "Nextflow version to use" + required: false + type: string + default: "24.04.4" + nf_test_version: + description: "nf-test version to use" + required: false + type: string + default: "0.9.0-rc1" workflow_call: inputs: profile: @@ -35,6 +45,16 @@ on: description: "Runner label to use for the test run" default: 'ubuntu-latest' type: string + nextflow_version: + description: "Nextflow version to use" + required: false + type: string + default: "24.04.4" + nf_test_version: + description: "nf-test version to use" + required: false + type: string + default: "0.9.0-rc1" env: NXF_SINGULARITY_CACHEDIR: /home/runner/.singularity_cache/.singularity @@ -48,63 +68,64 @@ jobs: if: inputs.paths != '' && inputs.profile != '' env: NXF_ANSI_LOG: false - NFTEST_VER: "0.9.0-rc1" - NFSCIL_TEST_DATA_HOME: /home/runner/_work - + NXF_DISABLE_CHECK_LATEST: true + NFTEST_VERSION: "${{ inputs.nf_test_version }}" + # nf-test directories need to be placed on an ephemeral disk accessible to docker. + # In our case, it's only the _work workspace for security reasons + NXF_WORKDIR: /home/runner/_work steps: - uses: actions/checkout@v4 - + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: 'pip' - uses: actions/setup-java@v4 with: distribution: "temurin" java-version: "17" - - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - + cache: 'gradle' + - uses: nf-core/setup-nextflow@v2 + with: + version: ${{ inputs.nextflow_version }} - name: Setup nf-test uses: AlexVCaron/setup-nf-test@2c5a31b90b34d97abf155b57ee0f79c7f5aa118a # Used to make the action executable on our runners. Related to how the home # and work directories are set up from ephemeral disks. See nf-core/setup-nf-test/issues/6 with: - version: ${{ env.NFTEST_VER }} + version: ${{ inputs.nf_test_version }} + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pdiff cryptography - name: Fix nf-test launching jvm with too much memory run: | sed -i 's/-Xmx10g//' $(which nf-test) - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - uses: actions/cache@v4 - id: cache-pip-pdiff - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pdiff - restore-keys: | - ${{ runner.os }}-pip-pdiff - - - uses: actions/github-script@v7 + - name: Compute test run UID + uses: actions/github-script@v7 id: test-run-identifier with: script: | identifier='${{ inputs.paths }}'.toLowerCase().replaceAll(/[/. ]+/g, '-').trim('-'); - core.setOutput('tag', identifier); + core.setOutput('uid', identifier); - - uses: actions/cache@v4 + - name: Cache test archives under test UID + uses: actions/cache@v4 with: - path: ${{ env.NFSCIL_TEST_DATA_HOME }}/nf-scil-test-archives - key: nf-scil-test-data-cache-${{ steps.test-run-identifier.outputs.tag }} - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pdiff cryptography + path: ${{ env.NXF_WORKDIR }}/nf-scil-test-archives + key: | + nf-scil-test-data-cache-${{ steps.test-run-identifier.outputs.uid }} + nf-scil-test-data-cache- - name: Run nf-test env: NFT_DIFF: "pdiff" NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + TMPDIR: ${{ env.NXF_WORKDIR }}/tmpdir + NFT_WORKDIR: ${{ env.NXF_WORKDIR }}/.nf-test + NXF_CACHE_DIR: ${{ env.NXF_WORKDIR }}/.nextflow + NXF_TEMP: ${{ env.NXF_WORKDIR }}/tmpdir + NFSCIL_TEST_DATA_HOME: ${{ env.NXF_WORKDIR }} run: | # use "docker_self_hosted" if it runs on self-hosted runner and inputs.profile=docker if [ "${{ inputs.profile }}" == "docker" ]; then @@ -113,12 +134,7 @@ jobs: PROFILE=${{ inputs.profile }} fi - # nf-test directories need to be placed on an ephemeral disk accessible to docker. - # In our case, it's only the _work workspace for security reasons - mkdir -p /home/runner/_work/tmpdir - - TMPDIR=/home/runner/_work/tmpdir \ - NFT_WORKDIR=/home/runner/_work \ + mkdir -p ${{ env.TMPDIR }} nf-test test \ --profile=$PROFILE \ --tap=test.tap \ @@ -134,26 +150,26 @@ jobs: - name: Tests logs if: failure() run: | - shopt -s globstar - for f in /home/runner/_work/tests/**/work/**/.command.log - do + shopt -s globstar + for f in ${{ env.NXF_WORKDIR }}/.nf-test/tests/**/work/**/.command.log + do - echo "$(sed '3q;d' $(dirname $f)/.command.run)" - cat $f + echo "$(sed '3q;d' $(dirname $f)/.command.run)" + cat $f - done + done - name: Upload images as artifacts if: always() uses: actions/upload-artifact@v4 with: - name: image_artifacts-${{ steps.test-run-identifier.outputs.tag }} - path: /home/runner/_work/tests/**/work/**/*.nii.gz + name: image_artifacts-${{ steps.test-run-identifier.outputs.uid }} + path: ${{ env.NXF_WORKDIR }}/.nf-test/tests/**/work/**/*.nii.gz overwrite: true - retention-days: 2 + retention-days: 1 compression-level: 9 - name: Clean up if: always() run: | - sudo rm -rf /home/ubuntu/tests/ + sudo rm -rf ${{ env.NXF_WORKDIR }}/.nf-test