use cache correctly and streamline versions through workflow reuse

.github/workflows/checks.yml

@@ -1,6 +1,22 @@
 name: nf-scil checks
 on:
   workflow_call:
+    inputs:
+      nextflow_version:
+        description: "Nextflow version to use"
+        required: false
+        type: string
+        default: "24.04.4"
+      nf_core_version:
+        description: "nf-core version to use"
+        required: false
+        type: string
+        default: "2.14.*"
+      nf_test_version:
+        description: "nf-test version to use"
+        required: false
+        type: string
+        default: "0.9.0-rc1"
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -11,6 +27,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
+        with:
+          cache: 'pip'
       - uses: pre-commit/[email protected]
         # FIXME Flip this off once we get to less than a couple hundred. Adding
         # this so it will only run against changed files. It'll make it much
@@ -21,13 +39,11 @@ jobs:
   prettier:
     runs-on: ubuntu-latest
     steps:
-      - name: Check out repository
-        uses: actions/checkout@v4
-
-      - name: Install NodeJS
-        uses: actions/setup-node@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
           node-version: "20"
+          cache: 'npm'
 
       - name: Install Prettier
         run: npm install -g [email protected]
@@ -39,10 +55,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
       - uses: actions/setup-node@v4
         with:
           node-version: "20"
+          cache: 'npm'
 
       - name: Install editorconfig-checker
         run: npm install -g editorconfig-checker
@@ -111,6 +127,8 @@ jobs:
     with:
       type: "module"
       component: ${{ matrix.module }}
+      nextflow_version: ${{ inputs.nextflow_version }}
+      nf_core_version: ${{ inputs.nf_core_version }}
     secrets: inherit
 
   lint-subworkflows:
@@ -127,6 +145,8 @@ jobs:
     with:
       type: "subworkflow"
       component: ${{ matrix.subworkflow }}
+      nextflow_version: ${{ inputs.nextflow_version }}
+      nf_core_version: ${{ inputs.nf_core_version }}
     secrets: inherit
 
   nf-test:
@@ -149,6 +169,8 @@ jobs:
       profile: ${{ matrix.profile }}
       paths: ${{ matrix.path }}
       runner: ${{ matrix.runner }}
+      nextflow_version: ${{ inputs.nextflow_version }}
+      nf_test_version: ${{ inputs.nf_test_version }}
     secrets: inherit
 
   confirm-pass:

.github/workflows/lint_module.yml

@@ -14,6 +14,16 @@ on:
         options:
           - module
           - subworkflow
+      nextflow_version:
+        description: "Nextflow version to use"
+        required: false
+        type: string
+        default: "24.04.4"
+      nf_core_version:
+        description: "nf-core version to use"
+        required: false
+        type: string
+        default: "2.14.*"
   workflow_call:
     inputs:
       component:
@@ -24,6 +34,16 @@ on:
         description: "Type of component"
         required: true
         type: string
+      nextflow_version:
+        description: "Nextflow version to use"
+        required: false
+        type: string
+        default: "24.04.4"
+      nf_core_version:
+        description: "nf-core version to use"
+        required: false
+        type: string
+        default: "2.14.*"
 
 run-name: Lint ${{ inputs.component }}
 jobs:
@@ -33,36 +53,31 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-
-      - uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip
-            ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-
-      - name: Install pip
-        run: python -m pip install --upgrade pip
-
+          cache: 'pip'
       - uses: actions/setup-java@v4
         with:
           distribution: "temurin"
           java-version: "17"
+          cache: 'gradle'
+      - uses: nf-core/setup-nextflow@v2
+        with:
+          version: ${{ inputs.nextflow_version }}
 
-      - name: Setup Nextflow
-        uses: nf-core/setup-nextflow@v2
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
 
       - name: Install nf-core tools
         run: |
-          python -m pip install --upgrade --force-reinstall nf-core==2.14.*
+          python -m pip install --upgrade --force-reinstall nf-core==${{ inputs.nf_core_version }}
 
       - name: Lint ${{ inputs.type }} ${{ inputs.component }}
         env:
           GIT_CLONE_PROTECTION_ACTIVE: false
-        run: nf-core --verbose ${{ inputs.type }}s --git-remote https://github.com/scilus/nf-scil.git lint ${{ inputs.component }}
+        run: |
+          nf-core --verbose ${{ inputs.type }}s \
+            --git-remote https://github.com/${{ github.repository }}.git \
+            --branch ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
+            lint ${{ inputs.component }}

.github/workflows/nf-test_module.yml

@@ -15,9 +15,19 @@ on:
           - docker
           - singularity
       runner:
-          description: "Runner label to use for the test run"
-          default: 'ubuntu-latest'
-          type: string
+        description: "Runner label to use for the test run"
+        default: 'ubuntu-latest'
+        type: string
+      nextflow_version:
+        description: "Nextflow version to use"
+        required: false
+        type: string
+        default: "24.04.4"
+      nf_test_version:
+        description: "nf-test version to use"
+        required: false
+        type: string
+        default: "0.9.0-rc1"
   workflow_call:
     inputs:
       profile:
@@ -35,6 +45,16 @@ on:
         description: "Runner label to use for the test run"
         default: 'ubuntu-latest'
         type: string
+      nextflow_version:
+        description: "Nextflow version to use"
+        required: false
+        type: string
+        default: "24.04.4"
+      nf_test_version:
+        description: "nf-test version to use"
+        required: false
+        type: string
+        default: "0.9.0-rc1"
 
 env:
   NXF_SINGULARITY_CACHEDIR: /home/runner/.singularity_cache/.singularity
@@ -48,63 +68,64 @@ jobs:
     if: inputs.paths != '' && inputs.profile != ''
     env:
       NXF_ANSI_LOG: false
-      NFTEST_VER: "0.9.0-rc1"
-      NFSCIL_TEST_DATA_HOME: /home/runner/_work
-
+      NXF_DISABLE_CHECK_LATEST: true
+      NFTEST_VERSION: "${{ inputs.nf_test_version }}"
+      # nf-test directories need to be placed on an ephemeral disk accessible to docker.
+      # In our case, it's only the _work workspace for security reasons
+      NXF_WORKDIR: /home/runner/_work
     steps:
       - uses: actions/checkout@v4
-
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: 'pip'
       - uses: actions/setup-java@v4
         with:
           distribution: "temurin"
           java-version: "17"
-
-      - name: Setup Nextflow
-        uses: nf-core/setup-nextflow@v2
-
+          cache: 'gradle'
+      - uses: nf-core/setup-nextflow@v2
+        with:
+          version: ${{ inputs.nextflow_version }}
       - name: Setup nf-test
         uses: AlexVCaron/setup-nf-test@2c5a31b90b34d97abf155b57ee0f79c7f5aa118a
         # Used to make the action executable on our runners. Related to how the home
         # and work directories are set up from ephemeral disks. See nf-core/setup-nf-test/issues/6
         with:
-          version: ${{ env.NFTEST_VER }}
+          version: ${{ inputs.nf_test_version }}
+
+      - name: Install Python dependencies
+        run: python -m pip install --upgrade pip pdiff cryptography
 
       - name: Fix nf-test launching jvm with too much memory
         run: |
           sed -i 's/-Xmx10g//' $(which nf-test)
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-
-      - uses: actions/cache@v4
-        id: cache-pip-pdiff
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-pdiff
-          restore-keys: |
-            ${{ runner.os }}-pip-pdiff
-
-      - uses: actions/github-script@v7
+      - name: Compute test run UID
+        uses: actions/github-script@v7
         id: test-run-identifier
         with:
           script: |
             identifier='${{ inputs.paths }}'.toLowerCase().replaceAll(/[/. ]+/g, '-').trim('-');
-            core.setOutput('tag', identifier);
+            core.setOutput('uid', identifier);
 
-      - uses: actions/cache@v4
+      - name: Cache test archives under test UID
+        uses: actions/cache@v4
         with:
-          path: ${{ env.NFSCIL_TEST_DATA_HOME }}/nf-scil-test-archives
-          key: nf-scil-test-data-cache-${{ steps.test-run-identifier.outputs.tag }}
-
-      - name: Install Python dependencies
-        run: python -m pip install --upgrade pip pdiff cryptography
+          path: ${{ env.NXF_WORKDIR }}/nf-scil-test-archives
+          key: |
+            nf-scil-test-data-cache-${{ steps.test-run-identifier.outputs.uid }}
+            nf-scil-test-data-cache-
 
       - name: Run nf-test
         env:
           NFT_DIFF: "pdiff"
           NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
+          TMPDIR: ${{ env.NXF_WORKDIR }}/tmpdir
+          NFT_WORKDIR: ${{ env.NXF_WORKDIR }}/.nf-test
+          NXF_CACHE_DIR: ${{ env.NXF_WORKDIR }}/.nextflow
+          NXF_TEMP: ${{ env.NXF_WORKDIR }}/tmpdir
+          NFSCIL_TEST_DATA_HOME: ${{ env.NXF_WORKDIR }}
         run: |
           # use "docker_self_hosted" if it runs on self-hosted runner and inputs.profile=docker
           if [ "${{ inputs.profile }}" == "docker" ]; then
@@ -113,12 +134,7 @@ jobs:
             PROFILE=${{ inputs.profile }}
           fi
 
-          # nf-test directories need to be placed on an ephemeral disk accessible to docker.
-          # In our case, it's only the _work workspace for security reasons
-          mkdir -p /home/runner/_work/tmpdir
-
-          TMPDIR=/home/runner/_work/tmpdir \
-          NFT_WORKDIR=/home/runner/_work \
+          mkdir -p ${{ env.TMPDIR }}
           nf-test test \
             --profile=$PROFILE \
             --tap=test.tap \
@@ -134,26 +150,26 @@ jobs:
       - name: Tests logs
         if: failure()
         run: |
-            shopt -s globstar
-            for f in /home/runner/_work/tests/**/work/**/.command.log
-            do
+          shopt -s globstar
+          for f in ${{ env.NXF_WORKDIR }}/.nf-test/tests/**/work/**/.command.log
+          do
 
-            echo "$(sed '3q;d' $(dirname $f)/.command.run)"
-            cat $f
+          echo "$(sed '3q;d' $(dirname $f)/.command.run)"
+          cat $f
 
-            done
+          done
 
       - name: Upload images as artifacts
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: image_artifacts-${{ steps.test-run-identifier.outputs.tag }}
-          path: /home/runner/_work/tests/**/work/**/*.nii.gz
+          name: image_artifacts-${{ steps.test-run-identifier.outputs.uid }}
+          path: ${{ env.NXF_WORKDIR }}/.nf-test/tests/**/work/**/*.nii.gz
           overwrite: true
-          retention-days: 2
+          retention-days: 1
           compression-level: 9
 
       - name: Clean up
         if: always()
         run: |
-          sudo rm -rf /home/ubuntu/tests/
+          sudo rm -rf ${{ env.NXF_WORKDIR }}/.nf-test