Merge branch 'dev' of https://github.com/nf-core/tools into subworkfl…

…ows-meta-yml-restructure

.editorconfig

@@ -21,7 +21,7 @@ indent_style = unset
 [**/Makefile]
 indent_style = unset
 
-[tests/__snapshots__/*]
+[tests/pipelines/__snapshots__/*]
 charset = unset
 end_of_line = unset
 insert_final_newline = unset

.github/workflows/changelog.py

@@ -1,3 +1,4 @@
+#! /usr/bin/env python3
 """
 Taken from https://github.com/MultiQC/MultiQC/blob/main/.github/workflows/changelog.py and updated for nf-core
 
@@ -18,7 +19,7 @@
 import re
 import sys
 from pathlib import Path
-from typing import List
+from typing import List, Tuple
 
 REPO_URL = "https://github.com/nf-core/tools"
 
@@ -32,7 +33,7 @@
 assert pr_number, pr_number
 
 # Trim the PR number added when GitHub squashes commits, e.g. "Template: Updated (#2026)"
-pr_title = pr_title.removesuffix(f" (#{pr_number})")
+pr_title = pr_title.removesuffix(f" (#{pr_number})")  # type: ignore
 
 changelog_path = workspace_path / "CHANGELOG.md"
 
@@ -50,7 +51,7 @@
     sys.exit(0)
 
 
-def _determine_change_type(pr_title) -> tuple[str, str]:
+def _determine_change_type(pr_title) -> Tuple[str, str]:
     """
     Determine the type of the PR: Template, Download, Linting, Modules, Subworkflows, or General
     Returns a tuple of the section name and the module info.
@@ -85,7 +86,7 @@ def _determine_change_type(pr_title) -> tuple[str, str]:
 pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))"
 
 # Handle manual changelog entries through comments.
-if comment := comment.removeprefix("@nf-core-bot changelog").strip():
+if comment := comment.removeprefix("@nf-core-bot changelog").strip():  # type: ignore
     print(f"Adding manual changelog entry: {comment}")
     pr_title = comment
 new_lines = [

.github/workflows/create-lint-wf.yml

@@ -38,7 +38,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "24.04.2"
           - "latest-everything"
     steps:
       - name: go to subdirectory and change nextflow workdir

.github/workflows/create-test-lint-wf-template.yml

@@ -31,27 +31,43 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
+  prepare-matrix:
+    name: Retrieve all template features
+    runs-on: ubuntu-latest
+    outputs:
+      all_features: ${{ steps.create_matrix.outputs.matrix }}
+    steps:
+      - name: 🏗 Set up yq
+        uses: frenck/action-setup-yq@v1
+      - name: checkout
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+      - name: Create Matrix
+        id: create_matrix
+        run: |
+          echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/template_features.yml)" >> $GITHUB_OUTPUT
+
   RunTestWorkflow:
     runs-on: ${{ matrix.runner }}
+    needs: prepare-matrix
     env:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
-        TEMPLATE:
-          - "template_skip_github_badges.yml"
-          - "template_skip_igenomes.yml"
-          - "template_skip_ci.yml"
+        TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }}
         runner:
           # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default
           - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }}
         profile: ["self_hosted_runner"]
         include:
-          - TEMPLATE: "template_skip_all.yml"
+          - TEMPLATE: all
             runner: ubuntu-latest
             profile: "docker"
-          - TEMPLATE: "template_skip_nf_core_configs.yml"
+          - TEMPLATE: nf_core_configs
             runner: ubuntu-latest
             profile: "docker"
+        exclude:
+          - TEMPLATE: nf_core_configs
+            profile: "self_hosted_runner"
       fail-fast: false
 
     steps:
@@ -80,33 +96,22 @@ jobs:
           version: latest-everything
 
       # Create template files
-      - name: Create template skip all (except github)
+      - name: Create template skip ${{ matrix.TEMPLATE }}
         run: |
           mkdir create-test-lint-wf
           export NXF_WORK=$(pwd)
-          printf "org: my-prefix\nskip: ['ci', 'github_badges', 'igenomes', 'nf_core_configs']" > create-test-lint-wf/template_skip_all.yml
-
-      - name: Create template skip github_badges
-        run: |
-          printf "org: my-prefix\nskip: github_badges" > create-test-lint-wf/template_skip_github_badges.yml
-
-      - name: Create template skip igenomes
-        run: |
-          printf "org: my-prefix\nskip: igenomes" > create-test-lint-wf/template_skip_igenomes.yml
-
-      - name: Create template skip ci
-        run: |
-          printf "org: my-prefix\nskip: ci" > create-test-lint-wf/template_skip_ci.yml
-
-      - name: Create template skip nf_core_configs
-        run: |
-          printf "org: my-prefix\nskip: nf_core_configs" > create-test-lint-wf/template_skip_nf_core_configs.yml
+          if [ ${{ matrix.TEMPLATE }} == "all" ]
+          then
+            printf "org: my-prefix\nskip_features: ${{ needs.prepare-matrix.outputs.all_features }}" > create-test-lint-wf/template_skip_all.yml
+          else
+            printf "org: my-prefix\nskip_features: [${{ matrix.TEMPLATE }}]" > create-test-lint-wf/template_skip_${{ matrix.TEMPLATE }}.yml
+          fi
 
       # Create a pipeline from the template
       - name: create a pipeline from the template ${{ matrix.TEMPLATE }}
         run: |
           cd create-test-lint-wf
-          nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml ${{ matrix.TEMPLATE }}
+          nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml template_skip_${{ matrix.TEMPLATE }}.yml
 
       - name: run the pipeline
         run: |

.github/workflows/create-test-wf.yml

@@ -39,7 +39,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "24.04.2"
           - "latest-everything"
     steps:
       - name: go to working directory
@@ -75,7 +75,7 @@ jobs:
           pwd
           # echo content of current directory
           ls -la
-          nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results
+          nextflow run nf-core-testpipeline -profile self_hosted_runner,test --outdir ./results
 
       - name: Upload log file artifact
         if: ${{ always() }}

.github/workflows/pytest.yml

@@ -67,7 +67,7 @@ jobs:
       - name: List tests
         id: list_tests
         run: |
-          echo "tests=$(find tests/test_* | tac | sed 's/tests\///g' | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT
+          echo "tests=$(find tests -type f -name "test_*.py" | tac | sed 's/tests\///g'  | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT
     outputs:
       tests: ${{ steps.list_tests.outputs.tests }}
 
@@ -149,10 +149,15 @@ jobs:
           name: Snapshot Report ${{ matrix.test }}
           path: ./snapshot_report.html
 
+      - name: remove slashes from test name
+        run: |
+          test=$(echo ${{ matrix.test }} | sed 's/\//__/g')
+          echo "test=${test}" >> $GITHUB_ENV
+
       - name: Upload coverage
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4
         with:
-          name: coverage_${{ matrix.test }}
+          name: coverage_${{ env.test }}
           path: .coverage
 
   coverage:

.github/workflows/update-textual-snapshots.yml

@@ -0,0 +1,93 @@
+name: Update Textual snapshots from a comment
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  update-snapshots:
+    # Only run if comment is on a PR with the main repo, and if it contains the magic keywords
+    if: >
+      contains(github.event.comment.html_url, '/pull/') &&
+      contains(github.event.comment.body, '@nf-core-bot update snapshots') &&
+      github.repository == 'nf-core/tools'
+    runs-on: ubuntu-latest
+    steps:
+      # Use the @nf-core-bot token to check out so we can push later
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        with:
+          token: ${{ secrets.nf_core_bot_auth_token }}
+
+      # indication that the command is running
+      - name: React on comment
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: eyes
+
+      # Action runs on the issue comment, so we don't get the PR by default
+      # Use the gh cli to check out the PR
+      - name: Checkout Pull Request
+        run: gh pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
+
+      # Install dependencies and run pytest
+      - name: Set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip -r requirements-dev.txt
+          pip install -e .
+
+      - name: Run pytest to update snapshots
+        id: pytest
+        run: |
+          python3 -m pytest tests/test_create_app.py --snapshot-update --color=yes  --durations=0
+        continue-on-error: true
+
+      # indication that the run has finished
+      - name: react if finished succesfully
+        if: steps.pytest.outcome == 'success'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: "+1"
+
+      - name: Commit & push changes
+        id: commit-and-push
+        if: steps.pytest.outcome == 'failure'
+        run: |
+          git config user.email "[email protected]"
+          git config user.name "nf-core-bot"
+          git config push.default upstream
+          git add .
+          git status
+          git commit -m "[automated] Update Textual snapshots"
+          git push
+
+      - name: react if snapshots were updated
+        id: react-if-updated
+        if: steps.commit-and-push.outcome == 'success'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: hooray
+
+      - name: react if snapshots were not updated
+        if: steps.commit-and-push.outcome == 'failure'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: confused
+
+      - name: react if snapshots were not updated
+        if: steps.commit-and-push.outcome  == 'failure'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          issue-number: ${{ github.event.issue.number }}
+          body: |
+            @${{ github.actor }} I tried to update the snapshots, but it didn't work. Please update them manually.

.gitpod.yml

@@ -6,17 +6,14 @@ tasks:
       python -m pip install -r requirements-dev.txt
       pre-commit install --install-hooks
       nextflow self-update
-  - name: unset JAVA_TOOL_OPTIONS
-    command: |
-      unset JAVA_TOOL_OPTIONS
 
 vscode:
-  extensions: # based on nf-core.nf-core-extensionpack
+  extensions:
     - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code
     - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files
     - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar
     - mechatroner.rainbow-csv # Highlight columns in csv files in different colors
-    # - nextflow.nextflow                    # Nextflow syntax highlighting
+    - nextflow.nextflow # Nextflow syntax highlighting
     - oderwat.indent-rainbow # Highlight indentation level
     - streetsidesoftware.code-spell-checker # Spelling checker for source code
     - charliermarsh.ruff # Code linter Ruff

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.0
+    rev: v0.6.0
     hooks:
       - id: ruff # linter
         args: [--fix, --exit-non-zero-on-fix] # sort imports and fix
@@ -10,7 +10,7 @@ repos:
     hooks:
       - id: prettier
         additional_dependencies:
-          - prettier@3.2.5
+          - prettier@3.3.3
 
   - repo: https://github.com/editorconfig-checker/editorconfig-checker.python
     rev: "2.7.3"
@@ -19,7 +19,7 @@ repos:
         alias: ec
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.1"
+    rev: "v1.11.1"
     hooks:
       - id: mypy
         additional_dependencies:
@@ -28,3 +28,4 @@ repos:
           - types-jsonschema
           - types-Markdown
           - types-setuptools
+          - pydantic

.prettierignore

@@ -5,6 +5,10 @@ docs/api/_build
 testing
 nf_core/module-template/meta.yml
 nf_core/pipeline-template/nextflow_schema.json
+nf_core/pipeline-template/modules.json
+nf_core/pipeline-template/tower.yml
+nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml
+tests/data/pipeline_create_template_skip.yml
 # don't run on things handled by ruff
 *.py
 *.pyc