Merge branch 'dev' into nf-core-template-merge-2.12

.devcontainer/devcontainer.json

@@ -18,11 +18,11 @@
                 "python.linting.flake8Path": "/opt/conda/bin/flake8",
                 "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle",
                 "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle",
-                "python.linting.pylintPath": "/opt/conda/bin/pylint"
+                "python.linting.pylintPath": "/opt/conda/bin/pylint",
             },
 
             // Add the IDs of extensions you want installed when the container is created.
-            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
-        }
-    }
+            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"],
+        },
+    },
 }

.gitattributes

@@ -2,3 +2,4 @@
 *.nf.test linguist-language=nextflow
 modules/nf-core/** linguist-generated
 subworkflows/nf-core/** linguist-generated
+tests/**/*nf.test.snap linguist-generated

.github/workflows/ci.yml

@@ -1,43 +1,194 @@
-name: nf-core CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+name: nf-core CI
 on:
   push:
     branches:
       - dev
   pull_request:
+    branches:
+      - dev
+      - master
   release:
-    types: [published]
+    types:
+      - "published"
 
 env:
   NXF_ANSI_LOG: false
+  NFTEST_VER: "0.7.3"
 
+# Cancel if a newer run is started
 concurrency:
-  group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: Run pipeline with test data
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/createpanelrefs') }}"
+  changes:
+    name: Check for changes
+    runs-on: ubuntu-latest
+    outputs:
+      # Expose matched filters as job 'tags' output variable
+      tags: ${{ steps.filter.outputs.changes }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: "tests/config/tags.yml"
+
+  define_nxf_versions:
+    name: Choose nextflow versions to test against depending on target branch
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.nxf_versions.outputs.matrix }}
+    steps:
+      - id: nxf_versions
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
+            echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT
+          else
+            echo matrix='["latest-everything", "23.04.0"]' | tee -a $GITHUB_OUTPUT
+          fi
+
+  test-dev:
+    if: (github.base_ref == 'dev' || (github.event_name == 'push'))
+    name: ${{ matrix.tags }} ${{ matrix.profile }} NXF ${{ matrix.NXF_VER }}
     runs-on: ubuntu-latest
+    needs: changes
     strategy:
+      fail-fast: false
       matrix:
-        NXF_VER:
-          - "23.04.0"
-          - "latest-everything"
+        # Run tests based on changes in code
+        tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
+        # Only run docker tests on dev branch
+        profile: ["docker"]
+        # Only test minimal version
+        NXF_VER: ["23.04.0"]
+        # Always run default test
+        include:
+          - tags: default
+
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v4
 
-      - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v1
+      - uses: actions/cache@v3
+        with:
+          path: /usr/local/bin/nextflow
+          key: ${{ runner.os }}
+          restore-keys: |
+            ${{ runner.os }}-nextflow-
+
+      - name: Install nf-test
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Install Nextflow ${{ matrix.NXF_VER }}
+        uses: nf-core/[email protected]
+        with:
+          version: "${{ matrix.NXF_VER }}"
+
+      - name: Run nf-test
+        uses: Wandalen/[email protected]
+        with:
+          command: nf-test test tests/pipeline/${{ matrix.tags }}.nf.test --profile "test,${{ matrix.profile }}" --tap=test.tap
+          attempt_limit: 3
+
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs-${{ matrix.profile }}
+          path: |
+            /home/runner/pytest_workflow_*/*/.nextflow.log
+            /home/runner/pytest_workflow_*/*/log.out
+            /home/runner/pytest_workflow_*/*/log.err
+            /home/runner/pytest_workflow_*/*/work
+
+  test-master:
+    # Only run on master branch or when a PR is opened against master
+    if: github.base_ref == 'master'
+    name: ${{ matrix.tags }} ${{ matrix.profile }} NXF ${{ matrix.NXF_VER }}
+    runs-on: ubuntu-latest
+    needs: [changes, define_nxf_versions]
+    strategy:
+      fail-fast: false
+      matrix:
+        # Run tests based on changes in code
+        tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
+        # Only run docker tests on dev branch
+        profile: ["docker", "singularity", "conda"]
+        NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }}
+        # Always run default test
+        include:
+          - tags: default
+
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v3
+
+      - uses: actions/cache@v3
+        with:
+          path: /usr/local/bin/nextflow
+          key: ${{ runner.os }}
+          restore-keys: |
+            ${{ runner.os }}-nextflow-
+
+      - name: Install nf-test
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Install Nextflow ${{ matrix.NXF_VER }}
+        uses: nf-core/[email protected]
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Run pipeline with test data
-        # TODO nf-core: You can customise CI pipeline run tests as required
-        # For example: adding multiple test runs with different parameters
-        # Remember that you can parallelise this by using strategy.matrix
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-singularity@v5
+        with:
+          singularity-version: 3.7.1
+
+      - name: Set up miniconda
+        if: matrix.profile == 'conda'
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          channels: conda-forge,bioconda,defaults
+          python-version: ${{ matrix.python-version }}
+
+      - name: Conda clean
+        if: matrix.profile == 'conda'
+        run: conda clean -a
+
+      - name: Run nf-test
+        uses: Wandalen/[email protected]
+        with:
+          command: nf-test test tests/pipeline/${{ matrix.tag }}.nf.test --profile "test,${{ matrix.profile }}" --tap=test.tap
+          attempt_limit: 3
+
+      - name: Output log on failure
+        if: failure()
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs-${{ matrix.profile }}
+          path: |
+            /home/runner/pytest_workflow_*/*/.nextflow.log
+            /home/runner/pytest_workflow_*/*/log.out
+            /home/runner/pytest_workflow_*/*/log.err
+            /home/runner/pytest_workflow_*/*/work
+            !/home/runner/pytest_workflow_*/*/work/conda
+            !/home/runner/pytest_workflow_*/*/work/singularity

.gitignore

@@ -1,8 +1,9 @@
+*.pyc
+.DS_Store
 .nextflow*
-work/
+.nf-test/
 data/
 results/
-.DS_Store
-testing/
 testing*
-*.pyc
+testing/
+work/

.nf-core.yml

@@ -1 +1,8 @@
 repository_type: pipeline
+lint:
+  actions_ci: False
+  files_exist:
+    - conf/modules.config
+  files_unchanged:
+    - .gitattributes
+    - .gitignore

.prettierignore

@@ -1,12 +1,13 @@
-email_template.html
-adaptivecard.json
-slackreport.json
+*.pyc
+.DS_Store
 .nextflow*
-work/
+.nf-test/
+adaptivecard.json
+bin/
 data/
+email_template.html
 results/
-.DS_Store
-testing/
+slackreport.json
 testing*
-*.pyc
-bin/
+testing/
+work/

CHANGELOG.md

@@ -9,6 +9,10 @@ Initial release of nf-core/createpanelrefs, created with the [nf-core](https://n
 
 ### `Added`
 
+- [#5](https://github.com/nf-core/createpanelrefs/pull/5) - `CNVKIT` can be used to create a PON
+- [#5](https://github.com/nf-core/createpanelrefs/pull/5) - Usage of nf-validation
+- [#5](https://github.com/nf-core/createpanelrefs/pull/5) - Usage of nf-test
+
 ### `Fixed`
 
 ### `Dependencies`

CITATIONS.md

@@ -10,9 +10,13 @@
 
 ## Pipeline tools
 
-- [FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)
+- [CNVKIT](https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1004873)
 
-  > Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online].
+  > Talevich E, Shain AH, Botton T, Bastian BC (2016) CNVkit: Genome-Wide Copy Number Detection and Visualization from Targeted DNA Sequencing. PLoS Comput Biol 12(4): e1004873. doi: 10.1371/journal.pcbi.1004873. PubMed PMID: 27100738. PubMed Central PMCID: PMC4839673.
+
+- [GATK] (https://genome.cshlp.org/content/20/9/1297)
+
+  > McKenna A, Hanna M, Banks E, et al. The Genome Analysis Toolkit: A MapReduce framework for analyzing next-generation DNA sequencing data. Genome Res. 2010;20(9):1297-1303. doi:10.1101/gr.107524.110
 
 - [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/)