Merge branch 'dev' into nf-core-template-merge-2.9

.github/workflows/awsfulltest.yml

@@ -12,12 +12,12 @@ jobs:
     name: Run AWS full tests
     if: github.repository == 'nf-core/methylseq'
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        aligner: ["bismark", "bismark_hisat", "bwameth"]
     steps:
       - name: Launch workflow via tower
         uses: seqeralabs/action-tower-launch@v2
-        # TODO nf-core: You can customise AWS full pipeline tests as required
-        # Add full size test data (but still relatively small datasets for few samples)
-        # on the `test_full.config` test runs with only one set of parameters
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
@@ -27,7 +27,8 @@ jobs:
           parameters: |
             {
               "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/methylseq/results-${{ github.sha }}"
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/methylseq/results-${{ github.sha }}/${{matrix.aligner}}"
+              "aligner": "${{ matrix.aligner }}",
             }
           profiles: test_full
 

.github/workflows/awstest.yml

@@ -9,6 +9,9 @@ jobs:
     name: Run AWS tests
     if: github.repository == 'nf-core/methylseq'
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        aligner: ["bismark", "bismark_hisat", "bwameth"]
     steps:
       # Launch workflow using Tower CLI tool action
       - name: Launch workflow via tower
@@ -21,7 +24,8 @@ jobs:
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/methylseq/work-${{ github.sha }}
           parameters: |
             {
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/methylseq/results-test-${{ github.sha }}"
+              "aligner": "${{ matrix.aligner }}",
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/methylseq/results-test-${{ github.sha }}/${{matrix.aligner}}"
             }
           profiles: test
 

.github/workflows/ci.yml

@@ -1,15 +1,20 @@
-name: nf-core CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+name: nf-core CI
 on:
   push:
     branches:
-      - dev
+      - "dev"
   pull_request:
+    branches:
+      - "dev"
+      - "master"
   release:
-    types: [published]
+    types:
+      - "published"
 
 env:
   NXF_ANSI_LOG: false
+  NFTEST_VER: "0.7.3"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
@@ -18,26 +23,85 @@ concurrency:
 jobs:
   test:
     name: Run pipeline with test data
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}"
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER:
           - "23.04.0"
           - "latest-everything"
+        aligner:
+          - "bismark"
+          - "bismark_hisat"
+          - "bwameth"
+        profile:
+          - "docker"
     steps:
+      # Skip if it's a pull_request to dev and NXF_VER is '22.10.1'
+      - name: Skip condition
+        id: condition
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
+            echo "skip=true" >> $GITHUB_OUTPUT
+          fi
+
       - name: Check out pipeline code
+        if: steps.condition.outputs.skip != 'true'
+        uses: actions/checkout@v3
+
+      - name: Hash Github Workspace
+        if: steps.condition.outputs.skip != 'true'
+        id: hash_workspace
+        run: |
+          echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
+
+      - name: Cache test data
+        if: steps.condition.outputs.skip != 'true'
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: ${{ steps.hash_workspace.outputs.digest }}
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true' && steps.condition.outputs.skip != 'true'
         uses: actions/checkout@v3
+        with:
+          repository: nf-core/test-datasets
+          ref: methylseq
+          path: test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        if: steps.condition.outputs.skip != 'true'
+        run: |
+          for f in ./test-datasets/samplesheet/*csv; do
+            sed -i 's=https://github.com/nf-core/test-datasets/raw/methylseq/=./test-datasets/=g' $f
+            sed -i 's=https://raw.githubusercontent.com/nf-core/test-datasets/methylseq/=./test-datasets/=g' $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
 
       - name: Install Nextflow
+        if: steps.condition.outputs.skip != 'true'
         uses: nf-core/setup-nextflow@v1
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Run pipeline with test data
-        # TODO nf-core: You can customise CI pipeline run tests as required
-        # For example: adding multiple test runs with different parameters
-        # Remember that you can parallelise this by using strategy.matrix
+      - name: Install nf-test
+        if: steps.condition.outputs.skip != 'true'
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Run nf-test
+        if: steps.condition.outputs.skip != 'true'
+        run: |
+          nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}" --tap=test.tap
+
+      # If the test fails, output the software_versions.yml using the 'batcat' utility
+      - name: Output log on failure
+        if: failure()
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/output/pipeline_info/software_versions.yml

.gitignore

@@ -6,3 +6,6 @@ results/
 testing/
 testing*
 *.pyc
+.vscode
+.nf-test
+test-datasets/

.nf-core.yml

@@ -1 +1,8 @@
+# FIXME - CI currently runs for dev and dsl2 branches, remove the latter before merging to dev
 repository_type: pipeline
+lint:
+  files_unchanged:
+    - .gitattributes
+  files_exist:
+    - bin/markdown_to_html.py
+  actions_ci: False

.prettierignore

@@ -10,3 +10,4 @@ testing/
 testing*
 *.pyc
 bin/
+*.nf.test.snap