NF_RCP-F_1.0.4-RC1

.codespellignore

@@ -0,0 +1,5 @@
+RNAseq
+OTU
+otu
+groupD
+groupd

.github/workflows/check_typos.yml

@@ -15,8 +15,9 @@ jobs:
             fail-fast: false
         steps:
           - uses: actions/checkout@v3
-          - uses: codespell-project/actions-codespell@master
+          - uses: codespell-project/codespell-problem-matcher@v1
+          - uses: codespell-project/[email protected]
             with:
               check_filenames: true
-              skip: "*.yml,*.cff,*.js,*.lock"
-              ignore_words_list: RNAseq
+              skip: "*.yml,*.cff,*.js,*.lock,*.pdf,*.ipynb"
+              ignore_words_file: ".codespellignore"

.github/workflows/ci.yml

@@ -0,0 +1,97 @@
+name: CI Updated Modules Testing
+# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+on:
+  push:
+    branches:
+      - DEV_NF_RCP-F
+
+env:
+  NXF_ANSI_LOG: false
+
+jobs:
+  changes:
+    name: Check for changes
+    runs-on: ubuntu-latest
+    outputs:
+      # Expose matched filters as job 'modules' output variable
+      modules: ${{ steps.filter.outputs.changes }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          base: 'DEV_NF_RCP-F'
+          filters: "RNAseq/Workflow_Documentation/NF_RCP-F/workflow_code/tests/config/nftest_modules.yml"
+
+  test:
+    name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.NXF_VER }}
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.modules != '[]'
+    strategy:
+      matrix:
+        NXF_VER:
+          - "22.10.1"
+          - "latest-everything"
+        profile:
+          - "docker"
+          - "singularity"
+        tags: ["${{ fromJson(needs.changes.outputs.modules) }}"]
+
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v3
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@be72b1dc0f932cea69aef64479ac863a86516c0c
+        with:
+          version: "${{ matrix.NXF_VER }}"
+
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-singularity@v5
+        with:
+          singularity-version: 3.7.1
+
+      - name: Install nf-test
+        id: nf-test
+        run: |
+          curl -fsSL https://code.askimed.com/install/nf-test | bash
+          chmod u+x nf-test
+          echo "bin_path=$(pwd)/nf-test" >> $GITHUB_OUTPUT
+
+
+      - name: Hash Github Workspace
+        id: hash_workspace
+        run: |
+          echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
+
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: RNAseq/Workflow_Documentation/NF_RCP-F/workflow_code/test-datasets
+          key: ${{ steps.hash_workspace.outputs.digest }}
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: J-81/test-datasets-extended
+          ref: NF_RCP-F
+          path: RNAseq/Workflow_Documentation/NF_RCP-F/workflow_code/test-datasets
+
+      # Test the module
+      - name: Run nf-test
+        run: |
+          cd ${GITHUB_WORKSPACE}/RNAseq/Workflow_Documentation/NF_RCP-F/workflow_code/
+          ${{ steps.nf-test.outputs.bin_path}} test \
+            --profile=${{ matrix.profile }} \
+            --tag ${{ matrix.tags }} \
+            --tap=test.tap
+
+      - uses: pcolby/tap-summary@v1
+        with:
+          path: >-
+            ${GITHUB_WORKSPACE}/RNAseq/Workflow_Documentation/NF_RCP-F/workflow_code/test.tap

.github/workflows/ci_minimal_full_pipeline.yml

@@ -0,0 +1,77 @@
+name: CI Minimal Dataset Full Pipeline
+# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+on: push
+
+env:
+  NXF_ANSI_LOG: false
+
+jobs:
+  Minimal_Dataset_Full_Pipeline:
+    name: ${{ matrix.tags }} ${{ matrix.profile }} ${{ matrix.NXF_VER }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        NXF_VER:
+          - "22.10.1"
+          - "latest-everything"
+        profile:
+          - "docker"
+          - "singularity"
+
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v3
+
+      - name: Install Nextflow
+        uses: nf-core/[email protected]
+        with:
+          version: "${{ matrix.NXF_VER }}"
+
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-singularity@v5
+        with:
+          singularity-version: 3.7.1
+
+      - name: Install nf-test
+        id: nf-test
+        run: |
+          curl -fsSL https://code.askimed.com/install/nf-test | bash
+          chmod u+x nf-test
+          echo "bin_path=$(pwd)/nf-test" >> $GITHUB_OUTPUT
+
+
+      - name: Hash Github Workspace
+        id: hash_workspace
+        run: |
+          echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
+
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: RNAseq/Workflow_Documentation/NF_RCP-F/workflow_code/test-datasets
+          key: ${{ steps.hash_workspace.outputs.digest }}
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: J-81/test-datasets-extended
+          ref: NF_RCP-F
+          path: RNAseq/Workflow_Documentation/NF_RCP-F/workflow_code/test-datasets
+
+      # Test the module
+      - name: Run nf-test on minimal core test datasets
+        run: |
+          cd ${GITHUB_WORKSPACE}/RNAseq/Workflow_Documentation/NF_RCP-F/workflow_code/
+          ${{ steps.nf-test.outputs.bin_path}} test \
+            --profile=${{ matrix.profile }} \
+            --tag core \
+            --tap=test.tap \
+            tests/*.test
+
+      - uses: pcolby/tap-summary@v1
+        with:
+          path: >-
+            ${GITHUB_WORKSPACE}/RNAseq/Workflow_Documentation/NF_RCP-F/workflow_code/test.tap

.github/workflows/markdown-link-check.yml

@@ -1,7 +1,8 @@
 name: Check Markdown links
 
-on: push
-
+on:
+  create: # runs when a reference (branch or tag) is created
+
 jobs:
   markdown-link-check:
     runs-on: ubuntu-latest

.gitignore

@@ -0,0 +1,2 @@
+.nextflow
+*.pyc

.gitpod.yml

@@ -0,0 +1,13 @@
+image: nfcore/gitpod:latest
+
+vscode:
+  extensions:
+    - ms-python.python
+    - eamodio.gitlens
+    - GitHub.copilot
+    - REditorSupport.r
+    - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code
+    - mechatroner.rainbow-csv # Highlight columns in csv files in different colors
+    - nextflow.nextflow # Nextflow syntax highlighting
+    - oderwat.indent-rainbow # Highlight indentation level
+    - streetsidesoftware.code-spell-checker # Spelling checker for source code

RNAseq/Pipeline_GL-DPPD-7101_Versions/GL-DPPD-7101-F.md

@@ -43,6 +43,7 @@ The DESeq2 Normalization and DGE step, [step 9](#9-normalize-read-counts-perform
 
 - Fixed rare edge case where groupwise mean and standard deviations could become misassociated to incorrect groups. This had affected [step 9f](#9f-prepare-genelab-dge-tables-with-annotations-on-datasets-with-ercc-spike-in) and [step 9i](#9i-prepare-genelab-dge-tables-with-annotations-on-datasets-without-ercc-spike-in).
 
+- [Step 2a](#2a-trimfilter-raw-data) adapter type argument removed in favor of using the built in TrimGalore! adapter [autodetection](https://github.com/FelixKrueger/TrimGalore/blob/0.6.7/Docs/Trim_Galore_User_Guide.md#adapter-auto-detection).
 ---
 
 # Table of contents  
@@ -122,7 +123,7 @@ The DESeq2 Normalization and DGE step, [step 9](#9-normalize-read-counts-perform
 |tximport|1.27.1|[https://github.com/mikelove/tximport](https://github.com/mikelove/tximport)|
 |tidyverse|1.3.1|[https://www.tidyverse.org](https://www.tidyverse.org)|
 |stringr|1.4.1|[https://github.com/tidyverse/stringr](https://github.com/tidyverse/stringr)|
-|dp_tools|1.1.8|[https://github.com/J-81/dp_tools](https://github.com/J-81/dp_tools)|
+|dp_tools|1.3.3|[https://github.com/J-81/dp_tools](https://github.com/J-81/dp_tools)|
 |pandas|1.5.0|[https://github.com/pandas-dev/pandas](https://github.com/pandas-dev/pandas)|
 |seaborn|0.12.0|[https://seaborn.pydata.org/](https://seaborn.pydata.org/)|
 |matplotlib|3.6.0|[https://matplotlib.org/stable](https://matplotlib.org/stable)|
@@ -204,7 +205,6 @@ trim_galore --gzip \
   --path_to_cutadapt /path/to/cutadapt \
   --cores NumberOfThreads \
   --phred33 \
-  --illumina \ # if adapters are not illumina, replace with adapters used
   --output_dir /path/to/TrimGalore/output/directory \
   --paired \ # only for PE studies, remove this parameter if raw data are SE
   sample1_R1_raw.fastq.gz sample1_R2_raw.fastq.gz sample2_R1_raw.fastq.gz sample2_R2_raw.fastq.gz
@@ -218,7 +218,6 @@ trim_galore --gzip \
 - `--path_to_cutadapt` - specify path to cutadapt software if it is not in your `$PATH`
 - `--cores` - specify the number of threads available on the server node to perform trimming
 - `--phred33` - instructs cutadapt to use ASCII+33 quality scores as Phred scores for quality trimming
-- `--illumina` - defines the adapter sequence to be trimmed as the first 13bp of the Illumina universal adapter `AGATCGGAAGAGC`
 - `--output_dir` - the output directory to store results
 - `--paired` - indicates paired-end reads - both reads, forward (R1) and reverse (R2) must pass length threshold or else both reads are removed
 - `sample1_R1_raw.fastq.gz sample1_R2_raw.fastq.gz sample2_R1_raw.fastq.gz sample2_R2_raw.fastq.gz` – the input reads are specified as a positional argument, paired-end read files are listed pairwise such that the forward reads (*R1_raw.fastq.gz) are immediately followed by the respective reverse reads (*R2_raw.fastq.gz) for each sample

RNAseq/Workflow_Documentation/NF_RCP-F/CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+- Github action support for CI testing (a83006ba91b1209e1857fefd96e9ff950ebb0cdc)
+
+### Fixed
+- Workflow usage files will all follow output directory set by workflow user (3e69f06432f62b7924d2e043ef4768c5d09bf614)
+### Changed 
+- TrimGalore! will now use autodetect for adaptor type (3b7e0bab4017e90481359c48f9cf7c8837ed54d2)
+- V&V migrated from dp_tools version 1.1.8 to 1.3.2 including:
+  - Migration of V&V protocol code to this codebase instead of dp_tools (b3684a4c1db5df06eab20916ef7e130c410c147c)
+  - Fix for sample wise checks reusing same samples (dca4fdad7518ac9ead3ee2e4c5f57ac0fe25c715)
+
 ## [1.0.3](https://github.com/nasa/GeneLab_Data_Processing/tree/NF_RCP-F_1.0.3/RNAseq/Workflow_Documentation/NF_RCP-F) - 2023-01-25
 
 ### Added

RNAseq/Workflow_Documentation/NF_RCP-F/README.md

@@ -101,9 +101,9 @@ All files required for utilizing the NF_RCP-F GeneLab workflow for processing RN
 copy of latest NF_RCP-F version on to your system, the code can be downloaded as a zip file from the release page then unzipped after downloading by running the following commands: 
 
 ```bash
-wget https://github.com/nasa/GeneLab_Data_Processing/releases/download/NF_RCP-F_1.0.3/NF_RCP-F_1.0.3.zip
+wget https://github.com/nasa/GeneLab_Data_Processing/releases/download/NF_RCP-F_1.0.4/NF_RCP-F_1.0.4.zip
 
-unzip NF_RCP-F_1.0.3.zip
+unzip NF_RCP-F_1.0.4.zip
 ```
 
 <br>
@@ -115,10 +115,10 @@ unzip NF_RCP-F_1.0.3.zip
 Although Nextflow can fetch Singularity images from a url, doing so may cause issues as detailed [here](https://github.com/nextflow-io/nextflow/issues/1210).
 
 To avoid this issue, run the following command to fetch the Singularity images prior to running the NF_RCP-F workflow:
-> Note: This command should be run in the location containing the `NF_RCP-F_1.0.3` directory that was downloaded in [step 2](#2-download-the-workflow-files) above. Depending on your network speed, fetching the images will take ~20 minutes.  
+> Note: This command should be run in the location containing the `NF_RCP-F_1.0.4` directory that was downloaded in [step 2](#2-download-the-workflow-files) above. Depending on your network speed, fetching the images will take ~20 minutes.  
 
 ```bash
-bash NF_RCP-F_1.0.3/bin/prepull_singularity.sh NF_RCP-F_1.0.3/config/software/by_docker_image.config
+bash NF_RCP-F_1.0.4/bin/prepull_singularity.sh NF_RCP-F_1.0.4/config/software/by_docker_image.config
 ```
 
 
@@ -134,15 +134,15 @@ export NXF_SINGULARITY_CACHEDIR=$(pwd)/singularity
 
 ### 4. Run the Workflow
 
-While in the location containing the `NF_RCP-F_1.0.3` directory that was downloaded in [step 2](#2-download-the-workflow-files), you are now able to run the workflow. Below are three examples of how to run the NF_RCP-F workflow:
+While in the location containing the `NF_RCP-F_1.0.4` directory that was downloaded in [step 2](#2-download-the-workflow-files), you are now able to run the workflow. Below are three examples of how to run the NF_RCP-F workflow:
 > Note: Nextflow commands use both single hyphen arguments (e.g. -help) that denote general nextflow arguments and double hyphen arguments (e.g. --ensemblVersion) that denote workflow specific parameters.  Take care to use the proper number of hyphens for each argument.
 
 <br>
 
 #### 4a. Approach 1: Run the workflow on a GeneLab RNAseq dataset with automatic retrieval of Ensembl reference fasta and gtf files
 
 ```bash
-nextflow run NF_RCP-F_1.0.3/main.nf \ 
+nextflow run NF_RCP-F_1.0.4/main.nf \ 
    -profile singularity \
    --gldsAccession GLDS-194 
 ```
@@ -154,7 +154,7 @@ nextflow run NF_RCP-F_1.0.3/main.nf \
 > Note: The `--ref_source` and `--ensemblVersion` parameters should match the reference source and version number of the local reference fasta and gtf files used
 
 ```bash
-nextflow run NF_RCP-F_1.0.3/main.nf \ 
+nextflow run NF_RCP-F_1.0.4/main.nf \ 
    -profile singularity \
    --gldsAccession GLDS-194 \
    --ensemblVersion 107 \
@@ -170,7 +170,7 @@ nextflow run NF_RCP-F_1.0.3/main.nf \
 > Note: Specifications for creating a runsheet manually are described [here](examples/runsheet/README.md).
 
 ```bash
-nextflow run NF_RCP-F_1.0.3/main.nf \ 
+nextflow run NF_RCP-F_1.0.4/main.nf \ 
    -profile singularity \
    --runsheetPath </path/to/runsheet> 
 ```
@@ -179,7 +179,7 @@ nextflow run NF_RCP-F_1.0.3/main.nf \
 
 **Required Parameters For All Approaches:**
 
-* `NF_RCP-F_1.0.3/main.nf` - Instructs Nextflow to run the NF_RCP-F workflow 
+* `NF_RCP-F_1.0.4/main.nf` - Instructs Nextflow to run the NF_RCP-F workflow 
 
 * `-profile` - Specifies the configuration profile(s) to load, `singularity` instructs Nextflow to setup and use singularity for all software called in the workflow
 
@@ -225,7 +225,7 @@ nextflow run NF_RCP-F_1.0.3/main.nf \
 All parameters listed above and additional optional arguments for the RCP workflow, including debug related options that may not be immediately useful for most users, can be viewed by running the following command:
 
 ```bash
-nextflow run NF_RCP-F_1.0.3/main.nf --help
+nextflow run NF_RCP-F_1.0.4/main.nf --help
 ```
 
 See `nextflow run -h` and [Nextflow's CLI run command documentation](https://nextflow.io/docs/latest/cli.html#run) for more options and details common to all nextflow workflows.