Merge pull request #197 from cidgoh/development

Merging new changes to master from development branch

.github/workflows/nextflow_CI.yml

@@ -1,16 +1,14 @@
 name: Nextflow CI
 
 on:
-  push:
-    branches:
-      - development
   pull_request:
     branches:
+      - development
       - master
 
 jobs:
   test_sarscov2_user:
-    name: Run pipeline test (user)
+    name: Run pipeline test (sarscov2_user)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -20,12 +18,72 @@ jobs:
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
 
-      - name: Run pipeline test (user)
+      - name: Run pipeline test (sarscov2_user)
         run: |
-          nextflow run main.nf -profile docker --prefix "covidmvp-user-$(date +%Y-%m-%d)" -params-file covidmvp_user.yaml
+          nextflow run main.nf -profile docker --prefix "covidmvp-user-$(date +%Y-%m-%d)" -params-file covidmvp_user_params.yaml
 
   test_sarscov2_reference:
-    name: Run pipeline test (reference)
+    name: Run pipeline test (sarscov2_reference)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Run pipeline test (sarscov2_reference)
+        run: |
+          nextflow run main.nf -profile docker --prefix "covidmvp-clinical-$(date +%Y-%m-%d)" --end_date $(date +%Y-%m-%d) -params-file covidmvp_clinical_params.yaml
+
+  test_sarscov2_timeseries:
+    name: Run pipeline test (sarscov2_timeseries)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Run pipeline test (sarscov2_timeseries)
+        run: |
+          nextflow run main.nf -profile docker --prefix "covidmvp-timeseries-$(date +%Y-%m-%d)" -params-file covidmvp_timeseries_params.yaml
+
+  test_sarscov2_wastewater:
+    name: Run pipeline test (sarscov2_wastewater)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Run pipeline test (sarscov2_wastewater)
+        run: |
+          nextflow run main.nf -profile docker --prefix "covidmvp-wastewater-$(date +%Y-%m-%d)" -params-file covidmvp_wastewater_params.yaml
+
+  test_mpox_user:
+    name: Run pipeline test (mpox_user)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Run pipeline test (mpox_user)
+        run: |
+          nextflow run main.nf -profile docker --prefix "poxmvp-user-$(date +%Y-%m-%d)" -params-file poxmvp_user_params.yaml
+
+  test_mpox_reference:
+    name: Run pipeline test (mpox_reference)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -35,13 +93,21 @@ jobs:
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
 
-      - name: Run pipeline test (reference)
+      - name: Run pipeline test (mpox_reference)
         run: |
-          nextflow run main.nf -profile docker --prefix "covidmvp-$(date +%Y-%m-%d)" --end_date $(date +%Y-%m-%d) -params-file covidmvp_clinical_params.yaml
+          nextflow run main.nf -profile docker --prefix "poxmvp-reference-$(date +%Y-%m-%d)" -params-file poxmvp_clinical_params.yaml
 
   check_success:
     name: Check if all tests passed
-    needs: [test_sarscov2_user, test_sarscov2_reference]
+    needs:
+      [
+        test_sarscov2_user,
+        test_sarscov2_reference,
+        test_sarscov2_timeseries,
+        test_sarscov2_wastewater,
+        test_mpox_reference,
+        test_mpox_user
+      ]
     runs-on: ubuntu-latest
     steps:
       - name: Check job status

.gitignore

@@ -3,16 +3,9 @@ results/*
 .idea/*
 .DS_store
 *.ipynb
-weekly*
 *.error
 *.out
 bin/__*
 *.sh
-latest*
-*.fa.xz
-*.gz
-hMPXV_*
 bin/web.log
 assets/config.ini
-*.xz
-*.log

assets/archive/metadata.json

@@ -0,0 +1,97 @@
+{
+    "Format": {
+      "sep": "tab",
+      "suffix": ".gz"
+    },
+    "Required": {
+      "isolate": true,
+      "lineage": true,
+      "sample_collection_date": true,
+      "country": true,
+      "state_province_territory": true
+    },
+    "Optional": {
+      "gene_name": false,
+      "organism": true,
+      "host_scientific_name": true,
+      "host_gender": false,
+      "host_age_bin": false,
+      "submission_date": false,
+      "gisaid_accession": true,
+      "length": false,
+      "purpose_of_sampling": false,
+      "purpose_of_sequencing": false,
+      "alias": true,
+      "Type": false,
+      "Subtype": false,
+      "SubLineageGroup": false,
+      "Clade": false,
+      "last_updated": true
+    },
+    "ViralAi_EpiCoV": {
+      "isolate": "isolate",
+      "lineage": "raw_lineage",
+      "organism": "organism",
+      "gisaid_accession": "gisaid_accession",
+      "host_scientific_name": "host_scientific_name",
+      "sample_collection_date": "sample_collection_date",
+      "country": "geo_loc_name_country",
+      "state_province_territory": "geo_loc_name_state_province_territory",
+      "alias": "lineage",
+      "purpose_of_sampling": "purpose_of_sampling_details",
+      "purpose_of_sequencing": "purpose_of_sequencing_details",
+      "last_updated": "last_updated"
+    },
+    "Gisaid_EpiCoV": {
+      "isolate": "isolate",
+      "lineage": "raw_lineage",
+      "organism": "organism",
+      "gisaid_accession": "gisaid_accession",
+      "host_scientific_name": "host_scientific_name",
+      "sample_collection_date": "sample_collection_date",
+      "country": "geo_loc_name_country",
+      "state_province_territory": "geo_loc_name_state_province_territory",
+      "alias": "lineage",
+      "purpose_of_sampling": "purpose_of_sampling_details",
+      "purpose_of_sequencing": "purpose_of_sequencing_details"
+    },
+    "Gisaid_EpiPox": {
+      "isolate": "isolate",
+      "lineage": "raw_lineage",
+      "organism": "organism",
+      "gisaid_accession": "gisaid_accession",
+      "host_scientific_name": "host_scientific_name",
+      "sample_collection_date": "sample_collection_date",
+      "country": "geo_loc_name_country",
+      "state_province_territory": "geo_loc_name_state_province_territory",
+      "alias": "lineage",
+      "purpose_of_sampling": "purpose_of_sampling_details",
+      "purpose_of_sequencing": "purpose_of_sequencing_details"
+    },
+    "NCBIVirus_EpiPox": {
+      "isolate": "isolate",
+      "lineage": "raw_lineage",
+      "organism": "organism",
+      "gisaid_accession": "gisaid_accession",
+      "host_scientific_name": "host_scientific_name",
+      "sample_collection_date": "sample_collection_date",
+      "country": "geo_loc_name_country",
+      "state_province_territory": "geo_loc_name_state_province_territory",
+      "alias": "lineage",
+      "purpose_of_sampling": "purpose_of_sampling_details",
+      "purpose_of_sequencing": "purpose_of_sequencing_details"
+    },
+    "Gisaid_EpiFlu": {
+      "isolate": "isolate",
+      "lineage": "raw_lineage",
+      "organism": "organism",
+      "gisaid_accession": "gisaid_accession",
+      "host_scientific_name": "host_scientific_name",
+      "sample_collection_date": "sample_collection_date",
+      "country": "geo_loc_name_country",
+      "state_province_territory": "geo_loc_name_state_province_territory",
+      "alias": "lineage",
+      "purpose_of_sampling": "purpose_of_sampling_details",
+      "purpose_of_sequencing": "purpose_of_sequencing_details"
+    }
+}

assets/metadata_conf/metadata_flu.yaml

@@ -0,0 +1,48 @@
+# This file contains a high-level summary of metadata columns across different sources.
+# general format of csv file is to be tab-demilimited, NO spaces in column headers: underscore (_) is used as best practice.
+# gz compressed file
+
+# Required metadata columns
+Format:
+  sep: "tab" # options tab
+  suffix: ".gz"
+
+Required:
+  isolate: True
+  lineage: True
+  sample_collection_date: True
+  country: True
+  state_province_territory: True
+
+# Option metadata columns
+Optional:
+  gene_name: False
+  organism: True
+  host_scientific_name: True
+  host_gender: False
+  host_age_bin: False
+  submission_date: False
+  gisaid_accession: True
+  length: False
+  purpose_of_sampling: False
+  purpose_of_sequencing: False
+  alias: True
+  Type: False
+  Subtype: False
+  SubLineageGroup: False
+  Clade: False
+  last_updated: True
+
+# Gisaid_EpiCoV metadata columns
+Gisaid_EpiFlu:
+  isolate: "isolate"
+  lineage: "raw_lineage"
+  organism: "organism"
+  gisaid_accession: "gisaid_accession"
+  host_scientific_name: "host_scientific_name"
+  sample_collection_date: "sample_collection_date"
+  country: "geo_loc_name_country"
+  state_province_territory: "geo_loc_name_state_province_territory"
+  alias: "lineage"
+  purpose_of_sampling: "purpose_of_sampling_details"
+  purpose_of_sequencing: "purpose_of_sequencing_details"

assets/metadata_conf/metadata_mpox.yaml

@@ -0,0 +1,76 @@
+# This file contains a high-level summary of metadata columns across different sources.
+# general format of csv file is to be tab-demilimited, NO spaces in column headers: underscore (_) is used as best practice.
+# gz compressed file
+
+# Required metadata columns
+Format:
+  sep: "tab" # options tab
+  suffix: ".gz"
+
+Required:
+  isolate: True
+  lineage: True
+  sample_collection_date: True
+  country: True
+  state_province_territory: True
+
+# Option metadata columns
+Optional:
+  gene_name: False
+  organism: True
+  host_scientific_name: True
+  host_gender: False
+  host_age_bin: False
+  submission_date: False
+  gisaid_accession: False
+  length: False
+  purpose_of_sampling: False
+  purpose_of_sequencing: False
+  alias: False
+  Type: False
+  Subtype: False
+  SubLineageGroup: False
+  clade: True
+  last_updated: False
+
+# Gisaid_mpox metadata columns
+Pathoplexus_mpox:
+  isolate: "accessionversion"
+  lineage: "lineage"
+  organism: "ncbivirusname"
+  host_scientific_name: "hostnamescientific"
+  sample_collection_date: "samplecollectiondate"
+  country: "geoloccountry"
+  state_province_territory: "geoloccity"
+  alias: "lineage"
+  purpose_of_sampling: "purposeofsampling"
+  purpose_of_sequencing: "purposeofsequencing"
+  clade: "clade"
+
+# Gisaid_EpiPox metadata columns
+Gisaid_EpiPox:
+  isolate: "isolate"
+  lineage: "raw_lineage"
+  organism: "organism"
+  gisaid_accession: "gisaid_accession"
+  host_scientific_name: "host_scientific_name"
+  sample_collection_date: "sample_collection_date"
+  country: "geo_loc_name_country"
+  state_province_territory: "geo_loc_name_state_province_territory"
+  alias: "lineage"
+  purpose_of_sampling: "purpose_of_sampling_details"
+  purpose_of_sequencing: "purpose_of_sequencing_details"
+
+# NCBIVirus_EpiPox metadata columns
+NCBIVirus_EpiPox:
+  isolate: "isolate"
+  lineage: "raw_lineage"
+  organism: "organism"
+  gisaid_accession: "gisaid_accession"
+  host_scientific_name: "host_scientific_name"
+  sample_collection_date: "sample_collection_date"
+  country: "geo_loc_name_country"
+  state_province_territory: "geo_loc_name_state_province_territory"
+  alias: "lineage"
+  purpose_of_sampling: "purpose_of_sampling_details"
+  purpose_of_sequencing: "purpose_of_sequencing_details"