Merge pull request #571 from 4dn-dcic/hic-update

Hi-C update to 0.3.0
4dn-dcic · May 24, 2024 · 7f9c8f2 · 7f9c8f2
2 parents f2c412d + f0b8237
commit 7f9c8f2
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 20 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -7,6 +7,12 @@ foursight
 Change Log
 ----------
 
+4.7.0
+=====
+
+# Add new version (0.3.0) of Hi-C pipeline and workflows to accepted versions
+`PR 571: Hi-C update to 0.3.0 <https://github.com/4dn-dcic/foursight/pull/571>`_
+
 4.6.0
 =====
 * Fix calls to get_es_metadata in checks/audit_checks.py to work when ES_HOST_LOCAL is set.

diff --git a/chalicelib_fourfront/checks/helpers/wfr_utils.py b/chalicelib_fourfront/checks/helpers/wfr_utils.py
@@ -31,19 +31,19 @@
     },
     "bwa-mem": {
         "run_time": 50,
-        "accepted_versions": ["0.2.6"]
+        "accepted_versions": ["0.2.6", "0.3.0"]
     },
     "pairsqc-single": {
         "run_time": 100,
         "accepted_versions": ["0.2.5", "0.2.6"]
     },
     "hi-c-processing-bam": {
         "run_time": 200,
-        "accepted_versions": ["0.2.6"]
+        "accepted_versions": ["0.2.6", "0.3.0"]
     },
     "hi-c-processing-pairs": {
         "run_time": 200,
-        "accepted_versions": ["0.2.6", "0.2.7"]
+        "accepted_versions": ["0.3.0"]
     },
     "hi-c-processing-pairs-nore": {
         "run_time": 200,
@@ -159,27 +159,27 @@
 # accepted versions for completed pipelines
 accepted_versions = {
     # OFFICIAL
-    'in situ Hi-C':  ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7"],
+    'in situ Hi-C':  ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # OFFICIAL
-    'Dilution Hi-C': ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7"],
+    'Dilution Hi-C': ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # OFFICIAL
-    'TCC':           ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7"],
+    'TCC':           ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # OFFICIAL  # NO-RE
-    'DNase Hi-C':    ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7"],
+    'DNase Hi-C':    ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # OFFICIAL  # NO-NORM
-    'Capture Hi-C':  ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7"],
+    'Capture Hi-C':  ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # OFFICIAL  # NO-RE
-    'Micro-C':       ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7"],
+    'Micro-C':       ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # Preliminary - Released to network  # NO-RE NO-NORM
-    'ChIA-PET':      ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7"],
+    'ChIA-PET':      ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # Preliminary - Released to network  # NO-RE NO-NORM
-    'in situ ChIA-PET': ["HiC_Pipeline_0.2.7"],
+    'in situ ChIA-PET': ["HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # Preliminary - Released to network  # NO-RE NO-NORM
-    'TrAC-loop':     ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7"],
+    'TrAC-loop':     ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # Preliminary - Released to network  # NO-NORM
-    'PLAC-seq':      ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7"],
+    'PLAC-seq':      ["HiC_Pipeline_0.2.6", "HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # Preliminary - Released to network  # NO-NORM
-    'HiChIP': ["HiC_Pipeline_0.2.7"],
+    'HiChIP': ["HiC_Pipeline_0.2.7", "HiC_Pipeline_0.3.0"],
     # bwa mem # handled manually for now
     'MARGI':         ['MARGI_Pipeline_1.1.1_dcic_4'],
     # Preliminary -  Don't release - (Released to network is pending approval from Belmont lab)
@@ -1410,14 +1410,23 @@ def patch_complete_data(patch_data, pipeline_type, auth, move_to_pc=False, pc_ap
             This is relevant for pipelines that produce files to be added to datasets upon which a pipeline (eg. Hi-C)
             has already been run - eg. Compartment Caller or Insulation Score/Boundaries
     """
-    titles = {"hic": "HiC Processing Pipeline - Preliminary Files",
+    titles = {"hic": "HiC Processing Pipeline - v0.3.0",
               "repliseq": "Repli-Seq Pipeline - Preliminary Files",
               'chip': "ENCODE ChIP-Seq Pipeline - Preliminary Files",
               'atac': "ENCODE ATAC-Seq Pipeline - Preliminary Files",
               'margi': "iMARGI Processing Pipeline - Preliminary Files",
               'rnaseq': "ENCODE RNA-Seq Pipeline - Preliminary Files",
               'insulation_scores_and_boundaries': "Insulation scores and boundaries calls - Preliminary Files",
               'compartments': "Compartments Signals - Preliminary Files"}
+
+    descriptions = {'hic': ("These are files generated using the updated Hi-C processing pipeline. "
+            "They should be largely similar to those available in the Processed Files tab, which were generated "
+            "with the previous version of the standard pipeline.  One potential difference of note is that the "
+            "version of cooler used to generate the mcool file has a bug fix to prevent a pixel duplication "
+            "issue which is observed in some files generated by the previous version of the pipeline.  Another "
+            "notable difference is that a filter is applied to remove reads with MAPQ scores below 30 prior "
+            "to mcool file generation.")}
+
     """move files to other processed_files field."""
     if not patch_data.get('patch_opf'):
         return ['no content in patch_opf, skipping']
@@ -1484,6 +1493,10 @@ def patch_complete_data(patch_data, pipeline_type, auth, move_to_pc=False, pc_ap
             new_data = {'title': pc_set_title,
                         'type': 'preliminary',
                         'files': list_pc}
+
+            if pipeline_type in descriptions:
+                new_data["description"] = descriptions[pipeline_type]
+
             patch_val.append(new_data)
             patch_body = {'other_processed_files': patch_val}
             ff_utils.patch_metadata(patch_body, obj_id=acc, key=auth)

diff --git a/chalicelib_fourfront/checks/helpers/wfrset_utils.py b/chalicelib_fourfront/checks/helpers/wfrset_utils.py
@@ -58,7 +58,7 @@ def step_settings(step_name, my_organism, attribution, overwrite=None):
     },
     {
         'app_name': 'bwa-mem',
-        'workflow_uuid': '3feedadc-50f9-4bb4-919b-09a8b731d0cc',
+        'workflow_uuid': '60952f14-693a-434b-9cb2-face065620f6',
         'parameters': {"nThreads": 16},
         'custom_pf_fields': {
             'out_bam': {
@@ -69,7 +69,7 @@ def step_settings(step_name, my_organism, attribution, overwrite=None):
     },
     {
         'app_name': 'hi-c-processing-bam',
-        'workflow_uuid': '023bfb3e-9a8b-42b9-a9d4-216079526f68',
+        'workflow_uuid': '6b2f1481-163e-4099-ae9b-ee49587107d3',
         'parameters': {"nthreads_merge": 16, "nthreads_parse_sort": 16},
         'custom_pf_fields': {
             'annotated_bam': {
@@ -84,8 +84,9 @@ def step_settings(step_name, my_organism, attribution, overwrite=None):
     },
     {
         'app_name': 'hi-c-processing-pairs',
-        'workflow_uuid': '4dn-dcic-lab:wf-hi-c-processing-pairs-0.2.7',
-        'parameters': {"nthreads": 4,
+        'workflow_uuid': '4dn-dcic-lab:wf-hi-c-processing-pairs-0.3.0',
+        'config': {"instance_type": 'r5a.2xlarge'},
+        'parameters': {"nthreads": 8,
                        "maxmem": "32g",
                        "max_split_cooler": 10,
                        "no_balance": False

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "foursight"
-version = "4.6.0"
+version = "4.7.0"
 description = "Serverless Chalice Application for Monitoring"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"