From 3cc616fd9eb6af04fda86ad0c48495939ba40f8b Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Thu, 2 Feb 2023 11:03:08 +0000 Subject: [PATCH 01/10] Add initial Nextflow schema for input parameters --- nextflow_schema.json | 97 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 nextflow_schema.json diff --git a/nextflow_schema.json b/nextflow_schema.json new file mode 100644 index 00000000..c00b894d --- /dev/null +++ b/nextflow_schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOUR_PIPELINE/master/nextflow_schema.json", + "title": "Nextflow pipeline parameters", + "description": "This pipeline uses Nextflow and processes some kind of data. The JSON Schema was built using the nf-core pipeline schema builder.", + "type": "object", + "properties": { + "in": { + "type": "string", + "format": "directory-path" + }, + "start_at": { + "type": "string", + "description": "Name of the first step to be executed by the pipeline", + "enum": [ + "illumination", + "registration", + "background", + "dearray", + "segmentation", + "watershed", + "quantification", + "downstream" + ], + "default": "registration" + }, + "stop_at": { + "type": "string", + "description": "Name of the final step to be executed by the pipeline", + "default": "quantification", + "enum": [ + "illumination", + "registration", + "background", + "dearray", + "segmentation", + "watershed", + "quantification", + "downstream" + ] + }, + "tma": { + "type": "boolean", + "description": "If true, MCMICRO treats input data as a TMA. If false, the input is assumed to be a whole-slide image" + }, + "segmentation_channel": { + "type": "string", + "description": "One or more channels to use for segmentation, specified using 1-based indexing; values will be forwarded to all segmentation modules", + "default": "1" + }, + "segmentation_recyze": { + "type": "boolean", + "description": "Whether the image should be reduced to the channels specified in segmentation-channel prior to being provided to the segmentation modules." + }, + "ilastik_model": { + "type": "string", + "description": "A custom .ilp file to be used as the classifier model for ilastik", + "pattern": "^.+\\.ilp$", + "format": "file-path" + }, + "segmentation": { + "type": "string", + "description": "A list of segmentation modules to run", + "default": "unmicst", + "enum": [ + "unmicst", + "ilastik" + ] + }, + "downstream": { + "type": "string", + "description": "A list of downstream modules to run", + "default": "scimap" + }, + "qc_files": { + "type": "string", + "description": "Wheter QC files should be copied, moved or symbolically linked from work directories to the project directory", + "enum": [ + "copy", + "move", + "symlink" + ], + "default": "copy" + }, + "viz": { + "type": "boolean", + "description": "Whether to generate a vizualization with Auto-Minerva" + }, + "background": { + "type": "boolean", + "description": "Wheter background subtraction should be performed, and the computed intermediates used in further processing" + } + }, + "required": [ + "in" + ] +} From bfa60a01bce7cf57b81152493d0858698ab7b427 Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Thu, 2 Feb 2023 11:06:01 +0000 Subject: [PATCH 02/10] Update nextflow_schema.json --- nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index c00b894d..8d77ac05 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/YOUR_PIPELINE/master/nextflow_schema.json", + "$id": "https://raw.githubusercontent.com/labsyspharm/mcmicro/master/nextflow_schema.json", "title": "Nextflow pipeline parameters", "description": "This pipeline uses Nextflow and processes some kind of data. The JSON Schema was built using the nf-core pipeline schema builder.", "type": "object", From 28036b76eed229455540d0180c91054e914701c2 Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Mon, 3 Apr 2023 19:15:40 +0100 Subject: [PATCH 03/10] Update nextflow schema w cache id 1680544409_a7171b6f8d1a --- nextflow_schema.json | 260 +++++++++++++++++++++++++++---------------- 1 file changed, 164 insertions(+), 96 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 8d77ac05..d8f780c0 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,97 +1,165 @@ { - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/labsyspharm/mcmicro/master/nextflow_schema.json", - "title": "Nextflow pipeline parameters", - "description": "This pipeline uses Nextflow and processes some kind of data. The JSON Schema was built using the nf-core pipeline schema builder.", - "type": "object", - "properties": { - "in": { - "type": "string", - "format": "directory-path" - }, - "start_at": { - "type": "string", - "description": "Name of the first step to be executed by the pipeline", - "enum": [ - "illumination", - "registration", - "background", - "dearray", - "segmentation", - "watershed", - "quantification", - "downstream" - ], - "default": "registration" - }, - "stop_at": { - "type": "string", - "description": "Name of the final step to be executed by the pipeline", - "default": "quantification", - "enum": [ - "illumination", - "registration", - "background", - "dearray", - "segmentation", - "watershed", - "quantification", - "downstream" - ] - }, - "tma": { - "type": "boolean", - "description": "If true, MCMICRO treats input data as a TMA. If false, the input is assumed to be a whole-slide image" - }, - "segmentation_channel": { - "type": "string", - "description": "One or more channels to use for segmentation, specified using 1-based indexing; values will be forwarded to all segmentation modules", - "default": "1" - }, - "segmentation_recyze": { - "type": "boolean", - "description": "Whether the image should be reduced to the channels specified in segmentation-channel prior to being provided to the segmentation modules." - }, - "ilastik_model": { - "type": "string", - "description": "A custom .ilp file to be used as the classifier model for ilastik", - "pattern": "^.+\\.ilp$", - "format": "file-path" - }, - "segmentation": { - "type": "string", - "description": "A list of segmentation modules to run", - "default": "unmicst", - "enum": [ - "unmicst", - "ilastik" - ] - }, - "downstream": { - "type": "string", - "description": "A list of downstream modules to run", - "default": "scimap" - }, - "qc_files": { - "type": "string", - "description": "Wheter QC files should be copied, moved or symbolically linked from work directories to the project directory", - "enum": [ - "copy", - "move", - "symlink" - ], - "default": "copy" - }, - "viz": { - "type": "boolean", - "description": "Whether to generate a vizualization with Auto-Minerva" - }, - "background": { - "type": "boolean", - "description": "Wheter background subtraction should be performed, and the computed intermediates used in further processing" - } - }, - "required": [ - "in" - ] -} + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/labsyspharm/mcmicro/master/nextflow_schema.json", + "title": "Nextflow pipeline parameters", + "description": "This pipeline uses Nextflow and processes some kind of data. The JSON Schema was built using the nf-core pipeline schema builder.", + "type": "object", + "definitions": { + "mandatory_parameters": { + "title": "Mandatory parameters", + "type": "object", + "description": "", + "default": "", + "properties": { + "in": { + "type": "string", + "format": "directory-path", + "description": "Location of the data" + } + }, + "required": [ + "in" + ] + }, + "optional_parameters": { + "title": "Optional parameters", + "type": "object", + "description": "", + "default": "", + "properties": { + "sample_name": { + "type": "string", + "description": "The name of the experiment/specimen" + }, + "start-at": { + "type": "string", + "description": "Name of the first step to be executed by the pipeline. Must be one of `illumination`, `registration`, `dearray` (TMA only), `probability-maps`, `segmentation`, `quantification`, `cell-states`", + "enum": [ + "illumination", + "registration", + "background", + "dearray", + "segmentation", + "watershed", + "quantification", + "downstream" + ], + "default": "registration" + }, + "stop-at": { + "type": "string", + "description": "Name of the final step to be executed by the pipeline. Spans the same vocabulary as `--start-at`.", + "default": "quantification", + "enum": [ + "illumination", + "registration", + "background", + "dearray", + "segmentation", + "watershed", + "quantification", + "downstream" + ] + }, + "tma": { + "type": "boolean", + "description": "If specified, mcmicro treats input data as a TMA. If omitted, the input is assumed to be a whole-slide image." + }, + "background": { + "type": "boolean", + "description": "If specified, mcmicro treats input data as if autofluorescence channels should be subtracted." + }, + "segmentation": { + "type": "string", + "description": "A list of segmentation modules to run", + "default": "unmicst", + "enum": [ + "unmicst", + "ilastik" + ] + }, + "downstream": { + "type": "string", + "description": "A list of downstream modules to run", + "default": "scimap" + }, + "viz": { + "type": "boolean", + "description": "Whether to generate a vizualization with Auto-Minerva" + }, + "qc-files": { + "type": "string", + "description": "Wheter QC files should be copied, moved or symbolically linked from work directories to the project directory", + "enum": [ + "copy", + "move", + "symlink" + ], + "default": "copy" + }, + "segmentation-channel": { + "type": "string", + "description": "One or more channels to use for segmentation, specified using 1-based indexing; values will be forwarded to all segmentation modules", + "default": "1" + }, + "segmentation-recyze": { + "type": "boolean", + "description": "Whether the image should be reduced to the channels specified in segmentation-channel prior to being provided to the segmentation modules." + } + } + }, + "parameters_for_individual_modules": { + "title": "Parameters for individual modules", + "type": "object", + "description": "Module-specific parameters can be specified using the various `opts` arguments, followed by the parameters enclosed inside single quotes `'`:", + "default": "", + "properties": { + "ashlar-opts": { + "type": "string", + "description": "Arguments to ASHLAR" + }, + "coreograph-opts": { + "type": "string", + "description": "Arguments to Coreograph" + }, + "unmicst-opts": { + "type": "string", + "description": "Arguments to UnMicst" + }, + "ilastik-opts": { + "type": "string", + "description": "Arguments to Ilastik" + }, + "s3seg-opts": { + "type": "string", + "description": "Arguments to S3Segmenter" + }, + "mcquant-opts": { + "type": "string", + "description": "Arguments to MCQUANT" + }, + "naievestates-opts": { + "type": "string", + "description": "Arguments to naievestates" + }, + "ilastik-model": { + "type": "string", + "description": "A custom `.ilp` file to be used as the classifier model for ilastik.", + "pattern": "^.+\\.ilp$", + "format": "file-path" + } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/mandatory_parameters" + }, + { + "$ref": "#/definitions/optional_parameters" + }, + { + "$ref": "#/definitions/parameters_for_individual_modules" + } + ] +} \ No newline at end of file From d5fd67a420f3faa01847661d863eb8d603e336ad Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Mon, 3 Apr 2023 19:37:24 +0100 Subject: [PATCH 04/10] Update schema valid values --- nextflow_schema.json | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index d8f780c0..171f5dfc 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -76,12 +76,22 @@ "enum": [ "unmicst", "ilastik" + "cypository", + "mesmer", + "cellpose" ] }, "downstream": { "type": "string", "description": "A list of downstream modules to run", - "default": "scimap" + "default": "scimap", + "enum": [ + "naivestates", + "scimap", + "fastpg", + "scanpy", + "flowsom" + ] }, "viz": { "type": "boolean", @@ -118,10 +128,6 @@ "type": "string", "description": "Arguments to ASHLAR" }, - "coreograph-opts": { - "type": "string", - "description": "Arguments to Coreograph" - }, "unmicst-opts": { "type": "string", "description": "Arguments to UnMicst" From 870dc9b76a7863dab6ee31ecb0ee3b322760f339 Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Mon, 3 Apr 2023 19:40:19 +0100 Subject: [PATCH 05/10] Missing comma in schema --- nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 171f5dfc..12daf982 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -75,7 +75,7 @@ "default": "unmicst", "enum": [ "unmicst", - "ilastik" + "ilastik", "cypository", "mesmer", "cellpose" From 5391268873bf6cedd1ea168bbe214707f522ea7c Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Mon, 3 Apr 2023 19:43:00 +0100 Subject: [PATCH 06/10] Add segmentation combinations --- nextflow_schema.json | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 12daf982..6e9ea84e 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -78,7 +78,33 @@ "ilastik", "cypository", "mesmer", - "cellpose" + "cellpose", + ["unmicst", "ilastik"], + ["unmicst", "cypository"], + ["unmicst", "mesmer"], + ["unmicst", "cellpose"], + ["ilastik", "cypository"], + ["ilastik", "mesmer"], + ["ilastik", "cellpose"], + ["cypository", "mesmer"], + ["cypository", "cellpose"], + ["mesmer", "cellpose"], + ["unmicst", "ilastik", "cypository"], + ["unmicst", "ilastik", "mesmer"], + ["unmicst", "ilastik", "cellpose"], + ["unmicst", "cypository", "mesmer"], + ["unmicst", "cypository", "cellpose"], + ["unmicst", "mesmer", "cellpose"], + ["ilastik", "cypository", "mesmer"], + ["ilastik", "cypository", "cellpose"], + ["ilastik", "mesmer", "cellpose"], + ["cypository", "mesmer", "cellpose"], + ["unmicst", "ilastik", "cypository", "mesmer"], + ["unmicst", "ilastik", "cypository", "cellpose"], + ["unmicst", "ilastik", "mesmer", "cellpose"], + ["unmicst", "cypository", "mesmer", "cellpose"], + ["ilastik", "cypository", "mesmer", "cellpose"], + ["unmicst", "ilastik", "cypository", "mesmer", "cellpose"] ] }, "downstream": { From 13e3085f6a214378b2d0d26f8ac1a536eb2d70fe Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Mon, 3 Apr 2023 19:45:35 +0100 Subject: [PATCH 07/10] Add downstream combinations --- nextflow_schema.json | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 6e9ea84e..112db30a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -116,7 +116,32 @@ "scimap", "fastpg", "scanpy", - "flowsom" + "flowsom", + ["naivestates", "scimap"], + ["naivestates", "fastpg"], + ["naivestates", "scanpy"], + ["naivestates", "flowsom"], + ["scimap", "fastpg"], + ["scimap", "scanpy"], + ["scimap", "flowsom"], + ["fastpg", "scanpy"], + ["fastpg", "flowsom"], + ["scanpy", "flowsom"], + ["naivestates", "scimap", "fastpg"], + ["naivestates", "scimap", "scanpy"], + ["naivestates", "scimap", "flowsom"], + ["naivestates", "fastpg", "scanpy"], + ["naivestates", "fastpg", "flowsom"], + ["naivestates", "scanpy", "flowsom"], + ["scimap", "fastpg", "scanpy"], + ["scimap", "fastpg", "flowsom"], + ["scimap", "scanpy", "flowsom"], + ["naivestates", "scimap", "fastpg", "scanpy"], + ["naivestates", "scimap", "fastpg", "flowsom"], + ["naivestates", "scimap", "scanpy", "flowsom"], + ["naivestates", "fastpg", "scanpy", "flowsom"], + ["scimap", "fastpg", "scanpy", "flowsom"], + ["naivestates", "scimap", "fastpg", "scanpy", "flowsom"] ] }, "viz": { From d9fd6cae5cad96894a8f94bbff0c32a08715622e Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Wed, 28 Jun 2023 11:16:23 +0100 Subject: [PATCH 08/10] Refinements to schema --- nextflow_schema.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 112db30a..ca3c7ee0 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema", "$id": "https://raw.githubusercontent.com/labsyspharm/mcmicro/master/nextflow_schema.json", - "title": "Nextflow pipeline parameters", - "description": "This pipeline uses Nextflow and processes some kind of data. The JSON Schema was built using the nf-core pipeline schema builder.", + "title": "MCMICRO pipeline parameters", + "description": "The MCMICRO pipeline uses Nextflow to transforms multi-channel whole-slide images into single-cell data. The JSON Schema was built using the nf-core pipeline schema builder.", "type": "object", "definitions": { "mandatory_parameters": { @@ -14,7 +14,7 @@ "in": { "type": "string", "format": "directory-path", - "description": "Location of the data" + "description": "Location of the input directory. Must contain markers.csv (comma-delimited file containing a column titled `marker_name`) and appropriate directory structure. See documentation at mcmicro.org" } }, "required": [ @@ -33,7 +33,7 @@ }, "start-at": { "type": "string", - "description": "Name of the first step to be executed by the pipeline. Must be one of `illumination`, `registration`, `dearray` (TMA only), `probability-maps`, `segmentation`, `quantification`, `cell-states`", + "description": "Name of the first step to be executed by the pipeline. Must be one of `illumination`, `registration`, `dearray` (TMA only), `probability-maps`, `segmentation`, `watershed`, `quantification`, or `downstream`", "enum": [ "illumination", "registration", @@ -48,7 +48,7 @@ }, "stop-at": { "type": "string", - "description": "Name of the final step to be executed by the pipeline. Spans the same vocabulary as `--start-at`.", + "description": "Name of the final step to be executed by the pipeline. Must be one of `illumination`, `registration`, `dearray` (TMA only), `probability-maps`, `segmentation`, `watershed`, `quantification`, or `downstream`", "default": "quantification", "enum": [ "illumination", @@ -63,11 +63,11 @@ }, "tma": { "type": "boolean", - "description": "If specified, mcmicro treats input data as a TMA. If omitted, the input is assumed to be a whole-slide image." + "description": "If selected, MCMICRO treats input data as a tisssue microarray and will attempt to de-array cores using `coreograph`. If omitted, the input is assumed to be a whole-slide image." }, "background": { "type": "boolean", - "description": "If specified, mcmicro treats input data as if autofluorescence channels should be subtracted." + "description": "If selected, MCMICRO treats input data as if background channels should be subtracted. Background channels to be specified in `markers.csv` as a column called `background` containing `true` OR `false`." }, "segmentation": { "type": "string", @@ -146,11 +146,11 @@ }, "viz": { "type": "boolean", - "description": "Whether to generate a vizualization with Auto-Minerva" + "description": "If selected a Minerva story will be generated in sequential groups of 4 channels using auto-Minerva" }, "qc-files": { "type": "string", - "description": "Wheter QC files should be copied, moved or symbolically linked from work directories to the project directory", + "description": "Should QC files should be copied, moved or symbolically linked from work directories to the project directory", "enum": [ "copy", "move", @@ -165,7 +165,7 @@ }, "segmentation-recyze": { "type": "boolean", - "description": "Whether the image should be reduced to the channels specified in segmentation-channel prior to being provided to the segmentation modules." + "description": "If selected the image will be reduced to the channels specified in segmentation-channel prior to being provided to the segmentation modules." } } }, From 477b80bd7405fc2b12502b1e469d28e2ec9ff0b2 Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Wed, 28 Jun 2023 11:17:56 +0100 Subject: [PATCH 09/10] Refine start-at and stop-at descriptions --- nextflow_schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index ca3c7ee0..a87c0bc6 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -33,7 +33,7 @@ }, "start-at": { "type": "string", - "description": "Name of the first step to be executed by the pipeline. Must be one of `illumination`, `registration`, `dearray` (TMA only), `probability-maps`, `segmentation`, `watershed`, `quantification`, or `downstream`", + "description": "Name of the first step to be executed by the pipeline.", "enum": [ "illumination", "registration", @@ -48,7 +48,7 @@ }, "stop-at": { "type": "string", - "description": "Name of the final step to be executed by the pipeline. Must be one of `illumination`, `registration`, `dearray` (TMA only), `probability-maps`, `segmentation`, `watershed`, `quantification`, or `downstream`", + "description": "Name of the final step to be executed by the pipeline.", "default": "quantification", "enum": [ "illumination", From 213b0471821d94744fe8bde2976177672c21b245 Mon Sep 17 00:00:00 2001 From: Adam Taylor Date: Wed, 28 Jun 2023 11:20:36 +0100 Subject: [PATCH 10/10] Refine background option description --- nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index a87c0bc6..cdbd6f8d 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -67,7 +67,7 @@ }, "background": { "type": "boolean", - "description": "If selected, MCMICRO treats input data as if background channels should be subtracted. Background channels to be specified in `markers.csv` as a column called `background` containing `true` OR `false`." + "description": "If selected, MCMICRO treats input data as if background channels should be subtracted using the `Backsub` module. See mcmicro.org documentation for details on additional `background` and `remove` columns required in the `markers.csv` file." }, "segmentation": { "type": "string",