diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile deleted file mode 100644 index a75392d9..00000000 --- a/.gitpod.Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM gitpod/workspace-full:latest - -USER root -# Install util tools. -RUN apt-get update \ - && apt-get install -y \ - apt-utils \ - sudo \ - git \ - less \ - wget - -RUN mkdir -p /workspace/data \ - && chown -R gitpod:gitpod /workspace/data - -RUN mkdir /home/gitpod/.conda -# Install conda -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh - -RUN chown -R gitpod:gitpod /opt/conda \ - && chmod -R 777 /opt/conda \ - && chown -R gitpod:gitpod /home/gitpod/.conda \ - && chmod -R 777 /home/gitpod/.conda - -# Give back control -USER root - -# Cleaning -RUN apt-get clean diff --git a/.gitpod.yml b/.gitpod.yml index f4549be3..0fa6d1f6 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -13,8 +13,8 @@ github: # add a "Review in Gitpod" button to pull requests (defaults to false) addBadge: false -image: - file: .gitpod.Dockerfile +# Old container: nfcore/gitpod:latest +image: nfcore/gitpod:latest # List the start up tasks. Learn more https://www.gitpod.io/docs/config-start-tasks/ tasks: @@ -22,18 +22,9 @@ tasks: command: gp await-port 23000 && gp preview https://training.seqera.io - name: Download Nextflow Tutorial - init: | - echo 'init script' # runs during prebuild - echo 'start script' - command: | - curl -s https://get.nextflow.io | bash - chmod +x nextflow - sudo mv nextflow /usr/local/bin/ - docker pull nextflow/rnaseq-nf - sudo apt install -y tree - sudo apt install -y graphviz - unset JAVA_TOOL_OPTIONS - alias conda_activate=". /opt/conda/etc/profile.d/conda.sh; conda activate base" cd nf-training + conda init bash + unset JAVA_TOOL_OPTIONS + docker pull nextflow/rnaseq-nf diff --git a/asciidocs/channels.adoc b/asciidocs/channels.adoc index 4a29bf8d..3c92c4ba 100644 --- a/asciidocs/channels.adoc +++ b/asciidocs/channels.adoc @@ -360,3 +360,301 @@ process fastqc { } ---- +=== Text files + +The `splitText` operator allows you to split multi-line strings or text file items, emitted by a source channel into chunks containing n lines, which will be emitted by the resulting channel. See: + +---- +Channel + .fromPath('data/meta/random.txt') // <1> + .splitText() // <2> + .view() // <3> +---- + +<1> Instructs Nextflow to make a channel from the path "data/meta/random.txt". +<2> The `splitText` operator splits each item into chunks of one line by default. +<3> View contents of the channel. + + +You can define the number of lines in each chunk by using the parameter `by`, as shown in the following example: + +---- +Channel + .fromPath('data/meta/random.txt') + .splitText( by: 2 ) + .subscribe { + print it; + print "--- end of the chunk ---\n" + } +---- + +TIP: The `subscribe` operator permits to execute a user defined function each time a new value is emitted by the source channel. + +An optional closure can be specified in order to transform the text chunks produced by the operator. The following example shows how to split text files into chunks of 10 lines and transform them into capital letters: + +---- +Channel + .fromPath('data/meta/random.txt') + .splitText( by: 10 ) { it.toUpperCase() } + .view() +---- + +You can also make counts for each line: + +---- +count=0 + +Channel + .fromPath('data/meta/random.txt') + .splitText() + .view { "${count++}: ${it.toUpperCase().trim()}" } + +---- + +Finally, you can also use the operator on plain files (outside of the channel context), as so: + +---- + def f = file('data/meta/random.txt') + def lines = f.splitText() + def count=0 + for( String row : lines ) { + log.info "${count++} ${row.toUpperCase()}" + } +---- + +=== Comma separate values (.csv) + +The `splitCsv` operator allows you to parse text items emitted by a channel, that are formatted using the CSV format. + +It then splits them into records or groups them into a list of records with a specified length. + +In the simplest case, just apply the `splitCsv` operator to a channel emitting a CSV formatted text files or text entries, to view only the first and fourth columns. For example: + +---- + Channel + .fromPath("data/meta/patients_1.csv") + .splitCsv() + // row is a list object + .view { row -> "${row[0]},${row[3]}" } +---- + +When the CSV begins with a header line defining the column names, you can specify the parameter `header: true` which allows you to reference each value by its name, as shown in the following example: + +---- + Channel + .fromPath("data/meta/patients_1.csv") + .splitCsv(header: true) + // row is a list object + .view { row -> "${row.patient_id},${row.num_samples}" } +---- + +Alternatively you can provide custom header names by specifying a the list of strings in the header parameter as shown below: + +---- + Channel + .fromPath("data/meta/patients_1.csv") + .splitCsv(header: ['col1', 'col2', 'col3', 'col4', 'col5'] ) + // row is a list object + .view { row -> "${row.col1},${row.col4}" } +---- + +You can also process multiple csv files at the same time: + +---- + Channel + .fromPath("data/meta/patients_*.csv") // <-- just use a pattern + .splitCsv(header:true) + .view { row -> "${row.patient_id}\t${row.num_samples}" } +---- + +TIP: Notice that you can change the output format simply by adding a different delimiter. + +Finally, you can also operate on csv files outside the channel context, as so: + +---- +def f = file('data/meta/patients_1.csv') + def lines = f.splitCsv() + for( List row : lines ) { + log.info "${row[0]} -- ${row[2]}" + } +---- + +[discrete] +=== Exercise + +Try inputting fastq reads to the RNA-Seq workflow from earlier using `.splitCSV`. + +.Click here for the answer: +[%collapsible] +==== +Add a csv text file containing the following, as example input with the name "fastq.csv": + +[source,nextflow,linenums] +---- +gut,/workspace/nf-training-public/nf-training/data/ggal/gut_1.fq,/workspace/nf-training-public/nf-training/data/ggal/gut_2.fq +---- + +Then replace the input channel for the reads in `script7.nf`. Changing the following lines: + +[source,nextflow,linenums] +---- +Channel + .fromFilePairs( params.reads, checkIfExists: true ) + .into { read_pairs_ch; read_pairs2_ch } +---- + +To a splitCsv channel factory input: + +[source,nextflow,linenums] +---- +Channel + .fromPath("fastq.csv") + .splitCsv() + .view () { row -> "${row[0]},${row[1]},${row[2]}" } + .into { read_pairs_ch; read_pairs2_ch } +---- + +Finally, change the cardinality of the processes that use the input data. For example, for the quantification process I change it from: + +[source,nextflow,linenums] +---- +process quantification { + tag "$sample_id" + + input: + path salmon_index from index_ch + tuple val(sample_id), path(reads) from read_pairs_ch + + output: + path sample_id into quant_ch + + script: + """ + salmon quant --threads $task.cpus --libType=U -i $salmon_index -1 ${reads[0]} -2 ${reads[1]} -o $sample_id + """ +} +---- + +To: + +[source,nextflow,linenums] +---- +process quantification { + tag "$sample_id" + + input: + path salmon_index from index_ch + tuple val(sample_id), path(reads1), path(reads2) from read_pairs_ch + + output: + path sample_id into quant_ch + + script: + """ + salmon quant --threads $task.cpus --libType=U -i $salmon_index -1 ${reads1} -2 ${reads2} -o $sample_id + """ +} +---- + +Repeat for the fastqc step. Now the workflow should run from a CSV file. +==== + +=== Tab separated values (.tsv) + +Parsing tsv files works in a similar way, just adding the `sep:'\t'` option in the `splitCsv` context: + +---- + Channel + .fromPath("data/meta/regions.tsv", checkIfExists:true) + // use `sep` option to parse TAB separated files + .splitCsv(sep:'\t') + // row is a list object + .view() +---- + +[discrete] +=== Exercise + +Try using the tab separation technique on the file "data/meta/regions.tsv", but print just the first column, and remove the header. + +.Answer: +[%collapsible] +==== + Channel + .fromPath("data/meta/regions.tsv", checkIfExists:true) + // use `sep` option to parse TAB separated files + .splitCsv(sep:'\t', header:true ) + // row is a list object + .view { row -> "${row.patient_id}" } +==== + +== More complex file formats + +=== JSON + +We can also easily parse the JSON file format using the following groovy schema: + +---- +import groovy.json.JsonSlurper + +def f = file('data/meta/regions.json') +def records = new JsonSlurper().parse(f) + + +for( def entry : records ) { + log.info "$entry.patient_id -- $entry.feature" +} +---- + +IMPORTANT: When using an older JSON version, you may need to replace `parse(f)` with `parseText(f.text)` + +=== YAML + +In a similar way, this is a way to parse YAML files: + +---- +import org.yaml.snakeyaml.Yaml + +def f = file('data/meta/regions.json') +def records = new Yaml().load(f) + + +for( def entry : records ) { + log.info "$entry.patient_id -- $entry.feature" +} +---- + +=== Storage of parsers into modules + +The best way to store parser scripts is to keep them in a nextflow module file. + +This follows the DSL2 way of working. + +See the following nextflow script: + +---- +nextflow.preview.dsl=2 + +include{ parseJsonFile } from './modules/parsers.nf' + +process foo { + input: + tuple val(meta), path(data_file) + + """ + echo your_command $meta.region_id $data_file + """ +} + +workflow { + Channel.fromPath('data/meta/regions*.json') \ + | flatMap { parseJsonFile(it) } \ + | map { entry -> tuple(entry,"/some/data/${entry.patient_id}.txt") } \ + | foo +} +---- + +To get this script to work, first we need to create a file called `parsers.nf`, and store it in the modules folder in the current directory. + +This file should have the `parseJsonFile` function present, then Nextflow will use this as a custom function within the workflow scope. + diff --git a/asciidocs/containers.adoc b/asciidocs/containers.adoc index a48ef739..bce3cecc 100644 --- a/asciidocs/containers.adoc +++ b/asciidocs/containers.adoc @@ -381,12 +381,7 @@ Conda is a popular package and environment manager. The built-in support for Con allows Nextflow pipelines to automatically create and activate the Conda environment(s), given the dependencies specified by each process. -For this Gitpod tutorial you need to activate conda by typing: - - -```bash -conda_activate -``` +For this Gitpod tutorial you need to open a new terminal to ensure that conda is activated (see the + button on the terminal). You should now see that the beginning of your command prompt has `(base)` written. If you wish to deactivate conda at any point, simply enter `conda deactivate`. diff --git a/asciidocs/index.adoc b/asciidocs/index.adoc index 1e06deeb..f09a15e5 100644 --- a/asciidocs/index.adoc +++ b/asciidocs/index.adoc @@ -30,5 +30,4 @@ include::config.adoc[] include::executors.adoc[] include::cache_and_resume.adoc[] include::debugging.adoc[] -include::parsing.adoc[] :leveloffset: -1 diff --git a/asciidocs/parsing.adoc b/asciidocs/parsing.adoc deleted file mode 100644 index c58e0eb3..00000000 --- a/asciidocs/parsing.adoc +++ /dev/null @@ -1,222 +0,0 @@ -= Parsing datasets into Nextflow - -An important skill in Nextflow is to know how to properly parse files into your processes. - -== Text files - -The `splitText` operator allows you to split multi-line strings or text file items, emitted by a source channel into chunks containing n lines, which will be emitted by the resulting channel. See: - ----- -Channel - .fromPath('data/meta/random.txt') // <1> - .splitText() // <2> - .view() // <3> ----- - -<1> Instructs Nextflow to make a channel from the path "data/meta/random.txt". -<2> The `splitText` operator splits each item into chunks of one line by default. -<3> View contents of the channel. - - -You can define the number of lines in each chunk by using the parameter `by`, as shown in the following example: - ----- -Channel - .fromPath('data/meta/random.txt') - .splitText( by: 2 ) - .subscribe { - print it; - print "--- end of the chunk ---\n" - } ----- - -TIP: The `subscribe` operator permits to execute a user defined function each time a new value is emitted by the source channel. - -An optional closure can be specified in order to transform the text chunks produced by the operator. The following example shows how to split text files into chunks of 10 lines and transform them into capital letters: - ----- -Channel - .fromPath('data/meta/random.txt') - .splitText( by: 10 ) { it.toUpperCase() } - .view() ----- - -You can also make counts for each line: - ----- -count=0 - -Channel - .fromPath('data/meta/random.txt') - .splitText() - .view { "${count++}: ${it.toUpperCase().trim()}" } - ----- - -Finally, you can also use the operator on plain files (outside of the channel context), as so: - ----- - def f = file('data/meta/random.txt') - def lines = f.splitText() - def count=0 - for( String row : lines ) { - log.info "${count++} ${row.toUpperCase()}" - } ----- - -== Comma separate values (.csv) - -The `splitCsv` operator allows you to parse text items emitted by a channel, that are formatted using the CSV format. - -It then splits them into records or groups them into a list of records with a specified length. - -In the simplest case, just apply the `splitCsv` operator to a channel emitting a CSV formatted text files or text entries, to view only the first and fourth columns. For example: - ----- - Channel - .fromPath("data/meta/patients_1.csv") - .splitCsv() - // row is a list object - .view { row -> "${row[0]},${row[3]}" } ----- - -When the CSV begins with a header line defining the column names, you can specify the parameter `header: true` which allows you to reference each value by its name, as shown in the following example: - ----- - Channel - .fromPath("data/meta/patients_1.csv") - .splitCsv(header: true) - // row is a list object - .view { row -> "${row.patient_id},${row.num_samples}" } ----- - -Alternatively you can provide custom header names by specifying a the list of strings in the header parameter as shown below: - ----- - Channel - .fromPath("data/meta/patients_1.csv") - .splitCsv(header: ['col1', 'col2', 'col3', 'col4', 'col5'] ) - // row is a list object - .view { row -> "${row.col1},${row.col4}" } ----- - -You can also process multiple csv files at the same time: - ----- - Channel - .fromPath("data/meta/patients_*.csv") // <-- just use a pattern - .splitCsv(header:true) - .view { row -> "${row.patient_id}\t${row.num_samples}" } ----- - -TIP: Notice that you can change the output format simply by adding a different delimiter. - -Finally, you can also operate on csv files outside the channel context, as so: - ----- -def f = file('data/meta/patients_1.csv') - def lines = f.splitCsv() - for( List row : lines ) { - log.info "${row[0]} -- ${row[2]}" - } ----- - -== Tab separated values (.tsv) - -Parsing tsv files works in a similar way, just adding the `sep:'\t'` option in the `splitCsv` context: - ----- - Channel - .fromPath("data/meta/regions.tsv", checkIfExists:true) - // use `sep` option to parse TAB separated files - .splitCsv(sep:'\t') - // row is a list object - .view() ----- - -[discrete] -=== Exercise - -Try using the tab separation technique on the file "data/meta/regions.tsv", but print just the first column, and remove the header. - -.Answer: -[%collapsible] -==== - Channel - .fromPath("data/meta/regions.tsv", checkIfExists:true) - // use `sep` option to parse TAB separated files - .splitCsv(sep:'\t', header:true ) - // row is a list object - .view { row -> "${row.patient_id}" } -==== - -== More complex file formats - -=== JSON - -We can also easily parse the JSON file format using the following groovy schema: - ----- -import groovy.json.JsonSlurper - -def f = file('data/meta/regions.json') -def records = new JsonSlurper().parse(f) - - -for( def entry : records ) { - log.info "$entry.patient_id -- $entry.feature" -} ----- - -IMPORTANT: When using an older JSON version, you may need to replace `parse(f)` with `parseText(f.text)` - -=== YAML - -In a similar way, this is a way to parse YAML files: - ----- -import org.yaml.snakeyaml.Yaml - -def f = file('data/meta/regions.json') -def records = new Yaml().load(f) - - -for( def entry : records ) { - log.info "$entry.patient_id -- $entry.feature" -} ----- - -=== Storage of parsers into modules - -The best way to store parser scripts is to keep them in a nextflow module file. - -This follows the DSL2 way of working. - -See the following nextflow script: - ----- -nextflow.preview.dsl=2 - -include{ parseJsonFile } from './modules/parsers.nf' - -process foo { - input: - tuple val(meta), path(data_file) - - """ - echo your_command $meta.region_id $data_file - """ -} - -workflow { - Channel.fromPath('data/meta/regions*.json') \ - | flatMap { parseJsonFile(it) } \ - | map { entry -> tuple(entry,"/some/data/${entry.patient_id}.txt") } \ - | foo -} ----- - -To get this script to work, first we need to create a file called `parsers.nf`, and store it in the modules folder in the current directory. - -This file should have the `parseJsonFile` function present, then Nextflow will use this as a custom function within the workflow scope. - diff --git a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.begin b/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.begin deleted file mode 100644 index e69de29b..00000000 diff --git a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.err b/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.err deleted file mode 100644 index e69de29b..00000000 diff --git a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.log b/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.log deleted file mode 100644 index a1c590d4..00000000 --- a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.log +++ /dev/null @@ -1 +0,0 @@ -WORLD! \ No newline at end of file diff --git a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.out b/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.out deleted file mode 100644 index a1c590d4..00000000 --- a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.out +++ /dev/null @@ -1 +0,0 @@ -WORLD! \ No newline at end of file diff --git a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.run b/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.run deleted file mode 100644 index e02dc40c..00000000 --- a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.run +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# NEXTFLOW TASK: convertToUpper (2) -set -e -set -u -NXF_DEBUG=${NXF_DEBUG:=0}; [[ $NXF_DEBUG > 1 ]] && set -x -NXF_ENTRY=${1:-nxf_main} - - -nxf_sleep() { - sleep $1 2>/dev/null || sleep 1; -} - -nxf_date() { - local ts=$(date +%s%3N); - if [[ ${#ts} == 10 ]]; then echo ${ts}000 - elif [[ $ts == *%3N ]]; then echo ${ts/\%3N/000} - elif [[ $ts == *3N ]]; then echo ${ts/3N/000} - elif [[ ${#ts} == 13 ]]; then echo $ts - else echo "Unexpected timestamp value: $ts"; exit 1 - fi -} - -nxf_env() { - echo '============= task environment =============' - env | sort | sed "s/\(.*\)AWS\(.*\)=\(.\{6\}\).*/\1AWS\2=\3xxxxxxxxxxxxx/" - echo '============= task output ==================' -} - -nxf_kill() { - declare -a children - while read P PP;do - children[$PP]+=" $P" - done < <(ps -e -o pid= -o ppid=) - - kill_all() { - [[ $1 != $$ ]] && kill $1 2>/dev/null || true - for i in ${children[$1]:=}; do kill_all $i; done - } - - kill_all $1 -} - -nxf_mktemp() { - local base=${1:-/tmp} - if [[ $(uname) = Darwin ]]; then mktemp -d $base/nxf.XXXXXXXXXX - else TMPDIR="$base" mktemp -d -t nxf.XXXXXXXXXX - fi -} - -nxf_fs_copy() { - local source=$1 - local target=$2 - local basedir=$(dirname $1) - mkdir -p $target/$basedir - cp -fRL $source $target/$basedir -} - -nxf_fs_move() { - local source=$1 - local target=$2 - local basedir=$(dirname $1) - mkdir -p $target/$basedir - mv -f $source $target/$basedir -} - -nxf_fs_rsync() { - rsync -rRl $1 $2 -} - -on_exit() { - exit_status=${nxf_main_ret:=$?} - printf $exit_status > /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.exitcode - set +u - [[ "$tee1" ]] && kill $tee1 2>/dev/null - [[ "$tee2" ]] && kill $tee2 2>/dev/null - [[ "$ctmp" ]] && rm -rf $ctmp || true - exit $exit_status -} - -on_term() { - set +e - [[ "$pid" ]] && nxf_kill $pid -} - -nxf_launch() { - /bin/bash -ue /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.sh -} - -nxf_stage() { - true - # stage input files - rm -f chunk_ab - ln -s /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_ab chunk_ab -} - -nxf_unstage() { - true - [[ ${nxf_main_ret:=0} != 0 ]] && return -} - -nxf_main() { - trap on_exit EXIT - trap on_term TERM INT USR2 - trap '' USR1 - - [[ "${NXF_CHDIR:-}" ]] && cd "$NXF_CHDIR" - NXF_SCRATCH='' - [[ $NXF_DEBUG > 0 ]] && nxf_env - touch /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.begin - set +u - set -u - [[ $NXF_SCRATCH ]] && echo "nxf-scratch-dir $HOSTNAME:$NXF_SCRATCH" && cd $NXF_SCRATCH - nxf_stage - - set +e - local ctmp=$(set +u; nxf_mktemp /dev/shm 2>/dev/null || nxf_mktemp $TMPDIR) - local cout=$ctmp/.command.out; mkfifo $cout - local cerr=$ctmp/.command.err; mkfifo $cerr - tee .command.out < $cout & - tee1=$! - tee .command.err < $cerr >&2 & - tee2=$! - ( nxf_launch ) >$cout 2>$cerr & - pid=$! - wait $pid || nxf_main_ret=$? - wait $tee1 $tee2 - nxf_unstage -} - -$NXF_ENTRY diff --git a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.sh b/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.sh deleted file mode 100644 index 579647ba..00000000 --- a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.command.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -ue -cat chunk_ab | tr '[a-z]' '[A-Z]' diff --git a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.exitcode b/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.exitcode deleted file mode 100644 index c2270834..00000000 --- a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/.exitcode +++ /dev/null @@ -1 +0,0 @@ -0 \ No newline at end of file diff --git a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/chunk_ab b/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/chunk_ab deleted file mode 120000 index 73dc9808..00000000 --- a/nf-training/work/2f/35e14b7b0c50555537cb67820c6dbc/chunk_ab +++ /dev/null @@ -1 +0,0 @@ -/Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_ab \ No newline at end of file diff --git a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.begin b/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.begin deleted file mode 100644 index e69de29b..00000000 diff --git a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.err b/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.err deleted file mode 100644 index e69de29b..00000000 diff --git a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.log b/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.log deleted file mode 100644 index e69de29b..00000000 diff --git a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.out b/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.out deleted file mode 100644 index e69de29b..00000000 diff --git a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.run b/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.run deleted file mode 100644 index a6c1dce6..00000000 --- a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.run +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash -# NEXTFLOW TASK: splitLetters (1) -set -e -set -u -NXF_DEBUG=${NXF_DEBUG:=0}; [[ $NXF_DEBUG > 1 ]] && set -x -NXF_ENTRY=${1:-nxf_main} - - -nxf_sleep() { - sleep $1 2>/dev/null || sleep 1; -} - -nxf_date() { - local ts=$(date +%s%3N); - if [[ ${#ts} == 10 ]]; then echo ${ts}000 - elif [[ $ts == *%3N ]]; then echo ${ts/\%3N/000} - elif [[ $ts == *3N ]]; then echo ${ts/3N/000} - elif [[ ${#ts} == 13 ]]; then echo $ts - else echo "Unexpected timestamp value: $ts"; exit 1 - fi -} - -nxf_env() { - echo '============= task environment =============' - env | sort | sed "s/\(.*\)AWS\(.*\)=\(.\{6\}\).*/\1AWS\2=\3xxxxxxxxxxxxx/" - echo '============= task output ==================' -} - -nxf_kill() { - declare -a children - while read P PP;do - children[$PP]+=" $P" - done < <(ps -e -o pid= -o ppid=) - - kill_all() { - [[ $1 != $$ ]] && kill $1 2>/dev/null || true - for i in ${children[$1]:=}; do kill_all $i; done - } - - kill_all $1 -} - -nxf_mktemp() { - local base=${1:-/tmp} - if [[ $(uname) = Darwin ]]; then mktemp -d $base/nxf.XXXXXXXXXX - else TMPDIR="$base" mktemp -d -t nxf.XXXXXXXXXX - fi -} - -nxf_fs_copy() { - local source=$1 - local target=$2 - local basedir=$(dirname $1) - mkdir -p $target/$basedir - cp -fRL $source $target/$basedir -} - -nxf_fs_move() { - local source=$1 - local target=$2 - local basedir=$(dirname $1) - mkdir -p $target/$basedir - mv -f $source $target/$basedir -} - -nxf_fs_rsync() { - rsync -rRl $1 $2 -} - -on_exit() { - exit_status=${nxf_main_ret:=$?} - printf $exit_status > /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.exitcode - set +u - [[ "$tee1" ]] && kill $tee1 2>/dev/null - [[ "$tee2" ]] && kill $tee2 2>/dev/null - [[ "$ctmp" ]] && rm -rf $ctmp || true - exit $exit_status -} - -on_term() { - set +e - [[ "$pid" ]] && nxf_kill $pid -} - -nxf_launch() { - /bin/bash -ue /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.sh -} - -nxf_stage() { - true -} - -nxf_unstage() { - true - [[ ${nxf_main_ret:=0} != 0 ]] && return -} - -nxf_main() { - trap on_exit EXIT - trap on_term TERM INT USR2 - trap '' USR1 - - [[ "${NXF_CHDIR:-}" ]] && cd "$NXF_CHDIR" - NXF_SCRATCH='' - [[ $NXF_DEBUG > 0 ]] && nxf_env - touch /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.begin - set +u - set -u - [[ $NXF_SCRATCH ]] && echo "nxf-scratch-dir $HOSTNAME:$NXF_SCRATCH" && cd $NXF_SCRATCH - nxf_stage - - set +e - local ctmp=$(set +u; nxf_mktemp /dev/shm 2>/dev/null || nxf_mktemp $TMPDIR) - local cout=$ctmp/.command.out; mkfifo $cout - local cerr=$ctmp/.command.err; mkfifo $cerr - tee .command.out < $cout & - tee1=$! - tee .command.err < $cerr >&2 & - tee2=$! - ( nxf_launch ) >$cout 2>$cerr & - pid=$! - wait $pid || nxf_main_ret=$? - wait $tee1 $tee2 - nxf_unstage -} - -$NXF_ENTRY diff --git a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.sh b/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.sh deleted file mode 100644 index 18882ecb..00000000 --- a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.command.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -ue -printf 'Hello world!' | split -b 6 - chunk_ diff --git a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.exitcode b/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.exitcode deleted file mode 100644 index c2270834..00000000 --- a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/.exitcode +++ /dev/null @@ -1 +0,0 @@ -0 \ No newline at end of file diff --git a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_aa b/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_aa deleted file mode 100644 index 54bf7eda..00000000 --- a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_aa +++ /dev/null @@ -1 +0,0 @@ -Hello \ No newline at end of file diff --git a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_ab b/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_ab deleted file mode 100644 index c944ebc2..00000000 --- a/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_ab +++ /dev/null @@ -1 +0,0 @@ -world! \ No newline at end of file diff --git a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.begin b/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.begin deleted file mode 100644 index e69de29b..00000000 diff --git a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.err b/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.err deleted file mode 100644 index e69de29b..00000000 diff --git a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.log b/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.log deleted file mode 100644 index baa3f654..00000000 --- a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.log +++ /dev/null @@ -1 +0,0 @@ -HELLO \ No newline at end of file diff --git a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.out b/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.out deleted file mode 100644 index baa3f654..00000000 --- a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.out +++ /dev/null @@ -1 +0,0 @@ -HELLO \ No newline at end of file diff --git a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.run b/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.run deleted file mode 100644 index e0540a4d..00000000 --- a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.run +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -# NEXTFLOW TASK: convertToUpper (1) -set -e -set -u -NXF_DEBUG=${NXF_DEBUG:=0}; [[ $NXF_DEBUG > 1 ]] && set -x -NXF_ENTRY=${1:-nxf_main} - - -nxf_sleep() { - sleep $1 2>/dev/null || sleep 1; -} - -nxf_date() { - local ts=$(date +%s%3N); - if [[ ${#ts} == 10 ]]; then echo ${ts}000 - elif [[ $ts == *%3N ]]; then echo ${ts/\%3N/000} - elif [[ $ts == *3N ]]; then echo ${ts/3N/000} - elif [[ ${#ts} == 13 ]]; then echo $ts - else echo "Unexpected timestamp value: $ts"; exit 1 - fi -} - -nxf_env() { - echo '============= task environment =============' - env | sort | sed "s/\(.*\)AWS\(.*\)=\(.\{6\}\).*/\1AWS\2=\3xxxxxxxxxxxxx/" - echo '============= task output ==================' -} - -nxf_kill() { - declare -a children - while read P PP;do - children[$PP]+=" $P" - done < <(ps -e -o pid= -o ppid=) - - kill_all() { - [[ $1 != $$ ]] && kill $1 2>/dev/null || true - for i in ${children[$1]:=}; do kill_all $i; done - } - - kill_all $1 -} - -nxf_mktemp() { - local base=${1:-/tmp} - if [[ $(uname) = Darwin ]]; then mktemp -d $base/nxf.XXXXXXXXXX - else TMPDIR="$base" mktemp -d -t nxf.XXXXXXXXXX - fi -} - -nxf_fs_copy() { - local source=$1 - local target=$2 - local basedir=$(dirname $1) - mkdir -p $target/$basedir - cp -fRL $source $target/$basedir -} - -nxf_fs_move() { - local source=$1 - local target=$2 - local basedir=$(dirname $1) - mkdir -p $target/$basedir - mv -f $source $target/$basedir -} - -nxf_fs_rsync() { - rsync -rRl $1 $2 -} - -on_exit() { - exit_status=${nxf_main_ret:=$?} - printf $exit_status > /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/92/315bcac48be0a8e001869c115d1596/.exitcode - set +u - [[ "$tee1" ]] && kill $tee1 2>/dev/null - [[ "$tee2" ]] && kill $tee2 2>/dev/null - [[ "$ctmp" ]] && rm -rf $ctmp || true - exit $exit_status -} - -on_term() { - set +e - [[ "$pid" ]] && nxf_kill $pid -} - -nxf_launch() { - /bin/bash -ue /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.sh -} - -nxf_stage() { - true - # stage input files - rm -f chunk_aa - ln -s /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_aa chunk_aa -} - -nxf_unstage() { - true - [[ ${nxf_main_ret:=0} != 0 ]] && return -} - -nxf_main() { - trap on_exit EXIT - trap on_term TERM INT USR2 - trap '' USR1 - - [[ "${NXF_CHDIR:-}" ]] && cd "$NXF_CHDIR" - NXF_SCRATCH='' - [[ $NXF_DEBUG > 0 ]] && nxf_env - touch /Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.begin - set +u - set -u - [[ $NXF_SCRATCH ]] && echo "nxf-scratch-dir $HOSTNAME:$NXF_SCRATCH" && cd $NXF_SCRATCH - nxf_stage - - set +e - local ctmp=$(set +u; nxf_mktemp /dev/shm 2>/dev/null || nxf_mktemp $TMPDIR) - local cout=$ctmp/.command.out; mkfifo $cout - local cerr=$ctmp/.command.err; mkfifo $cerr - tee .command.out < $cout & - tee1=$! - tee .command.err < $cerr >&2 & - tee2=$! - ( nxf_launch ) >$cout 2>$cerr & - pid=$! - wait $pid || nxf_main_ret=$? - wait $tee1 $tee2 - nxf_unstage -} - -$NXF_ENTRY diff --git a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.sh b/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.sh deleted file mode 100644 index 7b46a9d3..00000000 --- a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.command.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -ue -cat chunk_aa | tr '[a-z]' '[A-Z]' diff --git a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.exitcode b/nf-training/work/92/315bcac48be0a8e001869c115d1596/.exitcode deleted file mode 100644 index c2270834..00000000 --- a/nf-training/work/92/315bcac48be0a8e001869c115d1596/.exitcode +++ /dev/null @@ -1 +0,0 @@ -0 \ No newline at end of file diff --git a/nf-training/work/92/315bcac48be0a8e001869c115d1596/chunk_aa b/nf-training/work/92/315bcac48be0a8e001869c115d1596/chunk_aa deleted file mode 120000 index 5302ce2d..00000000 --- a/nf-training/work/92/315bcac48be0a8e001869c115d1596/chunk_aa +++ /dev/null @@ -1 +0,0 @@ -/Users/cwyatt/Desktop/Nextflow_practise/Update_docs_2/dsl2_review/nf-training/work/36/360a0bc96ba097bcd69fbae81f8fd8/chunk_aa \ No newline at end of file