fix issue when no stage out is defined

Signed-off-by: jorgee <[email protected]>
nextflow-io · Sep 30, 2024 · 3930ea1 · 3930ea1
1 parent 1699edc
commit 3930ea1
Show file tree

Hide file tree

Showing 9 changed files with 51 additions and 29 deletions.
diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy
@@ -372,6 +372,7 @@ class BashWrapperBuilder {
         binding.launch_cmd = getLaunchCommand(interpreter,env)
         binding.stage_cmd = getStageCommand()
         binding.unstage_cmd = getUnstageCommand()
+        binding.unstage_controls_cmd = getUnstageControlsCommand()
         binding.unstage_controls = changeDir || shouldUnstageOutputs() ? getUnstageControls() : null
 
         if( changeDir || shouldUnstageOutputs() ) {
@@ -752,6 +753,9 @@ class BashWrapperBuilder {
 
     protected String getUnstageCommand() { 'nxf_unstage' }
 
+    protected String getUnstageControlsCommand() { 'nxf_unstage_std_files' }
+
+
     protected String getUnstageControls() {
         def result = copyFileToWorkDir(TaskRun.CMD_OUTFILE) + ' || true' + ENDL
         result += copyFileToWorkDir(TaskRun.CMD_ERRFILE) + ' || true' + ENDL

diff --git a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt
@@ -103,9 +103,9 @@ on_exit() {
     ## Can be caused either by the task script, unstage script or after script if defined
     local last_err=$?
     ## capture the task error first or fallback to unstage error
-    exit_status=${nxf_main_ret:=$nxf_unstage_ret}
-    ## Check if an error happened in after script
-    [[ $exit_status -eq 0 && $last_err -ne 0 ]] && exit_status=$last_err
+    exit_status=${nxf_main_ret:=0}
+    [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:0} -ne 0 ]] && exit_status=${nxf_unstage_ret}
+    [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err}
     printf -- $exit_status {{exit_file}}
     set +u
     {{cleanup_cmd}}
@@ -129,11 +129,10 @@ nxf_stage() {
 
 nxf_unstage() {
     true
-    [[ ${nxf_main_ret:=0} != 0 ]] && return
     {{unstage_outputs}}
 }
 
-nxf_unstage_std_files(){
+nxf_unstage_std_files() {
     true
     {{unstage_controls}}
 }
@@ -165,11 +164,12 @@ nxf_main() {
     (set -o pipefail; (nxf_launch | tee {{stdout_file}}) 3>&1 1>&2 2>&3 | tee {{stderr_file}}) &
     pid=$!
     wait $pid || nxf_main_ret=$?
-
-    ## Data unstaging redirecting stdout and stderr with append mode
-    (set -e -o pipefail; ({{unstage_cmd}} | tee -a {{stdout_file}}) 3>&1 1>&2 2>&3 | tee -a {{stderr_file}})
-    nxf_unstage_ret=$?
-    nxf_unstage_std_files
+    if [[ ${nxf_main_ret:=0} == 0 ]]; then
+        ## Data unstaging redirecting stdout and stderr with append mode
+        (set -e -o pipefail; ({{unstage_cmd}} | tee -a {{stdout_file}}) 3>&1 1>&2 2>&3 | tee -a {{stderr_file}})
+        nxf_unstage_ret=$?
+    fi
+    {{unstage_controls_cmd}}
     {{after_script}}
 }
 

diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy
@@ -1311,12 +1311,14 @@ class BashWrapperBuilderTest extends Specification {
         then:
         builder.getStageCommand() == 'nxf_stage'
         builder.getUnstageCommand() == 'nxf_unstage'
+        builder.getUnstageControlsCommand() == 'nxf_unstage_std_files'
 
         when:
         def binding = builder.makeBinding()
         then:
         binding.stage_cmd == 'nxf_stage'
         binding.unstage_cmd == 'nxf_unstage'
+        binding.unstage_controls_cmd == 'nxf_unstage_std_files'
 
     }
 

diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt
@@ -271,8 +271,9 @@ nxf_fs_fcp() {
 
 on_exit() {
     local last_err=$?
-    exit_status=${nxf_main_ret:=$nxf_unstage_ret}
-    [[ $exit_status -eq 0 && $last_err -ne 0 ]] && exit_status=$last_err
+    exit_status=${nxf_main_ret:=0}
+    [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:0} -ne 0 ]] && exit_status=${nxf_unstage_ret}
+    [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err}
     printf -- $exit_status > {{folder}}/.exitcode
     set +u
     exit $exit_status
@@ -293,10 +294,9 @@ nxf_stage() {
 
 nxf_unstage() {
     true
-    [[ ${nxf_main_ret:=0} != 0 ]] && return
 }
 
-nxf_unstage_std_files(){
+nxf_unstage_std_files() {
     true
 }
 
@@ -319,9 +319,10 @@ nxf_main() {
     (set -o pipefail; (nxf_launch | tee .command.out) 3>&1 1>&2 2>&3 | tee .command.err) &
     pid=$!
     wait $pid || nxf_main_ret=$?
-
-    (set -e -o pipefail; (nxf_unstage | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err)
-    nxf_unstage_ret=$?
+    if [[ ${nxf_main_ret:=0} == 0 ]]; then
+        (set -e -o pipefail; (nxf_unstage | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err)
+        nxf_unstage_ret=$?
+    fi
     nxf_unstage_std_files
 }
 

diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt
@@ -82,8 +82,9 @@ nxf_fs_fcp() {
 
 on_exit() {
     local last_err=$?
-    exit_status=${nxf_main_ret:=$nxf_unstage_ret}
-    [[ $exit_status -eq 0 && $last_err -ne 0 ]] && exit_status=$last_err
+    exit_status=${nxf_main_ret:=0}
+    [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:0} -ne 0 ]] && exit_status=${nxf_unstage_ret}
+    [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err}
     printf -- $exit_status > {{folder}}/.exitcode
     set +u
     exit $exit_status
@@ -104,10 +105,9 @@ nxf_stage() {
 
 nxf_unstage() {
     true
-    [[ ${nxf_main_ret:=0} != 0 ]] && return
 }
 
-nxf_unstage_std_files(){
+nxf_unstage_std_files() {
     true
 }
 
@@ -136,9 +136,10 @@ nxf_main() {
     (set -o pipefail; (nxf_launch | tee .command.out) 3>&1 1>&2 2>&3 | tee .command.err) &
     pid=$!
     wait $pid || nxf_main_ret=$?
-
-    (set -e -o pipefail; (nxf_unstage | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err)
-    nxf_unstage_ret=$?
+    if [[ ${nxf_main_ret:=0} == 0 ]]; then
+        (set -e -o pipefail; (nxf_unstage | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err)
+        nxf_unstage_ret=$?
+    fi
     nxf_unstage_std_files
 }
 

diff --git a/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy
@@ -58,7 +58,7 @@ class BashWrapperBuilderWithS3Test extends Specification {
         binding.unstage_outputs == '''\
                     IFS=$'\\n'
                     for name in $(eval "ls -1d test.bam test.bai bla\\ nk.txt" | sort | uniq); do
-                        nxf_s3_upload $name s3://some/buck\\ et || true
+                        nxf_s3_upload $name s3://some/buck\\ et
                     done
                     unset IFS
                     '''.stripIndent().rightTrim()

diff --git a/...oogle/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncher.groovy b/...oogle/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncher.groovy
@@ -64,7 +64,10 @@ class GoogleLifeSciencesScriptLauncher extends BashWrapperBuilder {
     protected String getStageCommand() { null }
 
     @Override
-    protected String getUnstageCommand() { null }
+    protected String getUnstageCommand() { 'true' }
+
+    @Override
+    protected String getUnstageControlsCommand() { null }
 
     @Override
     String touchFile(Path file) {

diff --git a/...e/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy b/...e/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy
@@ -53,7 +53,8 @@ class GoogleLifeSciencesScriptLauncherTest extends GoogleSpecification {
         then:
         binding.touch_file == null
         binding.stage_cmd == null
-        binding.unstage_cmd == null
+        binding.unstage_cmd == 'true'
+        binding.unstage_controls_cmd == null
         binding.task_env == '''\
                 chmod +x /work/xx/yy/nextflow-bin/* || true
                 export PATH=/work/xx/yy/nextflow-bin:$PATH

diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt
@@ -168,7 +168,10 @@ nxf_fs_fcp() {
 }
 
 on_exit() {
-    exit_status=${nxf_main_ret:=$?}
+    local last_err=$?
+    exit_status=${nxf_main_ret:=0}
+    [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:0} -ne 0 ]] && exit_status=${nxf_unstage_ret}
+    [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err}
     printf -- $exit_status > {{folder}}/.exitcode
     set +u
     exit $exit_status
@@ -194,10 +197,13 @@ nxf_stage() {
 
 nxf_unstage() {
     true
+}
+
+nxf_unstage_std_files() {
+    true
     gsutil -m -q cp -R .command.out gs://bucket/work/dir/.command.out || true
     gsutil -m -q cp -R .command.err gs://bucket/work/dir/.command.err || true
     gsutil -m -q cp -R .exitcode gs://bucket/work/dir/.exitcode || true
-    [[ ${nxf_main_ret:=0} != 0 ]] && return
 }
 
 nxf_main() {
@@ -217,6 +223,10 @@ nxf_main() {
     (set -o pipefail; (nxf_launch | tee .command.out) 3>&1 1>&2 2>&3 | tee .command.err) &
     pid=$!
     wait $pid || nxf_main_ret=$?
+    if [[ ${nxf_main_ret:=0} == 0 ]]; then
+        (set -e -o pipefail; (true | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err)
+        nxf_unstage_ret=$?
+    fi
 }
 
 $NXF_ENTRY