including test failure in aws s3 cp

Signed-off-by: jorgee <[email protected]>

validation/awsbatch-unstage-fail.config

@@ -0,0 +1,12 @@
+/*
+ * do not include plugin requirements otherwise latest
+ * published version will be downloaded instead of using local build
+ */
+
+workDir = 's3://nextflow-ci/work'
+process.executor = 'awsbatch'
+process.queue = 'nextflow-ci'
+process.container = 'quay.io/nextflow/test-aws-unstage-fail:1.0'
+aws.region = 'eu-west-1'
+aws.batch.maxTransferAttempts = 3 
+aws.batch.delayBetweenAttempts = '5 sec'

validation/awsbatch.sh

@@ -7,6 +7,13 @@ get_abs_filename() {
 
 export NXF_CMD=${NXF_CMD:-$(get_abs_filename ../launch.sh)}
 
+# Execution should fail ignoring
+$NXF_CMD run test-aws-unstage-fail.nf -c awsbatch-unstage-fail.config || true
+[[ `grep -c "Error executing process > 'test (1)'" .nextflow.log` == 1 ]] || false
+[[ `grep -c "  Essential container in task exited" .nextflow.log` == 1 ]] || false
+[[ `grep -cozP "Command exit status:\n  1" .nextflow.log` == 1 ]] || false
+[[ `grep -c "Producing a failure in aws" .nextflow.log` == 2 ]] || false
+
 $NXF_CMD run test-complexpaths.nf -c awsbatch.config
 [[ -d foo ]] || false
 [[ -e 'foo/.alpha' ]] || false
@@ -73,4 +80,4 @@ $NXF_CMD run nextflow-io/hello \
     -process.array 10 \
     -with-wave \
     -with-fusion \
-    -c awsbatch.config
+    -c awsbatch.config

validation/test-aws-unstage-fail-container/Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu
+
+RUN apt-get update && apt-get -y install curl unzip && apt-get clean
+
+
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && ./aws/install && rm -rf aws*
+
+ADD fake_aws /fake_aws
+
+ENV PATH=/fake_aws/bin/:$PATH

validation/test-aws-unstage-fail-container/fake_aws/bin/aws

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [[ "$*" == *".command."* ]] || [[ "$*" == *".exitcode"* ]]; then
+   /usr/local/bin/aws $@
+else
+   >&2 echo "Producing a failure in aws $@"
+   exit 2
+fi
+

validation/test-aws-unstage-fail.nf

@@ -0,0 +1,16 @@
+process test {
+  input: 
+	val i
+  output: 
+        file("test${i}")
+        file("test_2_${i}")
+  script:
+  """
+  dd if=/dev/urandom of=test${i} bs=1K count=90
+  dd if=/dev/urandom of=test_2_${i} bs=1K count=90
+  """
+}
+
+workflow {
+  Channel.of(1) | test
+}