runtime-v2: out parameters for scripts (walmartlabs#540)

Author: brig

examples/runtime-v2/demo-flow/concord.yml

@@ -86,6 +86,7 @@ flows:
   # get and set script variables
   scriptFlow:
     - script: scripts/test-script.groovy
+      out: newVar
       meta:
         segmentName: "Processing ${myVar} Script Variable..."
       error:

examples/runtime-v2/demo-flow/scripts/test-script.groovy

@@ -1,6 +1,6 @@
 // get a variable
-def v = execution.variables().get('myVar')
-println('Hello ' + myVar)
+def v = myVar
+println('Hello ' + v)
 
 // set a variable
-execution.variables().set('newVar', 'Hello, world!')
+result.set('newVar', 'Hello, world!')

examples/runtime-v2/out_groovy/concord.yml

@@ -0,0 +1,22 @@
+configuration:
+  runtime: "concord-v2"
+  dependencies:
+  - "mvn://org.codehaus.groovy:groovy-all:pom:2.5.8"
+
+flows:
+  default:
+  - script: groovy
+    body: |
+      result.set("myVar", "myValue");
+    out: scriptResult
+
+  - log: "result: ${scriptResult}"    # result: {myVar=myValue}
+
+  - script: groovy
+    body: |
+      result.set("myVar", "myValue");
+    out:
+      myVar: ${result.myVar}
+
+  - log: "myVar: ${myVar}"    # myVar: myValue
+

examples/runtime-v2/out_groovy/run.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+SERVER_ADDR="$1"
+
+rm -rf target && mkdir target
+cp -R concord.yml target/
+
+cd target && zip -r payload.zip ./* > /dev/null && cd ..
+
+read -p "Username: " CURL_USER
+curl -v \
+-u ${CURL_USER} \
+-F archive=@target/payload.zip \
+-F out=myVar \
+http://${SERVER_ADDR}/api/v1/process

examples/runtime-v2/out_js/concord.yml

@@ -0,0 +1,19 @@
+configuration:
+  runtime: "concord-v2"
+
+flows:
+  default:
+  - script: js
+    body: |
+      result.set("myVar", "myValue");
+    out: scriptResult
+
+  - log: "result: ${scriptResult}"    # result: {myVar=myValue}
+
+  - script: js
+    body: |
+      result.set("myVar", "myValue");
+    out:
+      myVar: ${result.myVar}
+
+  - log: "myVar: ${myVar}"    # myVar: myValue

examples/runtime-v2/out_js/run.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+SERVER_ADDR="$1"
+
+rm -rf target && mkdir target
+cp -R concord.yml target/
+
+cd target && zip -r payload.zip ./* > /dev/null && cd ..
+
+read -p "Username: " CURL_USER
+curl -u ${CURL_USER} -F archive=@target/payload.zip http://${SERVER_ADDR}/api/v1/process

examples/runtime-v2/out_python/concord.yml

@@ -0,0 +1,21 @@
+configuration:
+  runtime: "concord-v2"
+  dependencies:
+  - "mvn://org.python:jython-standalone:2.7.0"
+
+flows:
+  default:
+  - script: python
+    body:
+      result.set("myVar", "myValue");
+    out: scriptResult
+
+  - log: "result: ${scriptResult}"    # result: {myVar=myValue}
+
+  - script: python
+    body:
+      result.set("myVar", "myValue");
+    out:
+      myVar: ${result.myVar}
+
+  - log: "myVar: ${myVar}"    # myVar: myValue

examples/runtime-v2/out_python/run.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+SERVER_ADDR="$1"
+
+rm -rf target && mkdir target
+cp -R concord.yml target/
+
+cd target && zip -r payload.zip ./* > /dev/null && cd ..
+
+read -p "Username: " CURL_USER
+curl -u ${CURL_USER} -F archive=@target/payload.zip http://${SERVER_ADDR}/api/v1/process

examples/runtime-v2/out_ruby/concord.yml

@@ -0,0 +1,21 @@
+configuration:
+  runtime: "concord-v2"
+  dependencies:
+  - "mvn://org.jruby:jruby:9.1.13.0"
+
+flows:
+  default:
+  - script: ruby
+    body: |
+      $result.set("myVar", "myValue");
+    out: scriptResult
+
+  - log: "result: ${scriptResult}"    # result: {myVar=myValue}
+
+  - script: ruby
+    body: |
+      $result.set("myVar", "myValue");
+    out:
+      myVar: ${result.myVar}
+
+  - log: "myVar: ${myVar}"    # myVar: myValue

examples/runtime-v2/out_ruby/run.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+SERVER_ADDR="$1"
+
+rm -rf target && mkdir target
+cp -R concord.yml target/
+
+cd target && zip -r payload.zip ./* > /dev/null && cd ..
+
+read -p "Username: " CURL_USER
+curl -u ${CURL_USER} -F archive=@target/payload.zip http://${SERVER_ADDR}/api/v1/process

runtime/v2/model/src/main/java/com/walmartlabs/concord/runtime/v2/model/ScriptCallOptions.java

@@ -45,6 +45,15 @@ default Map<String, Serializable> input() {
         return Collections.emptyMap();
     }
 
+    @Nullable
+    String out();
+
+    @Value.Default
+    @AllowNulls
+    default Map<String, Serializable> outExpr() {
+        return Collections.emptyMap();
+    }
+
     @Nullable
     String inputExpression();
 

runtime/v2/model/src/main/java/com/walmartlabs/concord/runtime/v2/parser/ScriptGrammar.java

@@ -38,11 +38,16 @@ private static Parser<Atom, ImmutableScriptCallOptions.Builder> scriptCallInOpti
         return orError(or(maybeMap.map(o::input), maybeExpression.map(o::inputExpression)), YamlValueType.SCRIPT_CALL_IN);
     }
 
+    private static Parser<Atom, ImmutableScriptCallOptions.Builder> scriptCallOutOption(ImmutableScriptCallOptions.Builder o) {
+        return orError(or(maybeMap.map(o::outExpr), maybeString.map(o::out)), YamlValueType.SCRIPT_CALL_OUT);
+    }
+
     private static Parser<Atom, ScriptCallOptions> scriptOptions(String stepName) {
         return with(() -> optionsBuilder(stepName),
                 o -> options(
                         optional("body", stringVal.map(o::body)),
                         optional("in", scriptCallInOption(o)),
+                        optional("out", scriptCallOutOption(o)),
                         optional("meta", mapVal.map(o::meta)),
                         optional("name", stringVal.map(v -> o.putMeta(Constants.SEGMENT_NAME, v))),
                         optional("withItems", nonNullVal.map(v -> o.withItems(WithItems.of(v, WithItems.Mode.SERIAL)))),

runtime/v2/model/src/main/java/com/walmartlabs/concord/runtime/v2/parser/YamlValueType.java

@@ -54,6 +54,7 @@ public final class YamlValueType<T> {
     public static final YamlValueType<SuspendStep> SUSPEND = type("SUSPEND");
     public static final YamlValueType<ScriptCall> SCRIPT = type("SCRIPT");
     public static final YamlValueType<ImmutableScriptCallOptions.Builder> SCRIPT_CALL_IN = type("OBJECT or EXPRESSION");
+    public static final YamlValueType<ImmutableScriptCallOptions.Builder> SCRIPT_CALL_OUT = type("STRING or OBJECT");
     public static final YamlValueType<Expression> EXPRESSION = type("EXPRESSION");
     public static final YamlValueType<String> EXPRESSION_VAL = type("EXPRESSION");
     public static final YamlValueType<Retry> RETRY = type("RETRY");

runtime/v2/model/src/main/java/com/walmartlabs/concord/runtime/v2/schema/ScriptCallMixIn.java

@@ -43,6 +43,10 @@ public interface ScriptCallMixIn extends StepMixIn {
     @JsonSchemaInject(json = "{\"oneOf\": [ {\"type\": \"string\"}, {\"type\": \"object\"} ]}", merge = false)
     Object input();
 
+    @JsonProperty("out")
+    @JsonSchemaInject(json = "{\"oneOf\": [ {\"type\": \"string\"}, {\"type\": \"object\"} ]}", merge = false)
+    Object out();
+
     @JsonProperty("withItems")
     WithItemsMixIn withItems();
 

runtime/v2/model/src/main/java/com/walmartlabs/concord/runtime/v2/serializer/ScriptCallStepSerializer.java

@@ -67,6 +67,9 @@ private static void serializeOptions(ScriptCallOptions options, JsonGenerator ge
         writeNotEmptyObjectField("in", options.input(), gen);
         writeNotEmptyObjectField("in", options.inputExpression(), gen);
 
+        writeNotEmptyObjectField("out", options.out(), gen);
+        writeNotEmptyObjectField("out", options.outExpr(), gen);
+
         if (options.withItems() != null) {
             WithItems items = Objects.requireNonNull(options.withItems());
             writeWithItems(items, gen);

runtime/v2/model/src/test/java/com/walmartlabs/concord/project/runtime/v2/parser/YamlErrorParserTest.java

@@ -2246,7 +2246,7 @@ public void test1701() throws Exception {
 
     @Test
     public void test1702() throws Exception {
-        String msg = "(002.yml): Error @ line: 4, col: 14. Unknown options: ['body1' [STRING] @ line: 4, col: 14], expected: [body, error, in, meta, name, parallelWithItems, retry, withItems]. Remove invalid options and/or fix indentation\n" +
+        String msg = "(002.yml): Error @ line: 4, col: 14. Unknown options: ['body1' [STRING] @ line: 4, col: 14], expected: [body, error, in, meta, name, out, parallelWithItems, retry, withItems]. Remove invalid options and/or fix indentation\n" +
                 "\twhile processing steps:\n" +
                 "\t'script' @ line: 3, col: 7\n" +
                 "\t\t'main' @ line: 2, col: 3\n" +

runtime/v2/model/src/test/java/com/walmartlabs/concord/project/runtime/v2/parser/YamlOkParserTest.java

@@ -437,6 +437,9 @@ public void test014() throws Exception {
         input.put("k", "v1");
         assertEquals(input, t.getOptions().input());
 
+        // out
+        assertEquals("result", t.getOptions().out());
+
         // retry
         assertNotNull(t.getOptions().retry());
         assertEquals(1, t.getOptions().retry().times());

runtime/v2/model/src/test/resources/014.yml

@@ -5,6 +5,7 @@ flows:
         throw new RuntimeException("kaboom!")
       in:
         k: "v1"
+      out: result
       error:
         - log: "Caught an error"
       withItems: 1

runtime/v2/runner/src/main/java/com/walmartlabs/concord/runtime/v2/runner/script/DefaultScriptEvaluator.java

@@ -59,27 +59,30 @@ public DefaultScriptEvaluator(TaskProviders taskProviders) {
     }
 
     @Override
-    public void eval(Context context, String language, Reader input, Map<String, Object> variables) {
+    public ScriptResult eval(Context context, String language, Reader input, Map<String, Object> variables) {
         ScriptEngine engine = getEngine(language);
 
         if (engine == null) {
             throw new RuntimeException("Script engine not found: " + language);
         }
 
-        ScriptContext ctx = new ScriptContext(context);
         Bindings b = engine.createBindings();
         b.put("polyglot.js.allowAllAccess", true);
 
+        ScriptResult scriptResult = new ScriptResult();
+        ScriptContext ctx = new ScriptContext(context);
         for (String ctxVar: CONTEXT_VARIABLE_NAMES) {
             b.put(ctxVar, ctx);
         }
         b.put("tasks", new TaskAccessor(taskProviders, ctx));
         b.put("log", log);
         b.putAll(context.variables().toMap());
         b.putAll(variables);
+        b.put("result", scriptResult);
 
         try {
             engine.eval(input, b);
+            return scriptResult;
         } catch (ScriptException e) {
             if (e.getCause() instanceof PolyglotException) {
                 throw new RuntimeException(e.getCause().getMessage());
@@ -114,7 +117,6 @@ public String getLanguage(String languageOrExtension) {
 
     @SuppressWarnings({"rawtypes", "unchecked"})
     private ScriptEngine getEngine(String language) {
-        // Javascript array is converted in Java to an empty map #214 (https://github.com/oracle/graaljs/issues/214)
         if (new GraalJSEngineFactory().getNames().contains(language)) {
             // Javascript array is converted in Java to an empty map #214 (https://github.com/oracle/graaljs/issues/214)
             HostAccess access = HostAccess.newBuilder(HostAccess.ALL)

runtime/v2/runner/src/main/java/com/walmartlabs/concord/runtime/v2/runner/script/ScriptEvaluator.java

@@ -27,7 +27,7 @@
 
 public interface ScriptEvaluator {
 
-    void eval(Context context, String language, Reader input, Map<String, Object> variables);
+    ScriptResult eval(Context context, String language, Reader input, Map<String, Object> variables);
 
     String getLanguage(String languageOrExtension);
 }

runtime/v2/runner/src/main/java/com/walmartlabs/concord/runtime/v2/runner/script/ScriptResult.java

@@ -0,0 +1,39 @@
+package com.walmartlabs.concord.runtime.v2.runner.script;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2022 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ScriptResult {
+
+    private final Map<String, Object> items = new ConcurrentHashMap<>();
+
+    public ScriptResult set(String key, Object value) {
+        Object sanitized = VariablesSanitizer.sanitize(value);
+        items.put(key, sanitized);
+        return this;
+    }
+
+    public Map<String, Object> items() {
+        return items;
+    }
+}

runtime/v2/runner/src/main/java/com/walmartlabs/concord/runtime/v2/runner/vm/OutputUtils.java

@@ -0,0 +1,47 @@
+package com.walmartlabs.concord.runtime.v2.runner.vm;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2022 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.walmartlabs.concord.runtime.v2.runner.el.EvalContextFactory;
+import com.walmartlabs.concord.runtime.v2.runner.el.ExpressionEvaluator;
+import com.walmartlabs.concord.runtime.v2.sdk.Context;
+import com.walmartlabs.concord.svm.Runtime;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Map;
+
+public final class OutputUtils {
+
+    public static void process(Runtime runtime, Context ctx, Map<String, Object> result, String out, Map<String, Serializable> outExpr) {
+        if (out != null) {
+            ctx.variables().set(out, result);
+        } else if (!outExpr.isEmpty()) {
+            ExpressionEvaluator expressionEvaluator = runtime.getService(ExpressionEvaluator.class);
+            Map<String, Object> vars = Collections.singletonMap("result", result);
+            Map<String, Serializable> evalOut = expressionEvaluator.evalAsMap(EvalContextFactory.global(ctx, vars), outExpr);
+            evalOut.forEach((k, v) -> ctx.variables().set(k, v));
+        }
+    }
+
+    private OutputUtils() {
+    }
+}