feat(*): new execution model

Cargo.toml

@@ -22,3 +22,5 @@ jaq-interpret = "1.2.1"
 jaq-parse = "1.0.2"
 jaq-core = "1.2.1"
 jaq-std = "1.2.1"
+derivative = "2.2.0"
+

datakit.meta.json

@@ -11,9 +11,21 @@
                   "type": { "type": "string" },
                   "name": { "type": "string" },
                   "input": { "type": "string" },
-                  "inputs": { "type": "array", "items": { "type": "string" } },
+                  "inputs": {
+                     "oneOf": [
+                        { "type": "array", "items": { "type": "string" } },
+                        { "type": "array", "items": { "type": "object", "additionalProperties": { "type": "string" } } },
+                        { "type": "object", "additionalProperties": { "type": "string" } }
+                     ]
+                  },
                   "output": { "type": "string" },
-                  "outputs": { "type": "array", "items": { "type": "string" } }
+                  "outputs": {
+                     "oneOf": [
+                        { "type": "array", "items": { "type": "string" } },
+                        { "type": "array", "items": { "type": "object", "additionalProperties": { "type": "string" } } },
+                        { "type": "object", "additionalProperties": { "type": "string" } }
+                     ]
+                  }
                }
             }
          }

docs/datakit.md

@@ -16,41 +16,148 @@ The data types are based on those of [serde-json], so representable value types
 
 ## The execution model
 
+Nodes can have input ports and output ports.
+Input ports consume data. Output ports produce data.
+
+You can link one node's output port to another node's input port.
+An input port can receive at most one link, that is, data can only arrive
+into an input via one other node. Therefore, there are no race conditions.
+
+An output port can be linked to multiple nodes. Therefore, one node can
+provide data to several other nodes.
+
 Each node triggers at most once.
 
-A node only triggers when all its inputs are available.
+A node only triggers when data is available to all its connected input ports;
+that is, only when all nodes connected to its inputs have finished
+executing.
 
 ## Node types
 
 The following node types are implemented:
 
-* `call`: an HTTP dispatch call
-* `template`: application of a string template
-* `response`: trigger a direct response, rather than forwarding a proxied response
+**Node type**        | **Input ports**   | **Output ports**  |  **Supported attributes**
+--------------------:|:-----------------:|:-----------------:|:-----------------------------
+`call                | `body`, `headers` | `body`, `headers` | `url`, `method`, `timeout`
+`jq`                 | user-defined      | user-defined      | `jq`
+`handlebars`         | user-defined      | `output`          | `template`, `content_type`
+`exit`               | `body`, `headers` |                   | `status`
+
+### `call` node type
+
+An HTTP dispatch call.
+
+#### Input ports:
+
+* `body`: body to use in the dispatch request.
+* `headers`: headers to use in the dispatch request.
+
+#### Output ports:
+
+* `body`: body returned as the dispatch response.
+* `headers`: headers returned as the dispatch response.
+
+#### Supported attributes:
+
+* `url`: the URL to use when dispatching.
+* `method`: the HTTP method (default is `GET`).
+* `timeout`: the dispatch timeout, in seconds (default is 60).
+
+### `jq` node type
+
+Execution of a JQ script for processing JSON. The JQ script is processed
+using the [jaq] implementation of the JQ language.
+
+#### Input ports:
+
+User-defined. Each input port declared by the user will correspond to a
+variable in the JQ execution context. A user can declare the name of the port
+explicitly, which is the name of the variable. If a port does not have a given
+name, it will get a default name based on the peer node and port to which it
+is connected, and the name will be normalized into a valid variable name (e.g.
+by replacing `.` to `_`).
+
+#### Output ports:
+
+User-defined. When the JQ script produces a JSON value, that is made available
+in the first output port of the node. If the JQ script produces multiple JSON
+values, each value will be routed to a separate output port.
+
+#### Supported attributes:
+
+* `jq`: the JQ script to execute when the node is triggered.
+
+### `handlebars` node type
+
+Application of a [Handlebars] template on a raw string, useful for producing
+arbitrary non-JSON content types.
+
+#### Input ports:
+
+User-defined. Each input port declared by the user will correspond to a
+variable in the Handlebars execution context. A user can declare the name of
+the port explicitly, which is the name of the variable. If a port does not
+have a given name, it will get a default name based on the peer node and port
+to which it is connected, and the name will be normalized into a valid
+variable name (e.g. by replacing `.` to `_`).
+
+#### Output ports:
+
+* `output`: the rendered template. The output payload will be in raw string
+  format, unless an alternative `content_type` triggers a conversion.
+
+#### Supported attributes:
+
+* `template`: the Handlebars template to apply when the node is triggered.
+* `content_type`: if set to a MIME type that matches one of DataKit's
+  supported payload types, such as `application/json`, the output payload will
+  be converted to that format, making its contents available for further
+  processing by other nodes (default is `text/plain`, which produces a raw
+  string).
+
+### `exit` node type
+
+Trigger an early exit that produces a direct response, rather than forwarding
+a proxied response.
+
+#### Input ports:
+
+* `body`: body to use in the early-exit response.
+* `headers`: headers to use in the early-exit response.
+
+#### Output ports:
+
+None.
+
+#### Supported attributes:
+
+* `status`: the HTTP status code to use in the early-exit response (default is
+  200).
 
 ## Implicit nodes
 
-DataKit defines a number of implicit nodes that can be used as inputs or outputs without being
-explicitly declared. These reserved node names cannot be used for user-defined nodes. These are:
-
-**Name**                    |  **Usage**     |  **Description**
----------------------------:|:--------------:|:------------------
-`request_headers`           | as input only  | headers from the incoming request
-`request_body`              | as input only  | body of the incoming request
-`service_request_headers`   | as output only | headers to be sent to the service being proxied to
-`service_request_body`      | as output only | body to be sent to the service being proxied to
-`service_response_headers`  | as input only  | headers from the response sent by the service being proxied to
-`service_response_body`     | as input only  | body of the response sent by the service being proxied to
-`response_headers`          | as output only | headers to be sent as a response to the incoming request
-`response_body`             | as output only | body to be sent as a response to the incoming request
-
-The `_headers` nodes produce maps from header names to their values.
+DataKit defines a number of implicit nodes that can be used without being
+explicitly declared. These reserved node names cannot be used for user-defined
+nodes. These are:
+
+**Node**             | **Input ports**   | **Output ports**  |  **Description**
+--------------------:|:-----------------:|:-----------------:|:------------------
+`request`            |                   | `body`, `headers` | the incoming request
+`service_request`    | `body`, `headers` |                   | request sent to the service being proxied to
+`service_response`   |                   | `body`, `headers` | response sent by the service being proxied to
+`response`           | `body`, `headers` |                   | response to be sent to the incoming request
+
+The `headers` ports produce and consume maps from header names to their values.
 Keys are header names are normalized to lowercase.
 Values are strings if there is a single instance of a header,
 or arrays of strings if there are multiple instances of the same header.
 
-The `_body` nodes produce either raw strings or JSON objects, depending on their corresponding
-`Content-Type` values.
+The `body` output ports produce either raw strings or JSON objects,
+depending on their corresponding `Content-Type` values.
+
+Likewise, the `body` input ports accept either raw strings or JSON objects,
+and both their `Content-Type` and `Content-Length` are automatically adjusted,
+according to the type and size of the incoming data.
 
 ## Debugging
 
@@ -69,3 +176,5 @@ as normal. Any other value will enable debug tracing.
 ---
 
 [serde-json]: https://docs.rs/serde_json/latest/serde_json/
+[Handlebars]: https://docs.rs/handlebars/latest/handlebars/
+[jaq]: https://lib.rs/crates/jaq

examples/call_headers/config/demo.yml

@@ -0,0 +1,33 @@
+_format_version: "3.0"
+services:
+- url: http://127.0.0.1:8001
+  name: my-service
+  routes:
+  - name: my-route
+    paths:
+    - /
+    strip_path: true
+  filter_chains:
+    - filters:
+      - name: datakit
+        config:
+          debug: true
+          nodes:
+          - name: MY_HEADERS
+            type: jq
+            inputs:
+            - req: request.headers
+            jq: |
+              {
+                "X-My-Call-Header": $req.apikey // "default value"
+              }
+          - name: CALL
+            type: call
+            inputs:
+            - headers: MY_HEADERS
+            url: https://httpbin.konghq.com/anything
+          - name: EXIT
+            type: exit
+            inputs:
+            - body: CALL.body
+            status: 200

test/demo.sh → examples/call_headers/demo.sh

@@ -26,16 +26,16 @@ fi
 message "Building the filter using cargo..."
 
 (
-    cd ..
+    cd ../..
     cargo build --target=wasm32-wasip1 --release || exit 1
 ) || exit 1
 
 ### Copy filter to wasm/ #######################################################
 
 mkdir -p wasm
 
-cp -a ../target/wasm32-wasip1/release/*.wasm wasm/
-cp ../*.meta.json wasm/
+cp -a ../../target/wasm32-wasip1/release/*.wasm wasm/
+cp ../../*.meta.json wasm/
 
 script_dir=$(dirname $(realpath $0))
 
@@ -46,18 +46,11 @@ message "Setting up the Kong Gateway container..."
 docker stop $DEMO_KONG_CONTAINER
 docker rm $DEMO_KONG_CONTAINER
 
-# Config trick to access localhost in a local Docker test,
-# in case you want to edit your config/demo.yml to target
-# a localhost server rather than httpbin.org:
-#
-# access_localhost="--add-host=host.docker.internal:$(ip -j address | jq -r '[ .[] | select(.ifname | test("^[ew]")) | .addr_info[] | select(.family == "inet") | .local ][0]')"
-access_localhost=""
-
 docker run -d --name "$DEMO_KONG_CONTAINER" \
     $access_localhost \
     -v "$script_dir/config:/kong/config/" \
     -v "$script_dir/wasm:/wasm" \
-    -e "KONG_LOG_LEVEL=info" \
+    -e "KONG_LOG_LEVEL=debug" \
     -e "KONG_DATABASE=off" \
     -e "KONG_DECLARATIVE_CONFIG=/kong/config/demo.yml" \
     -e "KONG_NGINX_WASM_SHM_KV_DATAKIT=12m" \
@@ -85,15 +78,12 @@ sleep 5
 
 message "Now let's send a request to see the filter in effect:"
 
-http :8000/anything
-http :8000/anything
-http :8000/anything
-http :8000/anything
-http :8000/anything
-http :8000/anything
+http :8000/
+http :8000/ apikey:mykey
+http :8000/ apikey:mykey x-datakit-debug-trace:1
 
 message "Finishing up!"
 
-docker stop $DEMO_KONG_CONTAINER
+#docker stop $DEMO_KONG_CONTAINER
 #docker rm $DEMO_KONG_CONTAINER
 

examples/call_headers/reconfigure.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+http :8001/config config=@config/demo.yml 
+

examples/call_headers/reset.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+rm -rf wasm