test: full integration

src/typegate/src/runtimes/substantial/filter_utils.ts

@@ -272,19 +272,19 @@ function inclusion(
 ) {
   const [left, right] = cp == "in" ? [l, r] : [r, l];
   if (Array.isArray(right)) {
-    // Note: Array.prototype.includes compare inner references
+    // Note: Array.prototype.includes compare item references, not the values
     const leftComp = JSON.stringify(left);
     return right.some((inner) => JSON.stringify(inner) === leftComp);
   } else if (typeof left == "string" && typeof right == "string") {
     return right.includes(left);
   } else if (
     typeof left == typeof right && typeof left == "object" && left != null
   ) {
-    // { a: { b: 1 } } in { a: { b: 1 }, c: 2 } := true
+    // Example: { a: { b: 1 } } in { a: { b: 1 }, c: 2 } => true
     const rightV = (right ?? {}) as Record<string, unknown>;
     for (const [k, leftVal] of Object.entries(left)) {
       const rightVal = rightV[k];
-      if (leftVal != rightVal) {
+      if (!testCompare(leftVal, rightVal)) {
         return false;
       }
     }

src/typegraph/core/src/runtimes/substantial/type_utils.rs

@@ -84,7 +84,11 @@ fn save(
 /// * special: "started_at", "status", ..
 fn filter_term_variants() -> Result<Vec<crate::types::TypeId>> {
     // FIXME: a generic json would have been helpful here vs json string
-    // let any = t::eitherx!(t::integer(), t::string(), t::boolean(), ...).build_named("AnyValue")?;
+    // let any_scalar = save("any_scalar", |n| {
+    //     t::eitherx!(t::integer(), t::string(), t::boolean(), t::struct_()).build_named(n)
+    // })?;
+    // let value_to_comp_against = t::eitherx!(any_scalar, t::listx(any_scalar)).build()?;
+
     let value_to_comp_against = t::string().format("json").build()?;
     let ops = ["eq", "lt", "lte", "gt", "gte", "in", "contains"]
         .into_iter()
@@ -95,7 +99,7 @@ fn filter_term_variants() -> Result<Vec<crate::types::TypeId>> {
         })
         .collect::<Result<Vec<_>>>()?;
 
-    let op_value = save("Op", |n| t::either(ops.clone().into_iter()).build_named(n))?;
+    let op_value = save("op", |n| t::either(ops.clone().into_iter()).build_named(n))?;
 
     let special = ["started_at", "ended_at", "status"]
         .into_iter()

tests/runtimes/substantial/common.ts

@@ -5,6 +5,7 @@ import { assertEquals, assertExists } from "@std/assert";
 import { connect, parseURL } from "redis";
 import { gql, Meta, sleep } from "../../utils/mod.ts";
 import { MetaTestCleanupFn } from "test-utils/test.ts";
+import { Expr } from "@metatype/typegate/runtimes/substantial/filter_utils.ts";
 
 export type BackendName = "fs" | "memory" | "redis";
 
@@ -563,7 +564,7 @@ export function childWorkflowTestTemplate(
 
       await sleep(delays.awaitCompleteSec * 1000);
 
-      t.should(`complete parent and all child workflows`, async () => {
+      await t.should(`complete parent and all child workflows`, async () => {
         await gql`
           query {
             children: results_raw(name: "bumpPackage") {
@@ -622,6 +623,40 @@ export function childWorkflowTestTemplate(
           })
           .on(e);
       });
+
+
+      await t.should(`filter the runs given a nested expr (${backendName})`, async () => {
+        await gql`
+          query {
+            search(name: "bumpPackage", filter: $filter) {
+              # started_at
+              # ended_at
+              status
+              value
+            }
+          }
+        `
+        .withVars({
+          filter: {
+            or: [
+              {
+                and: [
+                  { status: { contains: JSON.stringify("COMPL") }},
+                  { contains: JSON.stringify("substantial") }
+                ]
+              },
+              { not: { not: { eq: JSON.stringify("Bump typegraph v3 => v4") } } }
+            ]
+          } satisfies Expr
+        })
+          .expectData({
+            search: [
+              { status: "COMPLETED", value: '"Bump typegraph v3 => v4"' },
+              { status: "COMPLETED", value: '"Bump substantial v2 => v3"' }
+            ]
+          })
+          .on(e);
+      });
     },
   );
 }
@@ -660,7 +695,7 @@ export function inputMutationTemplate(
 
       let currentRunId: string | null = null;
       await t.should(
-        `start accidentialInputMutation workflow and return its run id (${backendName})`,
+        `start accidentalInputMutation workflow and return its run id (${backendName})`,
         async () => {
           await gql`
             mutation {
@@ -688,7 +723,7 @@ export function inputMutationTemplate(
         async () => {
           await gql`
             query {
-              results_raw(name: "accidentialInputMutation") {
+              results_raw(name: "accidentalInputMutation") {
                 ongoing {
                   count
                   runs {

tests/runtimes/substantial/filter_utils_test.ts

@@ -80,17 +80,17 @@ Meta.test("base filter logic", async (t) => {
   };
 
   // ------------------
-  await testShould("be able discriminate truthy values and 1)", {
+  await testShould("be able to discriminate truthy values and 1", {
     filter: { eq: val(1) },
     expected: [
-        {
+      {
         ended_at: "2024-01-06T00:00:00.000Z",
         run_id: "fakeUUID#7",
         started_at: "2024-01-05T00:00:00.000Z",
         status: "COMPLETED",
         value: "1",
-        },
-    ]
+      },
+    ],
   });
 
   await testShould('work with null and special values (e.g. "status")', {
@@ -115,13 +115,13 @@ Meta.test("base filter logic", async (t) => {
         status: "ONGOING",
         value: undefined,
       },
-    {
+      {
         ended_at: "2024-01-05T00:00:00.000Z",
         run_id: "fakeUUID#6",
         started_at: "2024-01-04T00:00:00.000Z",
         status: "COMPLETED",
         value: "null",
-    }
+      },
     ],
   });
 
@@ -134,7 +134,7 @@ Meta.test("base filter logic", async (t) => {
             { contains: val(["three"]) },
           ],
         },
-        { contains: val({ nested: { object: 1234 } })},
+        { contains: val({ nested: { object: 1234 } }) },
         { in: val("Fatal: error+ some other string") },
       ],
     },
@@ -153,6 +153,13 @@ Meta.test("base filter logic", async (t) => {
         status: "COMPLETED",
         value: '[1,2,["three"]]',
       },
+      {
+        ended_at: "2024-01-05T00:00:00.000Z",
+        run_id: "fakeUUID#5",
+        started_at: "2024-01-04T00:00:00.000Z",
+        status: "COMPLETED_WITH_ERROR",
+        value: '{"nested":{"object":1234},"b":4}',
+      },
     ],
   });
 
@@ -196,3 +203,10 @@ Meta.test("base filter logic", async (t) => {
     },
   );
 });
+
+// TODO: bench?
+// Each case should be relatively close as we traverse from start to end without any back and forth
+// 1. benchmark(listTrasersal, bigNoopExpr, items=100000)
+// 2. benchmark(arrayWithNegFilter, bigNoopExpr, items=100000)
+
+// then avg unit overhead := [time(bigNoopExpr) - time(listTrasersal)] / 100000

tests/runtimes/substantial/substantial.py

@@ -21,14 +21,16 @@ def substantial(g: Graph):
             backend = Backend.redis("SUB_REDIS")
 
     file = (
-        WorkflowFile.deno(file="workflow.ts", deps=["imports/common_types.ts"])
+        WorkflowFile.deno(
+            file="workflows/workflow.ts", deps=["imports/common_types.ts"]
+        )
         .import_(
             [
                 "saveAndSleepExample",
                 "eventsAndExceptionExample",
                 "retryExample",
                 "secretsExample",
-                "accidentialInputMutation",
+                "accidentalInputMutation",
             ]
         )
         .build()
@@ -80,6 +82,6 @@ def substantial(g: Graph):
                     )
                 }
             )
-        ).reduce({"name": "accidentialInputMutation"}),
+        ).reduce({"name": "accidentalInputMutation"}),
         **sub.internals(),
     )

tests/runtimes/substantial/substantial_child_workflow.py

@@ -20,7 +20,9 @@ def substantial_child_workflow(g: Graph):
             backend = Backend.redis("SUB_REDIS")
 
     file = (
-        WorkflowFile.deno(file="child_workflow.ts", deps=["imports/common_types.ts"])
+        WorkflowFile.deno(
+            file="workflows/child_workflow.ts", deps=["imports/common_types.ts"]
+        )
         .import_(["bumpPackage", "bumpAll"])
         .build()
     )
@@ -38,5 +40,6 @@ def substantial_child_workflow(g: Graph):
         start=sub.start(t.struct({"packages": t.list(package)})).reduce(
             {"name": "bumpAll"}
         ),
+        search=sub.advanced_filters(),
         **sub.internals(),
     )

tests/runtimes/substantial/child_workflow.ts → tests/runtimes/substantial/workflows/child_workflow.ts

@@ -1,7 +1,7 @@
 // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0.
 // SPDX-License-Identifier: MPL-2.0
 
-import { Context } from "./imports/common_types.ts";
+import { Context } from "../imports/common_types.ts";
 
 function apply(pkg: string, oldVersion: string, newVersion: string) {
   console.info(

tests/runtimes/substantial/workflow.ts → tests/runtimes/substantial/workflows/workflow.ts

@@ -7,7 +7,7 @@ import {
   sendSubscriptionEmail,
   sleep,
   Workflow,
-} from "./imports/common_types.ts";
+} from "../imports/common_types.ts";
 
 export const eventsAndExceptionExample: Workflow<string> = async (
   ctx: Context
@@ -112,7 +112,7 @@ export const secretsExample: Workflow<void> = (_, { secrets }) => {
   return Promise.resolve();
 };
 
-export async function accidentialInputMutation(ctx: Context) {
+export async function accidentalInputMutation(ctx: Context) {
   const { items } = ctx.kwargs;
 
   const copy = [];