[RFC] Filtering over nested arrays of scalars

justfile

@@ -1,3 +1,4 @@
+# Lists all targets
 default:
   just --list
 
@@ -9,14 +10,48 @@ docker +args:
   docker compose -f dev.docker-compose.yaml down
   exit $exit_code
 
+# Builds in docker
 ci-build:
   just docker cargo build
 
+# Runs the tests in docker
 ci-test:
   just docker cargo test
 
+# Runs linting checks in docker
 ci-lint:
   just docker cargo clippy
 
+# Runs benchmarks in docker
 ci-bench:
   just docker cargo bench
+
+# Runs the tests
+test *ARGS:
+  #!/usr/bin/env bash
+  if command -v cargo-nextest; then
+    COMMAND=(cargo nextest run)
+  else
+    COMMAND=(cargo test)
+  fi
+  COMMAND+=(--no-fail-fast "$@")
+  echo "${COMMAND[*]}"
+  "${COMMAND[@]}"
+
+# Formats all the Markdown, Rust, Nix etc
+fix-format: fix-format-prettier
+  cargo fmt --all
+  ! command -v nix || nix fmt
+
+# Formats Markdown, etc with prettier
+fix-format-prettier:
+  npx --yes prettier --write .
+
+# Runs the tests and updates all goldenfiles with the test output
+update-golden-files:
+  UPDATE_GOLDENFILES=1 just test
+  just fix-format-prettier
+
+# Starts the ndc-spec documentation webserver
+start-docs:
+  cd specification && mdbook serve

ndc-models/src/lib.rs

@@ -95,7 +95,7 @@ pub struct ExistsCapabilities {
 #[schemars(title = "Nested Field Capabilities")]
 pub struct NestedFieldCapabilities {
     /// Does the connector support filtering by values of nested fields
-    pub filter_by: Option<LeafCapability>,
+    pub filter_by: Option<NestedFieldFilterByCapabilities>,
     /// Does the connector support ordering by values of nested fields
     pub order_by: Option<LeafCapability>,
     /// Does the connector support aggregating values within nested fields
@@ -104,7 +104,35 @@ pub struct NestedFieldCapabilities {
     /// `NestedField::NestedCollection`
     pub nested_collections: Option<LeafCapability>,
 }
-// ANCHOR_END: NestedCollectionCapabilities
+// ANCHOR_END: NestedFieldCapabilities
+
+// ANCHOR: NestedFieldFilterByCapabilities
+#[skip_serializing_none]
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[schemars(title = "Nested Field Filter By Capabilities")]
+pub struct NestedFieldFilterByCapabilities {
+    /// Does the connector support filtering over nested arrays
+    pub nested_arrays: Option<NestedArrayFilterByCapabilities>,
+}
+// ANCHOR_END: NestedFieldFilterByCapabilities
+
+// ANCHOR: NestedArrayFilterByCapabilities
+#[skip_serializing_none]
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[schemars(title = "Nested Array Filter By Capabilities")]
+pub struct NestedArrayFilterByCapabilities {
+    /// Does the connector support filtering over nested arrays using existential quantification.
+    /// This means the connector must support ExistsInCollection::NestedScalarCollection.
+    pub exists: Option<LeafCapability>,
+    /// Does the connector support filtering over nested arrays by checking if the array contains a value.
+    /// This must be supported for all types that can be contained in an array that implement an 'eq'
+    /// comparison operator.
+    pub contains: Option<LeafCapability>,
+    /// Does the connector support filtering over nested arrays by checking if the array is empty.
+    /// This must be supported no matter what type is contained in the array.
+    pub is_empty: Option<LeafCapability>,
+}
+// ANCHOR_END: NestedArrayFilterByCapabilities
 
 // ANCHOR: AggregateCapabilities
 #[skip_serializing_none]
@@ -817,13 +845,29 @@ pub enum Expression {
         operator: ComparisonOperatorName,
         value: ComparisonValue,
     },
+    ArrayComparison {
+        column: ComparisonTarget,
+        comparison: ArrayComparison,
+    },
     Exists {
         in_collection: ExistsInCollection,
         predicate: Option<Box<Expression>>,
     },
 }
 // ANCHOR_END: Expression
 
+// ANCHOR: ArrayComparison
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[schemars(title = "Array Comparison")]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum ArrayComparison {
+    /// Check if the array contains the specified value
+    Contains { value: ComparisonValue },
+    /// Check is the array is empty
+    IsEmpty,
+}
+// ANCHOR_END: ArrayComparison
+
 // ANCHOR: UnaryComparisonOperator
 #[derive(
     Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, JsonSchema,
@@ -931,6 +975,18 @@ pub enum ExistsInCollection {
         #[serde(skip_serializing_if = "Vec::is_empty", default)]
         field_path: Vec<FieldName>,
     },
+    /// Specifies a column that contains a nested array of scalars. The
+    /// array will be brought into scope of the nested expression where
+    /// each element becomes an object with one '__value' column that
+    /// contains the element value.
+    NestedScalarCollection {
+        column_name: FieldName,
+        #[serde(skip_serializing_if = "BTreeMap::is_empty", default)]
+        arguments: BTreeMap<ArgumentName, Argument>,
+        /// Path to a nested collection via object columns
+        #[serde(skip_serializing_if = "Vec::is_empty", default)]
+        field_path: Vec<FieldName>,
+    },
 }
 // ANCHOR_END: ExistsInCollection
 

ndc-models/tests/json_schema/capabilities_response.jsonschema

@@ -178,7 +178,7 @@
           "description": "Does the connector support filtering by values of nested fields",
           "anyOf": [
             {
-              "$ref": "#/definitions/LeafCapability"
+              "$ref": "#/definitions/NestedFieldFilterByCapabilities"
             },
             {
               "type": "null"
@@ -220,6 +220,62 @@
         }
       }
     },
+    "NestedFieldFilterByCapabilities": {
+      "title": "Nested Field Filter By Capabilities",
+      "type": "object",
+      "properties": {
+        "nested_arrays": {
+          "description": "Does the connector support filtering over nested arrays",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/NestedArrayFilterByCapabilities"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        }
+      }
+    },
+    "NestedArrayFilterByCapabilities": {
+      "title": "Nested Array Filter By Capabilities",
+      "type": "object",
+      "properties": {
+        "exists": {
+          "description": "Does the connector support filtering over nested arrays using existential quantification. This means the connector must support ExistsInCollection::NestedScalarCollection.",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/LeafCapability"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "contains": {
+          "description": "Does the connector support filtering over nested arrays by checking if the array contains a value. This must be supported for all types that can be contained in an array that implement an 'eq' comparison operator.",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/LeafCapability"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "is_empty": {
+          "description": "Does the connector support filtering over nested arrays by checking if the array is empty. This must be supported no matter what type is contained in the array.",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/LeafCapability"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        }
+      }
+    },
     "ExistsCapabilities": {
       "title": "Exists Capabilities",
       "type": "object",

ndc-models/tests/json_schema/mutation_request.jsonschema

@@ -741,6 +741,28 @@
             }
           }
         },
+        {
+          "type": "object",
+          "required": [
+            "column",
+            "comparison",
+            "type"
+          ],
+          "properties": {
+            "type": {
+              "type": "string",
+              "enum": [
+                "array_comparison"
+              ]
+            },
+            "column": {
+              "$ref": "#/definitions/ComparisonTarget"
+            },
+            "comparison": {
+              "$ref": "#/definitions/ArrayComparison"
+            }
+          }
+        },
         {
           "type": "object",
           "required": [
@@ -942,6 +964,45 @@
         }
       ]
     },
+    "ArrayComparison": {
+      "title": "Array Comparison",
+      "oneOf": [
+        {
+          "description": "Check if the array contains the specified value",
+          "type": "object",
+          "required": [
+            "type",
+            "value"
+          ],
+          "properties": {
+            "type": {
+              "type": "string",
+              "enum": [
+                "contains"
+              ]
+            },
+            "value": {
+              "$ref": "#/definitions/ComparisonValue"
+            }
+          }
+        },
+        {
+          "description": "Check is the array is empty",
+          "type": "object",
+          "required": [
+            "type"
+          ],
+          "properties": {
+            "type": {
+              "type": "string",
+              "enum": [
+                "is_empty"
+              ]
+            }
+          }
+        }
+      ]
+    },
     "ExistsInCollection": {
       "title": "Exists In Collection",
       "oneOf": [
@@ -1028,6 +1089,38 @@
               }
             }
           }
+        },
+        {
+          "description": "Specifies a column that contains a nested array of scalars. The array will be brought into scope of the nested expression where each element becomes an object with one '__value' column that contains the element value.",
+          "type": "object",
+          "required": [
+            "column_name",
+            "type"
+          ],
+          "properties": {
+            "type": {
+              "type": "string",
+              "enum": [
+                "nested_scalar_collection"
+              ]
+            },
+            "column_name": {
+              "type": "string"
+            },
+            "arguments": {
+              "type": "object",
+              "additionalProperties": {
+                "$ref": "#/definitions/Argument"
+              }
+            },
+            "field_path": {
+              "description": "Path to a nested collection via object columns",
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          }
         }
       ]
     },