From 2cb81484126a703b3a629bd35847aa1c3ce53e51 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 24 Feb 2025 16:22:44 +0100 Subject: [PATCH 01/20] feat: add generic filtering capabalities to getPubsWithRelatedValues --- core/lib/__tests__/matchers.ts | 8 +- core/lib/server/pub-filters.db.test.ts | 552 +++++++++++++++++++++++ core/lib/server/pub-filters.ts | 239 ++++++++++ core/lib/server/pub.ts | 12 +- core/lib/server/validate-filters.ts | 121 +++++ packages/contracts/src/resources/site.ts | 126 ++++++ 6 files changed, 1055 insertions(+), 3 deletions(-) create mode 100644 core/lib/server/pub-filters.db.test.ts create mode 100644 core/lib/server/pub-filters.ts create mode 100644 core/lib/server/validate-filters.ts diff --git a/core/lib/__tests__/matchers.ts b/core/lib/__tests__/matchers.ts index 75f3d31c6..5de495a05 100644 --- a/core/lib/__tests__/matchers.ts +++ b/core/lib/__tests__/matchers.ts @@ -7,7 +7,11 @@ import type { db } from "~/kysely/database"; const deepSortValues = (pub: ProcessedPub): ProcessedPub => { pub.values - .sort((a, b) => (a.value as string).localeCompare(b.value as string)) + .sort((a, b) => { + const aValue = String(a.value); + const bValue = String(b.value); + return aValue.localeCompare(bValue); + }) .map((item) => ({ ...item, relatedPub: item.relatedPub?.values ? deepSortValues(item.relatedPub) : item.relatedPub, @@ -60,7 +64,7 @@ expect.extend({ message: () => pass ? `Expected pub ${isNot ? "not" : ""} to have values ${JSON.stringify(expected)}, and it does ${isNot ? "not" : ""}` - : `Expected pub ${isNot ? "not to" : "to"} match values ${this.utils.diff(sortedPubValues.values, expected)}`, + : `Expected pub ${isNot ? "not to" : "to"} match values ${this.utils.diff(expected, sortedPubValues.values)}`, }; }, }); diff --git a/core/lib/server/pub-filters.db.test.ts b/core/lib/server/pub-filters.db.test.ts new file mode 100644 index 000000000..58fee86ad --- /dev/null +++ b/core/lib/server/pub-filters.db.test.ts @@ -0,0 +1,552 @@ +import { describe, expect, it } from "vitest"; + +import type { Filter, Json } from "contracts"; +import type { CommunitiesId, PubsId } from "db/public"; +import { filterSchema } from "contracts"; +import { CoreSchemaType } from "db/public"; + +import { createSeed } from "~/prisma/seed/createSeed"; +import { mockServerCode } from "../__tests__/utils"; +import { applyFilters } from "./pub-filters"; + +const { createForEachMockedTransaction, testDb } = await mockServerCode(); + +const { getTrx, rollback } = createForEachMockedTransaction(); + +const communitySlug = `${new Date().toISOString()}:test-filter-pub`; + +const trueId = crypto.randomUUID() as PubsId; +const vector3Id = crypto.randomUUID() as PubsId; + +const twenty99 = new Date("2099-01-01"); + +const seed = createSeed({ + community: { + name: "test", + slug: communitySlug, + }, + pubFields: { + Title: { schemaName: CoreSchemaType.String }, + Number: { schemaName: CoreSchemaType.Number }, + Boolean: { schemaName: CoreSchemaType.Boolean }, + Date: { schemaName: CoreSchemaType.DateTime }, + Array: { schemaName: CoreSchemaType.StringArray }, + NumberArray: { schemaName: CoreSchemaType.NumericArray }, + Vector3: { schemaName: CoreSchemaType.Vector3 }, + File: { schemaName: CoreSchemaType.FileUpload }, + Relation: { schemaName: CoreSchemaType.Null, relation: true }, + }, + pubTypes: { + "Basic Pub": { + Title: { isTitle: true }, + Number: { isTitle: false }, + Boolean: { isTitle: false }, + Date: { isTitle: false }, + Array: { isTitle: false }, + Vector3: { isTitle: false }, + File: { isTitle: false }, + Relation: { isTitle: false }, + NumberArray: { isTitle: false }, + }, + }, + stages: {}, + pubs: [ + { + pubType: "Basic Pub", + values: { + Title: "Some title", + }, + }, + { + pubType: "Basic Pub", + values: { + Title: "Another title", + }, + }, + { + pubType: "Basic Pub", + values: { + Number: 42, + }, + }, + { + id: trueId, + pubType: "Basic Pub", + values: { + Boolean: true, + }, + }, + { + pubType: "Basic Pub", + values: { + Array: ["item1", "item2"], + }, + }, + { + id: vector3Id, + pubType: "Basic Pub", + values: { + Vector3: [0, 0, 0], + }, + }, + { + pubType: "Basic Pub", + values: { + NumberArray: [1, 2, 3], + Date: twenty99, + }, + }, + + { + pubType: "Basic Pub", + values: { + NumberArray: [10, 20, 30, 40], + }, + }, + { + pubType: "Basic Pub", + values: { + Relation: null, + }, + }, + ], +}); + +// let community: CommunitySeedOutput; + +const seedCommunity = async (trx = testDb) => { + const { seedCommunity } = await import("~/prisma/seed/seedCommunity"); + const community = await seedCommunity( + seed, + { + // no random slug like usual because we have to define `.each` tests statically + randomSlug: false, + }, + trx + ); + + // set the updatedAt for a pub to a weird date + const p = await trx + .updateTable("pubs") + .set({ createdAt: new Date("2024-01-01") }) + .where("id", "=", trueId) + .returningAll() + .execute(); + + await trx + .updateTable("pubs") + .set({ createdAt: twenty99 }) + .where("id", "=", vector3Id) + .execute(); + + return community; +}; + +const community = await seedCommunity(); + +// afterAll(async () => { +// const { deleteCommunity } = await import("~/prisma/seed/deleteCommunity"); +// await deleteCommunity(communityId); +// }); + +const coolQuery = (filter: Filter) => { + const trx = getTrx(); + + const q = trx + .selectFrom("pubs") + .selectAll() + .where((eb) => applyFilters(eb, filter)); + + return { + q, + compiled: q.compile(), + }; +}; + +const validateFilter = async (communityId: CommunitiesId, filter: Filter, trx = getTrx()) => { + const { validateFilter: valFilter } = await import("./validate-filters"); + return valFilter(communityId, filter, trx); +}; + +const slug = (str: string) => `${communitySlug}:${str}`; + +describe("pub-filters", () => { + describe("filter validation", () => { + describe("schema", () => { + it("successfully parses a filter", async () => { + const filter: Filter = { + [community.pubFields.Title.slug]: { $eq: "test" }, + }; + + const parsed = filterSchema.safeParse(filter); + expect(parsed.success).toBe(true); + expect(parsed.data).toEqual(filter); + }); + }); + + describe("pubField validation", () => { + it("rejects unknown fields", async () => { + const filter: Filter = { + [`${community.community.slug}:unknownField`]: { $eq: "test" }, + }; + + await expect(validateFilter(community.community.id, filter)).rejects.toThrow(); + }); + + it("only allows valid operators for a field", async () => { + const filter: Filter = { + [community.pubFields.Title.slug]: { $invalid: "test" }, + }; + + const parsed = filterSchema.safeParse(filter); + expect(parsed.success).toBe(false); + + await expect(validateFilter(community.community.id, filter)).rejects.toThrow(); + }); + + it("does not allow gte on a string field", async () => { + const filter: Filter = { + [community.pubFields.Title.slug]: { $gte: "test" }, + }; + + const parsed = filterSchema.safeParse(filter); + + expect(parsed.success).toBe(false); + + await expect(validateFilter(community.community.id, filter)).rejects.toThrow( + /Operators \[\$gte\] are not valid for schema type String/ + ); + }); + }); + }); + + const currentDate = new Date(); + + const validFilterCases: { + title: string; + filter: Filter; + sql: string; + parameters: (string | number)[]; + resultValues: { value: Json; fieldSlug?: string }[][]; + }[] = [ + { + title: "simple equality", + filter: { + [slug("title")]: { $eq: "Some title" }, + }, + sql: `"slug" = $1 and "value" = $2`, + parameters: [slug("title"), '"Some title"'], + resultValues: [[{ value: "Some title", fieldSlug: slug("title") }]], + }, + { + title: "simple inequality", + filter: { + [slug("title")]: { $ne: "Some title" }, + }, + sql: `"slug" = $1 and "value" != $2`, + parameters: [slug("title"), '"Some title"'], + resultValues: [[{ value: "Another title", fieldSlug: slug("title") }]], + }, + { + title: "case insensitive equality", + filter: { + [slug("title")]: { $eqi: "some title" }, + }, + sql: `"slug" = $1 and lower(value::text) = $2`, + parameters: [slug("title"), '"some title"'], + resultValues: [[{ value: "Some title", fieldSlug: slug("title") }]], + }, + { + title: "case insensitive inequality", + filter: { + [slug("title")]: { $nei: "some title" }, + }, + sql: `"slug" = $1 and lower(value::text) != $2`, + parameters: [slug("title"), '"some title"'], + resultValues: [[{ value: "Another title", fieldSlug: slug("title") }]], + }, + { + title: "string contains", + filter: { + [slug("title")]: { $contains: "Another" }, + }, + sql: `"slug" = $1 and value::text like $2`, + parameters: [slug("title"), "%Another%"], + resultValues: [[{ value: "Another title", fieldSlug: slug("title") }]], + }, + { + title: "string contains case insensitive", + filter: { + [slug("title")]: { $containsi: "another" }, + }, + sql: `"slug" = $1 and value::text ilike $2`, + parameters: [slug("title"), "%another%"], + resultValues: [[{ value: "Another title", fieldSlug: slug("title") }]], + }, + { + title: "string not contains", + filter: { + [slug("title")]: { $notContains: "Another" }, + }, + sql: `"slug" = $1 and value::text not like $2`, + parameters: [slug("title"), "%Another%"], + resultValues: [[{ value: "Some title", fieldSlug: slug("title") }]], + }, + { + title: "string not contains case insensitive", + filter: { + [slug("title")]: { $notContainsi: "another" }, + }, + sql: `"slug" = $1 and value::text not ilike $2`, + parameters: [slug("title"), "%another%"], + resultValues: [[{ value: "Some title", fieldSlug: slug("title") }]], + }, + + { + title: "array contains w/ json path", + filter: { + [slug("array")]: { $jsonPath: '$[*] == "item1"' }, + }, + sql: `"slug" = $1 and "value" @@ $2`, + parameters: [slug("array"), '$[*] == "item1"'], + resultValues: [[{ value: ["item1", "item2"], fieldSlug: slug("array") }]], + }, + { + title: "array specific index value check", + filter: { + [slug("numberarray")]: { $jsonPath: "$[1] > 10" }, + }, + sql: `"slug" = $1 and "value" @@ $2`, + parameters: [slug("numberarray"), "$[1] > 10"], + resultValues: [[{ value: [10, 20, 30, 40], fieldSlug: slug("numberarray") }]], + }, + { + title: "nested logical operators", + filter: { + $or: [ + { [slug("title")]: { $eq: "Some title" } }, + { + $and: [ + { [slug("number")]: { $gt: 40 } }, + { [slug("number")]: { $lt: 50 } }, + ], + }, + ], + }, + + sql: `(("slug" = $1 and "value" = $2) or (("slug" = $3 and "value" > $4) and ("slug" = $5 and "value" < $6)))`, + parameters: [slug("title"), '"Some title"', slug("number"), 40, slug("number"), 50], + resultValues: [ + [{ value: 42, fieldSlug: slug("number") }], + [{ value: "Some title", fieldSlug: slug("title") }], + ], + }, + { + title: "updatedAt & createdAt filters", + filter: { + $or: [ + { + createdAt: { + $lte: new Date("2025-01-01"), + }, + }, + { + createdAt: { + $gte: new Date("2090-01-01"), + }, + }, + ], + }, + sql: `"pubs"."createdAt" <= $1 or "pubs"."createdAt" >= $2`, + parameters: [ + `"${new Date("2025-01-01").toISOString()}"`, + `"${new Date("2090-01-01").toISOString()}"`, + ], + resultValues: [ + [ + { + value: true, + fieldSlug: slug("boolean"), + }, + ], + [{ value: [0, 0, 0], fieldSlug: slug("vector3") }], + ], + }, + { + title: "date filters", + filter: { + [slug("date")]: { + $eq: twenty99.toISOString(), + }, + }, + sql: `"slug" = $1 and "value" = $2`, + parameters: [slug("date"), `"${twenty99.toISOString()}"`], + resultValues: [ + [ + { + value: [1, 2, 3], + fieldSlug: slug("numberarray"), + }, + { value: twenty99.toISOString(), fieldSlug: slug("date") }, + ], + ], + }, + ]; + describe("SQL generation", () => { + it.concurrent.each(validFilterCases)( + "generates correct SQL for $title", + async ({ filter, sql, parameters }) => { + const trx = getTrx(); + + const q = coolQuery(filter).compiled; + + expect(q.sql).toMatch(sql); + expect(q.parameters).toEqual(parameters); + } + ); + }); + + const invalidFilterCases: { + title: string; + filter: Filter; + error: RegExp; + }[] = [ + { + title: "all invalid operators for strings", + filter: { + [slug("title")]: { + $eq: "test", + $eqi: "test", + $ne: "test", + $nei: "test", + $gt: "test", + $lt: "test", + $gte: "test", + $lte: "test", + $in: "test", + $notIn: "test", + $any: "test", + $all: "test", + }, + }, + error: /Operators \[\$gt, \$lt, \$gte, \$lte, \$in, \$notIn, \$any, \$all\] are not valid for schema type String/, + }, + { + title: "all invalid operators for numbers", + filter: { + [slug("number")]: { + $startsWith: "test", + $endsWith: "test", + $startsWithi: "test", + $endsWithi: "test", + $containsi: "test", + $notContainsi: "test", + $contains: "test", + $notContains: "test", + $any: "test", + $all: "test", + $size: "test", + }, + }, + error: /Operators \[\$startsWith, \$endsWith, \$startsWithi, \$endsWithi, \$containsi, \$notContainsi, \$contains, \$notContains, \$any, \$all, \$size\] are not valid for schema type Number/, + }, + { + title: "all invalid operators for booleans", + filter: { + [slug("boolean")]: { + $eq: true, + $eqi: true, + $ne: false, + $nei: false, + $lt: true, + $lte: true, + $gt: false, + $gte: false, + $contains: true, + $notContains: false, + $containsi: true, + $notContainsi: false, + $null: true, + $notNull: false, + $between: [true, false], + $startsWith: true, + $startsWithi: true, + $endsWith: true, + $endsWithi: true, + $size: 1, + }, + }, + error: /Operators \[\$eqi, \$nei, \$lt, \$lte, \$gt, \$gte, \$contains, \$notContains, \$containsi, \$notContainsi, \$between, \$startsWith, \$startsWithi, \$endsWith, \$endsWithi, \$size\] are not valid for schema type Boolean/, + }, + { + title: "unknown operators", + filter: { + [slug("title")]: { $invalid: "test" }, + }, + error: /Operators \[\$invalid\] are not valid for schema type String/, + }, + { + title: "unknown fields", + filter: { + [slug("unknownField")]: { $eq: "test" }, + }, + error: /Pub values contain fields that do not exist in the community: .*:unknownField/, + }, + ]; + + describe("validation", () => { + it.concurrent.each(validFilterCases)( + "correctly validates filter for $title", + async ({ filter }) => { + const trx = getTrx(); + // const community = await seedCommunity(trx); + await expect( + validateFilter(community.community.id, filter, trx) + ).resolves.not.toThrow(); + } + ); + + it.each(invalidFilterCases)("correctly rejects $title", async ({ filter, error }) => { + const trx = getTrx(); + // const community = await seedCommunity(trx); + await expect(validateFilter(community.community.id, filter, trx)).rejects.toThrow( + error + ); + }); + }); + + describe("filtering", async () => { + it.concurrent.each(validFilterCases)( + "filters by $title", + async ({ filter, resultValues }) => { + const trx = getTrx(); + + const { getPubsWithRelatedValuesAndChildren } = await import("~/lib/server/pub"); + const pubs = await getPubsWithRelatedValuesAndChildren( + { + communityId: community.community.id, + }, + { + trx, + filters: filter, + } + ); + + expect( + pubs, + "Expected the same number of pubs to be returned as the number of specified result values" + ).toHaveLength(resultValues.length); + + if (pubs.length === 0) { + return; + } + + pubs.sort((a, b) => a.values[0].schemaName.localeCompare(b.values[0].schemaName)); + + pubs.forEach((pub, idx) => { + expect(pub).toHaveValues(resultValues[idx]); + }); + } + ); + }); +}); diff --git a/core/lib/server/pub-filters.ts b/core/lib/server/pub-filters.ts new file mode 100644 index 000000000..b11e7f517 --- /dev/null +++ b/core/lib/server/pub-filters.ts @@ -0,0 +1,239 @@ +import type { ExpressionBuilder, ExpressionWrapper } from "kysely"; + +import { sql } from "kysely"; + +import type { BaseFilter, Filter, FilterOperator, LogicalFilter, LogicalOperator } from "contracts"; +import { logicalOperators } from "contracts"; +import { CoreSchemaType } from "db/public"; +import { assert } from "utils"; + +type PathSegment = string; // Regular property + +type Path = PathSegment[]; + +// Helper type to get the type of a value at a specific path + +type EntriedLogicalFilter = [ + ["$or", NonNullable], + ["$and", NonNullable], + ["$not", NonNullable], +][number]; +type EntriedArrayFilter = [["$any", Filter], ["$all", Filter]]; + +type EntriedFilter = [string, BaseFilter[keyof BaseFilter]] | EntriedLogicalFilter; + +const isLogicalFilter = (filter: EntriedFilter): filter is EntriedLogicalFilter => { + return ( + typeof filter === "object" && + filter !== null && + logicalOperators.includes(filter[0] as LogicalOperator) + ); +}; + +export const stringOperators = [ + "$eq", + "$eqi", + "$ne", + "$nei", + "$contains", + "$notContains", + "$containsi", + "$notContainsi", + "$startsWith", + "$startsWithi", + "$endsWith", + "$endsWithi", + "$null", + "$notNull", +] as const satisfies FilterOperator[]; + +export const coreSchemaTypeAllowedOperators = { + [CoreSchemaType.Boolean]: ["$eq", "$ne", "$null", "$notNull"], + [CoreSchemaType.String]: stringOperators, + [CoreSchemaType.Number]: [ + "$eq", + "$ne", + "$lt", + "$lte", + "$gt", + "$gte", + "$between", + "$null", + "$notNull", + ], + [CoreSchemaType.Vector3]: ["$eq", "$ne", "$null", "$notNull", "$jsonPath"], + [CoreSchemaType.NumericArray]: ["$contains", "$notContains", "$null", "$notNull", "$jsonPath"], + [CoreSchemaType.StringArray]: ["$contains", "$notContains", "$null", "$notNull", "$jsonPath"], + [CoreSchemaType.DateTime]: [ + "$eq", + "$ne", + "$lt", + "$lte", + "$gt", + "$gte", + "$between", + "$null", + "$notNull", + ], + [CoreSchemaType.Email]: stringOperators, + [CoreSchemaType.URL]: stringOperators, + [CoreSchemaType.MemberId]: ["$eq", "$ne", "$null", "$notNull", "$in", "$notIn"], + [CoreSchemaType.FileUpload]: ["$null", "$notNull", "$jsonPath"], + [CoreSchemaType.RichText]: [ + "$eq", + "$ne", + "$contains", + "$notContains", + "$containsi", + "$notContainsi", + "$startsWith", + "$startsWithi", + "$endsWith", + "$endsWithi", + "$null", + "$notNull", + ], + [CoreSchemaType.Null]: ["$null"], +} as const satisfies Record; + +const filterMap = { + $eq: (eb, column, value) => eb(column, "=", JSON.stringify(value)), + $eqi: (eb, column, value) => + eb(sql.raw(`lower(${column}::text)`), "=", JSON.stringify(String(value).toLowerCase())), + $ne: (eb, column, value) => eb(column, "!=", JSON.stringify(value)), + $nei: (eb, column, value) => + eb(sql.raw(`lower(${column}::text)`), "!=", JSON.stringify(String(value).toLowerCase())), + $null: (eb, column, value) => eb(column, "is", null), + $notNull: (eb, column, value) => eb(column, "is not", null), + $lt: (eb, column, value) => + eb(column, "<", typeof value === "string" ? JSON.stringify(value) : value), + $lte: (eb, column, value) => + eb(column, "<=", typeof value === "string" ? JSON.stringify(value) : value), + $gt: (eb, column, value) => + eb(column, ">", typeof value === "string" ? JSON.stringify(value) : value), + $gte: (eb, column, value) => + eb(column, ">=", typeof value === "string" ? JSON.stringify(value) : value), + $between: (eb, column, value) => { + assert(Array.isArray(value)); + return eb.and([ + eb(column, ">=", typeof value[0] === "string" ? JSON.stringify(value[0]) : value[0]), + eb(column, "<=", typeof value[1] === "string" ? JSON.stringify(value[1]) : value[1]), + ]); + }, + $in: (eb, column, value) => eb(column, "in", value), + $notIn: (eb, column, value) => eb(column, "not in", value), + $contains: (eb, column, value) => eb(sql.raw(`${column}::text`), "like", `%${String(value)}%`), + $notContains: (eb, column, value) => + eb(sql.raw(`${column}::text`), "not like", `%${String(value)}%`), + $containsi: (eb, column, value) => + eb(sql.raw(`${column}::text`), "ilike", `%${String(value).toLowerCase()}%`), + $notContainsi: (eb, column, value) => + eb(sql.raw(`${column}::text`), "not ilike", `%${String(value).toLowerCase()}%`), + $startsWith: (eb, column, value) => eb(column, "like", `${String(value)}%`), + $startsWithi: (eb, column, value) => eb(column, "ilike", `${String(value).toLowerCase()}%`), + $endsWith: (eb, column, value) => eb(column, "like", `%${String(value)}`), + $endsWithi: (eb, column, value) => eb(column, "ilike", `%${String(value).toLowerCase()}`), + $jsonPath: (eb, column, value) => { + assert(typeof value === "string"); + return eb("value", "@@", value); + }, +} as const satisfies Record< + FilterOperator, + ( + eb: ExpressionBuilder, + column: "value" | "pubs.updatedAt" | "pubs.createdAt", + value: unknown, + carriedOperator?: "any" | "all" + ) => ExpressionWrapper +>; + +export const isNonRecursiveFilter = ( + filter: BaseFilter[string] +): filter is Exclude => { + if (Object.keys(filter).every((k) => k.startsWith("$"))) { + return true; + } + return false; +}; + +export type FieldsWithFilters = { + [slug: string]: Set; +}; + +type ExpressionWrapped> = + E extends ExpressionBuilder ? ExpressionWrapper : never; +export function applyFilters>( + eb: K, + filters: Filter +): ExpressionWrapped { + const conditions = Object.entries(filters).map((filter: EntriedFilter) => { + if (isLogicalFilter(filter)) { + if (filter[0] === "$or") { + return eb.or(filter[1].map((f) => applyFilters(eb, f))); + } + if (filter[0] === "$and") { + return eb.and(filter[1].map((f) => applyFilters(eb, f))); + } + if (filter[0] === "$not") { + return eb.not(applyFilters(eb, filter[1])); + } + + throw new Error(`Unknown logical operator: ${filter[0]}`); + } + + const [field, val] = filter; + + const isDate = field === "updatedAt" || field === "createdAt"; + if ( + isDate && + !new Set(Object.keys(val)).isSubsetOf( + new Set(coreSchemaTypeAllowedOperators[CoreSchemaType.DateTime]) + ) + ) { + throw new Error(`Date filters must use date operators: ${JSON.stringify(val)}`); + } + + if (!isNonRecursiveFilter(val)) { + throw new Error(`Unknown filter: ${JSON.stringify(filter)}`); + } + + return eb.and([ + ...(isDate ? [] : [eb("slug", "=", field)]), + ...Object.entries(val).map((entry) => { + const [operator, value] = entry as [FilterOperator, unknown]; + + const whereFn = filterMap[operator]; + if (!whereFn) { + throw new Error(`Unknown operator: ${operator}`); + } + + const maybeStringifiedValue = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map((v) => maybeStringifiedValue(v)); + } + + if (value instanceof Date) { + return value.toISOString(); + } + + return value; + }; + + return whereFn( + eb, + isDate ? `pubs.${field}` : "value", + maybeStringifiedValue(value) + ); + }), + ]); + + // const validOperators = getValidOperatorsForSchema(getJsonSchemaByCoreSchemaType(field)); + // // naive check + + // if (!validOperators.includes(operator)) { + // throw new Error(`Operator ${operator} is not valid for schema type ${field}`); + // } + }); + + return eb.and(conditions) as ExpressionWrapped; +} diff --git a/core/lib/server/pub.ts b/core/lib/server/pub.ts index 3e87c05a2..9b6571a84 100644 --- a/core/lib/server/pub.ts +++ b/core/lib/server/pub.ts @@ -13,6 +13,7 @@ import partition from "lodash.partition"; import type { CreatePubRequestBodyWithNullsNew, + Filter, FTSReturn, GetPubResponseBody, Json, @@ -51,6 +52,7 @@ import { parseLastModifiedBy } from "../lastModifiedBy"; import { autoCache } from "./cache/autoCache"; import { autoRevalidate } from "./cache/autoRevalidate"; import { BadRequestError, NotFoundError } from "./errors"; +import { applyFilters } from "./pub-filters"; import { getPubFields } from "./pubFields"; import { getPubTypeBase } from "./pubtype"; import { movePub } from "./stages"; @@ -740,7 +742,7 @@ export const getPlainPub = (pubId: PubsId, trx = db) => * Validates that all provided slugs exist in the community. * @throws Error if any slugs don't exist in the community */ -const getFieldInfoForSlugs = async ({ +export const getFieldInfoForSlugs = async ({ slugs, communityId, includeRelations = true, @@ -1356,6 +1358,7 @@ interface GetPubsWithRelatedValuesAndChildrenOptions extends GetManyParams, Mayb fieldSlugs?: string[]; onlyTitles?: boolean; trx?: typeof db; + filters?: Filter; } // TODO: We allow calling getPubsWithRelatedValuesAndChildren with no userId so that event driven @@ -1779,6 +1782,13 @@ export async function getPubsWithRelatedValuesAndChildren< .$if(Boolean(props.pubTypeId), (qb) => qb.where("pubs.pubTypeId", "=", props.pubTypeId!) ) + .$if(Boolean(options?.filters), (qb) => + // TODO: maybe dedupe this + qb + .leftJoin("pub_values as pv", "pv.pubId", "pubs.id") + .innerJoin("pub_fields as pf", "pf.id", "pv.fieldId") + .where((eb) => applyFilters(eb, options!.filters!)) + ) .$if(Boolean(orderBy), (qb) => qb.orderBy(orderBy!, orderDirection ?? "desc")) .$if(Boolean(limit), (qb) => qb.limit(limit!)) .$if(Boolean(offset), (qb) => qb.offset(offset!)) diff --git a/core/lib/server/validate-filters.ts b/core/lib/server/validate-filters.ts new file mode 100644 index 000000000..60d81c79e --- /dev/null +++ b/core/lib/server/validate-filters.ts @@ -0,0 +1,121 @@ +import type { Filter, FilterOperator } from "contracts"; +import type { CommunitiesId } from "db/public"; +import { logicalOperators } from "contracts"; +import { CoreSchemaType } from "db/public"; + +import type { FieldsWithFilters } from "./pub-filters"; +import { db } from "~/kysely/database"; +import { getFieldInfoForSlugs } from "./pub"; +import { coreSchemaTypeAllowedOperators, isNonRecursiveFilter } from "./pub-filters"; + +const isDateAt = (field: string) => field === "updatedAt" || field === "createdAt"; + +export class InvalidFilterError extends Error { + constructor( + fieldInfo: string, + allowedOperators: FilterOperator[], + operators: FilterOperator[] + ) { + const incorrectOperators = new Set(operators).difference(new Set(allowedOperators)); + + const message = `Operators [${Array.from(incorrectOperators).join(", ")}] are not valid for ${fieldInfo}: Only [${allowedOperators.join(", ")}] are allowed`; + super(message); + } +} + +export class InvalidPubFieldFilterError extends InvalidFilterError { + constructor( + field: { schemaName: CoreSchemaType; slug: string }, + allowedOperators: FilterOperator[], + operators: FilterOperator[] + ) { + super( + `schema type ${field.schemaName} of field ${field.slug}`, + allowedOperators, + operators + ); + } +} + +export class InvalidDateFilterError extends InvalidFilterError { + constructor( + field: "updatedAt" | "createdAt", + allowedOperators: FilterOperator[], + operators: FilterOperator[] + ) { + super(`date field ${field}`, allowedOperators, operators); + } +} + +export async function validateFilter(communityId: CommunitiesId, filter: Filter, trx = db) { + // find all the fields in the filter and their operators as an array + + const findFields = ( + filter: Filter, + fieldsWithFilters: FieldsWithFilters = {} + ): FieldsWithFilters => { + for (const [field, val] of Object.entries(filter)) { + if (isDateAt(field)) { + if ( + Object.keys(val).some( + (k) => + !coreSchemaTypeAllowedOperators.DateTime.includes( + k as (typeof coreSchemaTypeAllowedOperators.DateTime)[number] + ) + ) + ) { + throw new InvalidDateFilterError( + field, + coreSchemaTypeAllowedOperators.DateTime, + Object.keys(val).map((k) => k as FilterOperator) + ); + } + continue; + } + + if (logicalOperators.includes(field as keyof Filter)) { + for (const f of val) { + findFields(f, fieldsWithFilters); + } + continue; + } + + if (isNonRecursiveFilter(val)) { + for (const [operator, value] of Object.entries(val)) { + fieldsWithFilters[field] = fieldsWithFilters[field] + ? fieldsWithFilters[field].add(operator as FilterOperator) + : new Set([operator as FilterOperator]); + } + continue; + } + + findFields(val, fieldsWithFilters); + } + return fieldsWithFilters; + }; + + const foundFields = findFields(filter); + const fields = await getFieldInfoForSlugs({ + communityId, + slugs: Object.keys(foundFields), + trx, + }); + + const mergedFields = fields.map((field) => { + return { + ...field, + operators: foundFields[field.slug], + }; + }); + + for (const field of mergedFields) { + const allowedOperators = coreSchemaTypeAllowedOperators[field.schemaName]; + if (!field.operators.isSubsetOf(new Set(allowedOperators))) { + throw new InvalidPubFieldFilterError( + { schemaName: field.schemaName, slug: field.slug }, + allowedOperators, + Array.from(field.operators) + ); + } + } +} diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index bf94bdb84..71468abcb 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -309,6 +309,123 @@ const preferRepresentationHeaderSchema = z.object({ .default("return=minimal"), }); +export const filterOperators = [ + "$eq", + "$eqi", + "$ne", + "$nei", + "$lt", + "$lte", + "$gt", + "$gte", + "$contains", + "$notContains", + "$containsi", + "$notContainsi", + "$null", + "$notNull", + "$in", + "$notIn", + "$between", + "$startsWith", + "$startsWithi", + "$endsWith", + "$endsWithi", + "$jsonPath", // json path (maybe dangerous), +] as const; + +export type FilterOperator = (typeof filterOperators)[number]; + +export const logicalOperators = ["$and", "$or", "$not"] as const; + +export type LogicalOperator = (typeof logicalOperators)[number]; + +export type BaseFilter = { + [slug: string]: + | { + [O in FilterOperator]?: unknown; + } + | Filter; +}; + +export type LogicalFilter = { + $and?: Filter[]; + $or?: Filter[]; + $not?: Filter; +}; + +export type Filter = BaseFilter | LogicalFilter; + +const allSchema = z.string().or(z.number()).or(z.boolean()).or(z.coerce.date()); + +const numberOrDateSchema = z.number().or(z.coerce.date()); + +const baseFilterSchema = z + .object({ + $eq: allSchema, + $eqi: z.string(), + $ne: allSchema, + $nei: z.string(), + $lt: numberOrDateSchema, + $lte: numberOrDateSchema, + $gt: numberOrDateSchema, + $gte: numberOrDateSchema, + $contains: z.string(), + $notContains: z.string(), + $containsi: z.string(), + $notContainsi: z.string(), + $null: z.never(), + $notNull: z.never(), + $in: z.array(allSchema), + $notIn: z.array(allSchema), + $between: z.tuple([numberOrDateSchema, numberOrDateSchema]), + $startsWith: z.string(), + $startsWithi: z.string(), + $endsWith: z.string(), + $endsWithi: z.string(), + $size: z.number(), + }) + .partial() + .refine((data) => { + if (!Object.keys(data).length) { + return false; + } + return true; + }, "Filter must have at least one operator") satisfies z.ZodType<{ + [K in FilterOperator]?: any; +}>; + +// this is a recursive type, so we need to use z.lazy() +export const filterSchema: z.ZodType = z.lazy(() => + z.union([ + // regular field filters + z.record( + z.union([ + // operator-value pairs + baseFilterSchema, + // nested filters (for object types) + filterSchema, + ]) + ), + // logical operators + z.object({ + $and: z.array(filterSchema).optional(), + $or: z.array(filterSchema).optional(), + $not: filterSchema, + }), + z.object({ + $and: z.array(filterSchema).optional(), + $or: z.array(filterSchema), + $not: filterSchema.optional(), + }), + z.object({ + $and: z.array(filterSchema), + $or: z.array(filterSchema).optional(), + $not: filterSchema.optional(), + }), + ]) +); + const getPubQuerySchema = z .object({ depth: z @@ -335,6 +452,15 @@ const getPubQuerySchema = z .describe( "Which field values to include in the response. Useful if you have very large pubs or want to save on bandwidth." ), + filters: filterSchema + .optional() + .describe( + "Filter criteria using Strapi-like syntax. Examples:\n" + + "- Basic: filters[fieldName][$eq]=value\n" + + "- Array: filters[tags][$contains]=important\n" + + "- Nested: filters[author][name][$eq]=John\n" + + "- Complex: filters[$or][0][date][$eq]=2020-01-01&filters[$or][1][date][$eq]=2020-01-02" + ), }) .passthrough(); From e1c6566a47a6b203fe789e828f522b1c764b8692 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 24 Feb 2025 16:26:37 +0100 Subject: [PATCH 02/20] docs: add a little bit of documentation --- packages/contracts/src/resources/site.ts | 58 +++++++++++++----------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index 71468abcb..f67da03bf 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -362,28 +362,35 @@ const numberOrDateSchema = z.number().or(z.coerce.date()); const baseFilterSchema = z .object({ - $eq: allSchema, - $eqi: z.string(), - $ne: allSchema, - $nei: z.string(), - $lt: numberOrDateSchema, - $lte: numberOrDateSchema, - $gt: numberOrDateSchema, - $gte: numberOrDateSchema, - $contains: z.string(), - $notContains: z.string(), - $containsi: z.string(), - $notContainsi: z.string(), - $null: z.never(), - $notNull: z.never(), - $in: z.array(allSchema), - $notIn: z.array(allSchema), - $between: z.tuple([numberOrDateSchema, numberOrDateSchema]), - $startsWith: z.string(), - $startsWithi: z.string(), - $endsWith: z.string(), - $endsWithi: z.string(), - $size: z.number(), + $eq: allSchema.describe("Equal to"), + $eqi: z.string().describe("Equal to (case insensitive)"), + $ne: allSchema.describe("Not equal to"), + $nei: z.string().describe("Not equal to (case insensitive)"), + $lt: numberOrDateSchema.describe("Less than"), + $lte: numberOrDateSchema.describe("Less than or equal to"), + $gt: numberOrDateSchema.describe("Greater than"), + $gte: numberOrDateSchema.describe("Greater than or equal to"), + $contains: z.string().describe("Contains"), + $notContains: z.string().describe("Does not contain"), + $containsi: z.string().describe("Contains (case insensitive)"), + $notContainsi: z.string().describe("Does not contain (case insensitive)"), + $null: z.never().describe("Is null"), + $notNull: z.never().describe("Is not null"), + $in: z.array(allSchema).describe("In"), + $notIn: z.array(allSchema).describe("Not in"), + $between: z.tuple([numberOrDateSchema, numberOrDateSchema]).describe("Between"), + $startsWith: z.string().describe("Starts with"), + $startsWithi: z.string().describe("Starts with (case insensitive)"), + $endsWith: z.string().describe("Ends with"), + $endsWithi: z.string().describe("Ends with (case insensitive)"), + $size: z.number().describe("Size"), + $jsonPath: z + .string() + .describe( + "You can use this to filter more complex json fields, like arrays. See the Postgres documentation for more detail.\n" + + "Example: filters[community-slug:jsonField][$jsonPath]=$[2] > 90\n" + + "This will filter the third element in the array, and check if it's greater than 90." + ), }) .partial() .refine((data) => { @@ -456,10 +463,9 @@ const getPubQuerySchema = z .optional() .describe( "Filter criteria using Strapi-like syntax. Examples:\n" + - "- Basic: filters[fieldName][$eq]=value\n" + - "- Array: filters[tags][$contains]=important\n" + - "- Nested: filters[author][name][$eq]=John\n" + - "- Complex: filters[$or][0][date][$eq]=2020-01-01&filters[$or][1][date][$eq]=2020-01-02" + "- Basic: filters[community-slug:fieldName][$eq]=value\n" + + "- Nested: filters[author][community-slug:name][$eq]=John\n" + + "- Complex: filters[$or][0][community-slug:date][$eq]=2020-01-01&filters[$or][1][community-slug:date][$eq]=2020-01-02" ), }) .passthrough(); From 9d23663b158ca0e6f70a0b968e0db46859f35072 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 24 Feb 2025 17:56:48 +0100 Subject: [PATCH 03/20] fix: properly pass through filters to api --- .../site/[...ts-rest]/route.ts | 19 +++++++++++++++++-- packages/contracts/src/resources/site.ts | 17 +++++++++-------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts b/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts index 7c4ae7948..daecfcf4c 100644 --- a/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts +++ b/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts @@ -29,6 +29,7 @@ import { userCan } from "~/lib/authorization/capabilities"; import { getStage } from "~/lib/db/queries"; import { createLastModifiedBy } from "~/lib/lastModifiedBy"; import { + BadRequestError, createPubRecursiveNew, deletePub, doesPubExist, @@ -50,6 +51,7 @@ import { findCommunityBySlug } from "~/lib/server/community"; import { getPubType, getPubTypesForCommunity } from "~/lib/server/pubtype"; import { getStages } from "~/lib/server/stages"; import { getMember, getSuggestedUsers, SAFE_USER_SELECT } from "~/lib/server/user"; +import { validateFilter } from "~/lib/server/validate-filters"; const baseAuthorizationObject = Object.fromEntries( Object.keys(ApiAccessScope).map( @@ -274,7 +276,15 @@ const handler = createNextHandler( cookies: false, }); - const { pubTypeId, stageId, ...rest } = query; + const { pubTypeId, stageId, filters, ...rest } = query; + + if (filters) { + try { + await validateFilter(community.id, filters); + } catch (e) { + throw new BadRequestError(e.message); + } + } const pubs = await getPubsWithRelatedValuesAndChildren( { @@ -283,9 +293,14 @@ const handler = createNextHandler( stageId, userId: user.id, }, - rest + { + ...rest, + filters, + } ); + console.log(pubs); + return { status: 200, body: pubs, diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index f67da03bf..2cc2aa739 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -393,6 +393,7 @@ const baseFilterSchema = z ), }) .partial() + .passthrough() .refine((data) => { if (!Object.keys(data).length) { return false; @@ -459,14 +460,6 @@ const getPubQuerySchema = z .describe( "Which field values to include in the response. Useful if you have very large pubs or want to save on bandwidth." ), - filters: filterSchema - .optional() - .describe( - "Filter criteria using Strapi-like syntax. Examples:\n" + - "- Basic: filters[community-slug:fieldName][$eq]=value\n" + - "- Nested: filters[author][community-slug:name][$eq]=John\n" + - "- Complex: filters[$or][0][community-slug:date][$eq]=2020-01-01&filters[$or][1][community-slug:date][$eq]=2020-01-02" - ), }) .passthrough(); @@ -584,6 +577,14 @@ export const siteApi = contract.router( offset: z.number().default(0).optional(), orderBy: z.enum(["createdAt", "updatedAt"]).optional(), orderDirection: z.enum(["asc", "desc"]).optional(), + filters: filterSchema + .optional() + .describe( + "Filter criteria using Strapi-like syntax. Examples:\n" + + "- Basic: filters[community-slug:fieldName][$eq]=value\n" + + "- Nested: filters[author][community-slug:name][$eq]=John\n" + + "- Complex: filters[$or][0][community-slug:date][$eq]=2020-01-01&filters[$or][1][community-slug:date][$eq]=2020-01-02" + ), }), responses: { 200: z.array(processedPubSchema), From e375ed31ed87ed2c012a6018486bb64377319297 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 24 Feb 2025 17:57:28 +0100 Subject: [PATCH 04/20] fix: add basic test for api filtering --- core/playwright/site-api.spec.ts | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/core/playwright/site-api.spec.ts b/core/playwright/site-api.spec.ts index 0225dc327..a695e8407 100644 --- a/core/playwright/site-api.spec.ts +++ b/core/playwright/site-api.spec.ts @@ -42,6 +42,8 @@ test.beforeAll(async ({ browser }) => { baseHeaders: { Authorization: `Bearer ${token}`, }, + // necessary else filters will not work + jsonQuery: true, }); }); @@ -86,6 +88,21 @@ test.describe("Site API", () => { ]) ); + const pubResponse2 = await client.pubs.create({ + headers: { + prefer: "return=representation", + }, + params: { + communitySlug: COMMUNITY_SLUG, + }, + body: { + pubTypeId: pubType.id, + values: { + [`${COMMUNITY_SLUG}:title`]: "Goodbye world", + }, + }, + }); + newPubId = pubResponse.body.id; }); @@ -103,5 +120,40 @@ test.describe("Site API", () => { expectStatus(response, 200); expect(response.body.id).toBe(newPubId); }); + + test("should be able to filter pubs", async () => { + const response = await client.pubs.getMany({ + params: { + communitySlug: COMMUNITY_SLUG, + }, + query: { + filters: { + [`${COMMUNITY_SLUG}:title`]: { + $containsi: "hello", + }, + }, + }, + }); + + expectStatus(response, 200); + expect(response.body).toHaveLength(1); + expect(response.body[0].id).toBe(newPubId); + + const response2 = await client.pubs.getMany({ + params: { + communitySlug: COMMUNITY_SLUG, + }, + query: { + filters: { + [`${COMMUNITY_SLUG}:title`]: { + $containsi: "farewell", + }, + }, + }, + }); + + expectStatus(response2, 200); + expect(response2.body).toHaveLength(0); + }); }); }); From 406001918c7936f7b892141191b7a1444194cf96 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 24 Feb 2025 18:03:19 +0100 Subject: [PATCH 05/20] fix: updatedAt/createdAt tests --- .../site/[...ts-rest]/route.ts | 2 - core/playwright/site-api.spec.ts | 57 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts b/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts index daecfcf4c..ed1ce450b 100644 --- a/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts +++ b/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts @@ -299,8 +299,6 @@ const handler = createNextHandler( } ); - console.log(pubs); - return { status: 200, body: pubs, diff --git a/core/playwright/site-api.spec.ts b/core/playwright/site-api.spec.ts index a695e8407..738fb1d05 100644 --- a/core/playwright/site-api.spec.ts +++ b/core/playwright/site-api.spec.ts @@ -50,6 +50,7 @@ test.beforeAll(async ({ browser }) => { test.describe("Site API", () => { test.describe("pubs", () => { let newPubId: PubsId; + let firstCreatedAt: Date; test("should be able to create a pub", async () => { const pubTypesResponse = await client.pubTypes.getMany({ params: { @@ -77,6 +78,8 @@ test.describe("Site API", () => { }, }); + firstCreatedAt = new Date(); + expectStatus(pubResponse, 201); expect(pubResponse.body.values).toEqual( @@ -155,5 +158,59 @@ test.describe("Site API", () => { expectStatus(response2, 200); expect(response2.body).toHaveLength(0); }); + + test("should be able to filter by createdAt", async () => { + const response = await client.pubs.getMany({ + params: { + communitySlug: COMMUNITY_SLUG, + }, + query: { + filters: { + createdAt: { + $gte: firstCreatedAt, + }, + }, + }, + }); + + expectStatus(response, 200); + expect(response.body).toHaveLength(1); + expect(response.body[0].id).not.toBe(newPubId); + }); + + test("should be able to filter by updatedAt", async () => { + const updatedAtDate = new Date(); + const updatedPub = await client.pubs.update({ + params: { + pubId: newPubId, + communitySlug: COMMUNITY_SLUG, + }, + body: { + [`${COMMUNITY_SLUG}:title`]: "Updated title", + }, + }); + + const response = await client.pubs.getMany({ + params: { + communitySlug: COMMUNITY_SLUG, + }, + query: { + filters: { + updatedAt: { + $gte: updatedAtDate, + }, + }, + }, + }); + expectStatus(response, 200); + expect(response.body).toHaveLength(1); + expect(response.body[0].id).toBe(newPubId); + expect(response.body[0].values).toMatchObject([ + expect.objectContaining({ + fieldSlug: `${COMMUNITY_SLUG}:title`, + value: "Updated title", + }), + ]); + }); }); }); From 2b2916d5efac4ac7cccf0c79a262d2d37cf70dd1 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 25 Feb 2025 10:43:23 +0100 Subject: [PATCH 06/20] fix: remove passthrough --- packages/contracts/src/resources/site.ts | 55 +++++++++++------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index 2cc2aa739..5793ba935 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -393,7 +393,6 @@ const baseFilterSchema = z ), }) .partial() - .passthrough() .refine((data) => { if (!Object.keys(data).length) { return false; @@ -434,34 +433,32 @@ export const filterSchema: z.ZodType = z.lazy(() => ]) ); -const getPubQuerySchema = z - .object({ - depth: z - .number() - .int() - .positive() - .default(2) - .describe( - "The depth to which to fetch children and related pubs. Defaults to 2, which means to fetch the top level pub and its children." - ), - withChildren: z.boolean().default(false).describe("Whether to fetch children."), - withRelatedPubs: z - .boolean() - .default(false) - .describe("Whether to include related pubs with the values"), - withPubType: z.boolean().default(false).describe("Whether to fetch the pub type."), - withStage: z.boolean().default(false).describe("Whether to fetch the stage."), - withMembers: z.boolean().default(false).describe("Whether to fetch the pub's members."), - fieldSlugs: z - .array(z.string()) - // this is necessary bc the query parser doesn't handle single string values as arrays - .or(z.string().transform((slug) => [slug])) - .optional() - .describe( - "Which field values to include in the response. Useful if you have very large pubs or want to save on bandwidth." - ), - }) - .passthrough(); +const getPubQuerySchema = z.object({ + depth: z + .number() + .int() + .positive() + .default(2) + .describe( + "The depth to which to fetch children and related pubs. Defaults to 2, which means to fetch the top level pub and its children." + ), + withChildren: z.boolean().default(false).describe("Whether to fetch children."), + withRelatedPubs: z + .boolean() + .default(false) + .describe("Whether to include related pubs with the values"), + withPubType: z.boolean().default(false).describe("Whether to fetch the pub type."), + withStage: z.boolean().default(false).describe("Whether to fetch the stage."), + withMembers: z.boolean().default(false).describe("Whether to fetch the pub's members."), + fieldSlugs: z + .array(z.string()) + // this is necessary bc the query parser doesn't handle single string values as arrays + .or(z.string().transform((slug) => [slug])) + .optional() + .describe( + "Which field values to include in the response. Useful if you have very large pubs or want to save on bandwidth." + ), +}); export type FTSReturn = { id: PubsId; From ffb0324d4dfff0f1171525a2ea2883a88b6ad6ce Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 25 Feb 2025 13:18:57 +0100 Subject: [PATCH 07/20] docs: add hardcoded docs for the filters rather than fixing zod-openapi --- packages/contracts/src/resources/site.ts | 38 ++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index 5793ba935..7cb38c222 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -388,7 +388,7 @@ const baseFilterSchema = z .string() .describe( "You can use this to filter more complex json fields, like arrays. See the Postgres documentation for more detail.\n" + - "Example: filters[community-slug:jsonField][$jsonPath]=$[2] > 90\n" + + 'Example: `filters[community-slug:jsonField][$jsonPath]="$[2] > 90"`\n' + "This will filter the third element in the array, and check if it's greater than 90." ), }) @@ -577,10 +577,38 @@ export const siteApi = contract.router( filters: filterSchema .optional() .describe( - "Filter criteria using Strapi-like syntax. Examples:\n" + - "- Basic: filters[community-slug:fieldName][$eq]=value\n" + - "- Nested: filters[author][community-slug:name][$eq]=John\n" + - "- Complex: filters[$or][0][community-slug:date][$eq]=2020-01-01&filters[$or][1][community-slug:date][$eq]=2020-01-02" + [ + "Filter pubs by their values or by `updatedAt` or `createdAt`.", + "", + "**Filters**", + "- `$eq`: Equal to. (strings, numbers, dates, booleans)", + "- `$eqi`: Equal to (case insensitive). (strings)", + "- `$ne`: Not equal to. (strings, numbers, dates, booleans)", + "- `$nei`: Not equal to (case insensitive). (strings)", + "- `$lt`: Less than. (numbers, dates)", + "- `$lte`: Less than or equal to. (numbers, dates)", + "- `$gt`: Greater than. (numbers, dates)", + "- `$gte`: Greater than or equal to. (numbers, dates)", + "- `$contains`: Contains. (strings)", + "- `$notContains`: Does not contain. (strings)", + "- `$containsi`: Contains (case insensitive). (strings)", + "- `$notContainsi`: Does not contain (case insensitive). (strings)", + "- `$null`: Is null. (strings, numbers, dates, booleans)", + "- `$notNull`: Is not null. (strings, numbers, dates, booleans)", + "- `$in`: In. (strings, numbers, dates, booleans)", + "- `$notIn`: Not in. (strings, numbers, dates, booleans)", + "- `$between`: Between. (numbers, dates)", + "- `$startsWith`: Starts with. (strings)", + "- `$startsWithi`: Starts with (case insensitive). (strings)", + "- `$endsWith`: Ends with. (strings)", + "- `$endsWithi`: Ends with (case insensitive). (strings)", + "- `$size`: Size. (numbers, dates)", + "- `$jsonPath`: JSON path. (strings, arrays, objects) You can use this to filter more complex json fields, like arrays. See the Postgres documentation for more detail. Example: `filters[community-slug:jsonField][$jsonPath]='$[2] > 90'` This will return all pubs where the `community:json-field` value's third element in the array is greater than 90.", + "", + "**Examples**", + "- Basic: `filters[community-slug:fieldName][$eq]=value`", + "- Complex: `filters[$or][0][updatedAt][$gte]=2020-01-01&filters[$or][1][createdAt][$gte]=2020-01-02`", + ].join("\n") ), }), responses: { From a23ee8e733af6b6bd05ae0c7bc7c1f5e167b6732 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 26 Feb 2025 14:44:09 +0100 Subject: [PATCH 08/20] fix: support more natural query syntax --- .../site/[...ts-rest]/route.ts | 91 +++++++++++- core/package.json | 2 + packages/contracts/src/resources/site.ts | 136 ++++++++++-------- pnpm-lock.yaml | 56 ++++---- 4 files changed, 188 insertions(+), 97 deletions(-) diff --git a/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts b/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts index ed1ce450b..0f64574fd 100644 --- a/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts +++ b/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts @@ -1,8 +1,9 @@ import type { User } from "lucia"; import { headers } from "next/headers"; -import { createNextHandler } from "@ts-rest/serverless/next"; +import { createNextHandler, RequestValidationError } from "@ts-rest/serverless/next"; import { jsonObjectFrom } from "kysely/helpers/postgres"; +import qs from "qs"; import { z } from "zod"; import type { @@ -19,8 +20,9 @@ import type { ApiAccessPermissionConstraintsInput, LastModifiedBy, } from "db/types"; -import { siteApi } from "contracts"; +import { baseFilterSchema, filterSchema, siteApi } from "contracts"; import { ApiAccessScope, ApiAccessType, Capabilities, MembershipType } from "db/public"; +import { assert } from "utils"; import type { CapabilityTarget } from "~/lib/authorization/capabilities"; import { db } from "~/kysely/database"; @@ -228,6 +230,81 @@ const shouldReturnRepresentation = async () => { return false; }; +/** + * manually parses the `?filters` query param. + * necessary because ts-rest only supports parsing object in query params + * if they're uri encoded. + * + * eg this does not fly + * ``` + * ?filters[community-slug:fieldName][$eq]=value + * ``` + * but this does + * ``` + * ?filters=%7B%22%7B%22updatedAt%22%3A%20%7B%22%24gte%22%3A%20%222025-01-01%22%7D%2C%22field-slug%22%3A%20%7B%22%24eq%22%3A%20%22some-value%22%7D%7D` + * ``` + * + * the latter is what a ts-rest client sends if `json-query: true`. we want to support both syntaxes. + * + */ +const manuallyParsePubFilterQueryParams = (url: string, query?: Record) => { + if (!query || Object.keys(query).length === 0) { + return query; + } + + // check if we already have properly structured filters + if (query.filters && typeof query.filters === "object") { + try { + const validatedFilters = filterSchema.parse(query.filters); + return { + ...query, + filters: validatedFilters, + }; + } catch (e) { + throw new RequestValidationError(null, null, e, null); + } + } + + // check if we have filter-like keys (using bracket notation) + const filterLikeKeys = Object.keys(query).filter((key) => key.startsWith("filters[")); + + if (filterLikeKeys.length === 0) { + return query; + } + + const queryString = url.split("?")[1]; + if (!queryString) { + return query; + } + + try { + // parse with qs + const parsedQuery = qs.parse(queryString, { + depth: 10, + arrayLimit: 100, + allowDots: false, // don't convert dots to objects (use brackets only) + ignoreQueryPrefix: true, // remove the leading '?' if present + }); + + if (!parsedQuery.filters) { + return query; // no filters found after parsing + } + + const validatedFilters = filterSchema.parse(parsedQuery.filters); + + return { + ...query, + ...parsedQuery, + filters: validatedFilters, + }; + } catch (e) { + if (e instanceof z.ZodError) { + throw new RequestValidationError(null, null, e, null); + } + throw new BadRequestError(`Error parsing filters: ${e.message}`); + } +}; + const handler = createNextHandler( siteApi, { @@ -269,7 +346,7 @@ const handler = createNextHandler( body: pub, }; }, - getMany: async ({ query }) => { + getMany: async ({ query }, { request }) => { const { user, community } = await checkAuthorization({ token: { scope: ApiAccessScope.pub, type: ApiAccessType.read }, // TODO: figure out capability here @@ -278,9 +355,11 @@ const handler = createNextHandler( const { pubTypeId, stageId, filters, ...rest } = query; - if (filters) { + const manuallyParsedFilters = manuallyParsePubFilterQueryParams(request.url, query); + + if (manuallyParsedFilters?.filters) { try { - await validateFilter(community.id, filters); + await validateFilter(community.id, manuallyParsedFilters.filters); } catch (e) { throw new BadRequestError(e.message); } @@ -295,7 +374,7 @@ const handler = createNextHandler( }, { ...rest, - filters, + filters: manuallyParsedFilters?.filters, } ); diff --git a/core/package.json b/core/package.json index 56f62dd6a..54a2fb05b 100644 --- a/core/package.json +++ b/core/package.json @@ -115,6 +115,7 @@ "pg": "^8.11.3", "prosemirror-markdown": "^1.12.0", "prosemirror-model": "^1.24.1", + "qs": "^6.14.0", "react": "catalog:react19", "react-dom": "catalog:react19", "react-hook-form": "catalog:", @@ -163,6 +164,7 @@ "@types/node": "catalog:", "@types/nodemailer": "^6.4.9", "@types/pg": "^8.11.6", + "@types/qs": "^6.9.18", "@types/react": "catalog:react19", "@types/react-dom": "catalog:react19", "@types/unist": "^3.0.2", diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index 7cb38c222..eb5e517dc 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -356,11 +356,11 @@ export type LogicalFilter = { export type Filter = BaseFilter | LogicalFilter; -const allSchema = z.string().or(z.number()).or(z.boolean()).or(z.coerce.date()); +const allSchema = z.string().or(z.coerce.number()).or(z.boolean()).or(z.coerce.date()); -const numberOrDateSchema = z.number().or(z.coerce.date()); +const numberOrDateSchema = z.coerce.number().or(z.coerce.date()); -const baseFilterSchema = z +export const baseFilterSchema = z .object({ $eq: allSchema.describe("Equal to"), $eqi: z.string().describe("Equal to (case insensitive)"), @@ -393,6 +393,7 @@ const baseFilterSchema = z ), }) .partial() + .passthrough() .refine((data) => { if (!Object.keys(data).length) { return false; @@ -556,7 +557,7 @@ export const siteApi = contract.router( pathParams: z.object({ pubId: z.string().uuid(), }), - query: getPubQuerySchema, + query: getPubQuerySchema.optional(), responses: { 200: processedPubSchema, }, @@ -567,50 +568,61 @@ export const siteApi = contract.router( summary: "Gets a list of pubs", description: "Get a list of pubs by ID. This endpoint is used by the PubPub site builder to get a list of pubs.", - query: getPubQuerySchema.extend({ - pubTypeId: pubTypesIdSchema.optional().describe("Filter by pub type ID."), - stageId: stagesIdSchema.optional().describe("Filter by stage ID."), - limit: z.number().default(10), - offset: z.number().default(0).optional(), - orderBy: z.enum(["createdAt", "updatedAt"]).optional(), - orderDirection: z.enum(["asc", "desc"]).optional(), - filters: filterSchema - .optional() - .describe( - [ - "Filter pubs by their values or by `updatedAt` or `createdAt`.", - "", - "**Filters**", - "- `$eq`: Equal to. (strings, numbers, dates, booleans)", - "- `$eqi`: Equal to (case insensitive). (strings)", - "- `$ne`: Not equal to. (strings, numbers, dates, booleans)", - "- `$nei`: Not equal to (case insensitive). (strings)", - "- `$lt`: Less than. (numbers, dates)", - "- `$lte`: Less than or equal to. (numbers, dates)", - "- `$gt`: Greater than. (numbers, dates)", - "- `$gte`: Greater than or equal to. (numbers, dates)", - "- `$contains`: Contains. (strings)", - "- `$notContains`: Does not contain. (strings)", - "- `$containsi`: Contains (case insensitive). (strings)", - "- `$notContainsi`: Does not contain (case insensitive). (strings)", - "- `$null`: Is null. (strings, numbers, dates, booleans)", - "- `$notNull`: Is not null. (strings, numbers, dates, booleans)", - "- `$in`: In. (strings, numbers, dates, booleans)", - "- `$notIn`: Not in. (strings, numbers, dates, booleans)", - "- `$between`: Between. (numbers, dates)", - "- `$startsWith`: Starts with. (strings)", - "- `$startsWithi`: Starts with (case insensitive). (strings)", - "- `$endsWith`: Ends with. (strings)", - "- `$endsWithi`: Ends with (case insensitive). (strings)", - "- `$size`: Size. (numbers, dates)", - "- `$jsonPath`: JSON path. (strings, arrays, objects) You can use this to filter more complex json fields, like arrays. See the Postgres documentation for more detail. Example: `filters[community-slug:jsonField][$jsonPath]='$[2] > 90'` This will return all pubs where the `community:json-field` value's third element in the array is greater than 90.", - "", - "**Examples**", - "- Basic: `filters[community-slug:fieldName][$eq]=value`", - "- Complex: `filters[$or][0][updatedAt][$gte]=2020-01-01&filters[$or][1][createdAt][$gte]=2020-01-02`", - ].join("\n") - ), - }), + query: getPubQuerySchema + .extend({ + pubTypeId: pubTypesIdSchema.optional().describe("Filter by pub type ID."), + stageId: stagesIdSchema.optional().describe("Filter by stage ID."), + limit: z.number().default(10), + offset: z.number().default(0).optional(), + orderBy: z.enum(["createdAt", "updatedAt"]).optional(), + orderDirection: z.enum(["asc", "desc"]).optional(), + /** + * The parsing of `filters` is handled in the route itself instead, + * because ts-rest cannot parse nested objects in query strings. + * eg `?filters[community-slug:fieldName][$eq]=value` becomes + * `{ filters['community-slug:fieldName']['$eq']: 'value'}`, + * rather than `{ filters: { 'community-slug:fieldName': { $eq: 'value' } } }`. + */ + filters: z + .record(z.any()) + .optional() + .describe( + [ + "Filter pubs by their values or by `updatedAt` or `createdAt`.", + "", + "**Filters**", + "- `$eq`: Equal to. (strings, numbers, dates, booleans)", + "- `$eqi`: Equal to (case insensitive). (strings)", + "- `$ne`: Not equal to. (strings, numbers, dates, booleans)", + "- `$nei`: Not equal to (case insensitive). (strings)", + "- `$lt`: Less than. (numbers, dates)", + "- `$lte`: Less than or equal to. (numbers, dates)", + "- `$gt`: Greater than. (numbers, dates)", + "- `$gte`: Greater than or equal to. (numbers, dates)", + "- `$contains`: Contains. (strings)", + "- `$notContains`: Does not contain. (strings)", + "- `$containsi`: Contains (case insensitive). (strings)", + "- `$notContainsi`: Does not contain (case insensitive). (strings)", + "- `$null`: Is null. (strings, numbers, dates, booleans)", + "- `$notNull`: Is not null. (strings, numbers, dates, booleans)", + "- `$in`: In. (strings, numbers, dates, booleans)", + "- `$notIn`: Not in. (strings, numbers, dates, booleans)", + "- `$between`: Between. (numbers, dates)", + "- `$startsWith`: Starts with. (strings)", + "- `$startsWithi`: Starts with (case insensitive). (strings)", + "- `$endsWith`: Ends with. (strings)", + "- `$endsWithi`: Ends with (case insensitive). (strings)", + "- `$size`: Size. (numbers, dates)", + "- `$jsonPath`: JSON path. (strings, arrays, objects) You can use this to filter more complex json fields, like arrays. See the Postgres documentation for more detail. Example: `filters[community-slug:jsonField][$jsonPath]='$[2] > 90'` This will return all pubs where the `community:json-field` value's third element in the array is greater than 90.", + "", + "**Examples**", + "- Basic: `filters[community-slug:fieldName][$eq]=value`", + "- Complex: `filters[$or][0][updatedAt][$gte]=2020-01-01&filters[$or][1][createdAt][$gte]=2020-01-02`", + ].join("\n") + ), + }) + .passthrough() + .optional(), responses: { 200: z.array(processedPubSchema), }, @@ -715,12 +727,14 @@ export const siteApi = contract.router( summary: "Gets a list of pub types", description: "Get a list of pub types by ID. This endpoint is used by the PubPub site builder to get a list of pub types.", - query: z.object({ - limit: z.number().default(10), - offset: z.number().default(0).optional(), - orderBy: z.enum(["createdAt", "updatedAt"]).optional(), - orderDirection: z.enum(["asc", "desc"]).optional(), - }), + query: z + .object({ + limit: z.number().default(10), + offset: z.number().default(0).optional(), + orderBy: z.enum(["createdAt", "updatedAt"]).optional(), + orderDirection: z.enum(["asc", "desc"]).optional(), + }) + .optional(), responses: { 200: pubTypesSchema.array(), }, @@ -746,12 +760,14 @@ export const siteApi = contract.router( summary: "Gets a list of stages", description: "Get a list of stages by ID. This endpoint is used by the PubPub site builder to get a list of stages.", - query: z.object({ - limit: z.number().default(10), - offset: z.number().default(0).optional(), - orderBy: z.enum(["createdAt", "updatedAt"]).optional(), - orderDirection: z.enum(["asc", "desc"]).optional(), - }), + query: z + .object({ + limit: z.number().default(10), + offset: z.number().default(0).optional(), + orderBy: z.enum(["createdAt", "updatedAt"]).optional(), + orderDirection: z.enum(["asc", "desc"]).optional(), + }) + .optional(), responses: { 200: stagesSchema.array(), }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0f1a8566e..2518d415c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -415,6 +415,9 @@ importers: prosemirror-model: specifier: ^1.24.1 version: 1.24.1 + qs: + specifier: ^6.14.0 + version: 6.14.0 react: specifier: catalog:react19 version: 19.0.0 @@ -554,6 +557,9 @@ importers: '@types/pg': specifier: ^8.11.6 version: 8.11.8 + '@types/qs': + specifier: ^6.9.18 + version: 6.9.18 '@types/react': specifier: catalog:react19 version: 19.0.6 @@ -6387,6 +6393,9 @@ packages: '@types/prosemirror-dev-tools@3.0.6': resolution: {integrity: sha512-zARROV118nwc+sX7W+0ea4cffqUeRNOSac0jttSpJ921aS6w++Be+RakAgGiTqoRpPV+J+wKomMR/RuKBAlEMg==} + '@types/qs@6.9.18': + resolution: {integrity: sha512-kK7dgTYDyGqS+e2Q4aK9X3D7q234CIZ1Bv0q/7Z5IwRDoADNU81xXJK/YVyLbLTZCoIwUoDoffFeF+p/eIklAA==} + '@types/react-dom@19.0.3': resolution: {integrity: sha512-0Knk+HJiMP/qOZgMyNFamlIjw9OFCsyC2ZbigmEEyXXixgre6IQpm/4V+r3qH4GC1JPvRJKInw+on2rV6YZLeA==} peerDependencies: @@ -9768,10 +9777,6 @@ packages: resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} engines: {node: '>= 6'} - object-inspect@1.13.2: - resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==} - engines: {node: '>= 0.4'} - object-inspect@1.13.3: resolution: {integrity: sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==} engines: {node: '>= 0.4'} @@ -10425,8 +10430,8 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} - qs@6.13.0: - resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} + qs@6.14.0: + resolution: {integrity: sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==} engines: {node: '>=0.6'} queue-microtask@1.2.3: @@ -10949,10 +10954,6 @@ packages: resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} engines: {node: '>= 0.4'} - side-channel@1.0.6: - resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} - engines: {node: '>= 0.4'} - side-channel@1.1.0: resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} engines: {node: '>= 0.4'} @@ -18528,6 +18529,8 @@ snapshots: dependencies: prosemirror-view: 1.34.3 + '@types/qs@6.9.18': {} + '@types/react-dom@19.0.3(@types/react@19.0.6)': dependencies: '@types/react': 19.0.6 @@ -19189,7 +19192,7 @@ snapshots: define-properties: 1.2.1 es-abstract: 1.23.3 es-errors: 1.3.0 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 is-array-buffer: 3.0.4 is-shared-array-buffer: 1.0.3 @@ -20091,7 +20094,7 @@ snapshots: is-string: 1.0.7 is-typed-array: 1.1.13 is-weakref: 1.0.2 - object-inspect: 1.13.2 + object-inspect: 1.13.3 object-keys: 1.1.1 object.assign: 4.1.5 regexp.prototype.flags: 1.5.2 @@ -20163,7 +20166,7 @@ snapshots: es-define-property@1.0.0: dependencies: - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 es-define-property@1.0.1: {} @@ -20836,7 +20839,7 @@ snapshots: dependencies: call-bind: 1.0.7 es-errors: 1.3.0 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 get-symbol-description@1.1.0: dependencies: @@ -20980,7 +20983,7 @@ snapshots: extend: 3.0.2 gaxios: 6.7.1 google-auth-library: 9.15.0 - qs: 6.13.0 + qs: 6.14.0 url-template: 2.0.8 uuid: 9.0.1 transitivePeerDependencies: @@ -20997,7 +21000,7 @@ snapshots: gopd@1.0.1: dependencies: - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 gopd@1.2.0: {} @@ -21400,7 +21403,7 @@ snapshots: dependencies: es-errors: 1.3.0 hasown: 2.0.2 - side-channel: 1.0.6 + side-channel: 1.1.0 internal-slot@1.1.0: dependencies: @@ -21432,7 +21435,7 @@ snapshots: is-array-buffer@3.0.4: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 is-array-buffer@3.0.5: dependencies: @@ -22651,8 +22654,6 @@ snapshots: object-hash@3.0.0: {} - object-inspect@1.13.2: {} - object-inspect@1.13.3: {} object-keys@1.1.1: {} @@ -23414,9 +23415,9 @@ snapshots: punycode@2.3.1: {} - qs@6.13.0: + qs@6.14.0: dependencies: - side-channel: 1.0.6 + side-channel: 1.1.0 queue-microtask@1.2.3: {} @@ -23970,7 +23971,7 @@ snapshots: safe-array-concat@1.1.2: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 has-symbols: 1.0.3 isarray: 2.0.5 @@ -24063,7 +24064,7 @@ snapshots: define-data-property: 1.1.4 es-errors: 1.3.0 function-bind: 1.1.2 - get-intrinsic: 1.2.4 + get-intrinsic: 1.2.7 gopd: 1.0.1 has-property-descriptors: 1.0.2 @@ -24139,13 +24140,6 @@ snapshots: object-inspect: 1.13.3 side-channel-map: 1.0.1 - side-channel@1.0.6: - dependencies: - call-bind: 1.0.7 - es-errors: 1.3.0 - get-intrinsic: 1.2.4 - object-inspect: 1.13.2 - side-channel@1.1.0: dependencies: es-errors: 1.3.0 From 0935bb5342b8aaedba251a6a4c47e9ed81a3271c Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 26 Feb 2025 14:56:17 +0100 Subject: [PATCH 09/20] dev: add integration test for parsing behavior --- core/playwright/site-api.spec.ts | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/core/playwright/site-api.spec.ts b/core/playwright/site-api.spec.ts index 738fb1d05..0a590f8aa 100644 --- a/core/playwright/site-api.spec.ts +++ b/core/playwright/site-api.spec.ts @@ -17,6 +17,8 @@ let page: Page; let client: ReturnType>; +let token: string; + test.beforeAll(async ({ browser }) => { page = await browser.newPage(); @@ -31,11 +33,13 @@ test.beforeAll(async ({ browser }) => { const apiTokenPage = new ApiTokenPage(page, COMMUNITY_SLUG); await apiTokenPage.goto(); - const token = await apiTokenPage.createToken({ + const createdToken = await apiTokenPage.createToken({ name: "test token", description: "test description", permissions: true, }); + expect(createdToken).not.toBeNull(); + token = createdToken!; client = initClient(siteApi, { baseUrl: `http://localhost:3000/`, @@ -56,6 +60,7 @@ test.describe("Site API", () => { params: { communitySlug: COMMUNITY_SLUG, }, + query: {}, }); expectStatus(pubTypesResponse, 200); @@ -212,5 +217,25 @@ test.describe("Site API", () => { }), ]); }); + + /** + * this is to test that ?filters[x][y]=z works + */ + test("should be able to filter by manually supplying query params", async () => { + const response = await fetch( + `http://localhost:3000/api/v0/c/${COMMUNITY_SLUG}/site/pubs?filters[createdAt][$gte]=${firstCreatedAt.toISOString()}`, + { + headers: { + Authorization: `Bearer ${token}`, + }, + } + ); + + const responseBody = await response.json(); + + expect(response.status).toBe(200); + expect(responseBody).toHaveLength(1); + expect(responseBody[0].id).not.toBe(newPubId); + }); }); }); From 55b9d24590a0cbc3345027fa364a9df08629c7c6 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 26 Feb 2025 15:00:21 +0100 Subject: [PATCH 10/20] dev: add explicit jsonquery test with client --- core/playwright/site-api.spec.ts | 39 +++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/core/playwright/site-api.spec.ts b/core/playwright/site-api.spec.ts index 0a590f8aa..f3b97f5a3 100644 --- a/core/playwright/site-api.spec.ts +++ b/core/playwright/site-api.spec.ts @@ -19,6 +19,16 @@ let client: ReturnType>; let token: string; +const createClient = (token: string, jsonQuery: boolean = false) => { + return initClient(siteApi, { + baseUrl: `http://localhost:3000/`, + baseHeaders: { + Authorization: `Bearer ${token}`, + }, + jsonQuery, + }); +}; + test.beforeAll(async ({ browser }) => { page = await browser.newPage(); @@ -41,14 +51,7 @@ test.beforeAll(async ({ browser }) => { expect(createdToken).not.toBeNull(); token = createdToken!; - client = initClient(siteApi, { - baseUrl: `http://localhost:3000/`, - baseHeaders: { - Authorization: `Bearer ${token}`, - }, - // necessary else filters will not work - jsonQuery: true, - }); + client = createClient(token, true); }); test.describe("Site API", () => { @@ -183,6 +186,26 @@ test.describe("Site API", () => { expect(response.body[0].id).not.toBe(newPubId); }); + test("should be able to filter by without jsonQuery", async () => { + const client = createClient(token, false); + const response = await client.pubs.getMany({ + params: { + communitySlug: COMMUNITY_SLUG, + }, + query: { + filters: { + createdAt: { + $gte: firstCreatedAt, + }, + }, + }, + }); + + expectStatus(response, 200); + expect(response.body).toHaveLength(1); + expect(response.body[0].id).not.toBe(newPubId); + }); + test("should be able to filter by updatedAt", async () => { const updatedAtDate = new Date(); const updatedPub = await client.pubs.update({ From 7429d81f9bb93a1c71bb7790f0ea2f87458cc431 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 26 Feb 2025 15:00:34 +0100 Subject: [PATCH 11/20] fix: fix type error --- core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts b/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts index 0f64574fd..4f4790373 100644 --- a/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts +++ b/core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts @@ -353,7 +353,7 @@ const handler = createNextHandler( cookies: false, }); - const { pubTypeId, stageId, filters, ...rest } = query; + const { pubTypeId, stageId, filters, ...rest } = query ?? {}; const manuallyParsedFilters = manuallyParsePubFilterQueryParams(request.url, query); From 5a91c8b8c9f65a4fad4d63299ef9b8cfcb941023 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 26 Feb 2025 15:10:53 +0100 Subject: [PATCH 12/20] fix: allow booleans --- packages/contracts/src/resources/site.ts | 64 ++++++++++++++---------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index eb5e517dc..db9c56168 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -356,7 +356,11 @@ export type LogicalFilter = { export type Filter = BaseFilter | LogicalFilter; -const allSchema = z.string().or(z.coerce.number()).or(z.boolean()).or(z.coerce.date()); +const allSchema = z + .string() + .or(z.coerce.number()) + .or(z.enum(["true", "false"]).transform((val) => val === "true")) + .or(z.coerce.date()); const numberOrDateSchema = z.coerce.number().or(z.coerce.date()); @@ -591,33 +595,41 @@ export const siteApi = contract.router( "Filter pubs by their values or by `updatedAt` or `createdAt`.", "", "**Filters**", - "- `$eq`: Equal to. (strings, numbers, dates, booleans)", - "- `$eqi`: Equal to (case insensitive). (strings)", - "- `$ne`: Not equal to. (strings, numbers, dates, booleans)", - "- `$nei`: Not equal to (case insensitive). (strings)", - "- `$lt`: Less than. (numbers, dates)", - "- `$lte`: Less than or equal to. (numbers, dates)", - "- `$gt`: Greater than. (numbers, dates)", - "- `$gte`: Greater than or equal to. (numbers, dates)", - "- `$contains`: Contains. (strings)", - "- `$notContains`: Does not contain. (strings)", - "- `$containsi`: Contains (case insensitive). (strings)", - "- `$notContainsi`: Does not contain (case insensitive). (strings)", - "- `$null`: Is null. (strings, numbers, dates, booleans)", - "- `$notNull`: Is not null. (strings, numbers, dates, booleans)", - "- `$in`: In. (strings, numbers, dates, booleans)", - "- `$notIn`: Not in. (strings, numbers, dates, booleans)", - "- `$between`: Between. (numbers, dates)", - "- `$startsWith`: Starts with. (strings)", - "- `$startsWithi`: Starts with (case insensitive). (strings)", - "- `$endsWith`: Ends with. (strings)", - "- `$endsWithi`: Ends with (case insensitive). (strings)", - "- `$size`: Size. (numbers, dates)", - "- `$jsonPath`: JSON path. (strings, arrays, objects) You can use this to filter more complex json fields, like arrays. See the Postgres documentation for more detail. Example: `filters[community-slug:jsonField][$jsonPath]='$[2] > 90'` This will return all pubs where the `community:json-field` value's third element in the array is greater than 90.", + "- `$eq`: Equal to. Works with strings, numbers, dates, booleans.", + "- `$eqi`: Equal to (case insensitive). Works with strings.", + "- `$ne`: Not equal to. Works with strings, numbers, dates, booleans.", + "- `$nei`: Not equal to (case insensitive). Works with strings.", + "- `$lt`: Less than. Works with numbers, dates.", + "- `$lte`: Less than or equal to. Works with numbers, dates.", + "- `$gt`: Greater than. Works with numbers, dates.", + "- `$gte`: Greater than or equal to. Works with numbers, dates.", + "- `$contains`: Contains substring. Works with strings.", + "- `$notContains`: Does not contain substring. Works with strings.", + "- `$containsi`: Contains substring (case insensitive). Works with strings.", + "- `$notContainsi`: Does not contain substring (case insensitive). Works with strings.", + "- `$null`: Is null. No value needed - use `filters[field][$null]=true`.", + "- `$notNull`: Is not null. No value needed - use `filters[field][$notNull]=true`.", + "- `$in`: Value is in array. Format: `filters[field][$in]=value1,value2,value3`.", + "- `$notIn`: Value is not in array. Format: `filters[field][$notIn]=value1,value2,value3`.", + "- `$between`: Value is between two values. Format: `filters[field][$between]=min,max`.", + "- `$startsWith`: String starts with. Works with strings.", + "- `$startsWithi`: String starts with (case insensitive). Works with strings.", + "- `$endsWith`: String ends with. Works with strings.", + "- `$endsWithi`: String ends with (case insensitive). Works with strings.", + "- `$jsonPath`: JSON path query for complex JSON fields. Example: `filters[field][$jsonPath]='$[2] > 90'`", + "", + "**Logical Operators**", + "- `$and`: All conditions must match. Format: `filters[$and][0][field][$eq]=value&filters[$and][1][field2][$eq]=value2`", + "- `$or`: Any condition can match. Format: `filters[$or][0][field][$eq]=value&filters[$or][1][field2][$eq]=value2`", + "- `$not`: Negate a condition. Format: `filters[$not][field][$eq]=value`", "", "**Examples**", - "- Basic: `filters[community-slug:fieldName][$eq]=value`", - "- Complex: `filters[$or][0][updatedAt][$gte]=2020-01-01&filters[$or][1][createdAt][$gte]=2020-01-02`", + "- Basic equality: `filters[community-slug:fieldName][$eq]=value`", + "- Date range: `filters[createdAt][$gte]=2023-01-01&filters[createdAt][$lte]=2023-12-31`", + "- Logical OR: `filters[$or][0][updatedAt][$gte]=2020-01-01&filters[$or][1][createdAt][$gte]=2020-01-02`", + "- Case-insensitive search: `filters[title][$containsi]=search term`", + "- Null check: `filters[assigneeId][$null]=true`", + "- JSON array filter: `filters[community-slug:jsonField][$jsonPath]='$[2] > 90'`", ].join("\n") ), }) From a6e99aec31b956f3e63ae0db7946de0ffa4d83c6 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 26 Feb 2025 16:55:42 +0100 Subject: [PATCH 13/20] fix: remove passthrough --- packages/contracts/src/resources/site.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index db9c56168..de9202db7 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -397,7 +397,6 @@ export const baseFilterSchema = z ), }) .partial() - .passthrough() .refine((data) => { if (!Object.keys(data).length) { return false; From 59e33f92d365a979a62bc0a54d93701bf95e02d6 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 26 Feb 2025 18:19:41 +0100 Subject: [PATCH 14/20] fix: add a ton of tests for filter parsing, and make it work as expected --- core/lib/server/pub-filters.db.test.ts | 304 +++++++++++++++++++++++ packages/contracts/src/resources/site.ts | 100 +++++--- 2 files changed, 365 insertions(+), 39 deletions(-) diff --git a/core/lib/server/pub-filters.db.test.ts b/core/lib/server/pub-filters.db.test.ts index 58fee86ad..f111c5842 100644 --- a/core/lib/server/pub-filters.db.test.ts +++ b/core/lib/server/pub-filters.db.test.ts @@ -1,3 +1,4 @@ +import QueryString from "qs"; import { describe, expect, it } from "vitest"; import type { Filter, Json } from "contracts"; @@ -69,6 +70,18 @@ const seed = createSeed({ Number: 42, }, }, + { + pubType: "Basic Pub", + values: { + Number: 24, + }, + }, + { + pubType: "Basic Pub", + values: { + Number: 54, + }, + }, { id: trueId, pubType: "Basic Pub", @@ -179,6 +192,7 @@ describe("pub-filters", () => { }; const parsed = filterSchema.safeParse(filter); + expect(parsed.error).toBeUndefined(); expect(parsed.success).toBe(true); expect(parsed.data).toEqual(filter); }); @@ -341,6 +355,28 @@ describe("pub-filters", () => { [{ value: "Some title", fieldSlug: slug("title") }], ], }, + { + title: "nested logical operators", + filter: { + [slug("number")]: { + $or: [ + { + $lt: 40, + }, + { + $gt: 50, + }, + ], + }, + }, + + sql: `("slug" = $1 and "value" < $2) or ("slug" = $3 and "value" > $4)`, + parameters: [slug("number"), 40, slug("number"), 50], + resultValues: [ + [{ value: 54, fieldSlug: slug("number") }], + [{ value: 24, fieldSlug: slug("number") }], + ], + }, { title: "updatedAt & createdAt filters", filter: { @@ -515,6 +551,274 @@ describe("pub-filters", () => { }); }); + const querystringCases: { + title: string; + querystring: string; + filter: Filter; + }[] = [ + { + title: "simple equality", + querystring: "filters[title][$eq]=Some title", + filter: { + title: { $eq: "Some title" }, + }, + }, + { + title: "multiple operators on same field", + querystring: "filters[number][$gt]=10&filters[number][$lt]=50", + filter: { + number: { $gt: 10, $lt: 50 }, + }, + }, + { + title: "multiple fields", + querystring: "filters[title][$eq]=Test&filters[number][$eq]=42", + filter: { + title: { $eq: "Test" }, + number: { $eq: 42 }, + }, + }, + { + title: "boolean coercion", + querystring: "filters[boolean][$eq]=true", + filter: { + boolean: { $eq: true }, + }, + }, + { + title: "number coercion", + querystring: "filters[number][$eq]=42", + filter: { + number: { $eq: 42 }, + }, + }, + { + title: "date coercion", + querystring: "filters[date][$eq]=2023-01-01T00:00:00.000Z", + filter: { + date: { $eq: new Date("2023-01-01T00:00:00.000Z") }, + }, + }, + { + title: "array in operator", + querystring: + "filters[number][$in][]=1&filters[number][$in][]=2&filters[number][$in][]=3", + filter: { + number: { $in: [1, 2, 3] }, + }, + }, + { + title: "between operator", + querystring: "filters[number][$between][0]=10&filters[number][$between][1]=20", + filter: { + number: { $between: [10, 20] }, + }, + }, + { + title: "case insensitive operators", + querystring: "filters[title][$containsi]=test&filters[title][$eqi]=another test", + filter: { + title: { $containsi: "test", $eqi: "another test" }, + }, + }, + { + title: "null and notNull operators", + querystring: "filters[title][$null]&filters[number][$notNull]", + filter: { + title: { $null: true }, + number: { $notNull: true }, + }, + }, + { + title: "jsonPath operator", + querystring: 'filters[array][$jsonPath]=$[*] == "item1"', + filter: { + array: { $jsonPath: '$[*] == "item1"' }, + }, + }, + { + title: "top-level logical OR", + querystring: "filters[$or][0][title][$eq]=Test&filters[$or][1][number][$eq]=42", + filter: { + $or: [{ title: { $eq: "Test" } }, { number: { $eq: 42 } }], + }, + }, + { + title: "top-level logical AND", + querystring: "filters[$and][0][title][$eq]=Test&filters[$and][1][number][$gt]=10", + filter: { + $and: [{ title: { $eq: "Test" } }, { number: { $gt: 10 } }], + }, + }, + { + title: "top-level logical NOT", + querystring: "filters[$not][title][$eq]=Test", + filter: { + $not: { title: { $eq: "Test" } }, + }, + }, + { + title: "nested logical operators", + querystring: + "filters[$or][0][title][$eq]=Test&filters[$or][1][$and][0][number][$gt]=10&filters[$or][1][$and][1][number][$lt]=50", + filter: { + $or: [ + { title: { $eq: "Test" } }, + { + $and: [{ number: { $gt: 10 } }, { number: { $lt: 50 } }], + }, + ], + }, + }, + { + title: "field-level logical OR", + querystring: "filters[number][$or][0][$lt]=10&filters[number][$or][1][$gt]=50", + filter: { + number: { + $or: [{ $lt: 10 }, { $gt: 50 }], + }, + }, + }, + { + title: "complex nested structure", + querystring: + "filters[$or][0][$and][0][title][$containsi]=test&filters[$or][0][$and][1][boolean][$eq]=true&filters[$or][1][$not][number][$between][0]=10&filters[$or][1][$not][number][$between][1]=20", + filter: { + $or: [ + { + $and: [{ title: { $containsi: "test" } }, { boolean: { $eq: true } }], + }, + { + $not: { + number: { $between: [10, 20] }, + }, + }, + ], + }, + }, + { + title: "multiple array values with coercion", + querystring: + "filters[numberArray][$in][]=1&filters[numberArray][$in][]=2&filters[numberArray][$in][]=3&filters[dateArray][$in][]=2023-01-01T00:00:00.000Z&filters[dateArray][$in][]=2023-01-02T00:00:00.000Z", + filter: { + numberArray: { $in: [1, 2, 3] }, + dateArray: { + $in: [ + new Date("2023-01-01T00:00:00.000Z"), + new Date("2023-01-02T00:00:00.000Z"), + ], + }, + }, + }, + { + title: "URL encoded special characters", + querystring: "filters[title][$contains]=special%20characters%20%26%20symbols", + filter: { + title: { $contains: "special characters & symbols" }, + }, + }, + { + title: "mixed type coercion in arrays", + querystring: + "filters[mixedArray][$in][]=string&filters[mixedArray][$in][]=42&filters[mixedArray][$in][]=true", + filter: { + mixedArray: { $in: ["string", 42, true] }, + }, + }, + { + title: "deeply nested logical operators with multiple field types", + querystring: + "filters[$and][0][$or][0][title][$containsi]=test&filters[$and][0][$or][1][number][$gt]=50&filters[$and][1][$not][$or][0][boolean][$eq]=false&filters[$and][1][$not][$or][1][date][$lt]=2023-01-01T00:00:00.000Z", + filter: { + $and: [ + { + $or: [{ title: { $containsi: "test" } }, { number: { $gt: 50 } }], + }, + { + $not: { + $or: [ + { boolean: { $eq: false } }, + { date: { $lt: new Date("2023-01-01T00:00:00.000Z") } }, + ], + }, + }, + ], + }, + }, + { + title: "complex filter with all operator types", + querystring: + 'filters[$or][0][title][$eq]=Test&filters[$or][0][title][$containsi]=important&filters[$or][1][number][$between][0]=10&filters[$or][1][number][$between][1]=50&filters[$or][2][date][$gt]=2023-01-01T00:00:00.000Z&filters[$or][2][boolean][$eq]=true&filters[$or][3][array][$jsonPath]=$[*] == "item1"', + filter: { + $or: [ + { title: { $eq: "Test", $containsi: "important" } }, + { number: { $between: [10, 50] } }, + { date: { $gt: new Date("2023-01-01T00:00:00.000Z") }, boolean: { $eq: true } }, + { array: { $jsonPath: '$[*] == "item1"' } }, + ], + }, + }, + ]; + describe("querystring parsing", () => { + it.concurrent.each(querystringCases)( + "correctly parses $title", + async ({ title, querystring, filter }) => { + const parsed = QueryString.parse(querystring, { + depth: 10, + }); + + const validatedFilter = filterSchema.safeParse(parsed.filters); + + expect(validatedFilter.error).toBeUndefined(); + expect(validatedFilter.success).toBe(true); + + expect(validatedFilter.data).toEqual(filter); + } + ); + + it("handles empty filters", async () => { + const querystring = ""; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); + + expect(validatedFilter.success).toBe(true); + expect(validatedFilter.data).toEqual({}); + }); + + it("rejects invalid operators", async () => { + const querystring = "filters[title][$invalid]=test"; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); + + expect(validatedFilter.success).toBe(false); + }); + + it("rejects invalid logical operators", async () => { + const querystring = "filters[$invalid][0][title][$eq]=test"; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); + + expect(validatedFilter.success).toBe(false); + }); + + it("handles malformed between operator", async () => { + const querystring = "filters[number][$between]=10"; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); + + expect(validatedFilter.success).toBe(false); + }); + + it("handles malformed array syntax", async () => { + const querystring = "filters[number][$in]=1,2,3"; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); + + // This should fail because $in expects an array + expect(validatedFilter.success).toBe(false); + }); + }); + describe("filtering", async () => { it.concurrent.each(validFilterCases)( "filters by $title", diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index de9202db7..7b02d6f0e 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -341,11 +341,10 @@ export const logicalOperators = ["$and", "$or", "$not"] as const; export type LogicalOperator = (typeof logicalOperators)[number]; export type BaseFilter = { - [slug: string]: - | { - [O in FilterOperator]?: unknown; - } - | Filter; + [O in FilterOperator]?: unknown; +}; +export type FieldLevelFilter = { + [slug: string]: BaseFilter | { $and?: BaseFilter[]; $or?: BaseFilter[]; $not?: BaseFilter }; }; export type LogicalFilter = { @@ -354,13 +353,13 @@ export type LogicalFilter = { $not?: Filter; }; -export type Filter = BaseFilter | LogicalFilter; +export type Filter = FieldLevelFilter | LogicalFilter; -const allSchema = z - .string() - .or(z.coerce.number()) +const allSchema = z.coerce + .number() .or(z.enum(["true", "false"]).transform((val) => val === "true")) - .or(z.coerce.date()); + .or(z.coerce.date()) + .or(z.string()); const numberOrDateSchema = z.coerce.number().or(z.coerce.date()); @@ -378,8 +377,14 @@ export const baseFilterSchema = z $notContains: z.string().describe("Does not contain"), $containsi: z.string().describe("Contains (case insensitive)"), $notContainsi: z.string().describe("Does not contain (case insensitive)"), - $null: z.never().describe("Is null"), - $notNull: z.never().describe("Is not null"), + $null: z + .string() + .transform(() => true) + .describe("Is null"), + $notNull: z + .string() + .transform(() => true) + .describe("Is not null"), $in: z.array(allSchema).describe("In"), $notIn: z.array(allSchema).describe("Not in"), $between: z.tuple([numberOrDateSchema, numberOrDateSchema]).describe("Between"), @@ -399,43 +404,60 @@ export const baseFilterSchema = z .partial() .refine((data) => { if (!Object.keys(data).length) { + console.log(data); return false; } return true; - }, "Filter must have at least one operator") satisfies z.ZodType<{ + }, "Filter must have at least one operator (base filter)") satisfies z.ZodType<{ [K in FilterOperator]?: any; }>; +const fieldSlugSchema = z + .string() + .regex(/^[a-zA-Z0-9_.:-]+$/, "At this level, you can only use field slugs"); + // this is a recursive type, so we need to use z.lazy() -export const filterSchema: z.ZodType = z.lazy(() => - z.union([ - // regular field filters - z.record( - z.union([ - // operator-value pairs - baseFilterSchema, - // nested filters (for object types) - filterSchema, - ]) - ), - // logical operators - z.object({ - $and: z.array(filterSchema).optional(), - $or: z.array(filterSchema).optional(), - $not: filterSchema, - }), - z.object({ +export const filterSchema: z.ZodType = z.lazy(() => { + const topLevelLogicalOperators = z + .object({ $and: z.array(filterSchema).optional(), - $or: z.array(filterSchema), - $not: filterSchema.optional(), - }), - z.object({ - $and: z.array(filterSchema), $or: z.array(filterSchema).optional(), $not: filterSchema.optional(), - }), - ]) -); + }) + .refine((data) => { + if (!Object.keys(data).length) { + return false; + } + return true; + }, "Filter must have at least one operator. A logical operator (e.g. $and, $or, $not) was expected here, but not found."); + + const fieldLevelThings = z + .object({ + $and: z.array(baseFilterSchema).optional(), + $or: z.array(baseFilterSchema).optional(), + $not: baseFilterSchema.optional(), + }) + .refine((data) => { + if (!Object.keys(data).length) { + return false; + } + return true; + }, "Filter must have at least one operator (field level)") + .or(baseFilterSchema); + + //z.union([ + // regular field filters + + // field -> operator -> value + // field -> logical operator -> operator -> value + // field -> logical operator -> logical operator -> ... -> operator -> value + // logical operator -> field -> operator -> value + // logical operator -> logical operator -> ... -> field -> operator -> value + + return z.union([z.record(fieldSlugSchema, fieldLevelThings), topLevelLogicalOperators]); //, + // logicalOperators, + // ]); +}); const getPubQuerySchema = z.object({ depth: z From 44eebdbb27197500dbf10867452b2f33e6993cfe Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 26 Feb 2025 18:19:55 +0100 Subject: [PATCH 15/20] fix: actually make logical filters work correctly --- core/lib/server/pub-filters.ts | 47 +++++++++++++++++++++++----- core/log.txt | 56 ++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 core/log.txt diff --git a/core/lib/server/pub-filters.ts b/core/lib/server/pub-filters.ts index b11e7f517..8fe6da386 100644 --- a/core/lib/server/pub-filters.ts +++ b/core/lib/server/pub-filters.ts @@ -150,6 +150,14 @@ const filterMap = { export const isNonRecursiveFilter = ( filter: BaseFilter[string] ): filter is Exclude => { + // Check if this is a logical operator within a field filter + if (filter && typeof filter === "object" && !Array.isArray(filter)) { + const keys = Object.keys(filter); + if (keys.some((k) => logicalOperators.includes(k as LogicalOperator))) { + return false; + } + } + if (Object.keys(filter).every((k) => k.startsWith("$"))) { return true; } @@ -194,6 +202,38 @@ export function applyFilters>( } if (!isNonRecursiveFilter(val)) { + // Handle logical operators within field filters + const logicalOps = Object.entries(val).filter(([key]) => + logicalOperators.includes(key as LogicalOperator) + ); + + if (logicalOps.length > 0) { + const [op, subFilters] = logicalOps[0] as [LogicalOperator, Filter[]]; + + if (op === "$or") { + // Transform field-level $or into top-level $or with field constraints + return eb.or( + subFilters.map((subFilter) => { + // Create a new filter with the field and subfilter + const newFilter = { [field]: subFilter }; + return applyFilters(eb, newFilter); + }) + ); + } else if (op === "$and") { + // Transform field-level $and into top-level $and with field constraints + return eb.and( + subFilters.map((subFilter) => { + const newFilter = { [field]: subFilter }; + return applyFilters(eb, newFilter); + }) + ); + } else if (op === "$not") { + // Handle $not operator + const newFilter = { [field]: subFilters }; + return eb.not(applyFilters(eb, newFilter)); + } + } + throw new Error(`Unknown filter: ${JSON.stringify(filter)}`); } @@ -226,13 +266,6 @@ export function applyFilters>( ); }), ]); - - // const validOperators = getValidOperatorsForSchema(getJsonSchemaByCoreSchemaType(field)); - // // naive check - - // if (!validOperators.includes(operator)) { - // throw new Error(`Operator ${operator} is not valid for schema type ${field}`); - // } }); return eb.and(conditions) as ExpressionWrapped; diff --git a/core/log.txt b/core/log.txt new file mode 100644 index 000000000..3a31b3af8 --- /dev/null +++ b/core/log.txt @@ -0,0 +1,56 @@ +Base + + 127.14 real 210.20 user 32.80 sys + +Parallel traces + 93.87 real 157.80 user 28.09 sys + 0.00 real 0.00 user 0.00 sys + 0.00 real 0.00 user 0.00 sys + 0.00 real 0.00 user 0.00 sys + 0.57 real 0.47 user 0.07 sys + +Worker only + 102.50 real 183.38 user 31.65 sys + +Main - preconstruct + 118.19 real 198.11 user 32.31 sys + +Parallel traces -preconstruct + 85.63 real 152.23 user 26.83 sys + 84.88 real 154.92 user 26.42 sys + +Parallel traces +preconstruct + 84.87 real 157.53 user 26.99 sys + 85.96 real 151.66 user 26.79 sys + +Parallel traces +sentry update + 84.15 real 148.79 user 26.30 sys + 83.87 real 148.66 user 26.21 sys + +Parallel traces +sentry update +sentry plugin disable + 82.49 real 145.19 user 26.07 sys + +Parallel traces +sentry update +sentry plugin disable +otel reuse + 81.94 real 143.09 user 25.82 sys + 88.20 real 152.83 user 26.53 sys + +Parallel traces +sentry update +webpack freeze + 76.65 real 107.46 user 16.51 sys + 75.88 real 107.77 user 16.19 sys + + +Parallel traces +sentry update -webpack freeze + custom webpack config + 90.16 real 153.39 user 26.37 sys + + +Parallel traces +sentry update -webpack freeze -custom webpack config + 89.74 real 153.69 user 25.84 sys + + +Parallel traces +webpack freeze +swcTraceProfiling + 90.16 real 153.39 user 26.37 sys + 77.43 real 113.78 user 17.20 sys + 74.43 real 111.62 user 17.38 sys + +Parallel traces +webpack freeze +workerThreads 58.92 real 121.60 user 16.89 sys + 74.22 real 166.89 user 27.58 sys From 5d01997e3f9e41eb92aa29a444224ca2f704e47b Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Thu, 27 Feb 2025 15:27:32 +0100 Subject: [PATCH 16/20] feat: add little typesafe mapping lib --- core/lib/mapping.ts | 71 +++++++++++++++++++++++++++++++++++++++++++++ core/lib/types.ts | 20 +++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 core/lib/mapping.ts diff --git a/core/lib/mapping.ts b/core/lib/mapping.ts new file mode 100644 index 000000000..328fde4e1 --- /dev/null +++ b/core/lib/mapping.ts @@ -0,0 +1,71 @@ +import type { Prettify, TupleToCustomObject } from "./types"; + +/** + * Type-safe version of Object.entries() + * Returns an array of tuples containing key-value pairs with proper typing + * Includes all properties that exist in the object, even if undefined + */ +export function entries< + const T extends Record, + const KeepUndefined extends boolean = false, +>( + obj: T, + keepUndefined?: KeepUndefined +): T extends T + ? { [K in keyof T]-?: [K, KeepUndefined extends true ? T[K] : NonNullable] }[keyof T][] + : never { + const newObj = Object.entries(obj); + + if (keepUndefined) { + return newObj as any; + } + + return newObj.filter(([_, v]) => v !== undefined) as any; +} + +/** + * Type-safe version of Object.fromEntries() + * Creates an object from an array of key-value pairs with proper typing + */ +export function fromEntries( + entries: T +): { [K in T[number][0]]: Extract[1] } { + return Object.fromEntries(entries) as any; +} + +type MapToEntries< + T extends readonly unknown[], + M extends readonly string[], + C extends any[] = [], + Buffer extends Record = {}, +> = C["length"] extends T["length"] + ? Buffer + : MapToEntries< + T, + M, + [...C, C["length"]], + Prettify< + Buffer & { + [NewKey in M[C["length"]]]: T[C["length"]]; + } + > + >; + +export function mapToEntries< + const T extends readonly unknown[], + const M extends readonly string[], + C extends any[] = [], + Buffer extends Record = {}, +>(obj: T, mapping: M): T extends T ? MapToEntries : never { + const result: Record = {}; + for (let i = 0; i < Math.min(obj.length, mapping.length); i++) { + result[mapping[i]] = obj[i]; + } + return result as any; +} + +export function keys>( + obj: T +): T extends T ? (keyof T)[] : never { + return Object.keys(obj) as any; +} diff --git a/core/lib/types.ts b/core/lib/types.ts index cb163be3c..d67428a5e 100644 --- a/core/lib/types.ts +++ b/core/lib/types.ts @@ -120,3 +120,23 @@ export type AutoReturnType DirectAutoOutput> export type UnionOmit = T extends T ? Omit : never; export type UnionPick = T extends T ? Pick : never; + +/** + * Maps a union of tuples to a union of objects + * TupleToObject<["a", number] | ["b", string]> = { key: "a", value: number } | { key: "b", value: string } + */ +export type TupleToObject = T extends [infer K extends string, infer V] + ? { [key in K]: V } + : never; + +/** + * Converts a tuple [K, V] to an object with custom property names + * Example: TupleToCustomObject<["$or", Filter[]], "operator", "filters"> = { operator: "$or", filters: Filter[] } + */ +export type TupleToCustomObject< + T extends [string, any], + KeyProp extends string = "key", + ValueProp extends string = "value", +> = T extends [infer K extends string, infer V] + ? { [key in KeyProp]: K } & { [key in ValueProp]: V } + : never; From eedd289a0f834895a2f0c5b702987c20c59baaa1 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Thu, 27 Feb 2025 15:28:09 +0100 Subject: [PATCH 17/20] chore: remove comment --- packages/contracts/src/resources/site.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index 7b02d6f0e..0978280ca 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -404,7 +404,6 @@ export const baseFilterSchema = z .partial() .refine((data) => { if (!Object.keys(data).length) { - console.log(data); return false; } return true; From 5a2f49770be4127635fc5ec062d6aac8a07c5be0 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Thu, 27 Feb 2025 15:28:35 +0100 Subject: [PATCH 18/20] fix: make typing of pub-filters better --- core/lib/server/pub-filters.ts | 120 +++++++++++++++++---------------- 1 file changed, 62 insertions(+), 58 deletions(-) diff --git a/core/lib/server/pub-filters.ts b/core/lib/server/pub-filters.ts index 8fe6da386..0bfc637f5 100644 --- a/core/lib/server/pub-filters.ts +++ b/core/lib/server/pub-filters.ts @@ -2,25 +2,27 @@ import type { ExpressionBuilder, ExpressionWrapper } from "kysely"; import { sql } from "kysely"; -import type { BaseFilter, Filter, FilterOperator, LogicalFilter, LogicalOperator } from "contracts"; +import type { + BaseFilter, + FieldLevelFilter, + Filter, + FilterOperator, + LogicalFilter, + LogicalOperator, +} from "contracts"; import { logicalOperators } from "contracts"; import { CoreSchemaType } from "db/public"; import { assert } from "utils"; -type PathSegment = string; // Regular property - -type Path = PathSegment[]; - -// Helper type to get the type of a value at a specific path +import { entries, fromEntries, keys, mapToEntries } from "../mapping"; type EntriedLogicalFilter = [ ["$or", NonNullable], ["$and", NonNullable], ["$not", NonNullable], ][number]; -type EntriedArrayFilter = [["$any", Filter], ["$all", Filter]]; -type EntriedFilter = [string, BaseFilter[keyof BaseFilter]] | EntriedLogicalFilter; +type EntriedFilter = [string, FieldLevelFilter[keyof FieldLevelFilter]] | EntriedLogicalFilter; const isLogicalFilter = (filter: EntriedFilter): filter is EntriedLogicalFilter => { return ( @@ -120,8 +122,10 @@ const filterMap = { eb(column, "<=", typeof value[1] === "string" ? JSON.stringify(value[1]) : value[1]), ]); }, - $in: (eb, column, value) => eb(column, "in", value), - $notIn: (eb, column, value) => eb(column, "not in", value), + $in: (eb, column, value) => + eb(column, "in", typeof value === "string" ? JSON.stringify(value) : value), + $notIn: (eb, column, value) => + eb(column, "not in", typeof value === "string" ? JSON.stringify(value) : value), $contains: (eb, column, value) => eb(sql.raw(`${column}::text`), "like", `%${String(value)}%`), $notContains: (eb, column, value) => eb(sql.raw(`${column}::text`), "not like", `%${String(value)}%`), @@ -147,13 +151,11 @@ const filterMap = { ) => ExpressionWrapper >; -export const isNonRecursiveFilter = ( - filter: BaseFilter[string] -): filter is Exclude => { +export const isNonRecursiveFilter = (filter: FieldLevelFilter[string]): filter is BaseFilter => { // Check if this is a logical operator within a field filter if (filter && typeof filter === "object" && !Array.isArray(filter)) { - const keys = Object.keys(filter); - if (keys.some((k) => logicalOperators.includes(k as LogicalOperator))) { + const ks = keys(filter); + if (ks.some((k) => logicalOperators.includes(k as LogicalOperator))) { return false; } } @@ -174,19 +176,12 @@ export function applyFilters>( eb: K, filters: Filter ): ExpressionWrapped { - const conditions = Object.entries(filters).map((filter: EntriedFilter) => { + const conditions = entries(filters).map((filter) => { + // Handle top-level logical operators if (isLogicalFilter(filter)) { - if (filter[0] === "$or") { - return eb.or(filter[1].map((f) => applyFilters(eb, f))); - } - if (filter[0] === "$and") { - return eb.and(filter[1].map((f) => applyFilters(eb, f))); - } - if (filter[0] === "$not") { - return eb.not(applyFilters(eb, filter[1])); - } + const operatorFilter = mapToEntries(filter, ["operator", "filters"]); - throw new Error(`Unknown logical operator: ${filter[0]}`); + return applyLogicalOperation(eb, operatorFilter); } const [field, val] = filter; @@ -201,46 +196,35 @@ export function applyFilters>( throw new Error(`Date filters must use date operators: ${JSON.stringify(val)}`); } + // Handle field-level logical operators if (!isNonRecursiveFilter(val)) { - // Handle logical operators within field filters - const logicalOps = Object.entries(val).filter(([key]) => - logicalOperators.includes(key as LogicalOperator) - ); - - if (logicalOps.length > 0) { - const [op, subFilters] = logicalOps[0] as [LogicalOperator, Filter[]]; + const logicalOps = entries(val).filter(([key]) => logicalOperators.includes(key)); - if (op === "$or") { - // Transform field-level $or into top-level $or with field constraints - return eb.or( - subFilters.map((subFilter) => { - // Create a new filter with the field and subfilter - const newFilter = { [field]: subFilter }; - return applyFilters(eb, newFilter); - }) - ); - } else if (op === "$and") { - // Transform field-level $and into top-level $and with field constraints - return eb.and( - subFilters.map((subFilter) => { - const newFilter = { [field]: subFilter }; - return applyFilters(eb, newFilter); - }) - ); - } else if (op === "$not") { - // Handle $not operator - const newFilter = { [field]: subFilters }; - return eb.not(applyFilters(eb, newFilter)); - } + if (logicalOps.length === 0) { + throw new Error(`Unknown filter: ${JSON.stringify(filter)}`); } - throw new Error(`Unknown filter: ${JSON.stringify(filter)}`); + const [operator, subFilters] = logicalOps[0]; + + // For field-level operators, we need to apply the field constraint to each subfilter + if (operator === "$not") { + // Special case for $not since it takes a single filter, not an array + const newFilter = { [field]: subFilters }; + return eb.not(applyFilters(eb, newFilter)); + } else { + // For $or and $and, map each subfilter to include the field + const fieldConstrainedFilters = subFilters.map((subFilter) => ({ + [field]: subFilter, + })); + return applyLogicalOperation(eb, { operator, filters: fieldConstrainedFilters }); + } } + // Handle regular field filters return eb.and([ ...(isDate ? [] : [eb("slug", "=", field)]), - ...Object.entries(val).map((entry) => { - const [operator, value] = entry as [FilterOperator, unknown]; + ...entries(val).map((entry) => { + const [operator, value] = entry; const whereFn = filterMap[operator]; if (!whereFn) { @@ -270,3 +254,23 @@ export function applyFilters>( return eb.and(conditions) as ExpressionWrapped; } + +// Helper function to apply logical operations +function applyLogicalOperation>( + eb: K, + operatorFilters: + | { operator: Exclude; filters: Filter[] } + | { operator: "$not"; filters: Filter } +): ExpressionWrapper { + switch (operatorFilters.operator) { + case "$or": + return eb.or(operatorFilters.filters.map((f) => applyFilters(eb, f))); + case "$and": + return eb.and(operatorFilters.filters.map((f) => applyFilters(eb, f))); + case "$not": + // $not should only have one filter + return eb.not(applyFilters(eb, operatorFilters.filters)); + default: + throw new Error(`Unknown logical operator: ${operatorFilters}`); + } +} From 64329ac76f71212c542a513854c05f6dae883fd7 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Thu, 27 Feb 2025 15:29:06 +0100 Subject: [PATCH 19/20] dev: rework the tests, expose not working filtering (despair) --- core/lib/server/pub-filters.db.test.ts | 1325 ++++++++++++------------ 1 file changed, 634 insertions(+), 691 deletions(-) diff --git a/core/lib/server/pub-filters.db.test.ts b/core/lib/server/pub-filters.db.test.ts index f111c5842..f2a04744b 100644 --- a/core/lib/server/pub-filters.db.test.ts +++ b/core/lib/server/pub-filters.db.test.ts @@ -1,3 +1,4 @@ +import { jsonArrayFrom } from "kysely/helpers/postgres"; import QueryString from "qs"; import { describe, expect, it } from "vitest"; @@ -18,6 +19,21 @@ const communitySlug = `${new Date().toISOString()}:test-filter-pub`; const trueId = crypto.randomUUID() as PubsId; const vector3Id = crypto.randomUUID() as PubsId; +const titleId = crypto.randomUUID() as PubsId; +const title2Id = crypto.randomUUID() as PubsId; +const anotherId = crypto.randomUUID() as PubsId; +const number42Id = crypto.randomUUID() as PubsId; +const number24Id = crypto.randomUUID() as PubsId; +const number54Id = crypto.randomUUID() as PubsId; +const arrayId = crypto.randomUUID() as PubsId; +const numberArrayId = crypto.randomUUID() as PubsId; +const numberArray2Id = crypto.randomUUID() as PubsId; +const relationId = crypto.randomUUID() as PubsId; +const testTitleId = crypto.randomUUID() as PubsId; +const testCaseId = crypto.randomUUID() as PubsId; +const specialCharsId = crypto.randomUUID() as PubsId; +const arrayItem1Id = crypto.randomUUID() as PubsId; +const importantDocId = crypto.randomUUID() as PubsId; const twenty99 = new Date("2099-01-01"); @@ -28,6 +44,7 @@ const seed = createSeed({ }, pubFields: { Title: { schemaName: CoreSchemaType.String }, + Email: { schemaName: CoreSchemaType.Email }, Number: { schemaName: CoreSchemaType.Number }, Boolean: { schemaName: CoreSchemaType.Boolean }, Date: { schemaName: CoreSchemaType.DateTime }, @@ -40,6 +57,7 @@ const seed = createSeed({ pubTypes: { "Basic Pub": { Title: { isTitle: true }, + Email: { isTitle: false }, Number: { isTitle: false }, Boolean: { isTitle: false }, Date: { isTitle: false }, @@ -53,75 +71,134 @@ const seed = createSeed({ stages: {}, pubs: [ { + id: titleId, pubType: "Basic Pub", values: { Title: "Some title", - }, - }, - { - pubType: "Basic Pub", - values: { - Title: "Another title", - }, - }, - { + Email: "test@test.com", + }, + }, + // { + // id: title2Id, + // pubType: "Basic Pub", + // values: { + // Title: "some Title", + // Email: "Test@Test.com", + // }, + // }, + // { + // id: anotherId, + // pubType: "Basic Pub", + // values: { + // Title: "Another title", + // Email: "test2@test.com", + // }, + // }, + { + id: number42Id, pubType: "Basic Pub", values: { Number: 42, }, }, - { - pubType: "Basic Pub", - values: { - Number: 24, - }, - }, - { - pubType: "Basic Pub", - values: { - Number: 54, - }, - }, - { - id: trueId, - pubType: "Basic Pub", - values: { - Boolean: true, - }, - }, - { - pubType: "Basic Pub", - values: { - Array: ["item1", "item2"], - }, - }, - { - id: vector3Id, - pubType: "Basic Pub", - values: { - Vector3: [0, 0, 0], - }, - }, - { - pubType: "Basic Pub", - values: { - NumberArray: [1, 2, 3], - Date: twenty99, - }, - }, - - { - pubType: "Basic Pub", - values: { - NumberArray: [10, 20, 30, 40], - }, - }, - { - pubType: "Basic Pub", - values: { - Relation: null, - }, - }, + // { + // id: number24Id, + // pubType: "Basic Pub", + // values: { + // Number: 24, + // }, + // }, + // { + // id: number54Id, + // pubType: "Basic Pub", + // values: { + // Number: 54, + // }, + // }, + // { + // id: trueId, + // pubType: "Basic Pub", + // values: { + // Boolean: true, + // }, + // }, + // { + // id: arrayId, + // pubType: "Basic Pub", + // values: { + // Array: ["item1", "item2"], + // }, + // }, + // { + // id: vector3Id, + // pubType: "Basic Pub", + // values: { + // Vector3: [0, 0, 0], + // }, + // }, + // { + // id: numberArrayId, + // pubType: "Basic Pub", + // values: { + // NumberArray: [1, 2, 3], + // Date: twenty99, + // }, + // }, + // { + // id: numberArray2Id, + // pubType: "Basic Pub", + // values: { + // NumberArray: [10, 20, 30, 40], + // }, + // }, + // { + // id: relationId, + // pubType: "Basic Pub", + // values: { + // Relation: null, + // Number: 50, + // }, + // }, + // { + // id: testTitleId, + // pubType: "Basic Pub", + // values: { + // Title: "Test", + // Number: 99, + // }, + // }, + // { + // id: testCaseId, + // pubType: "Basic Pub", + // values: { + // Boolean: true, + // Title: "Test case", + // }, + // }, + // { + // id: specialCharsId, + // pubType: "Basic Pub", + // values: { + // Title: "Some title with special characters & symbols", + // }, + // }, + // { + // id: arrayItem1Id, + // pubType: "Basic Pub", + // values: { + // Array: ["item1", "item2"], + // }, + // }, + // { + // id: importantDocId, + // pubType: "Basic Pub", + // values: { + // Title: "Test important document", + // Number: 40, + // Date: new Date("2023-02-01T00:00:00.000Z"), + // Boolean: true, + // }, + // }, ], }); @@ -183,674 +260,540 @@ const validateFilter = async (communityId: CommunitiesId, filter: Filter, trx = const slug = (str: string) => `${communitySlug}:${str}`; -describe("pub-filters", () => { - describe("filter validation", () => { - describe("schema", () => { - it("successfully parses a filter", async () => { - const filter: Filter = { - [community.pubFields.Title.slug]: { $eq: "test" }, - }; - - const parsed = filterSchema.safeParse(filter); - expect(parsed.error).toBeUndefined(); - expect(parsed.success).toBe(true); - expect(parsed.data).toEqual(filter); - }); - }); - - describe("pubField validation", () => { - it("rejects unknown fields", async () => { - const filter: Filter = { - [`${community.community.slug}:unknownField`]: { $eq: "test" }, - }; - - await expect(validateFilter(community.community.id, filter)).rejects.toThrow(); - }); - - it("only allows valid operators for a field", async () => { - const filter: Filter = { - [community.pubFields.Title.slug]: { $invalid: "test" }, - }; - - const parsed = filterSchema.safeParse(filter); - expect(parsed.success).toBe(false); - - await expect(validateFilter(community.community.id, filter)).rejects.toThrow(); - }); - - it("does not allow gte on a string field", async () => { - const filter: Filter = { - [community.pubFields.Title.slug]: { $gte: "test" }, - }; - - const parsed = filterSchema.safeParse(filter); - - expect(parsed.success).toBe(false); - - await expect(validateFilter(community.community.id, filter)).rejects.toThrow( - /Operators \[\$gte\] are not valid for schema type String/ - ); - }); - }); - }); - - const currentDate = new Date(); - - const validFilterCases: { - title: string; - filter: Filter; - sql: string; - parameters: (string | number)[]; - resultValues: { value: Json; fieldSlug?: string }[][]; - }[] = [ - { - title: "simple equality", - filter: { - [slug("title")]: { $eq: "Some title" }, - }, - sql: `"slug" = $1 and "value" = $2`, - parameters: [slug("title"), '"Some title"'], - resultValues: [[{ value: "Some title", fieldSlug: slug("title") }]], - }, - { - title: "simple inequality", - filter: { - [slug("title")]: { $ne: "Some title" }, - }, - sql: `"slug" = $1 and "value" != $2`, - parameters: [slug("title"), '"Some title"'], - resultValues: [[{ value: "Another title", fieldSlug: slug("title") }]], - }, - { - title: "case insensitive equality", - filter: { - [slug("title")]: { $eqi: "some title" }, - }, - sql: `"slug" = $1 and lower(value::text) = $2`, - parameters: [slug("title"), '"some title"'], - resultValues: [[{ value: "Some title", fieldSlug: slug("title") }]], - }, - { - title: "case insensitive inequality", - filter: { - [slug("title")]: { $nei: "some title" }, - }, - sql: `"slug" = $1 and lower(value::text) != $2`, - parameters: [slug("title"), '"some title"'], - resultValues: [[{ value: "Another title", fieldSlug: slug("title") }]], - }, - { - title: "string contains", - filter: { - [slug("title")]: { $contains: "Another" }, - }, - sql: `"slug" = $1 and value::text like $2`, - parameters: [slug("title"), "%Another%"], - resultValues: [[{ value: "Another title", fieldSlug: slug("title") }]], - }, - { - title: "string contains case insensitive", - filter: { - [slug("title")]: { $containsi: "another" }, - }, - sql: `"slug" = $1 and value::text ilike $2`, - parameters: [slug("title"), "%another%"], - resultValues: [[{ value: "Another title", fieldSlug: slug("title") }]], - }, - { - title: "string not contains", - filter: { - [slug("title")]: { $notContains: "Another" }, - }, - sql: `"slug" = $1 and value::text not like $2`, - parameters: [slug("title"), "%Another%"], - resultValues: [[{ value: "Some title", fieldSlug: slug("title") }]], - }, - { - title: "string not contains case insensitive", - filter: { - [slug("title")]: { $notContainsi: "another" }, - }, - sql: `"slug" = $1 and value::text not ilike $2`, - parameters: [slug("title"), "%another%"], - resultValues: [[{ value: "Some title", fieldSlug: slug("title") }]], - }, - - { - title: "array contains w/ json path", - filter: { - [slug("array")]: { $jsonPath: '$[*] == "item1"' }, - }, - sql: `"slug" = $1 and "value" @@ $2`, - parameters: [slug("array"), '$[*] == "item1"'], - resultValues: [[{ value: ["item1", "item2"], fieldSlug: slug("array") }]], +const unifiedTestCases: { + title: string; + filter: Filter; + querystring: string; + sql: string; + parameters: (string | number)[]; + foundIds: PubsId[]; +}[] = [ + { + title: "simple equality", + filter: { + [slug("title")]: { $eq: "Some title" }, + }, + querystring: `filters[${slug("title")}][$eq]=Some title`, + sql: `"slug" = $1 and "value" = $2`, + parameters: [slug("title"), '"Some title"'], + foundIds: [titleId], + }, + { + title: "simple inequality", + filter: { + [slug("email")]: { $ne: "test@test.com" }, + }, + querystring: `filters[${slug("email")}][$ne]=test@test.com`, + sql: `"slug" = $1 and "value" != $2`, + parameters: [slug("email"), '"test@test.com"'], + foundIds: [anotherId, title2Id], + }, + { + title: "case insensitive equality", + filter: { + [slug("title")]: { $eqi: "some title" }, + }, + querystring: `filters[${slug("title")}][$eqi]=some title`, + sql: `"slug" = $1 and lower(value::text) = $2`, + parameters: [slug("title"), '"some title"'], + foundIds: [titleId, title2Id], + }, + { + title: "case insensitive inequality", + filter: { + [slug("email")]: { $nei: "test@test.com" }, + }, + querystring: `filters[${slug("email")}][$nei]=test@test.com`, + sql: `"slug" = $1 and lower(value::text) != $2`, + parameters: [slug("email"), '"test@test.com"'], + foundIds: [anotherId], + }, + { + title: "string contains", + filter: { + [slug("title")]: { $contains: "Another" }, + }, + querystring: `filters[${slug("title")}][$contains]=Another`, + sql: `"slug" = $1 and value::text like $2`, + parameters: [slug("title"), "%Another%"], + foundIds: [anotherId], + }, + { + title: "string contains case insensitive", + filter: { + [slug("title")]: { $containsi: "another" }, + }, + querystring: `filters[${slug("title")}][$containsi]=another`, + sql: `"slug" = $1 and value::text ilike $2`, + parameters: [slug("title"), "%another%"], + foundIds: [anotherId], + }, + { + title: "string not contains", + filter: { + [slug("email")]: { $notContains: "Test" }, + }, + querystring: `filters[${slug("email")}][$notContains]=Test`, + sql: `"slug" = $1 and value::text not like $2`, + parameters: [slug("email"), "%Test%"], + foundIds: [titleId, anotherId], + }, + { + title: "string not contains case insensitive", + filter: { + [slug("email")]: { $notContainsi: "Test@" }, + }, + querystring: `filters[${slug("email")}][$notContainsi]=Test@`, + sql: `"slug" = $1 and value::text not ilike $2`, + parameters: [slug("email"), "%Test@%"], + foundIds: [anotherId], + }, + { + title: "array contains w/ json path", + filter: { + [slug("array")]: { $jsonPath: '$[*] == "item1"' }, + }, + querystring: `filters[${slug("array")}][$jsonPath]=$[*] == "item1"`, + sql: `"slug" = $1 and "value" @@ $2`, + parameters: [slug("array"), '$[*] == "item1"'], + foundIds: [arrayId, arrayItem1Id], + }, + { + title: "array specific index value check", + filter: { + [slug("numberarray")]: { $jsonPath: "$[1] > 10" }, + }, + querystring: `filters[${slug("numberarray")}][$jsonPath]=$[1] > 10`, + sql: `"slug" = $1 and "value" @@ $2`, + parameters: [slug("numberarray"), "$[1] > 10"], + foundIds: [numberArray2Id], + }, + { + title: "nested logical operators", + filter: { + $or: [ + { [slug("title")]: { $eq: "Some title" } }, + { + $and: [{ [slug("number")]: { $gt: 40 } }, { [slug("number")]: { $lt: 50 } }], + }, + ], }, - { - title: "array specific index value check", - filter: { - [slug("numberarray")]: { $jsonPath: "$[1] > 10" }, + querystring: `filters[$or][0][${slug("title")}][$eq]=Some title&filters[$or][1][$and][0][${slug("number")}][$gt]=40&filters[$or][1][$and][1][${slug("number")}][$lt]=50`, + sql: `(("slug" = $1 and "value" = $2) or (("slug" = $3 and "value" > $4) and ("slug" = $5 and "value" < $6)))`, + parameters: [slug("title"), '"Some title"', slug("number"), 40, slug("number"), 50], + foundIds: [titleId, number42Id], + }, + { + title: "field-level logical operators", + filter: { + [slug("number")]: { + $or: [{ $lt: 40 }, { $gt: 50 }], }, - sql: `"slug" = $1 and "value" @@ $2`, - parameters: [slug("numberarray"), "$[1] > 10"], - resultValues: [[{ value: [10, 20, 30, 40], fieldSlug: slug("numberarray") }]], }, - { - title: "nested logical operators", - filter: { - $or: [ - { [slug("title")]: { $eq: "Some title" } }, - { - $and: [ - { [slug("number")]: { $gt: 40 } }, - { [slug("number")]: { $lt: 50 } }, - ], + querystring: `filters[${slug("number")}][$or][0][$lt]=40&filters[${slug("number")}][$or][1][$gt]=50`, + sql: `("slug" = $1 and "value" < $2) or ("slug" = $3 and "value" > $4)`, + parameters: [slug("number"), 40, slug("number"), 50], + foundIds: [number24Id, number54Id, testTitleId], + }, + { + title: "updatedAt & createdAt filters", + filter: { + $or: [ + { + createdAt: { + $lte: new Date("2025-01-01"), + }, + }, + { + createdAt: { + $gte: new Date("2090-01-01"), }, - ], - }, - - sql: `(("slug" = $1 and "value" = $2) or (("slug" = $3 and "value" > $4) and ("slug" = $5 and "value" < $6)))`, - parameters: [slug("title"), '"Some title"', slug("number"), 40, slug("number"), 50], - resultValues: [ - [{ value: 42, fieldSlug: slug("number") }], - [{ value: "Some title", fieldSlug: slug("title") }], - ], - }, - { - title: "nested logical operators", - filter: { - [slug("number")]: { - $or: [ - { - $lt: 40, - }, - { - $gt: 50, - }, - ], }, - }, - - sql: `("slug" = $1 and "value" < $2) or ("slug" = $3 and "value" > $4)`, - parameters: [slug("number"), 40, slug("number"), 50], - resultValues: [ - [{ value: 54, fieldSlug: slug("number") }], - [{ value: 24, fieldSlug: slug("number") }], ], }, - { - title: "updatedAt & createdAt filters", - filter: { - $or: [ - { - createdAt: { - $lte: new Date("2025-01-01"), - }, - }, - { - createdAt: { - $gte: new Date("2090-01-01"), - }, - }, - ], + querystring: + "filters[$or][0][createdAt][$lte]=2025-01-01T00:00:00.000Z&filters[$or][1][createdAt][$gte]=2090-01-01T00:00:00.000Z", + sql: `"pubs"."createdAt" <= $1 or "pubs"."createdAt" >= $2`, + parameters: [ + `"${new Date("2025-01-01").toISOString()}"`, + `"${new Date("2090-01-01").toISOString()}"`, + ], + foundIds: [trueId, vector3Id], + }, + { + title: "date filters", + filter: { + [slug("date")]: { + $eq: twenty99, }, - sql: `"pubs"."createdAt" <= $1 or "pubs"."createdAt" >= $2`, - parameters: [ - `"${new Date("2025-01-01").toISOString()}"`, - `"${new Date("2090-01-01").toISOString()}"`, - ], - resultValues: [ - [ - { - value: true, - fieldSlug: slug("boolean"), - }, - ], - [{ value: [0, 0, 0], fieldSlug: slug("vector3") }], - ], }, - { - title: "date filters", - filter: { - [slug("date")]: { - $eq: twenty99.toISOString(), + querystring: `filters[${slug("date")}][$eq]=${twenty99.toISOString()}`, + sql: `"slug" = $1 and "value" = $2`, + parameters: [slug("date"), `"${twenty99.toISOString()}"`], + foundIds: [numberArrayId], + }, + { + title: "multiple operators on same field should be treated as AND", + filter: { + [slug("number")]: { $gt: 40, $lt: 50 }, + }, + querystring: `filters[${slug("number")}][$gt]=40&filters[${slug("number")}][$lt]=50`, + sql: `"slug" = $1 and "value" > $2 and "value" < $3`, + parameters: [slug("number"), 40, 50], + foundIds: [number42Id], + }, + { + title: "multiple fields should be treated as AND", + filter: { + [slug("title")]: { $eq: "Test" }, + [slug("number")]: { $eq: 99 }, + }, + querystring: `filters[${slug("title")}][$eq]=Test&filters[${slug("number")}][$eq]=99`, + sql: `("slug" = $1 and "value" = $2) and ("slug" = $3 and "value" = $4)`, + parameters: [slug("title"), '"Test"', slug("number"), "99"], + foundIds: [testTitleId], + }, + { + title: "boolean coercion", + filter: { + [slug("boolean")]: { $eq: true }, + }, + querystring: `filters[${slug("boolean")}][$eq]=true`, + sql: `"slug" = $1 and "value" = $2`, + parameters: [slug("boolean"), "true"], + foundIds: [testCaseId], + }, + { + title: "array in operator", + filter: { + [slug("number")]: { $in: [1, 2, 3] }, + }, + querystring: `filters[${slug("number")}][$in][]=1&filters[${slug("number")}][$in][]=2&filters[${slug("number")}][$in][]=3`, + sql: `"slug" = $1 and "value" in ($2, $3, $4)`, + parameters: [slug("number"), 1, 2, 3], + foundIds: [number42Id], + }, + { + title: "between operator", + filter: { + [slug("number")]: { $between: [10, 20] }, + }, + querystring: `filters[${slug("number")}][$between][0]=10&filters[${slug("number")}][$between][1]=20`, + sql: `"slug" = $1 and ("value" >= $2 and "value" <= $3)`, + parameters: [slug("number"), 10, 20], + foundIds: [number42Id], + }, + { + title: "null and notNull operators", + filter: { + [slug("title")]: { $null: true }, + [slug("number")]: { $notNull: true }, + }, + querystring: `filters[${slug("title")}][$null]&filters[${slug("number")}][$notNull]`, + sql: `("slug" = $1 and "value" is null) and ("slug" = $2 and "value" is not null)`, + parameters: [slug("title"), slug("number")], + foundIds: [relationId], + }, + { + title: "top-level logical AND", + filter: { + $and: [{ [slug("title")]: { $eq: "Test" } }, { [slug("number")]: { $gt: 10 } }], + }, + querystring: `filters[$and][0][${slug("title")}][$eq]=Test&filters[$and][1][${slug("number")}][$gt]=10`, + sql: `("slug" = $1 and "value" = $2) and ("slug" = $3 and "value" > $4)`, + parameters: [slug("title"), '"Test"', slug("number"), 10], + foundIds: [testTitleId], + }, + { + title: "top-level logical NOT", + filter: { + $not: { [slug("title")]: { $eq: "Test" } }, + }, + querystring: `filters[$not][${slug("title")}][$eq]=Test`, + sql: `not ("slug" = $1 and "value" = $2)`, + parameters: [slug("title"), '"Test"'], + foundIds: [titleId, anotherId, specialCharsId, importantDocId], + }, + { + title: "complex nested structure", + filter: { + $or: [ + { + $and: [ + { [slug("title")]: { $containsi: "test" } }, + { [slug("boolean")]: { $eq: true } }, + ], }, - }, - sql: `"slug" = $1 and "value" = $2`, - parameters: [slug("date"), `"${twenty99.toISOString()}"`], - resultValues: [ - [ - { - value: [1, 2, 3], - fieldSlug: slug("numberarray"), + { + $not: { + [slug("number")]: { $between: [10, 20] }, }, - { value: twenty99.toISOString(), fieldSlug: slug("date") }, - ], + }, ], }, - ]; - describe("SQL generation", () => { - it.concurrent.each(validFilterCases)( - "generates correct SQL for $title", - async ({ filter, sql, parameters }) => { - const trx = getTrx(); - - const q = coolQuery(filter).compiled; - - expect(q.sql).toMatch(sql); - expect(q.parameters).toEqual(parameters); - } - ); - }); - - const invalidFilterCases: { - title: string; - filter: Filter; - error: RegExp; - }[] = [ - { - title: "all invalid operators for strings", - filter: { - [slug("title")]: { - $eq: "test", - $eqi: "test", - $ne: "test", - $nei: "test", - $gt: "test", - $lt: "test", - $gte: "test", - $lte: "test", - $in: "test", - $notIn: "test", - $any: "test", - $all: "test", + querystring: `filters[$or][0][$and][0][${slug("title")}][$containsi]=test&filters[$or][0][$and][1][${slug("boolean")}][$eq]=true&filters[$or][1][$not][${slug("number")}][$between][0]=10&filters[$or][1][$not][${slug("number")}][$between][1]=20`, + sql: `(("slug" = $1 and value::text ilike $2) and ("slug" = $3 and "value" = $4)) or not ("slug" = $5 and ("value" >= $6 and "value" <= $7))`, + parameters: [slug("title"), "%test%", slug("boolean"), "true", slug("number"), 10, 20], + foundIds: [testCaseId], + }, + { + title: "multiple array values with coercion", + filter: { + [slug("numberarray")]: { $in: [1, 2, 3] }, + [slug("date")]: { + $in: [new Date("2023-01-01T00:00:00.000Z"), new Date("2023-01-02T00:00:00.000Z")], + }, + }, + querystring: `filters[${slug("numberarray")}][$in][]=1&filters[${slug("numberarray")}][$in][]=2&filters[${slug("numberarray")}][$in][]=3&filters[${slug("date")}][$in][]=2023-01-01T00:00:00.000Z&filters[${slug("date")}][$in][]=2023-01-02T00:00:00.000Z`, + sql: `("slug" = $1 and "value" in ($2, $3, $4)) and ("slug" = $5 and "value" in ($6, $7))`, + parameters: [ + slug("numberarray"), + 1, + 2, + 3, + slug("date"), + new Date("2023-01-01T00:00:00.000Z").toISOString(), + new Date("2023-01-02T00:00:00.000Z").toISOString(), + ], + foundIds: [numberArrayId], + }, + { + title: "URL encoded special characters", + filter: { + [slug("title")]: { $contains: "special characters & symbols" }, + }, + querystring: `filters[${slug("title")}][$contains]=special%20characters%20%26%20symbols`, + sql: `"slug" = $1 and value::text like $2`, + parameters: [slug("title"), "%special characters & symbols%"], + foundIds: [specialCharsId], + }, + { + title: "mixed type coercion in arrays", + filter: { + [slug("mixedarray")]: { $in: ["string", 42, true] }, + }, + querystring: `filters[${slug("mixedarray")}][$in][]=string&filters[${slug("mixedarray")}][$in][]=42&filters[${slug("mixedarray")}][$in][]=true`, + sql: `"slug" = $1 and "value" in ($2, $3, $4)`, + parameters: [slug("mixedarray"), "string", 42, true], + foundIds: [number42Id], + }, + { + title: "deeply nested logical operators with multiple field types", + filter: { + $and: [ + { + $or: [ + { [slug("title")]: { $containsi: "test" } }, + { [slug("number")]: { $gt: 50 } }, + ], }, - }, - error: /Operators \[\$gt, \$lt, \$gte, \$lte, \$in, \$notIn, \$any, \$all\] are not valid for schema type String/, - }, - { - title: "all invalid operators for numbers", - filter: { - [slug("number")]: { - $startsWith: "test", - $endsWith: "test", - $startsWithi: "test", - $endsWithi: "test", - $containsi: "test", - $notContainsi: "test", - $contains: "test", - $notContains: "test", - $any: "test", - $all: "test", - $size: "test", + { + $not: { + $or: [ + { [slug("boolean")]: { $eq: false } }, + { [slug("date")]: { $lt: new Date("2023-01-01T00:00:00.000Z") } }, + ], + }, }, - }, - error: /Operators \[\$startsWith, \$endsWith, \$startsWithi, \$endsWithi, \$containsi, \$notContainsi, \$contains, \$notContains, \$any, \$all, \$size\] are not valid for schema type Number/, + ], }, - { - title: "all invalid operators for booleans", - filter: { - [slug("boolean")]: { - $eq: true, - $eqi: true, - $ne: false, - $nei: false, - $lt: true, - $lte: true, - $gt: false, - $gte: false, - $contains: true, - $notContains: false, - $containsi: true, - $notContainsi: false, - $null: true, - $notNull: false, - $between: [true, false], - $startsWith: true, - $startsWithi: true, - $endsWith: true, - $endsWithi: true, - $size: 1, + querystring: `filters[$and][0][$or][0][${slug("title")}][$containsi]=test&filters[$and][0][$or][1][${slug("number")}][$gt]=50&filters[$and][1][$not][$or][0][${slug("boolean")}][$eq]=false&filters[$and][1][$not][$or][1][${slug("date")}][$lt]=2023-01-01T00:00:00.000Z`, + sql: `(("slug" = $1 and value::text ilike $2) or ("slug" = $3 and "value" > $4)) and not (("slug" = $5 and "value" = $6) or ("slug" = $7 and "value" < $8))`, + parameters: [ + slug("title"), + "%test%", + slug("number"), + 50, + slug("boolean"), + "false", + slug("date"), + `"${new Date("2023-01-01T00:00:00.000Z").toISOString()}"`, + ], + foundIds: [number54Id], + }, + { + title: "complex filter with all operator types", + filter: { + $or: [ + { [slug("title")]: { $eq: "Test", $containsi: "important" } }, + { [slug("number")]: { $between: [10, 50] } }, + { + [slug("date")]: { $gt: new Date("2023-01-01T00:00:00.000Z") }, + [slug("boolean")]: { $eq: true }, }, - }, - error: /Operators \[\$eqi, \$nei, \$lt, \$lte, \$gt, \$gte, \$contains, \$notContains, \$containsi, \$notContainsi, \$between, \$startsWith, \$startsWithi, \$endsWith, \$endsWithi, \$size\] are not valid for schema type Boolean/, - }, - { - title: "unknown operators", - filter: { - [slug("title")]: { $invalid: "test" }, - }, - error: /Operators \[\$invalid\] are not valid for schema type String/, + { [slug("array")]: { $jsonPath: '$[*] == "item1"' } }, + ], }, - { - title: "unknown fields", - filter: { - [slug("unknownField")]: { $eq: "test" }, - }, - error: /Pub values contain fields that do not exist in the community: .*:unknownField/, - }, - ]; - - describe("validation", () => { - it.concurrent.each(validFilterCases)( - "correctly validates filter for $title", - async ({ filter }) => { - const trx = getTrx(); - // const community = await seedCommunity(trx); - await expect( - validateFilter(community.community.id, filter, trx) - ).resolves.not.toThrow(); - } - ); + querystring: `filters[$or][0][${slug("title")}][$eq]=Test&filters[$or][0][${slug("title")}][$containsi]=important&filters[$or][1][${slug("number")}][$between][0]=10&filters[$or][1][${slug("number")}][$between][1]=50&filters[$or][2][${slug("date")}][$gt]=2023-01-01T00:00:00.000Z&filters[$or][2][${slug("boolean")}][$eq]=true&filters[$or][3][${slug("array")}][$jsonPath]=$[*] == "item1"`, + sql: `(("slug" = $1 and "value" = $2 and value::text ilike $3) or ("slug" = $4 and ("value" >= $5 and "value" <= $6)) or (("slug" = $7 and "value" > $8) and ("slug" = $9 and "value" = $10)) or ("slug" = $11 and "value" @@ $12))`, + parameters: [ + slug("title"), + '"Test"', + "%important%", + slug("number"), + 10, + 50, + slug("date"), + `"${new Date("2023-01-01T00:00:00.000Z").toISOString()}"`, + slug("boolean"), + "true", + slug("array"), + '$[*] == "item1"', + ], + foundIds: [testTitleId, testCaseId, specialCharsId, importantDocId], + }, +]; - it.each(invalidFilterCases)("correctly rejects $title", async ({ filter, error }) => { +describe("SQL generation", () => { + it.concurrent.each(unifiedTestCases)( + "generates correct SQL for $title", + async ({ filter, sql, parameters }) => { const trx = getTrx(); - // const community = await seedCommunity(trx); - await expect(validateFilter(community.community.id, filter, trx)).rejects.toThrow( - error - ); - }); - }); + const q = coolQuery(filter).compiled; + expect(q.sql).toMatch(sql); + expect(q.parameters).toEqual(parameters); + } + ); +}); - const querystringCases: { - title: string; - querystring: string; - filter: Filter; - }[] = [ - { - title: "simple equality", - querystring: "filters[title][$eq]=Some title", - filter: { - title: { $eq: "Some title" }, - }, - }, - { - title: "multiple operators on same field", - querystring: "filters[number][$gt]=10&filters[number][$lt]=50", - filter: { - number: { $gt: 10, $lt: 50 }, - }, - }, - { - title: "multiple fields", - querystring: "filters[title][$eq]=Test&filters[number][$eq]=42", - filter: { - title: { $eq: "Test" }, - number: { $eq: 42 }, - }, - }, - { - title: "boolean coercion", - querystring: "filters[boolean][$eq]=true", - filter: { - boolean: { $eq: true }, - }, - }, - { - title: "number coercion", - querystring: "filters[number][$eq]=42", - filter: { - number: { $eq: 42 }, - }, - }, - { - title: "date coercion", - querystring: "filters[date][$eq]=2023-01-01T00:00:00.000Z", - filter: { - date: { $eq: new Date("2023-01-01T00:00:00.000Z") }, - }, - }, - { - title: "array in operator", - querystring: - "filters[number][$in][]=1&filters[number][$in][]=2&filters[number][$in][]=3", - filter: { - number: { $in: [1, 2, 3] }, - }, - }, - { - title: "between operator", - querystring: "filters[number][$between][0]=10&filters[number][$between][1]=20", - filter: { - number: { $between: [10, 20] }, - }, - }, - { - title: "case insensitive operators", - querystring: "filters[title][$containsi]=test&filters[title][$eqi]=another test", - filter: { - title: { $containsi: "test", $eqi: "another test" }, - }, - }, - { - title: "null and notNull operators", - querystring: "filters[title][$null]&filters[number][$notNull]", - filter: { - title: { $null: true }, - number: { $notNull: true }, - }, - }, - { - title: "jsonPath operator", - querystring: 'filters[array][$jsonPath]=$[*] == "item1"', - filter: { - array: { $jsonPath: '$[*] == "item1"' }, - }, - }, - { - title: "top-level logical OR", - querystring: "filters[$or][0][title][$eq]=Test&filters[$or][1][number][$eq]=42", - filter: { - $or: [{ title: { $eq: "Test" } }, { number: { $eq: 42 } }], - }, - }, - { - title: "top-level logical AND", - querystring: "filters[$and][0][title][$eq]=Test&filters[$and][1][number][$gt]=10", - filter: { - $and: [{ title: { $eq: "Test" } }, { number: { $gt: 10 } }], - }, - }, - { - title: "top-level logical NOT", - querystring: "filters[$not][title][$eq]=Test", - filter: { - $not: { title: { $eq: "Test" } }, - }, - }, - { - title: "nested logical operators", - querystring: - "filters[$or][0][title][$eq]=Test&filters[$or][1][$and][0][number][$gt]=10&filters[$or][1][$and][1][number][$lt]=50", - filter: { - $or: [ - { title: { $eq: "Test" } }, - { - $and: [{ number: { $gt: 10 } }, { number: { $lt: 50 } }], - }, - ], - }, - }, - { - title: "field-level logical OR", - querystring: "filters[number][$or][0][$lt]=10&filters[number][$or][1][$gt]=50", - filter: { - number: { - $or: [{ $lt: 10 }, { $gt: 50 }], - }, - }, - }, - { - title: "complex nested structure", - querystring: - "filters[$or][0][$and][0][title][$containsi]=test&filters[$or][0][$and][1][boolean][$eq]=true&filters[$or][1][$not][number][$between][0]=10&filters[$or][1][$not][number][$between][1]=20", - filter: { - $or: [ - { - $and: [{ title: { $containsi: "test" } }, { boolean: { $eq: true } }], - }, - { - $not: { - number: { $between: [10, 20] }, - }, - }, - ], - }, - }, - { - title: "multiple array values with coercion", - querystring: - "filters[numberArray][$in][]=1&filters[numberArray][$in][]=2&filters[numberArray][$in][]=3&filters[dateArray][$in][]=2023-01-01T00:00:00.000Z&filters[dateArray][$in][]=2023-01-02T00:00:00.000Z", - filter: { - numberArray: { $in: [1, 2, 3] }, - dateArray: { - $in: [ - new Date("2023-01-01T00:00:00.000Z"), - new Date("2023-01-02T00:00:00.000Z"), - ], - }, - }, - }, - { - title: "URL encoded special characters", - querystring: "filters[title][$contains]=special%20characters%20%26%20symbols", - filter: { - title: { $contains: "special characters & symbols" }, - }, - }, - { - title: "mixed type coercion in arrays", - querystring: - "filters[mixedArray][$in][]=string&filters[mixedArray][$in][]=42&filters[mixedArray][$in][]=true", - filter: { - mixedArray: { $in: ["string", 42, true] }, - }, - }, - { - title: "deeply nested logical operators with multiple field types", - querystring: - "filters[$and][0][$or][0][title][$containsi]=test&filters[$and][0][$or][1][number][$gt]=50&filters[$and][1][$not][$or][0][boolean][$eq]=false&filters[$and][1][$not][$or][1][date][$lt]=2023-01-01T00:00:00.000Z", - filter: { - $and: [ - { - $or: [{ title: { $containsi: "test" } }, { number: { $gt: 50 } }], - }, - { - $not: { - $or: [ - { boolean: { $eq: false } }, - { date: { $lt: new Date("2023-01-01T00:00:00.000Z") } }, - ], - }, - }, - ], - }, - }, - { - title: "complex filter with all operator types", - querystring: - 'filters[$or][0][title][$eq]=Test&filters[$or][0][title][$containsi]=important&filters[$or][1][number][$between][0]=10&filters[$or][1][number][$between][1]=50&filters[$or][2][date][$gt]=2023-01-01T00:00:00.000Z&filters[$or][2][boolean][$eq]=true&filters[$or][3][array][$jsonPath]=$[*] == "item1"', - filter: { - $or: [ - { title: { $eq: "Test", $containsi: "important" } }, - { number: { $between: [10, 50] } }, - { date: { $gt: new Date("2023-01-01T00:00:00.000Z") }, boolean: { $eq: true } }, - { array: { $jsonPath: '$[*] == "item1"' } }, - ], - }, - }, - ]; - describe("querystring parsing", () => { - it.concurrent.each(querystringCases)( - "correctly parses $title", - async ({ title, querystring, filter }) => { - const parsed = QueryString.parse(querystring, { - depth: 10, - }); +describe("querystring parsing", () => { + it.concurrent.each(unifiedTestCases)( + "correctly parses $title", + async ({ querystring, filter }) => { + const parsed = QueryString.parse(querystring, { + depth: 10, + }); - const validatedFilter = filterSchema.safeParse(parsed.filters); + const validatedFilter = filterSchema.safeParse(parsed.filters); - expect(validatedFilter.error).toBeUndefined(); - expect(validatedFilter.success).toBe(true); + expect(validatedFilter.error).toBeUndefined(); + expect(validatedFilter.success).toBe(true); - expect(validatedFilter.data).toEqual(filter); - } - ); + expect(validatedFilter.data).toEqual(filter); + } + ); - it("handles empty filters", async () => { - const querystring = ""; - const parsed = QueryString.parse(querystring); - const validatedFilter = filterSchema.safeParse(parsed); + it("handles empty filters", async () => { + const querystring = ""; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); - expect(validatedFilter.success).toBe(true); - expect(validatedFilter.data).toEqual({}); - }); + expect(validatedFilter.success).toBe(true); + expect(validatedFilter.data).toEqual({}); + }); - it("rejects invalid operators", async () => { - const querystring = "filters[title][$invalid]=test"; - const parsed = QueryString.parse(querystring); - const validatedFilter = filterSchema.safeParse(parsed); + it("rejects invalid operators", async () => { + const querystring = `filters[${slug("title")}][$invalid]=test`; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); - expect(validatedFilter.success).toBe(false); - }); + expect(validatedFilter.success).toBe(false); + }); - it("rejects invalid logical operators", async () => { - const querystring = "filters[$invalid][0][title][$eq]=test"; - const parsed = QueryString.parse(querystring); - const validatedFilter = filterSchema.safeParse(parsed); + it("rejects invalid logical operators", async () => { + const querystring = `filters[$invalid][0][${slug("title")}][$eq]=test`; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); - expect(validatedFilter.success).toBe(false); - }); + expect(validatedFilter.success).toBe(false); + }); - it("handles malformed between operator", async () => { - const querystring = "filters[number][$between]=10"; - const parsed = QueryString.parse(querystring); - const validatedFilter = filterSchema.safeParse(parsed); + it("handles malformed between operator", async () => { + const querystring = `filters[${slug("number")}][$between]=10`; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); - expect(validatedFilter.success).toBe(false); - }); + expect(validatedFilter.success).toBe(false); + }); - it("handles malformed array syntax", async () => { - const querystring = "filters[number][$in]=1,2,3"; - const parsed = QueryString.parse(querystring); - const validatedFilter = filterSchema.safeParse(parsed); + it("handles malformed array syntax", async () => { + const querystring = `filters[${slug("number")}][$in]=1,2,3`; + const parsed = QueryString.parse(querystring); + const validatedFilter = filterSchema.safeParse(parsed); - // This should fail because $in expects an array - expect(validatedFilter.success).toBe(false); - }); + // This should fail because $in expects an array + expect(validatedFilter.success).toBe(false); }); +}); - describe("filtering", async () => { - it.concurrent.each(validFilterCases)( - "filters by $title", - async ({ filter, resultValues }) => { - const trx = getTrx(); +describe("filtering", async () => { + it.concurrent.each(unifiedTestCases)("filters by $title", async ({ sql, filter, foundIds }) => { + const trx = getTrx(); - const { getPubsWithRelatedValuesAndChildren } = await import("~/lib/server/pub"); - const pubs = await getPubsWithRelatedValuesAndChildren( - { - communityId: community.community.id, - }, - { - trx, - filters: filter, - } - ); - - expect( - pubs, - "Expected the same number of pubs to be returned as the number of specified result values" - ).toHaveLength(resultValues.length); - - if (pubs.length === 0) { - return; - } - - pubs.sort((a, b) => a.values[0].schemaName.localeCompare(b.values[0].schemaName)); - - pubs.forEach((pub, idx) => { - expect(pub).toHaveValues(resultValues[idx]); - }); + const { getPubsWithRelatedValuesAndChildren } = await import("~/lib/server/pub"); + const pubs = await getPubsWithRelatedValuesAndChildren( + { + communityId: community.community.id, + }, + { + trx, + filters: filter, } ); + + const testTitlePub = community.pubs.find((pub) => pub.id === testTitleId); + const tttpub = await trx + .with( + "all_pubs_and_values", + (db) => + db + .selectFrom("pubs") + .leftJoin("pub_values as pv", "pv.pubId", "pubs.id") + .innerJoin("pub_fields as pf", "pf.id", "pv.fieldId") + .selectAll() + .select("pubs.id as pId") + .where("pubs.communityId", "=", community.community.id) + // .where((eb) => applyFilters(eb, filter)) + // .where("pubs.id", "=", testTitleId) + ) + .selectFrom("all_pubs_and_values") + .selectAll("all_pubs_and_values") + .where((eb) => + eb.and([ + eb.exists( + eb + .selectFrom("all_pubs_and_values as p") + .select(eb.lit(1).as("x")) + .where((eb) => + eb.and([ + eb("all_pubs_and_values.slug", "=", slug("email")), + eb("all_pubs_and_values.value", "=", '"test@test.com"'), + ]) + ) + ), + // eb.exists( + // eb + // .selectFrom("all_pubs_and_values as p") + // .select(eb.lit(1).as("x")) + // .where("all_pubs_and_values.slug", "=", slug("title")) + // ), + ]) + ) + .distinctOn("all_pubs_and_values.pId") + .execute(); + console.log(sql); + console.log(tttpub); + console.log(pubs, filter); + + expect( + pubs, + "Expected the same number of pubs to be returned as the number of specified foundIds" + ).toHaveLength(foundIds.length); + + if (pubs.length === 0) { + return; + } + + // Create a set of expected IDs for easier comparison + const expectedIds = new Set(foundIds); + + // Check that each returned pub has an ID in our expected set + pubs.forEach((pub) => { + expect( + expectedIds.has(pub.id), + `Pub with ID ${pub.id} was not expected in the results`, + `Expected to find Pub ${community.pubs.find((cPub) => cPub.id === pub.id)}, but found ` + ).toBe(true); + }); }); }); From 5300d204912b2f1316bbc12c5835a85545994bb7 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 3 Mar 2025 19:49:20 +0100 Subject: [PATCH 20/20] fix: IT WORKS --- core/lib/server/pub-filters.db.test.ts | 530 ++++++++++++----------- core/lib/server/pub-filters.ts | 244 ++++++----- core/lib/server/pub.ts | 15 +- core/lib/server/validate-filters.ts | 128 ++++-- packages/contracts/src/resources/site.ts | 65 +-- 5 files changed, 565 insertions(+), 417 deletions(-) diff --git a/core/lib/server/pub-filters.db.test.ts b/core/lib/server/pub-filters.db.test.ts index f2a04744b..55fb95d61 100644 --- a/core/lib/server/pub-filters.db.test.ts +++ b/core/lib/server/pub-filters.db.test.ts @@ -2,14 +2,14 @@ import { jsonArrayFrom } from "kysely/helpers/postgres"; import QueryString from "qs"; import { describe, expect, it } from "vitest"; -import type { Filter, Json } from "contracts"; +import type { Filter, Json, ProcessedPub } from "contracts"; import type { CommunitiesId, PubsId } from "db/public"; import { filterSchema } from "contracts"; import { CoreSchemaType } from "db/public"; import { createSeed } from "~/prisma/seed/createSeed"; import { mockServerCode } from "../__tests__/utils"; -import { applyFilters } from "./pub-filters"; +import { applyFieldOperators, applyFilters } from "./pub-filters"; const { createForEachMockedTransaction, testDb } = await mockServerCode(); @@ -78,22 +78,22 @@ const seed = createSeed({ Email: "test@test.com", }, }, - // { - // id: title2Id, - // pubType: "Basic Pub", - // values: { - // Title: "some Title", - // Email: "Test@Test.com", - // }, - // }, - // { - // id: anotherId, - // pubType: "Basic Pub", - // values: { - // Title: "Another title", - // Email: "test2@test.com", - // }, - // }, + { + id: title2Id, + pubType: "Basic Pub", + values: { + Title: "some Title", + Email: "Test@Test.com", + }, + }, + { + id: anotherId, + pubType: "Basic Pub", + values: { + Title: "Another title", + Email: "test2@test.com", + }, + }, { id: number42Id, pubType: "Basic Pub", @@ -101,104 +101,99 @@ const seed = createSeed({ Number: 42, }, }, - // { - // id: number24Id, - // pubType: "Basic Pub", - // values: { - // Number: 24, - // }, - // }, - // { - // id: number54Id, - // pubType: "Basic Pub", - // values: { - // Number: 54, - // }, - // }, - // { - // id: trueId, - // pubType: "Basic Pub", - // values: { - // Boolean: true, - // }, - // }, - // { - // id: arrayId, - // pubType: "Basic Pub", - // values: { - // Array: ["item1", "item2"], - // }, - // }, - // { - // id: vector3Id, - // pubType: "Basic Pub", - // values: { - // Vector3: [0, 0, 0], - // }, - // }, - // { - // id: numberArrayId, - // pubType: "Basic Pub", - // values: { - // NumberArray: [1, 2, 3], - // Date: twenty99, - // }, - // }, - // { - // id: numberArray2Id, - // pubType: "Basic Pub", - // values: { - // NumberArray: [10, 20, 30, 40], - // }, - // }, - // { - // id: relationId, - // pubType: "Basic Pub", - // values: { - // Relation: null, - // Number: 50, - // }, - // }, - // { - // id: testTitleId, - // pubType: "Basic Pub", - // values: { - // Title: "Test", - // Number: 99, - // }, - // }, - // { - // id: testCaseId, - // pubType: "Basic Pub", - // values: { - // Boolean: true, - // Title: "Test case", - // }, - // }, - // { - // id: specialCharsId, - // pubType: "Basic Pub", - // values: { - // Title: "Some title with special characters & symbols", - // }, - // }, - // { - // id: arrayItem1Id, - // pubType: "Basic Pub", - // values: { - // Array: ["item1", "item2"], - // }, - // }, - // { - // id: importantDocId, - // pubType: "Basic Pub", - // values: { - // Title: "Test important document", - // Number: 40, - // Date: new Date("2023-02-01T00:00:00.000Z"), - // Boolean: true, - // }, - // }, + { + id: number24Id, + pubType: "Basic Pub", + values: { + Number: 24, + Date: twenty99, + }, + }, + { + id: number54Id, + pubType: "Basic Pub", + values: { + Number: 54, + }, + }, + { + id: trueId, + pubType: "Basic Pub", + values: { + Boolean: true, + }, + }, + { + id: arrayId, + pubType: "Basic Pub", + values: { + Array: ["item1", "item2"], + }, + }, + { + id: vector3Id, + pubType: "Basic Pub", + values: { + Vector3: [0, 0, 0], + }, + }, + { + id: numberArrayId, + pubType: "Basic Pub", + values: { + NumberArray: [1, 2, 3], + Date: twenty99, + }, + }, + { + id: numberArray2Id, + pubType: "Basic Pub", + values: { + NumberArray: [10, 20, 30, 40], + }, + }, + { + id: relationId, + pubType: "Basic Pub", + values: { + Relation: null, + Number: 50, + }, + }, + { + id: testTitleId, + pubType: "Basic Pub", + values: { + Title: "Test", + Number: 99, + }, + }, + { + id: testCaseId, + pubType: "Basic Pub", + values: { + Date: new Date("2023-02-01T00:00:00.000Z"), + Boolean: true, + Title: "Test case", + }, + }, + { + id: specialCharsId, + pubType: "Basic Pub", + values: { + Title: "Some title with special characters & symbols", + }, + }, + { + id: importantDocId, + pubType: "Basic Pub", + values: { + Title: "Test important document", + Number: 40, + Date: new Date("2023-02-01T00:00:00.000Z"), + Boolean: true, + }, + }, ], }); @@ -264,9 +259,9 @@ const unifiedTestCases: { title: string; filter: Filter; querystring: string; - sql: string; + sql: string | RegExp | (string | RegExp)[]; parameters: (string | number)[]; - foundIds: PubsId[]; + foundIds: PubsId[] | ((pubs: ProcessedPub[]) => PubsId[]); }[] = [ { title: "simple equality", @@ -345,7 +340,7 @@ const unifiedTestCases: { }, querystring: `filters[${slug("email")}][$notContainsi]=Test@`, sql: `"slug" = $1 and value::text not ilike $2`, - parameters: [slug("email"), "%Test@%"], + parameters: [slug("email"), "%test@%"], foundIds: [anotherId], }, { @@ -356,7 +351,7 @@ const unifiedTestCases: { querystring: `filters[${slug("array")}][$jsonPath]=$[*] == "item1"`, sql: `"slug" = $1 and "value" @@ $2`, parameters: [slug("array"), '$[*] == "item1"'], - foundIds: [arrayId, arrayItem1Id], + foundIds: [arrayId], }, { title: "array specific index value check", @@ -379,7 +374,7 @@ const unifiedTestCases: { ], }, querystring: `filters[$or][0][${slug("title")}][$eq]=Some title&filters[$or][1][$and][0][${slug("number")}][$gt]=40&filters[$or][1][$and][1][${slug("number")}][$lt]=50`, - sql: `(("slug" = $1 and "value" = $2) or (("slug" = $3 and "value" > $4) and ("slug" = $5 and "value" < $6)))`, + sql: /exists \(.*"slug" = \$1 and "value" = \$2\).* or \(.*"slug" = \$3 and "value" > \$4\).* and exists \(.*"slug" = \$5 and "value" < \$6\).*\)/, parameters: [slug("title"), '"Some title"', slug("number"), 40, slug("number"), 50], foundIds: [titleId, number42Id], }, @@ -387,12 +382,12 @@ const unifiedTestCases: { title: "field-level logical operators", filter: { [slug("number")]: { - $or: [{ $lt: 40 }, { $gt: 50 }], + $or: { $lt: 40, $gt: 50 }, //[{ $lt: 40 }, { $gt: 50 }], }, }, - querystring: `filters[${slug("number")}][$or][0][$lt]=40&filters[${slug("number")}][$or][1][$gt]=50`, - sql: `("slug" = $1 and "value" < $2) or ("slug" = $3 and "value" > $4)`, - parameters: [slug("number"), 40, slug("number"), 50], + querystring: `filters[${slug("number")}][$or][$lt]=40&filters[${slug("number")}][$or][$gt]=50`, + sql: `"slug" = $1 and ("value" < $2 or "value" > $3)`, + parameters: [slug("number"), 40, 50], foundIds: [number24Id, number54Id, testTitleId], }, { @@ -430,7 +425,7 @@ const unifiedTestCases: { querystring: `filters[${slug("date")}][$eq]=${twenty99.toISOString()}`, sql: `"slug" = $1 and "value" = $2`, parameters: [slug("date"), `"${twenty99.toISOString()}"`], - foundIds: [numberArrayId], + foundIds: [numberArrayId, number24Id], }, { title: "multiple operators on same field should be treated as AND", @@ -438,7 +433,7 @@ const unifiedTestCases: { [slug("number")]: { $gt: 40, $lt: 50 }, }, querystring: `filters[${slug("number")}][$gt]=40&filters[${slug("number")}][$lt]=50`, - sql: `"slug" = $1 and "value" > $2 and "value" < $3`, + sql: `"slug" = $1 and ("value" > $2 and "value" < $3)`, parameters: [slug("number"), 40, 50], foundIds: [number42Id], }, @@ -449,7 +444,7 @@ const unifiedTestCases: { [slug("number")]: { $eq: 99 }, }, querystring: `filters[${slug("title")}][$eq]=Test&filters[${slug("number")}][$eq]=99`, - sql: `("slug" = $1 and "value" = $2) and ("slug" = $3 and "value" = $4)`, + sql: /exists \(.*"slug" = \$1 and "value" = \$2\).* and exists \(.*"slug" = \$3 and "value" = \$4\).*/, parameters: [slug("title"), '"Test"', slug("number"), "99"], foundIds: [testTitleId], }, @@ -461,58 +456,79 @@ const unifiedTestCases: { querystring: `filters[${slug("boolean")}][$eq]=true`, sql: `"slug" = $1 and "value" = $2`, parameters: [slug("boolean"), "true"], - foundIds: [testCaseId], + foundIds: [testCaseId, importantDocId, trueId], }, { title: "array in operator", filter: { - [slug("number")]: { $in: [1, 2, 3] }, + [slug("number")]: { $in: [42, 24] }, }, - querystring: `filters[${slug("number")}][$in][]=1&filters[${slug("number")}][$in][]=2&filters[${slug("number")}][$in][]=3`, - sql: `"slug" = $1 and "value" in ($2, $3, $4)`, - parameters: [slug("number"), 1, 2, 3], - foundIds: [number42Id], + querystring: `filters[${slug("number")}][$in][]=42&filters[${slug("number")}][$in][]=24`, + sql: `"slug" = $1 and "value" in ($2, $3)`, + parameters: [slug("number"), 42, 24], + foundIds: [number42Id, number24Id], }, { title: "between operator", filter: { - [slug("number")]: { $between: [10, 20] }, + [slug("number")]: { $between: [20, 25] }, }, - querystring: `filters[${slug("number")}][$between][0]=10&filters[${slug("number")}][$between][1]=20`, + querystring: `filters[${slug("number")}][$between][0]=20&filters[${slug("number")}][$between][1]=25`, sql: `"slug" = $1 and ("value" >= $2 and "value" <= $3)`, - parameters: [slug("number"), 10, 20], - foundIds: [number42Id], - }, - { - title: "null and notNull operators", - filter: { - [slug("title")]: { $null: true }, - [slug("number")]: { $notNull: true }, - }, - querystring: `filters[${slug("title")}][$null]&filters[${slug("number")}][$notNull]`, - sql: `("slug" = $1 and "value" is null) and ("slug" = $2 and "value" is not null)`, - parameters: [slug("title"), slug("number")], - foundIds: [relationId], + parameters: [slug("number"), 20, 25], + foundIds: [number24Id], }, + // { + // title: "null and notNull operators", + // filter: { + // [slug("relation")]: { $null: true }, + // }, + // querystring: `filters[${slug("relation")}][$null]`, + // sql: `"slug" = $1 and "value" is null`, + // parameters: [slug("relation")], + // foundIds: [relationId], + // }, { title: "top-level logical AND", filter: { $and: [{ [slug("title")]: { $eq: "Test" } }, { [slug("number")]: { $gt: 10 } }], }, querystring: `filters[$and][0][${slug("title")}][$eq]=Test&filters[$and][1][${slug("number")}][$gt]=10`, - sql: `("slug" = $1 and "value" = $2) and ("slug" = $3 and "value" > $4)`, + sql: /exists \(.*"slug" = \$1 and "value" = \$2\).* and exists \(.*"slug" = \$3 and "value" > \$4\).*/, parameters: [slug("title"), '"Test"', slug("number"), 10], foundIds: [testTitleId], }, { title: "top-level logical NOT", filter: { + // this works slightly differently than you'd expect $not: { [slug("title")]: { $eq: "Test" } }, }, querystring: `filters[$not][${slug("title")}][$eq]=Test`, - sql: `not ("slug" = $1 and "value" = $2)`, + sql: /not exists \(.*"slug" = \$1 and "value" = \$2\).*/, + parameters: [slug("title"), '"Test"'], + foundIds: (pubs) => + pubs + .filter( + (p) => + !p.values.some((v) => v.fieldSlug === slug("title") && v.value === "Test") + ) + .map((p) => p.id), + }, + { + title: "field-level logical NOT", + filter: { + [slug("title")]: { $not: { $eq: "Test" } }, + }, + querystring: `filters[${slug("title")}][$not][$eq]=Test`, + sql: '"slug" = $1 and not "value" = $2', parameters: [slug("title"), '"Test"'], - foundIds: [titleId, anotherId, specialCharsId, importantDocId], + foundIds: (pubs) => + pubs + .filter((p) => + p.values.some((v) => v.fieldSlug === slug("title") && v.value !== "Test") + ) + .map((p) => p.id), }, { title: "complex nested structure", @@ -525,37 +541,61 @@ const unifiedTestCases: { ], }, { - $not: { - [slug("number")]: { $between: [10, 20] }, - }, + [slug("number")]: { $not: { $between: [30, 50] } }, }, ], }, - querystring: `filters[$or][0][$and][0][${slug("title")}][$containsi]=test&filters[$or][0][$and][1][${slug("boolean")}][$eq]=true&filters[$or][1][$not][${slug("number")}][$between][0]=10&filters[$or][1][$not][${slug("number")}][$between][1]=20`, - sql: `(("slug" = $1 and value::text ilike $2) and ("slug" = $3 and "value" = $4)) or not ("slug" = $5 and ("value" >= $6 and "value" <= $7))`, - parameters: [slug("title"), "%test%", slug("boolean"), "true", slug("number"), 10, 20], - foundIds: [testCaseId], + querystring: `filters[$or][0][$and][0][${slug("title")}][$containsi]=test&filters[$or][0][$and][1][${slug("boolean")}][$eq]=true&filters[$or][1][${slug("number")}][$not][$between][0]=30&filters[$or][1][${slug("number")}][$not][$between][1]=50`, + sql: [ + /\(exists \(.*"slug" = \$1 and value::text ilike \$2\) and exists \(.*"slug" = \$3 and "value" = \$4\)/, + /or exists \(.*"slug" = \$5 and not \("value" >= \$6 and "value" <= \$7\)/, + ], + parameters: [slug("title"), "%test%", slug("boolean"), "true", slug("number"), 30, 50], + foundIds: community.pubs + .filter( + (p) => + (p.values.some( + (v) => v.fieldSlug === slug("title") && /test/i.test(v.value as string) + ) && + p.values.some( + (v) => v.fieldSlug === slug("boolean") && v.value === true + )) || + p.values.some( + (v) => + v.fieldSlug === slug("number") && + !((v.value as number) >= 30 && (v.value as number) <= 50) + ) + ) + .map((p) => p.id), }, { title: "multiple array values with coercion", filter: { - [slug("numberarray")]: { $in: [1, 2, 3] }, + [slug("number")]: { $in: [20, 24, 45] }, [slug("date")]: { - $in: [new Date("2023-01-01T00:00:00.000Z"), new Date("2023-01-02T00:00:00.000Z")], + $in: [ + new Date("2023-01-01T00:00:00.000Z"), + new Date("2023-01-02T00:00:00.000Z"), + twenty99, + ], }, }, - querystring: `filters[${slug("numberarray")}][$in][]=1&filters[${slug("numberarray")}][$in][]=2&filters[${slug("numberarray")}][$in][]=3&filters[${slug("date")}][$in][]=2023-01-01T00:00:00.000Z&filters[${slug("date")}][$in][]=2023-01-02T00:00:00.000Z`, - sql: `("slug" = $1 and "value" in ($2, $3, $4)) and ("slug" = $5 and "value" in ($6, $7))`, + querystring: `filters[${slug("number")}][$in][]=20&filters[${slug("number")}][$in][]=24&filters[${slug("number")}][$in][]=45&filters[${slug("date")}][$in][]=2023-01-01T00:00:00.000Z&filters[${slug("date")}][$in][]=2023-01-02T00:00:00.000Z&filters[${slug("date")}][$in][]=${twenty99.toISOString()}`, + sql: [ + `"slug" = $1 and "value" in ($2, $3, $4)) and exists`, + `"slug" = $5 and "value" in ($6, $7, $8))`, + ], parameters: [ - slug("numberarray"), - 1, - 2, - 3, + slug("number"), + 20, + 24, + 45, slug("date"), - new Date("2023-01-01T00:00:00.000Z").toISOString(), - new Date("2023-01-02T00:00:00.000Z").toISOString(), + `"${new Date("2023-01-01T00:00:00.000Z").toISOString()}"`, + `"${new Date("2023-01-02T00:00:00.000Z").toISOString()}"`, + `"${twenty99.toISOString()}"`, ], - foundIds: [numberArrayId], + foundIds: [number24Id], }, { title: "URL encoded special characters", @@ -567,16 +607,6 @@ const unifiedTestCases: { parameters: [slug("title"), "%special characters & symbols%"], foundIds: [specialCharsId], }, - { - title: "mixed type coercion in arrays", - filter: { - [slug("mixedarray")]: { $in: ["string", 42, true] }, - }, - querystring: `filters[${slug("mixedarray")}][$in][]=string&filters[${slug("mixedarray")}][$in][]=42&filters[${slug("mixedarray")}][$in][]=true`, - sql: `"slug" = $1 and "value" in ($2, $3, $4)`, - parameters: [slug("mixedarray"), "string", 42, true], - foundIds: [number42Id], - }, { title: "deeply nested logical operators with multiple field types", filter: { @@ -598,7 +628,7 @@ const unifiedTestCases: { ], }, querystring: `filters[$and][0][$or][0][${slug("title")}][$containsi]=test&filters[$and][0][$or][1][${slug("number")}][$gt]=50&filters[$and][1][$not][$or][0][${slug("boolean")}][$eq]=false&filters[$and][1][$not][$or][1][${slug("date")}][$lt]=2023-01-01T00:00:00.000Z`, - sql: `(("slug" = $1 and value::text ilike $2) or ("slug" = $3 and "value" > $4)) and not (("slug" = $5 and "value" = $6) or ("slug" = $7 and "value" < $8))`, + sql: `select * from "pubs" where ((exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $1 and value::text ilike $2) or exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $3 and "value" > $4)) and not (exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $5 and "value" = $6) or exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $7 and "value" < $8)))`, parameters: [ slug("title"), "%test%", @@ -609,14 +639,37 @@ const unifiedTestCases: { slug("date"), `"${new Date("2023-01-01T00:00:00.000Z").toISOString()}"`, ], - foundIds: [number54Id], + foundIds: (pubs) => + pubs + .filter( + (p) => + (p.values.some( + (v) => v.fieldSlug === slug("title") && /test/i.test(v.value as string) + ) || + p.values.some( + (v) => v.fieldSlug === slug("number") && (v.value as number) > 50 + )) && + !( + p.values.some( + (v) => + v.fieldSlug === slug("boolean") && + (v.value as boolean) === false + ) || + p.values.some( + (v) => + v.fieldSlug === slug("date") && + (v.value as Date) < new Date("2023-01-01T00:00:00.000Z") + ) + ) + ) + .map((p) => p.id), }, { title: "complex filter with all operator types", filter: { $or: [ - { [slug("title")]: { $eq: "Test", $containsi: "important" } }, - { [slug("number")]: { $between: [10, 50] } }, + { [slug("title")]: { $containsi: "important" } }, + { [slug("number")]: { $between: [40, 45] } }, { [slug("date")]: { $gt: new Date("2023-01-01T00:00:00.000Z") }, [slug("boolean")]: { $eq: true }, @@ -624,15 +677,19 @@ const unifiedTestCases: { { [slug("array")]: { $jsonPath: '$[*] == "item1"' } }, ], }, - querystring: `filters[$or][0][${slug("title")}][$eq]=Test&filters[$or][0][${slug("title")}][$containsi]=important&filters[$or][1][${slug("number")}][$between][0]=10&filters[$or][1][${slug("number")}][$between][1]=50&filters[$or][2][${slug("date")}][$gt]=2023-01-01T00:00:00.000Z&filters[$or][2][${slug("boolean")}][$eq]=true&filters[$or][3][${slug("array")}][$jsonPath]=$[*] == "item1"`, - sql: `(("slug" = $1 and "value" = $2 and value::text ilike $3) or ("slug" = $4 and ("value" >= $5 and "value" <= $6)) or (("slug" = $7 and "value" > $8) and ("slug" = $9 and "value" = $10)) or ("slug" = $11 and "value" @@ $12))`, + querystring: `filters[$or][0][${slug("title")}][$containsi]=important&filters[$or][1][${slug("number")}][$between][0]=40&filters[$or][1][${slug("number")}][$between][1]=45&filters[$or][2][${slug("date")}][$gt]=2023-01-01T00:00:00.000Z&filters[$or][2][${slug("boolean")}][$eq]=true&filters[$or][3][${slug("array")}][$jsonPath]=$[*] == "item1"`, + sql: [ + '"slug" = $1 and value::text ilike $2) or exists', + '"slug" = $3 and ("value" >= $4 and "value" <= $5)) or (exists', + /exists \(.*"slug" = \$6 and "value" > \$7\).* and exists \(.*"slug" = \$8 and "value" = \$9\).* or exists/, + '"slug" = $10 and "value" @@ $11', + ], parameters: [ slug("title"), - '"Test"', "%important%", slug("number"), - 10, - 50, + 40, + 45, slug("date"), `"${new Date("2023-01-01T00:00:00.000Z").toISOString()}"`, slug("boolean"), @@ -640,7 +697,16 @@ const unifiedTestCases: { slug("array"), '$[*] == "item1"', ], - foundIds: [testTitleId, testCaseId, specialCharsId, importantDocId], + foundIds: [ + // bc important doc title + importantDocId, + // bc number 42 is between 40 and 45 + number42Id, + // array selector + arrayId, + // boolean true and date after 2023-01-01 + testCaseId, + ], }, ]; @@ -648,9 +714,14 @@ describe("SQL generation", () => { it.concurrent.each(unifiedTestCases)( "generates correct SQL for $title", async ({ filter, sql, parameters }) => { - const trx = getTrx(); const q = coolQuery(filter).compiled; - expect(q.sql).toMatch(sql); + if (Array.isArray(sql)) { + sql.forEach((sqlSnippet) => { + expect(q.sql).toMatch(sqlSnippet); + }); + } else { + expect(q.sql).toMatch(sql); + } expect(q.parameters).toEqual(parameters); } ); @@ -660,16 +731,27 @@ describe("querystring parsing", () => { it.concurrent.each(unifiedTestCases)( "correctly parses $title", async ({ querystring, filter }) => { + const trx = getTrx(); const parsed = QueryString.parse(querystring, { depth: 10, }); + // this is a quick check to make sure the querystring is parsed as we think it should be + expect( + JSON.stringify(parsed.filters) + .replace(/([^\\])""/, "$1true") + .replace(/"/g, ""), + "Querystring filter should match the defined filter" + ).toEqual(JSON.stringify(filter).replace(/"/g, "")); + const validatedFilter = filterSchema.safeParse(parsed.filters); expect(validatedFilter.error).toBeUndefined(); expect(validatedFilter.success).toBe(true); expect(validatedFilter.data).toEqual(filter); + + await validateFilter(community.community.id, validatedFilter.data!, trx); } ); @@ -717,10 +799,11 @@ describe("querystring parsing", () => { }); describe("filtering", async () => { - it.concurrent.each(unifiedTestCases)("filters by $title", async ({ sql, filter, foundIds }) => { + it.concurrent.each(unifiedTestCases)("filters by $title", async ({ filter, foundIds }) => { const trx = getTrx(); const { getPubsWithRelatedValuesAndChildren } = await import("~/lib/server/pub"); + const pubs = await getPubsWithRelatedValuesAndChildren( { communityId: community.community.id, @@ -731,69 +814,26 @@ describe("filtering", async () => { } ); - const testTitlePub = community.pubs.find((pub) => pub.id === testTitleId); - const tttpub = await trx - .with( - "all_pubs_and_values", - (db) => - db - .selectFrom("pubs") - .leftJoin("pub_values as pv", "pv.pubId", "pubs.id") - .innerJoin("pub_fields as pf", "pf.id", "pv.fieldId") - .selectAll() - .select("pubs.id as pId") - .where("pubs.communityId", "=", community.community.id) - // .where((eb) => applyFilters(eb, filter)) - // .where("pubs.id", "=", testTitleId) - ) - .selectFrom("all_pubs_and_values") - .selectAll("all_pubs_and_values") - .where((eb) => - eb.and([ - eb.exists( - eb - .selectFrom("all_pubs_and_values as p") - .select(eb.lit(1).as("x")) - .where((eb) => - eb.and([ - eb("all_pubs_and_values.slug", "=", slug("email")), - eb("all_pubs_and_values.value", "=", '"test@test.com"'), - ]) - ) - ), - // eb.exists( - // eb - // .selectFrom("all_pubs_and_values as p") - // .select(eb.lit(1).as("x")) - // .where("all_pubs_and_values.slug", "=", slug("title")) - // ), - ]) - ) - .distinctOn("all_pubs_and_values.pId") - .execute(); - console.log(sql); - console.log(tttpub); - console.log(pubs, filter); + const expectedIds = typeof foundIds === "function" ? foundIds(pubs) : foundIds; expect( pubs, "Expected the same number of pubs to be returned as the number of specified foundIds" - ).toHaveLength(foundIds.length); + ).toHaveLength(expectedIds.length); if (pubs.length === 0) { return; } - // Create a set of expected IDs for easier comparison - const expectedIds = new Set(foundIds); + const expectedIdsSet = new Set(expectedIds); - // Check that each returned pub has an ID in our expected set - pubs.forEach((pub) => { + Array.from(expectedIdsSet).forEach((id) => { + const expectedPub = community.pubs.find((p) => p.id === id); + const foundPub = pubs.find((p) => p.id === id); expect( - expectedIds.has(pub.id), - `Pub with ID ${pub.id} was not expected in the results`, - `Expected to find Pub ${community.pubs.find((cPub) => cPub.id === pub.id)}, but found ` - ).toBe(true); + foundPub, + `Expected to find Pub with values ${JSON.stringify(expectedPub?.values.map((v) => v.value))} but found pubs with values ${JSON.stringify(pubs.map((p) => p.values.map((v) => v.value)))}` + ).toBeDefined(); }); }); }); diff --git a/core/lib/server/pub-filters.ts b/core/lib/server/pub-filters.ts index 0bfc637f5..32da74a60 100644 --- a/core/lib/server/pub-filters.ts +++ b/core/lib/server/pub-filters.ts @@ -3,7 +3,6 @@ import type { ExpressionBuilder, ExpressionWrapper } from "kysely"; import { sql } from "kysely"; import type { - BaseFilter, FieldLevelFilter, Filter, FilterOperator, @@ -14,8 +13,6 @@ import { logicalOperators } from "contracts"; import { CoreSchemaType } from "db/public"; import { assert } from "utils"; -import { entries, fromEntries, keys, mapToEntries } from "../mapping"; - type EntriedLogicalFilter = [ ["$or", NonNullable], ["$and", NonNullable], @@ -47,6 +44,8 @@ export const stringOperators = [ "$endsWithi", "$null", "$notNull", + "$in", + "$notIn", ] as const satisfies FilterOperator[]; export const coreSchemaTypeAllowedOperators = { @@ -62,6 +61,8 @@ export const coreSchemaTypeAllowedOperators = { "$between", "$null", "$notNull", + "$in", + "$notIn", ], [CoreSchemaType.Vector3]: ["$eq", "$ne", "$null", "$notNull", "$jsonPath"], [CoreSchemaType.NumericArray]: ["$contains", "$notContains", "$null", "$notNull", "$jsonPath"], @@ -76,6 +77,8 @@ export const coreSchemaTypeAllowedOperators = { "$between", "$null", "$notNull", + "$in", + "$notIn", ], [CoreSchemaType.Email]: stringOperators, [CoreSchemaType.URL]: stringOperators, @@ -123,7 +126,15 @@ const filterMap = { ]); }, $in: (eb, column, value) => - eb(column, "in", typeof value === "string" ? JSON.stringify(value) : value), + eb( + column, + "in", + typeof value === "string" + ? JSON.stringify(value) + : Array.isArray(value) + ? value.map((val) => (typeof val === "string" ? JSON.stringify(val) : val)) + : value + ), $notIn: (eb, column, value) => eb(column, "not in", typeof value === "string" ? JSON.stringify(value) : value), $contains: (eb, column, value) => eb(sql.raw(`${column}::text`), "like", `%${String(value)}%`), @@ -151,126 +162,147 @@ const filterMap = { ) => ExpressionWrapper >; -export const isNonRecursiveFilter = (filter: FieldLevelFilter[string]): filter is BaseFilter => { - // Check if this is a logical operator within a field filter - if (filter && typeof filter === "object" && !Array.isArray(filter)) { - const ks = keys(filter); - if (ks.some((k) => logicalOperators.includes(k as LogicalOperator))) { - return false; - } +/** + * Helper function to stringify values as needed + */ +const maybeStringifiedValue = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map((v) => maybeStringifiedValue(v)); } - if (Object.keys(filter).every((k) => k.startsWith("$"))) { - return true; + if (value instanceof Date) { + return value.toISOString(); } - return false; -}; -export type FieldsWithFilters = { - [slug: string]: Set; + return value; }; -type ExpressionWrapped> = - E extends ExpressionBuilder ? ExpressionWrapper : never; -export function applyFilters>( - eb: K, - filters: Filter -): ExpressionWrapped { - const conditions = entries(filters).map((filter) => { - // Handle top-level logical operators - if (isLogicalFilter(filter)) { - const operatorFilter = mapToEntries(filter, ["operator", "filters"]); +/** + * Checks if an object is a field-level logical operator + */ +const isFieldLevelLogicalOperator = (obj: any): boolean => { + if (typeof obj !== "object" || obj === null) { + return false; + } - return applyLogicalOperation(eb, operatorFilter); - } + const keys = Object.keys(obj); + return keys.length === 1 && logicalOperators.includes(keys[0] as LogicalOperator); +}; - const [field, val] = filter; +/** + * Apply operators for a single field + */ +export const applyFieldOperators = >( + eb: K, + operators: Record, + column: "value" | "pubs.updatedAt" | "pubs.createdAt" +): ExpressionWrapper => { + const keys = Object.keys(operators); + // check if this is a field-level logical operator like { $or: {...} } + // TODO: doesn't per se need to be length 1 + if (keys.length === 1 && logicalOperators.includes(keys[0] as LogicalOperator)) { + const operator = keys[0] as LogicalOperator; + const conditions = operators[operator]; - const isDate = field === "updatedAt" || field === "createdAt"; - if ( - isDate && - !new Set(Object.keys(val)).isSubsetOf( - new Set(coreSchemaTypeAllowedOperators[CoreSchemaType.DateTime]) - ) - ) { - throw new Error(`Date filters must use date operators: ${JSON.stringify(val)}`); + if (operator === "$or") { + return eb.or( + Object.entries(conditions).map(([op, value]) => { + if (isFieldLevelLogicalOperator({ [op]: value })) { + return applyFieldOperators(eb, { [op]: value }, column); + } + const whereFn = filterMap[op as FilterOperator]; + if (!whereFn) { + throw new Error(`Unknown operator: ${op}`); + } + return whereFn(eb, column, maybeStringifiedValue(value)); + }) + ); + } else if (operator === "$and") { + return eb.and( + Object.entries(conditions).map(([op, value]) => { + if (isFieldLevelLogicalOperator({ [op]: value })) { + return applyFieldOperators(eb, { [op]: value }, column); + } + const whereFn = filterMap[op as FilterOperator]; + if (!whereFn) { + throw new Error(`Unknown operator: ${op}`); + } + return whereFn(eb, column, maybeStringifiedValue(value)); + }) + ); + } else if (operator === "$not") { + return eb.not(applyFieldOperators(eb, conditions, column)); } + } - // Handle field-level logical operators - if (!isNonRecursiveFilter(val)) { - const logicalOps = entries(val).filter(([key]) => logicalOperators.includes(key)); - - if (logicalOps.length === 0) { - throw new Error(`Unknown filter: ${JSON.stringify(filter)}`); + // Regular field operators + return eb.and( + Object.entries(operators).map(([op, value]) => { + if (isFieldLevelLogicalOperator({ [op]: value })) { + return applyFieldOperators(eb, { [op]: value }, column); } - - const [operator, subFilters] = logicalOps[0]; - - // For field-level operators, we need to apply the field constraint to each subfilter - if (operator === "$not") { - // Special case for $not since it takes a single filter, not an array - const newFilter = { [field]: subFilters }; - return eb.not(applyFilters(eb, newFilter)); - } else { - // For $or and $and, map each subfilter to include the field - const fieldConstrainedFilters = subFilters.map((subFilter) => ({ - [field]: subFilter, - })); - return applyLogicalOperation(eb, { operator, filters: fieldConstrainedFilters }); + const whereFn = filterMap[op as FilterOperator]; + if (!whereFn) { + throw new Error(`Unknown operator: ${op}`); } - } + return whereFn(eb, column, maybeStringifiedValue(value)); + }) + ); +}; - // Handle regular field filters - return eb.and([ - ...(isDate ? [] : [eb("slug", "=", field)]), - ...entries(val).map((entry) => { - const [operator, value] = entry; +type ExpressionWrapped> = + E extends ExpressionBuilder ? ExpressionWrapper : never; - const whereFn = filterMap[operator]; - if (!whereFn) { - throw new Error(`Unknown operator: ${operator}`); - } +/** + * main filter application function + */ +export function applyFilters>( + eb: K, + filters: Filter +): ExpressionWrapped { + // top-level logical operators + if (filters.$and) { + return eb.and( + // @ts-expect-error TODO: FIX + filters.$and.map((condition) => applyFilters(eb, condition)) + ) as ExpressionWrapped; + } else if (filters.$or) { + return eb.or( + // @ts-expect-error TODO: FIX + filters.$or.map((condition) => applyFilters(eb, condition)) + ) as ExpressionWrapped; + } else if (filters.$not) { + return eb.not(applyFilters(eb, filters.$not)) as ExpressionWrapped; + } - const maybeStringifiedValue = (value: unknown): unknown => { - if (Array.isArray(value)) { - return value.map((v) => maybeStringifiedValue(v)); - } + // handle field filters - we treat multiple fields as an implicit AND + const fieldConditions = Object.entries(filters); - if (value instanceof Date) { - return value.toISOString(); - } + if (fieldConditions.length === 0) { + return eb.val(true) as ExpressionWrapped; + } - return value; - }; + return eb.and( + fieldConditions.map(([field, operators]) => { + const isDate = field === "updatedAt" || field === "createdAt"; - return whereFn( - eb, - isDate ? `pubs.${field}` : "value", - maybeStringifiedValue(value) + if (isDate) { + // handle date fields directly on the pubs table + return applyFieldOperators(eb, operators, `pubs.${field}`); + } else { + // for regular fields, use EXISTS subquery + return eb.exists( + eb + .selectFrom("pub_values") + .innerJoin("pub_fields", "pub_fields.id", "pub_values.fieldId") + .select(eb.lit(1).as("exists_check")) + .where("pub_values.pubId", "=", eb.ref("pubs.id")) + .where("pub_fields.slug", "=", field) + .where((innerEb) => { + return applyFieldOperators(innerEb, operators, "value"); + }) ); - }), - ]); - }); - - return eb.and(conditions) as ExpressionWrapped; -} - -// Helper function to apply logical operations -function applyLogicalOperation>( - eb: K, - operatorFilters: - | { operator: Exclude; filters: Filter[] } - | { operator: "$not"; filters: Filter } -): ExpressionWrapper { - switch (operatorFilters.operator) { - case "$or": - return eb.or(operatorFilters.filters.map((f) => applyFilters(eb, f))); - case "$and": - return eb.and(operatorFilters.filters.map((f) => applyFilters(eb, f))); - case "$not": - // $not should only have one filter - return eb.not(applyFilters(eb, operatorFilters.filters)); - default: - throw new Error(`Unknown logical operator: ${operatorFilters}`); - } + } + }) + ) as ExpressionWrapped; } diff --git a/core/lib/server/pub.ts b/core/lib/server/pub.ts index f6daa854c..4fc32350a 100644 --- a/core/lib/server/pub.ts +++ b/core/lib/server/pub.ts @@ -1774,12 +1774,15 @@ export async function getPubsWithRelatedValuesAndChildren< .$if(Boolean(props.pubTypeId), (qb) => qb.where("pubs.pubTypeId", "=", props.pubTypeId!) ) - .$if(Boolean(options?.filters), (qb) => - // TODO: maybe dedupe this - qb - .leftJoin("pub_values as pv", "pv.pubId", "pubs.id") - .innerJoin("pub_fields as pf", "pf.id", "pv.fieldId") - .where((eb) => applyFilters(eb, options!.filters!)) + .$if( + Boolean(options?.filters), + (qb) => + // TODO: maybe dedupe this + qb.where((eb) => applyFilters(eb, options!.filters!)) + + // .leftJoin("pub_values as pv", "pv.pubId", "pubs.id") + // .innerJoin("pub_fields as pf", "pf.id", "pv.fieldId") + // .where((eb) => applyFilters(eb, options!.filters!)) ) .$if(Boolean(orderBy), (qb) => qb.orderBy(orderBy!, orderDirection ?? "desc")) .$if(Boolean(limit), (qb) => qb.limit(limit!)) diff --git a/core/lib/server/validate-filters.ts b/core/lib/server/validate-filters.ts index 60d81c79e..b5b1a9810 100644 --- a/core/lib/server/validate-filters.ts +++ b/core/lib/server/validate-filters.ts @@ -1,12 +1,11 @@ -import type { Filter, FilterOperator } from "contracts"; +import type { Filter, FilterOperator, LogicalOperator } from "contracts"; import type { CommunitiesId } from "db/public"; import { logicalOperators } from "contracts"; import { CoreSchemaType } from "db/public"; -import type { FieldsWithFilters } from "./pub-filters"; import { db } from "~/kysely/database"; import { getFieldInfoForSlugs } from "./pub"; -import { coreSchemaTypeAllowedOperators, isNonRecursiveFilter } from "./pub-filters"; +import { coreSchemaTypeAllowedOperators } from "./pub-filters"; const isDateAt = (field: string) => field === "updatedAt" || field === "createdAt"; @@ -47,49 +46,108 @@ export class InvalidDateFilterError extends InvalidFilterError { } } -export async function validateFilter(communityId: CommunitiesId, filter: Filter, trx = db) { - // find all the fields in the filter and their operators as an array +type FieldsWithFilters = Record>; + +/** + * Extracts operators from a field-level condition, including those nested in logical operators + */ +const extractOperatorsFromValue = ( + value: any, + accumulatedOperators: Set = new Set() +): Set => { + if (typeof value !== "object" || value === null) { + return accumulatedOperators; + } + + const keys = Object.keys(value); + if (keys.length > 1 || !logicalOperators.includes(keys[0] as LogicalOperator)) { + keys.forEach((key) => { + if (logicalOperators.includes(key as LogicalOperator)) { + extractOperatorsFromValue({ [key]: value[key] }, accumulatedOperators); + return; + } + + accumulatedOperators.add(key as FilterOperator); + }); + + return accumulatedOperators; + } + + const logicalOperator = keys[0] as LogicalOperator; + const conditions = value[logicalOperator]; + + // were done + if (typeof conditions !== "object" || conditions === null) { + return accumulatedOperators; + } + + // process each condition inside the logical operator + // add the logical operator itself + accumulatedOperators.add(logicalOperator as FilterOperator); + + // process nested conditions + Object.entries(conditions).forEach(([op, val]) => { + if (logicalOperators.includes(op as LogicalOperator)) { + // nested logical operators + extractOperatorsFromValue({ [op]: val }, accumulatedOperators); + return; + } + + accumulatedOperators.add(op as FilterOperator); + }); + + return accumulatedOperators; +}; +export async function validateFilter(communityId: CommunitiesId, filter: Filter, trx = db) { const findFields = ( filter: Filter, fieldsWithFilters: FieldsWithFilters = {} ): FieldsWithFilters => { for (const [field, val] of Object.entries(filter)) { + if (logicalOperators.includes(field as any)) { + if (Array.isArray(val)) { + for (const f of val) { + findFields(f, fieldsWithFilters); + } + continue; + } + if (typeof val === "object" && val !== null) { + findFields(val, fieldsWithFilters); + } + continue; + } + if (isDateAt(field)) { - if ( - Object.keys(val).some( - (k) => - !coreSchemaTypeAllowedOperators.DateTime.includes( - k as (typeof coreSchemaTypeAllowedOperators.DateTime)[number] - ) - ) - ) { + const operators = extractOperatorsFromValue(val); + + // check if all operators are valid for DateTime + const invalidOperators = Array.from(operators).filter( + (op) => + !coreSchemaTypeAllowedOperators.DateTime.includes(op as any) && + !logicalOperators.includes(op as any) + ); + + if (invalidOperators.length > 0) { throw new InvalidDateFilterError( - field, + field as "updatedAt" | "createdAt", coreSchemaTypeAllowedOperators.DateTime, - Object.keys(val).map((k) => k as FilterOperator) + invalidOperators as FilterOperator[] ); } continue; } - if (logicalOperators.includes(field as keyof Filter)) { - for (const f of val) { - findFields(f, fieldsWithFilters); - } - continue; - } + // handle regular fields + if (typeof val === "object" && val !== null) { + // extract all operators, including those nested in logical operators + const operators = extractOperatorsFromValue(val); - if (isNonRecursiveFilter(val)) { - for (const [operator, value] of Object.entries(val)) { - fieldsWithFilters[field] = fieldsWithFilters[field] - ? fieldsWithFilters[field].add(operator as FilterOperator) - : new Set([operator as FilterOperator]); - } - continue; + // add operators to the field + fieldsWithFilters[field] = fieldsWithFilters[field] + ? new Set([...fieldsWithFilters[field], ...operators]) + : operators; } - - findFields(val, fieldsWithFilters); } return fieldsWithFilters; }; @@ -110,11 +168,17 @@ export async function validateFilter(communityId: CommunitiesId, filter: Filter, for (const field of mergedFields) { const allowedOperators = coreSchemaTypeAllowedOperators[field.schemaName]; - if (!field.operators.isSubsetOf(new Set(allowedOperators))) { + + // just to be sure + const fieldOperators = new Set( + Array.from(field.operators).filter((op) => !logicalOperators.includes(op as any)) + ); + + if (!fieldOperators.isSubsetOf(new Set(allowedOperators))) { throw new InvalidPubFieldFilterError( { schemaName: field.schemaName, slug: field.slug }, allowedOperators, - Array.from(field.operators) + Array.from(fieldOperators) as FilterOperator[] ); } } diff --git a/packages/contracts/src/resources/site.ts b/packages/contracts/src/resources/site.ts index 0978280ca..391ccd292 100644 --- a/packages/contracts/src/resources/site.ts +++ b/packages/contracts/src/resources/site.ts @@ -343,14 +343,16 @@ export type LogicalOperator = (typeof logicalOperators)[number]; export type BaseFilter = { [O in FilterOperator]?: unknown; }; -export type FieldLevelFilter = { - [slug: string]: BaseFilter | { $and?: BaseFilter[]; $or?: BaseFilter[]; $not?: BaseFilter }; -}; export type LogicalFilter = { - $and?: Filter[]; - $or?: Filter[]; - $not?: Filter; + $and?: (BaseFilterWithAndOr | FieldLevelFilter) | (BaseFilterWithAndOr | FieldLevelFilter)[]; + $or?: (BaseFilterWithAndOr | FieldLevelFilter) | (BaseFilterWithAndOr | FieldLevelFilter)[]; + $not?: BaseFilterWithAndOr | FieldLevelFilter; +}; +type BaseFilterWithAndOr = BaseFilter & LogicalFilter; + +export type FieldLevelFilter = { + [slug: string]: BaseFilterWithAndOr; }; export type Filter = FieldLevelFilter | LogicalFilter; @@ -401,13 +403,7 @@ export const baseFilterSchema = z "This will filter the third element in the array, and check if it's greater than 90." ), }) - .partial() - .refine((data) => { - if (!Object.keys(data).length) { - return false; - } - return true; - }, "Filter must have at least one operator (base filter)") satisfies z.ZodType<{ + .partial() satisfies z.ZodType<{ [K in FilterOperator]?: any; }>; @@ -415,6 +411,30 @@ const fieldSlugSchema = z .string() .regex(/^[a-zA-Z0-9_.:-]+$/, "At this level, you can only use field slugs"); +const baseFilterSchemaWithAndOr: z.ZodType = z + .lazy(() => + baseFilterSchema + .extend({ + $and: baseFilterSchemaWithAndOr.or(z.array(baseFilterSchemaWithAndOr)), + $or: baseFilterSchemaWithAndOr.or(z.array(baseFilterSchemaWithAndOr)), + $not: baseFilterSchemaWithAndOr, + }) + .partial() + ) + .superRefine((data, ctx) => { + console.log(ctx.path, data); + if (!Object.keys(data).length) { + ctx.addIssue({ + path: ctx.path, + code: z.ZodIssueCode.custom, + message: "Filter must have at least one operator (base filter)", + }); + return false; + } + return true; + }); +// "Filter must have at least one operator (base filter)"); + // this is a recursive type, so we need to use z.lazy() export const filterSchema: z.ZodType = z.lazy(() => { const topLevelLogicalOperators = z @@ -430,20 +450,6 @@ export const filterSchema: z.ZodType = z.lazy(() => { return true; }, "Filter must have at least one operator. A logical operator (e.g. $and, $or, $not) was expected here, but not found."); - const fieldLevelThings = z - .object({ - $and: z.array(baseFilterSchema).optional(), - $or: z.array(baseFilterSchema).optional(), - $not: baseFilterSchema.optional(), - }) - .refine((data) => { - if (!Object.keys(data).length) { - return false; - } - return true; - }, "Filter must have at least one operator (field level)") - .or(baseFilterSchema); - //z.union([ // regular field filters @@ -453,7 +459,10 @@ export const filterSchema: z.ZodType = z.lazy(() => { // logical operator -> field -> operator -> value // logical operator -> logical operator -> ... -> field -> operator -> value - return z.union([z.record(fieldSlugSchema, fieldLevelThings), topLevelLogicalOperators]); //, + return z.union([ + z.record(fieldSlugSchema, baseFilterSchemaWithAndOr), + topLevelLogicalOperators, + ]); //, // logicalOperators, // ]); });