feat: allow detailed filtering of pubvalues and updatedAt/createdAt through the api

core/.env.test

@@ -16,7 +16,7 @@ MAILGUN_SMTP_USERNAME="xxx"
 OTEL_SERVICE_NAME="pubpub-v7-dev" # should be shared across components but not environments
 HONEYCOMB_API_KEY="xxx"
 
-KYSELY_DEBUG="true"
+# KYSELY_DEBUG="true"
 
 GCLOUD_KEY_FILE='xxx'
 

core/app/api/v0/c/[communitySlug]/site/[...ts-rest]/route.ts

@@ -1,8 +1,9 @@
 import type { User } from "lucia";
 
 import { headers } from "next/headers";
-import { createNextHandler } from "@ts-rest/serverless/next";
+import { createNextHandler, RequestValidationError } from "@ts-rest/serverless/next";
 import { jsonObjectFrom } from "kysely/helpers/postgres";
+import qs from "qs";
 import { z } from "zod";
 
 import type {
@@ -19,8 +20,9 @@ import type {
 	ApiAccessPermissionConstraintsInput,
 	LastModifiedBy,
 } from "db/types";
-import { siteApi } from "contracts";
+import { baseFilterSchema, filterSchema, siteApi } from "contracts";
 import { ApiAccessScope, ApiAccessType, Capabilities, MembershipType } from "db/public";
+import { assert } from "utils";
 
 import type { CapabilityTarget } from "~/lib/authorization/capabilities";
 import { db } from "~/kysely/database";
@@ -29,6 +31,7 @@ import { userCan } from "~/lib/authorization/capabilities";
 import { getStage } from "~/lib/db/queries";
 import { createLastModifiedBy } from "~/lib/lastModifiedBy";
 import {
+	BadRequestError,
 	createPubRecursiveNew,
 	deletePub,
 	doesPubExist,
@@ -47,6 +50,7 @@ import {
 import { validateApiAccessToken } from "~/lib/server/apiAccessTokens";
 import { getCommunitySlug } from "~/lib/server/cache/getCommunitySlug";
 import { findCommunityBySlug } from "~/lib/server/community";
+import { validateFilter } from "~/lib/server/pub-filters-validate";
 import { getPubType, getPubTypesForCommunity } from "~/lib/server/pubtype";
 import { getStages } from "~/lib/server/stages";
 import { getMember, getSuggestedUsers, SAFE_USER_SELECT } from "~/lib/server/user";
@@ -226,6 +230,81 @@ const shouldReturnRepresentation = async () => {
 	return false;
 };
 
+/**
+ * manually parses the `?filters` query param.
+ * necessary because ts-rest only supports parsing object in query params
+ * if they're uri encoded.
+ *
+ * eg this does not fly
+ *  ```
+ * ?filters[community-slug:fieldName][$eq]=value
+ * ```
+ * but this does
+ *  ```
+ * ?filters=%7B%22%7B%22updatedAt%22%3A%20%7B%22%24gte%22%3A%20%222025-01-01%22%7D%2C%22field-slug%22%3A%20%7B%22%24eq%22%3A%20%22some-value%22%7D%7D`
+ * ```
+ *
+ * the latter is what a ts-rest client sends if `json-query: true`. we want to support both syntaxes.
+ *
+ */
+const manuallyParsePubFilterQueryParams = (url: string, query?: Record<string, any>) => {
+	if (!query || Object.keys(query).length === 0) {
+		return query;
+	}
+
+	// check if we already have properly structured filters
+	if (query.filters && typeof query.filters === "object") {
+		try {
+			const validatedFilters = filterSchema.parse(query.filters);
+			return {
+				...query,
+				filters: validatedFilters,
+			};
+		} catch (e) {
+			throw new RequestValidationError(null, null, e, null);
+		}
+	}
+
+	// check if we have filter-like keys (using bracket notation)
+	const filterLikeKeys = Object.keys(query).filter((key) => key.startsWith("filters["));
+
+	if (filterLikeKeys.length === 0) {
+		return query;
+	}
+
+	const queryString = url.split("?")[1];
+	if (!queryString) {
+		return query;
+	}
+
+	try {
+		// parse with qs
+		const parsedQuery = qs.parse(queryString, {
+			depth: 10,
+			arrayLimit: 100,
+			allowDots: false, // don't convert dots to objects (use brackets only)
+			ignoreQueryPrefix: true, // remove the leading '?' if present
+		});
+
+		if (!parsedQuery.filters) {
+			return query; // no filters found after parsing
+		}
+
+		const validatedFilters = filterSchema.parse(parsedQuery.filters);
+
+		return {
+			...query,
+			...parsedQuery,
+			filters: validatedFilters,
+		};
+	} catch (e) {
+		if (e instanceof z.ZodError) {
+			throw new RequestValidationError(null, null, e, null);
+		}
+		throw new BadRequestError(`Error parsing filters: ${e.message}`);
+	}
+};
+
 const handler = createNextHandler(
 	siteApi,
 	{
@@ -267,14 +346,24 @@ const handler = createNextHandler(
 					body: pub,
 				};
 			},
-			getMany: async ({ query }) => {
+			getMany: async ({ query }, { request }) => {
 				const { user, community } = await checkAuthorization({
 					token: { scope: ApiAccessScope.pub, type: ApiAccessType.read },
 					// TODO: figure out capability here
 					cookies: false,
 				});
 
-				const { pubTypeId, stageId, ...rest } = query;
+				const { pubTypeId, stageId, filters, ...rest } = query ?? {};
+
+				const manuallyParsedFilters = manuallyParsePubFilterQueryParams(request.url, query);
+
+				if (manuallyParsedFilters?.filters) {
+					try {
+						await validateFilter(community.id, manuallyParsedFilters.filters);
+					} catch (e) {
+						throw new BadRequestError(e.message);
+					}
+				}
 
 				const pubs = await getPubsWithRelatedValues(
 					{
@@ -283,7 +372,10 @@ const handler = createNextHandler(
 						stageId,
 						userId: user.id,
 					},
-					rest
+					{
+						...rest,
+						filters: manuallyParsedFilters?.filters,
+					}
 				);
 
 				return {

core/lib/__tests__/globalSetup.ts

@@ -19,15 +19,16 @@ export const setup = async () => {
 
 	logger.info("Resetting database...");
 	const result = spawnSync(
-		"MINIMAL_SEED=true pnpm --filter core exec dotenv -e ./.env.test -e ./.env.test.local prisma migrate reset -- --preview-feature --force",
+		"pnpm --filter core exec dotenv -e ./.env.test -e ./.env.test.local prisma migrate reset -- --preview-feature --force",
 		{
 			shell: true,
-			stdio: "inherit",
+			// stdio: "inherit",
 		}
 	);
 	const { stderr, error } = result;
-
-	if (error) {
+	if (!error) {
+		logger.info("Database reset successful");
+	} else {
 		logger.error(
 			"Something went wrong while trying to reset the database before running tests."
 		);

core/lib/__tests__/matchers.ts

@@ -64,7 +64,7 @@ expect.extend({
 			message: () =>
 				pass
 					? `Expected pub ${isNot ? "not" : ""} to have values ${JSON.stringify(expected)}, and it does ${isNot ? "not" : ""}`
-					: `Expected pub ${isNot ? "not to" : "to"} match values ${this.utils.diff(sortedPubValues.values, expected)}`,
+					: `Expected pub ${isNot ? "not to" : "to"} match values ${this.utils.diff(expected, sortedPubValues.values)}`,
 		};
 	},
 });

core/lib/env/env.mjs

@@ -33,7 +33,7 @@ export const env = createEnv({
 		DATABASE_URL: z.string().url(),
 		KYSELY_DEBUG: z.string().optional(),
 		KYSELY_ARTIFICIAL_LATENCY: z.coerce.number().optional(),
-		LOG_LEVEL: z.enum(["debug", "info", "warn", "error"]).optional(),
+		LOG_LEVEL: z.enum(["benchmark", "debug", "info", "warn", "error"]).optional(),
 		MAILGUN_SMTP_PASSWORD: selfHostedOptional(z.string()),
 		MAILGUN_SMTP_USERNAME: selfHostedOptional(z.string()),
 		MAILGUN_SMTP_HOST: selfHostedOptional(z.string()),

core/lib/mapping.ts

@@ -0,0 +1,71 @@
+import type { Prettify, TupleToCustomObject } from "./types";
+
+/**
+ * Type-safe version of Object.entries()
+ * Returns an array of tuples containing key-value pairs with proper typing
+ * Includes all properties that exist in the object, even if undefined
+ */
+export function entries<
+	const T extends Record<string, unknown>,
+	const KeepUndefined extends boolean = false,
+>(
+	obj: T,
+	keepUndefined?: KeepUndefined
+): T extends T
+	? { [K in keyof T]-?: [K, KeepUndefined extends true ? T[K] : NonNullable<T[K]>] }[keyof T][]
+	: never {
+	const newObj = Object.entries(obj);
+
+	if (keepUndefined) {
+		return newObj as any;
+	}
+
+	return newObj.filter(([_, v]) => v !== undefined) as any;
+}
+
+/**
+ * Type-safe version of Object.fromEntries()
+ * Creates an object from an array of key-value pairs with proper typing
+ */
+export function fromEntries<const T extends [PropertyKey, any][]>(
+	entries: T
+): { [K in T[number][0]]: Extract<T[number], [K, any]>[1] } {
+	return Object.fromEntries(entries) as any;
+}
+
+type MapToEntries<
+	T extends readonly unknown[],
+	M extends readonly string[],
+	C extends any[] = [],
+	Buffer extends Record<string, any> = {},
+> = C["length"] extends T["length"]
+	? Buffer
+	: MapToEntries<
+			T,
+			M,
+			[...C, C["length"]],
+			Prettify<
+				Buffer & {
+					[NewKey in M[C["length"]]]: T[C["length"]];
+				}
+			>
+		>;
+
+export function mapToEntries<
+	const T extends readonly unknown[],
+	const M extends readonly string[],
+	C extends any[] = [],
+	Buffer extends Record<string, any> = {},
+>(obj: T, mapping: M): T extends T ? MapToEntries<T, M, C, Buffer> : never {
+	const result: Record<string, any> = {};
+	for (let i = 0; i < Math.min(obj.length, mapping.length); i++) {
+		result[mapping[i]] = obj[i];
+	}
+	return result as any;
+}
+
+export function keys<const T extends Record<string, any>>(
+	obj: T
+): T extends T ? (keyof T)[] : never {
+	return Object.keys(obj) as any;
+}

core/lib/server/__snapshots__/pub-filters.db.test.ts.snap

@@ -0,0 +1,7 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`SQL generation > generates correct SQL for 'mixed array and object syntax with de…' 1`] = `"select * from "pubs" where ((exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $1 and (value::text ilike $2 and not value::text ilike $3)) or exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $4 and ("value" >= $5 or "value" = $6))) and not (exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $7 and "value" = $8) or exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $9 and not "value" > $10)))"`;
+
+exports[`SQL generation > generates correct SQL for 'multiple field-level logical operator…' 1`] = `"select * from "pubs" where (exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $1 and (value::text ilike $2 or not (value::text ilike $3 and value::text like $4))) and exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $5 and (not "value" < $6 and ("value" = $7 or "value" = $8))))"`;
+
+exports[`SQL generation > generates correct SQL for 'object syntax: complex mixture of fie…' 1`] = `"select * from "pubs" where ((exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $1 and (value::text ilike $2 or value::text ilike $3)) and exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $4 and not "value" < $5)) or not (exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $6 and not ("value" >= $7 and "value" <= $8)) and exists (select 1 as "exists_check" from "pub_values" inner join "pub_fields" on "pub_fields"."id" = "pub_values"."fieldId" where "pub_values"."pubId" = "pubs"."id" and "pub_fields"."slug" = $9 and "value" = $10)))"`;