From 8372531460c36284c6e606b0cbfee7418e2428ae Mon Sep 17 00:00:00 2001 From: igalklebanov Date: Mon, 28 Oct 2024 00:22:18 +0200 Subject: [PATCH] introduce Serialized introduce ValueNode.serialized. introduce eb.valSerialized. introduce sql.valSerialized. fix json-traversal test suite. fix null handling @ compiler. rename to `valJson`. add instructions in errors. typings test inserts. call the new type `Json` instead, to not introduce a breaking change. add missing json column @ Getting Started. add `appendSerializedValue`. --- site/docs/getting-started/Summary.tsx | 16 ++- site/docs/getting-started/_types.mdx | 12 +- src/expression/expression-builder.ts | 49 +++++++- src/operation-node/value-node.ts | 4 +- src/query-compiler/default-query-compiler.ts | 10 ++ src/raw-builder/sql.ts | 21 ++++ src/util/column-type.ts | 28 +++++ test/node/src/json-traversal.test.ts | 24 ++-- test/typings/shared.d.ts | 21 ++-- test/typings/test-d/insert.test-d.ts | 125 ++++++++++++++++++- 10 files changed, 270 insertions(+), 40 deletions(-) diff --git a/site/docs/getting-started/Summary.tsx b/site/docs/getting-started/Summary.tsx index 6fed7f893..555c0869a 100644 --- a/site/docs/getting-started/Summary.tsx +++ b/site/docs/getting-started/Summary.tsx @@ -2,11 +2,7 @@ import Admonition from '@theme/Admonition' import CodeBlock from '@theme/CodeBlock' import Link from '@docusaurus/Link' import { IUseADifferentDatabase } from './IUseADifferentDatabase' -import { - PRETTY_DIALECT_NAMES, - type Dialect, - type PropsWithDialect, -} from './shared' +import { type Dialect, type PropsWithDialect } from './shared' const dialectSpecificCodeSnippets: Record = { postgresql: ` await db.schema.createTable('person') @@ -17,6 +13,7 @@ const dialectSpecificCodeSnippets: Record = { .addColumn('created_at', 'timestamp', (cb) => cb.notNull().defaultTo(sql\`now()\`) ) + .addColumn('metadata', 'jsonb', (cb) => cb.notNull()) .execute()`, mysql: ` await db.schema.createTable('person') .addColumn('id', 'integer', (cb) => cb.primaryKey().autoIncrement()) @@ -26,6 +23,7 @@ const dialectSpecificCodeSnippets: Record = { .addColumn('created_at', 'timestamp', (cb) => cb.notNull().defaultTo(sql\`now()\`) ) + .addColumn('metadata', 'json', (cb) => cb.notNull()) .execute()`, // TODO: Update line 42's IDENTITY once identity(1,1) is added to core. mssql: ` await db.schema.createTable('person') @@ -36,6 +34,7 @@ const dialectSpecificCodeSnippets: Record = { .addColumn('created_at', 'datetime', (cb) => cb.notNull().defaultTo(sql\`GETDATE()\`) ) + .addColumn('metadata', sql\`nvarchar(max)\`, (cb) => cb.notNull()) .execute()`, sqlite: ` await db.schema.createTable('person') .addColumn('id', 'integer', (cb) => cb.primaryKey().autoIncrement().notNull()) @@ -45,6 +44,7 @@ const dialectSpecificCodeSnippets: Record = { .addColumn('created_at', 'timestamp', (cb) => cb.notNull().defaultTo(sql\`current_timestamp\`) ) + .addColumn('metadata', 'text', (cb) => cb.notNull()) .execute()`, } @@ -107,6 +107,12 @@ ${dialectSpecificCodeSnippet} first_name: 'Jennifer', last_name: 'Aniston', gender: 'woman', + metadata: sql.valJson({ + login_at: new Date().toISOString(), + ip: null, + agent: null, + plan: 'free', + }), }) }) diff --git a/site/docs/getting-started/_types.mdx b/site/docs/getting-started/_types.mdx index 42cf066be..1f4b98b58 100644 --- a/site/docs/getting-started/_types.mdx +++ b/site/docs/getting-started/_types.mdx @@ -10,7 +10,7 @@ import { ColumnType, Generated, Insertable, - JSONColumnType, + Json, Selectable, Updateable, } from 'kysely' @@ -45,12 +45,10 @@ export interface PersonTable { // can never be updated: created_at: ColumnType - // You can specify JSON columns using the `JSONColumnType` wrapper. - // It is a shorthand for `ColumnType`, where T - // is the type of the JSON object/array retrieved from the database, - // and the insert and update types are always `string` since you're - // always stringifying insert/update values. - metadata: JSONColumnType<{ + // You can specify JSON columns using the `Json` wrapper. + // When inserting/updating values of such columns, you're required to wrap the + // values with `eb.valJson` or `sql.valJson`. + metadata: Json<{ login_at: string ip: string | null agent: string | null diff --git a/src/expression/expression-builder.ts b/src/expression/expression-builder.ts index 938040882..856f16426 100644 --- a/src/expression/expression-builder.ts +++ b/src/expression/expression-builder.ts @@ -69,7 +69,7 @@ import { ValTuple5, } from '../parser/tuple-parser.js' import { TupleNode } from '../operation-node/tuple-node.js' -import { Selectable } from '../util/column-type.js' +import { Selectable, Serialized } from '../util/column-type.js' import { JSONPathNode } from '../operation-node/json-path-node.js' import { KyselyTypeError } from '../util/type-error.js' import { @@ -78,6 +78,7 @@ import { } from '../parser/data-type-parser.js' import { CastNode } from '../operation-node/cast-node.js' import { SelectFrom } from '../parser/select-from-parser.js' +import { ValueNode } from '../operation-node/value-node.js' export interface ExpressionBuilder { /** @@ -511,6 +512,44 @@ export interface ExpressionBuilder { value: VE, ): ExpressionWrapper> + /** + * Returns a value expression that will be serialized before being passed to the database. + * + * This can be used to pass in an object/array value when inserting/updating a + * value to a column defined with `Json`. + * + * Default serializer function is `JSON.stringify`. + * + * ### Example + * + * ```ts + * import { GeneratedAlways, Json } from 'kysely' + * + * interface Database { + * person: { + * id: GeneratedAlways + * name: string + * experience: Json<{ title: string; company: string }[]> + * preferences: Json<{ locale: string; timezone: string }> + * profile: Json<{ email_verified: boolean }> + * } + * } + * + * const result = await db + * .insertInto('person') + * .values(({ valJson }) => ({ + * name: 'Jennifer Aniston', + * experience: valJson([{ title: 'Software Engineer', company: 'Google' }]), // ✔️ + * preferences: valJson({ locale: 'en' }), // ❌ missing `timezone` + * profile: JSON.stringify({ email_verified: true }), // ❌ doesn't match `Serialized<{ email_verified }>` + * })) + * .execute() + * ``` + */ + valJson( + obj: O, + ): ExpressionWrapper> + /** * Creates a tuple expression. * @@ -1140,6 +1179,14 @@ export function createExpressionBuilder( return new ExpressionWrapper(parseValueExpression(value)) }, + valJson( + value: O, + ): ExpressionWrapper> { + return new ExpressionWrapper( + ValueNode.create(value, { serialized: true }), + ) + }, + refTuple( ...values: ReadonlyArray> ): ExpressionWrapper { diff --git a/src/operation-node/value-node.ts b/src/operation-node/value-node.ts index 2c811d0dd..b492fd46b 100644 --- a/src/operation-node/value-node.ts +++ b/src/operation-node/value-node.ts @@ -5,6 +5,7 @@ export interface ValueNode extends OperationNode { readonly kind: 'ValueNode' readonly value: unknown readonly immediate?: boolean + readonly serialized?: boolean } /** @@ -15,9 +16,10 @@ export const ValueNode = freeze({ return node.kind === 'ValueNode' }, - create(value: unknown): ValueNode { + create(value: unknown, props?: { serialized?: boolean }): ValueNode { return freeze({ kind: 'ValueNode', + ...props, value, }) }, diff --git a/src/query-compiler/default-query-compiler.ts b/src/query-compiler/default-query-compiler.ts index 5a04313d6..bc124a5ae 100644 --- a/src/query-compiler/default-query-compiler.ts +++ b/src/query-compiler/default-query-compiler.ts @@ -502,6 +502,8 @@ export class DefaultQueryCompiler protected override visitValue(node: ValueNode): void { if (node.immediate) { this.appendImmediateValue(node.value) + } else if (node.serialized) { + this.appendSerializedValue(node.value) } else { this.appendValue(node.value) } @@ -1667,6 +1669,14 @@ export class DefaultQueryCompiler this.append(this.getCurrentParameterPlaceholder()) } + protected appendSerializedValue(parameter: unknown): void { + if (parameter === null) { + this.appendValue(null) + } else { + this.appendValue(JSON.stringify(parameter)) + } + } + protected getLeftIdentifierWrapper(): string { return '"' } diff --git a/src/raw-builder/sql.ts b/src/raw-builder/sql.ts index 6e115b32a..48642fb79 100644 --- a/src/raw-builder/sql.ts +++ b/src/raw-builder/sql.ts @@ -6,6 +6,7 @@ import { ValueNode } from '../operation-node/value-node.js' import { parseStringReference } from '../parser/reference-parser.js' import { parseTable } from '../parser/table-parser.js' import { parseValueExpression } from '../parser/value-parser.js' +import { Serialized } from '../util/column-type.js' import { createQueryId } from '../util/query-id.js' import { RawBuilder, createRawBuilder } from './raw-builder.js' @@ -120,6 +121,17 @@ export interface Sql { */ val(value: V): RawBuilder + /** + * `sql.valJson(value)` is a shortcut for: + * + * ```ts + * sql>`${serializerFn(obj)}` + * ``` + * + * Default serializer function is `JSON.stringify`. + */ + valJson(value: O): RawBuilder> + /** * @deprecated Use {@link Sql.val} instead. */ @@ -398,6 +410,15 @@ export const sql: Sql = Object.assign( }) }, + valJson(value: O): RawBuilder> { + return createRawBuilder({ + queryId: createQueryId(), + rawNode: RawNode.createWithChild( + ValueNode.create(value, { serialized: true }), + ), + }) + }, + value(value: V): RawBuilder { return this.val(value) }, diff --git a/src/util/column-type.ts b/src/util/column-type.ts index c8614bd05..98727ee0a 100644 --- a/src/util/column-type.ts +++ b/src/util/column-type.ts @@ -67,9 +67,37 @@ export type Generated = ColumnType */ export type GeneratedAlways = ColumnType +/** + * A shortcut for defining type-safe JSON columns. Inserts/updates require passing + * values that are wrapped with `eb.valJson` or `sql.valJson` instead of `JSON.stringify`. + */ +export type Json< + SelectType extends object | null, + InsertType extends Serialized | Extract = + | Serialized + | Extract, + UpdateType extends Serialized | Extract = + | Serialized + | Extract, +> = ColumnType + +/** + * A symbol that is used to brand serialized objects/arrays. + * @internal + */ +declare const SerializedBrand: unique symbol + +/** + * A type that is used to brand serialized objects/arrays. + */ +export type Serialized = O & { + readonly [SerializedBrand]: '⚠️ When you insert into or update columns of type `Json` (or similar), you should wrap your JSON value with `eb.valJson` or `sql.valJson`, instead of `JSON.stringify`. ⚠️' +} + /** * A shortcut for defining JSON columns, which are by default inserted/updated * as stringified JSON strings. + * @deprecated Use {@link Json} instead. */ export type JSONColumnType< SelectType extends object | null, diff --git a/test/node/src/json-traversal.test.ts b/test/node/src/json-traversal.test.ts index 363aeb33e..8ae957282 100644 --- a/test/node/src/json-traversal.test.ts +++ b/test/node/src/json-traversal.test.ts @@ -1,6 +1,6 @@ import { ColumnDefinitionBuilder, - JSONColumnType, + Json, ParseJSONResultsPlugin, SqlBool, sql, @@ -732,9 +732,9 @@ async function initJSONTest( let db = testContext.db.withTables<{ person_metadata: { person_id: number - website: JSONColumnType<{ url: string }> - nicknames: JSONColumnType - profile: JSONColumnType<{ + website: Json<{ url: string }> + nicknames: Json + profile: Json<{ auth: { roles: string[] last_login?: { device: string } @@ -744,12 +744,12 @@ async function initJSONTest( avatar: string | null tags: string[] }> - experience: JSONColumnType< + experience: Json< { establishment: string }[] > - schedule: JSONColumnType<{ name: string; time: string }[][][]> + schedule: Json<{ name: string; time: string }[][][]> } }>() @@ -798,20 +798,20 @@ async function insertDefaultJSONDataSet(ctx: TestContext) { await ctx.db .insertInto('person_metadata') - .values( + .values((eb) => people .filter((person) => person.first_name && person.last_name) .map((person, index) => ({ person_id: person.id, - website: JSON.stringify({ + website: eb.valJson({ url: `https://www.${person.first_name!.toLowerCase()}${person.last_name!.toLowerCase()}.com`, }), - nicknames: JSON.stringify([ + nicknames: eb.valJson([ `${person.first_name![0]}.${person.last_name![0]}.`, `${person.first_name} the Great`, `${person.last_name} the Magnificent`, ]), - profile: JSON.stringify({ + profile: eb.valJson({ tags: ['awesome'], auth: { roles: ['contributor', 'moderator'], @@ -823,12 +823,12 @@ async function insertDefaultJSONDataSet(ctx: TestContext) { }, avatar: null, }), - experience: JSON.stringify([ + experience: eb.valJson([ { establishment: 'The University of Life', }, ]), - schedule: JSON.stringify([[[{ name: 'Gym', time: '12:15' }]]]), + schedule: sql.valJson([[[{ name: 'Gym', time: '12:15' }]]]), })), ) .execute() diff --git a/test/typings/shared.d.ts b/test/typings/shared.d.ts index 9a7bc1afd..741453df2 100644 --- a/test/typings/shared.d.ts +++ b/test/typings/shared.d.ts @@ -1,9 +1,4 @@ -import { - ColumnType, - Generated, - GeneratedAlways, - JSONColumnType, -} from '../../dist/cjs' +import { ColumnType, Generated, GeneratedAlways, Json } from '../../dist/cjs' export interface Pet { id: Generated @@ -71,21 +66,21 @@ export interface Person { export interface PersonMetadata { id: Generated person_id: number - website: JSONColumnType<{ url: string }> - nicknames: JSONColumnType - profile: JSONColumnType<{ + website: Json<{ url: string }> + nicknames: Json + profile: Json<{ auth: { roles: string[] last_login?: { device: string } } tags: string[] }> - experience: JSONColumnType< + experience: Json< { establishment: string }[] > - schedule: JSONColumnType<{ name: string; time: string }[][][]> - record: JSONColumnType> - array: JSONColumnType> + schedule: Json<{ name: string; time: string }[][][]> + record: Json> + array: Json | null> } diff --git a/test/typings/test-d/insert.test-d.ts b/test/typings/test-d/insert.test-d.ts index e8f049ba1..f4325c99e 100644 --- a/test/typings/test-d/insert.test-d.ts +++ b/test/typings/test-d/insert.test-d.ts @@ -1,5 +1,5 @@ import { expectError, expectType } from 'tsd' -import { InsertResult, Kysely, sql } from '..' +import { ExpressionBuilder, InsertObject, InsertResult, Kysely, sql } from '..' import { Database } from '../shared' async function testInsert(db: Kysely) { @@ -268,3 +268,126 @@ async function testOutput(db: Kysely) { expectError(db.insertInto('person').output('deleted.age').values(person)) expectError(db.insertInto('person').outputAll('deleted').values(person)) } + +async function testValJson(db: Kysely) { + const getValues = < + O extends Partial>, + >( + { valJson }: ExpressionBuilder, + overrides?: O, + ) => ({ + array: valJson(['123']), + experience: valJson([{ establishment: 'New York Times' }]), + nicknames: valJson(['Jenny']), + person_id: 1, + profile: valJson({ + auth: { + roles: ['admin'], + }, + tags: ['important'], + }), + website: valJson({ url: 'http://example.com' }), + record: valJson({ key: 'value' }), + schedule: valJson([ + [ + [ + { + name: 'foo', + time: '2024-01-01T00:00:00.000Z', + }, + ], + ], + ]), + ...overrides, + }) + + db.insertInto('person_metadata').values(getValues).execute() + + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + array: null, + }), + ) + + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + array: eb.valJson(null), + }), + ) + + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + array: sql.valJson(null), + }), + ) + + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + website: sql.valJson({ url: 'http://example.com' }), + }), + ) + + expectError( + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + array: ['123'], // expects `valJson(Array | null)` + }), + ), + ) + + expectError( + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + array: eb.val(['123']), // expects `valJson(Array | null)` + }), + ), + ) + + expectError( + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + array: eb.valJson({}), // expects `valJson(Array | null)` + }), + ), + ) + + expectError( + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + array: eb.valJson([123]), // expects `valJson(Array | null)` + }), + ), + ) + + expectError( + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + experience: [{ establishment: 'New York Times' }], // expects `valJson({ establishment: string }[])` + }), + ), + ) + + expectError( + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + experience: eb.valJson({ establishment: 'New York Times' }), // expects `valJson({ establishment: string }[])` + }), + ), + ) + + expectError( + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + experience: eb.valJson([{}]), // expects `valJson({ establishment: string }[])` + }), + ), + ) + + expectError( + db.insertInto('person_metadata').values((eb) => + getValues(eb, { + experience: eb.valJson([{ establishment: 2 }]), // expects `valJson({ establishment: string }[])` + }), + ), + ) +}