From b2885a525215c0fb69e4761f899a8cdbe3227058 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 28 Apr 2025 13:02:15 +0200 Subject: [PATCH 1/5] Encode new schema options --- packages/common/src/db/schema/Table.ts | 74 ++++++++++++++----- .../common/tests/db/schema/Schema.test.ts | 8 ++ packages/common/tests/db/schema/Table.test.ts | 44 ++++++++++- .../common/tests/db/schema/TableV2.test.ts | 4 + 4 files changed, 112 insertions(+), 18 deletions(-) diff --git a/packages/common/src/db/schema/Table.ts b/packages/common/src/db/schema/Table.ts index 5f91afaf2..ef4c9ca6b 100644 --- a/packages/common/src/db/schema/Table.ts +++ b/packages/common/src/db/schema/Table.ts @@ -10,16 +10,22 @@ import { Index } from './Index.js'; import { IndexedColumn } from './IndexedColumn.js'; import { TableV2 } from './TableV2.js'; -export interface TableOptions { +interface SharedTableOptions { + localOnly?: boolean; + insertOnly?: boolean; + viewName?: string; + includeOld?: boolean | 'when-changed'; + includeMetadata?: boolean; + ignoreEmptyUpdate?: boolean; +} + +export interface TableOptions extends SharedTableOptions { /** * The synced table name, matching sync rules */ name: string; columns: Column[]; indexes?: Index[]; - localOnly?: boolean; - insertOnly?: boolean; - viewName?: string; } export type RowType> = { @@ -30,17 +36,17 @@ export type RowType> = { export type IndexShorthand = Record; -export interface TableV2Options { +export interface TableV2Options extends SharedTableOptions { indexes?: IndexShorthand; - localOnly?: boolean; - insertOnly?: boolean; - viewName?: string; } export const DEFAULT_TABLE_OPTIONS = { indexes: [], insertOnly: false, - localOnly: false + localOnly: false, + includeOld: false, + includeMetadata: false, + ignoreEmptyUpdate: false, }; export const InvalidSQLCharacters = /["'%,.#\s[\]]/; @@ -144,10 +150,9 @@ export class Table { private initTableV1(options: TableOptions) { this.options = { ...options, - indexes: options.indexes || [], - insertOnly: options.insertOnly ?? DEFAULT_TABLE_OPTIONS.insertOnly, - localOnly: options.localOnly ?? DEFAULT_TABLE_OPTIONS.localOnly + indexes: options.indexes || [] }; + this.applyDefaultOptions(); } private initTableV2(columns: Columns, options?: TableV2Options) { @@ -173,14 +178,26 @@ export class Table { name: '', columns: convertedColumns, indexes: convertedIndexes, - insertOnly: options?.insertOnly ?? DEFAULT_TABLE_OPTIONS.insertOnly, - localOnly: options?.localOnly ?? DEFAULT_TABLE_OPTIONS.localOnly, - viewName: options?.viewName + viewName: options?.viewName, + insertOnly: options?.insertOnly, + localOnly: options?.localOnly, + includeOld: options?.includeOld, + includeMetadata: options?.includeMetadata, + ignoreEmptyUpdate: options?.ignoreEmptyUpdate }; + this.applyDefaultOptions(); this._mappedColumns = columns; } + private applyDefaultOptions() { + this.options.insertOnly ??= DEFAULT_TABLE_OPTIONS.insertOnly; + this.options.localOnly ??= DEFAULT_TABLE_OPTIONS.localOnly; + this.options.includeOld ??= DEFAULT_TABLE_OPTIONS.includeOld; + this.options.includeMetadata ??= DEFAULT_TABLE_OPTIONS.includeMetadata; + this.options.ignoreEmptyUpdate ??= DEFAULT_TABLE_OPTIONS.ignoreEmptyUpdate; + } + get name() { return this.options.name; } @@ -212,11 +229,23 @@ export class Table { } get localOnly() { - return this.options.localOnly ?? false; + return this.options.localOnly!; } get insertOnly() { - return this.options.insertOnly ?? false; + return this.options.insertOnly!; + } + + get includeOld() { + return this.options.includeOld!; + } + + get includeMetadata() { + return this.options.includeMetadata!; + } + + get ignoreEmptyUpdate() { + return this.options.ignoreEmptyUpdate!; } get internalName() { @@ -250,6 +279,13 @@ export class Table { throw new Error(`Table has too many columns. The maximum number of columns is ${MAX_AMOUNT_OF_COLUMNS}.`); } + if (this.includeMetadata && this.localOnly) { + throw new Error(`Can't include metadata for local-only tables.`); + } + if (this.includeOld != false && this.localOnly) { + throw new Error(`Can't include old values for local-only tables.`); + } + const columnNames = new Set(); columnNames.add('id'); for (const column of this.columns) { @@ -291,6 +327,10 @@ export class Table { view_name: this.viewName, local_only: this.localOnly, insert_only: this.insertOnly, + include_old: this.includeOld != false, + include_old_only_when_changed: this.includeOld == 'when-changed', + include_metadata: this.includeMetadata, + ignore_empty_update: this.ignoreEmptyUpdate, columns: this.columns.map((c) => c.toJSON()), indexes: this.indexes.map((e) => e.toJSON(this)) }; diff --git a/packages/common/tests/db/schema/Schema.test.ts b/packages/common/tests/db/schema/Schema.test.ts index 51bbf5aa9..c9f303535 100644 --- a/packages/common/tests/db/schema/Schema.test.ts +++ b/packages/common/tests/db/schema/Schema.test.ts @@ -90,6 +90,10 @@ describe('Schema', () => { view_name: 'users', local_only: false, insert_only: false, + ignore_empty_update: false, + include_metadata: false, + include_old: false, + include_old_only_when_changed: false, columns: [ { name: 'name', type: 'TEXT' }, { name: 'age', type: 'INTEGER' } @@ -101,6 +105,10 @@ describe('Schema', () => { view_name: 'posts', local_only: false, insert_only: false, + ignore_empty_update: false, + include_metadata: false, + include_old: false, + include_old_only_when_changed: false, columns: [ { name: 'title', type: 'TEXT' }, { name: 'content', type: 'TEXT' } diff --git a/packages/common/tests/db/schema/Table.test.ts b/packages/common/tests/db/schema/Table.test.ts index 3f213517c..acc33f0cc 100644 --- a/packages/common/tests/db/schema/Table.test.ts +++ b/packages/common/tests/db/schema/Table.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { Table } from '../../../src/db/schema/Table'; +import { Table, TableV2Options } from '../../../src/db/schema/Table'; import { column, Column, ColumnType } from '../../../src/db/schema/Column'; import { Index } from '../../../src/db/schema/Index'; import { IndexedColumn } from '../../../src/db/schema/IndexedColumn'; @@ -103,6 +103,10 @@ describe('Table', () => { view_name: 'customView', local_only: false, insert_only: false, + ignore_empty_update: false, + include_metadata: false, + include_old: false, + include_old_only_when_changed: false, columns: [ { name: 'name', type: 'TEXT' }, { name: 'age', type: 'INTEGER' } @@ -126,6 +130,22 @@ describe('Table', () => { expect(table.indexes[0].columns[0].ascending).toBe(false); }); + it('should handle options', () => { + function createTable(options: TableV2Options) { + return new Table({name: column.text}, options); + } + + expect(createTable({}).toJSON().include_metadata).toBe(false); + expect(createTable({includeMetadata: true}).toJSON().include_metadata).toBe(true); + + expect(createTable({includeOld: true}).toJSON().include_old).toBe(true); + expect(createTable({includeOld: true}).toJSON().include_old_only_when_changed).toBe(false); + expect(createTable({includeOld: 'when-changed'}).toJSON().include_old).toBe(true); + expect(createTable({includeOld: 'when-changed'}).toJSON().include_old_only_when_changed).toBe(true); + + expect(createTable({ignoreEmptyUpdate: true}).toJSON().ignore_empty_update).toBe(true); + }); + describe('validate', () => { it('should throw an error for invalid view names', () => { expect(() => { @@ -173,5 +193,27 @@ describe('Table', () => { }).validate() ).toThrowError('Invalid characters in column name: #invalid-name'); }); + + it('should throw an error for local-only tables with metadata', () => { + expect(() => + new Table({ + name: column.text, + }, { localOnly: true, includeMetadata: true }).validate() + ).toThrowError("Can't include metadata for local-only tables."); + }); + + it('should throw an error for local-only tables tracking old values', () => { + expect(() => + new Table({ + name: column.text, + }, { localOnly: true, includeOld: true }).validate() + ).toThrowError("Can't include old values for local-only tables."); + + expect(() => + new Table({ + name: column.text, + }, { localOnly: true, includeOld: 'when-changed' }).validate() + ).toThrowError("Can't include old values for local-only tables."); + }); }); }); diff --git a/packages/common/tests/db/schema/TableV2.test.ts b/packages/common/tests/db/schema/TableV2.test.ts index 6e42f07b5..55895b63d 100644 --- a/packages/common/tests/db/schema/TableV2.test.ts +++ b/packages/common/tests/db/schema/TableV2.test.ts @@ -79,6 +79,10 @@ describe('TableV2', () => { view_name: 'customView', local_only: false, insert_only: false, + ignore_empty_update: false, + include_metadata: false, + include_old: false, + include_old_only_when_changed: false, columns: [ { name: 'name', type: 'TEXT' }, { name: 'age', type: 'INTEGER' } From 2949d585aa918f9eee4986d826e39ab8fd2f1056 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 28 Apr 2025 14:49:20 +0200 Subject: [PATCH 2/5] Add tests and changeset entry --- .changeset/giant-ladybugs-dress.md | 8 ++ .../src/client/sync/bucket/CrudEntry.ts | 34 +++++++- packages/common/src/db/schema/Schema.ts | 10 +-- packages/common/src/db/schema/Table.ts | 9 ++- packages/common/tests/db/schema/Table.test.ts | 41 ++++++---- packages/node/tests/crud.test.ts | 78 +++++++++++++++++++ 6 files changed, 152 insertions(+), 28 deletions(-) create mode 100644 .changeset/giant-ladybugs-dress.md create mode 100644 packages/node/tests/crud.test.ts diff --git a/.changeset/giant-ladybugs-dress.md b/.changeset/giant-ladybugs-dress.md new file mode 100644 index 000000000..07f45b985 --- /dev/null +++ b/.changeset/giant-ladybugs-dress.md @@ -0,0 +1,8 @@ +--- +'@powersync/common': minor +--- + +- Add `includeOld` option on `Table` which sets `CrudEntry.oldData` to previous values on updates. +- Add `includeMetadata` option on `Table` which adds a `_metadata` column that can be used for updates. + The configured metadata is available through `CrudEntry.metadata`. +- Add `ignoreEmptyUpdate` option which skips creating CRUD entries for updates that don't change any values. diff --git a/packages/common/src/client/sync/bucket/CrudEntry.ts b/packages/common/src/client/sync/bucket/CrudEntry.ts index ae862aca4..edc1bdeec 100644 --- a/packages/common/src/client/sync/bucket/CrudEntry.ts +++ b/packages/common/src/client/sync/bucket/CrudEntry.ts @@ -25,9 +25,11 @@ export type CrudEntryJSON = { type CrudEntryDataJSON = { data: Record; + old?: Record; op: UpdateType; type: string; id: string; + metadata?: string; }; /** @@ -62,6 +64,13 @@ export class CrudEntry { * Data associated with the change. */ opData?: Record; + + /** + * For tables where the `includeOld` option has been enabled, this tracks previous values for + * `UPDATE` and `DELETE` statements. + */ + oldData?: Record; + /** * Table that contained the change. */ @@ -71,9 +80,26 @@ export class CrudEntry { */ transactionId?: number; + /** + * Client-side metadata attached with this write. + * + * This field is only available when the `includeMetadata` option was set to `true` when creating a table + * and the insert or update statement set the `_metadata` column. + */ + metadata?: string; + static fromRow(dbRow: CrudEntryJSON) { const data: CrudEntryDataJSON = JSON.parse(dbRow.data); - return new CrudEntry(parseInt(dbRow.id), data.op, data.type, data.id, dbRow.tx_id, data.data); + return new CrudEntry( + parseInt(dbRow.id), + data.op, + data.type, + data.id, + dbRow.tx_id, + data.data, + data.old, + data.metadata + ); } constructor( @@ -82,7 +108,9 @@ export class CrudEntry { table: string, id: string, transactionId?: number, - opData?: Record + opData?: Record, + oldData?: Record, + metadata?: string ) { this.clientId = clientId; this.id = id; @@ -90,6 +118,8 @@ export class CrudEntry { this.opData = opData; this.table = table; this.transactionId = transactionId; + this.oldData = oldData; + this.metadata = metadata; } /** diff --git a/packages/common/src/db/schema/Schema.ts b/packages/common/src/db/schema/Schema.ts index 8d283a080..0aa175c3a 100644 --- a/packages/common/src/db/schema/Schema.ts +++ b/packages/common/src/db/schema/Schema.ts @@ -53,15 +53,7 @@ export class Schema { private convertToClassicTables(props: S) { return Object.entries(props).map(([name, table]) => { - const convertedTable = new Table({ - name, - columns: table.columns, - indexes: table.indexes, - localOnly: table.localOnly, - insertOnly: table.insertOnly, - viewName: table.viewNameOverride || name - }); - return convertedTable; + return table.copyWithName(name); }); } } diff --git a/packages/common/src/db/schema/Table.ts b/packages/common/src/db/schema/Table.ts index ef4c9ca6b..8c51fb31e 100644 --- a/packages/common/src/db/schema/Table.ts +++ b/packages/common/src/db/schema/Table.ts @@ -46,7 +46,7 @@ export const DEFAULT_TABLE_OPTIONS = { localOnly: false, includeOld: false, includeMetadata: false, - ignoreEmptyUpdate: false, + ignoreEmptyUpdate: false }; export const InvalidSQLCharacters = /["'%,.#\s[\]]/; @@ -143,6 +143,13 @@ export class Table { } } + copyWithName(name: string): Table { + return new Table({ + ...this.options, + name + }); + } + private isTableV1(arg: TableOptions | Columns): arg is TableOptions { return 'columns' in arg && Array.isArray(arg.columns); } diff --git a/packages/common/tests/db/schema/Table.test.ts b/packages/common/tests/db/schema/Table.test.ts index acc33f0cc..bc8ccd880 100644 --- a/packages/common/tests/db/schema/Table.test.ts +++ b/packages/common/tests/db/schema/Table.test.ts @@ -132,18 +132,18 @@ describe('Table', () => { it('should handle options', () => { function createTable(options: TableV2Options) { - return new Table({name: column.text}, options); + return new Table({ name: column.text }, options); } expect(createTable({}).toJSON().include_metadata).toBe(false); - expect(createTable({includeMetadata: true}).toJSON().include_metadata).toBe(true); + expect(createTable({ includeMetadata: true }).toJSON().include_metadata).toBe(true); - expect(createTable({includeOld: true}).toJSON().include_old).toBe(true); - expect(createTable({includeOld: true}).toJSON().include_old_only_when_changed).toBe(false); - expect(createTable({includeOld: 'when-changed'}).toJSON().include_old).toBe(true); - expect(createTable({includeOld: 'when-changed'}).toJSON().include_old_only_when_changed).toBe(true); + expect(createTable({ includeOld: true }).toJSON().include_old).toBe(true); + expect(createTable({ includeOld: true }).toJSON().include_old_only_when_changed).toBe(false); + expect(createTable({ includeOld: 'when-changed' }).toJSON().include_old).toBe(true); + expect(createTable({ includeOld: 'when-changed' }).toJSON().include_old_only_when_changed).toBe(true); - expect(createTable({ignoreEmptyUpdate: true}).toJSON().ignore_empty_update).toBe(true); + expect(createTable({ ignoreEmptyUpdate: true }).toJSON().ignore_empty_update).toBe(true); }); describe('validate', () => { @@ -196,23 +196,32 @@ describe('Table', () => { it('should throw an error for local-only tables with metadata', () => { expect(() => - new Table({ - name: column.text, - }, { localOnly: true, includeMetadata: true }).validate() + new Table( + { + name: column.text + }, + { localOnly: true, includeMetadata: true } + ).validate() ).toThrowError("Can't include metadata for local-only tables."); }); it('should throw an error for local-only tables tracking old values', () => { expect(() => - new Table({ - name: column.text, - }, { localOnly: true, includeOld: true }).validate() + new Table( + { + name: column.text + }, + { localOnly: true, includeOld: true } + ).validate() ).toThrowError("Can't include old values for local-only tables."); expect(() => - new Table({ - name: column.text, - }, { localOnly: true, includeOld: 'when-changed' }).validate() + new Table( + { + name: column.text + }, + { localOnly: true, includeOld: 'when-changed' } + ).validate() ).toThrowError("Can't include old values for local-only tables."); }); }); diff --git a/packages/node/tests/crud.test.ts b/packages/node/tests/crud.test.ts new file mode 100644 index 000000000..15fb8c6c5 --- /dev/null +++ b/packages/node/tests/crud.test.ts @@ -0,0 +1,78 @@ +import { expect } from 'vitest'; +import { column, Schema, Table } from '@powersync/common'; +import { databaseTest } from './utils'; + +databaseTest('include metadata', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text + }, + { includeMetadata: true } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name, _metadata) VALUES (uuid(), ?, ?);', ['entry', 'so meta']); + + const batch = await database.getNextCrudTransaction(); + expect(batch?.crud[0].metadata).toBe('so meta'); +}); + +databaseTest('include old values', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text + }, + { includeOld: true } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name) VALUES (uuid(), ?);', ['entry']); + await database.execute('DELETE FROM ps_crud;'); + await database.execute('UPDATE lists SET name = ?', ['new name']); + + const batch = await database.getNextCrudTransaction(); + expect(batch?.crud[0].oldData).toStrictEqual({name: 'entry'}); +}); + +databaseTest('include old values when changed', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text, + content: column.text + }, + { includeOld: 'when-changed' } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?);', ['name', 'content']); + await database.execute('DELETE FROM ps_crud;'); + await database.execute('UPDATE lists SET name = ?', ['new name']); + + const batch = await database.getNextCrudTransaction(); + expect(batch?.crud[0].oldData).toStrictEqual({name: 'name'}); +}); + +databaseTest('ignore empty update', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text + }, + { ignoreEmptyUpdate: true } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name) VALUES (uuid(), ?);', ['name']); + await database.execute('DELETE FROM ps_crud;'); + await database.execute('UPDATE lists SET name = ?', ['name']); + + const batch = await database.getNextCrudTransaction(); + expect(batch).toBeNull(); +}); From 85b39bd17d5f6d55f84ca1ed68839e8e8936b630 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 28 Apr 2025 15:02:42 +0200 Subject: [PATCH 3/5] Support column filters --- packages/common/src/db/schema/Table.ts | 20 ++++++++++++++--- packages/common/tests/db/schema/Table.test.ts | 11 +++++++--- packages/node/tests/crud.test.ts | 22 ++++++++++++++++++- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/packages/common/src/db/schema/Table.ts b/packages/common/src/db/schema/Table.ts index 8c51fb31e..637094e2f 100644 --- a/packages/common/src/db/schema/Table.ts +++ b/packages/common/src/db/schema/Table.ts @@ -14,11 +14,23 @@ interface SharedTableOptions { localOnly?: boolean; insertOnly?: boolean; viewName?: string; - includeOld?: boolean | 'when-changed'; + includeOld?: boolean | IncludeOldOptions; includeMetadata?: boolean; ignoreEmptyUpdate?: boolean; } +/** Whether to include old columns when PowerSync tracks local changes. + * + * Including old columns may be helpful for some backend connector implementations, which is + * why it can be enabled on per-table or per-columm basis. + */ +export interface IncludeOldOptions { + /** When defined, a list of column names for which old values should be tracked. */ + columns?: string[]; + /** When enabled, only include values that have actually been changed by an update. */ + onlyWhenChanged?: boolean; +} + export interface TableOptions extends SharedTableOptions { /** * The synced table name, matching sync rules @@ -329,13 +341,15 @@ export class Table { } toJSON() { + const includeOld = this.includeOld; + return { name: this.name, view_name: this.viewName, local_only: this.localOnly, insert_only: this.insertOnly, - include_old: this.includeOld != false, - include_old_only_when_changed: this.includeOld == 'when-changed', + include_old: includeOld && ((includeOld as any).columns ?? true), + include_old_only_when_changed: typeof includeOld == 'object' && includeOld.onlyWhenChanged == true, include_metadata: this.includeMetadata, ignore_empty_update: this.ignoreEmptyUpdate, columns: this.columns.map((c) => c.toJSON()), diff --git a/packages/common/tests/db/schema/Table.test.ts b/packages/common/tests/db/schema/Table.test.ts index bc8ccd880..3678719a1 100644 --- a/packages/common/tests/db/schema/Table.test.ts +++ b/packages/common/tests/db/schema/Table.test.ts @@ -140,8 +140,13 @@ describe('Table', () => { expect(createTable({ includeOld: true }).toJSON().include_old).toBe(true); expect(createTable({ includeOld: true }).toJSON().include_old_only_when_changed).toBe(false); - expect(createTable({ includeOld: 'when-changed' }).toJSON().include_old).toBe(true); - expect(createTable({ includeOld: 'when-changed' }).toJSON().include_old_only_when_changed).toBe(true); + + const complexIncldueOld = createTable({ includeOld: { + columns: ['foo', 'bar'], + onlyWhenChanged: true, + } }); + expect(complexIncldueOld.toJSON().include_old).toStrictEqual(['foo', 'bar']); + expect(complexIncldueOld.toJSON().include_old_only_when_changed).toBe(true); expect(createTable({ ignoreEmptyUpdate: true }).toJSON().ignore_empty_update).toBe(true); }); @@ -220,7 +225,7 @@ describe('Table', () => { { name: column.text }, - { localOnly: true, includeOld: 'when-changed' } + { localOnly: true, includeOld: { onlyWhenChanged: false } } ).validate() ).toThrowError("Can't include old values for local-only tables."); }); diff --git a/packages/node/tests/crud.test.ts b/packages/node/tests/crud.test.ts index 15fb8c6c5..b2e50e066 100644 --- a/packages/node/tests/crud.test.ts +++ b/packages/node/tests/crud.test.ts @@ -38,6 +38,26 @@ databaseTest('include old values', async ({ database }) => { expect(batch?.crud[0].oldData).toStrictEqual({name: 'entry'}); }); +databaseTest('include old values with column filter', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text, + content: column.text + }, + { includeOld: { columns: ['name'] } } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?);', ['name', 'content']); + await database.execute('DELETE FROM ps_crud;'); + await database.execute('UPDATE lists SET name = ?, content = ?', ['new name', 'new content']); + + const batch = await database.getNextCrudTransaction(); + expect(batch?.crud[0].oldData).toStrictEqual({name: 'name'}); +}); + databaseTest('include old values when changed', async ({ database }) => { await database.init(); const schema = new Schema({ @@ -46,7 +66,7 @@ databaseTest('include old values when changed', async ({ database }) => { name: column.text, content: column.text }, - { includeOld: 'when-changed' } + { includeOld: { onlyWhenChanged: true } } ) }); await database.updateSchema(schema); From 32b2413b742418ee1949273a62cd74f373768856 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 5 May 2025 14:14:05 +0200 Subject: [PATCH 4/5] Apply new naming scheme --- .changeset/giant-ladybugs-dress.md | 6 +-- .../src/client/sync/bucket/CrudEntry.ts | 4 +- packages/common/src/db/schema/Table.ts | 52 +++++++++---------- packages/common/tests/db/schema/Table.test.ts | 16 +++--- packages/node/tests/crud.test.ts | 10 ++-- 5 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.changeset/giant-ladybugs-dress.md b/.changeset/giant-ladybugs-dress.md index 07f45b985..92c919230 100644 --- a/.changeset/giant-ladybugs-dress.md +++ b/.changeset/giant-ladybugs-dress.md @@ -2,7 +2,7 @@ '@powersync/common': minor --- -- Add `includeOld` option on `Table` which sets `CrudEntry.oldData` to previous values on updates. -- Add `includeMetadata` option on `Table` which adds a `_metadata` column that can be used for updates. +- Add `trackOld` option on `Table` which sets `CrudEntry.oldData` to previous values on updates. +- Add `trackMetadata` option on `Table` which adds a `_metadata` column that can be used for updates. The configured metadata is available through `CrudEntry.metadata`. -- Add `ignoreEmptyUpdate` option which skips creating CRUD entries for updates that don't change any values. +- Add `ignoreEmptyUpdates` option which skips creating CRUD entries for updates that don't change any values. diff --git a/packages/common/src/client/sync/bucket/CrudEntry.ts b/packages/common/src/client/sync/bucket/CrudEntry.ts index edc1bdeec..4007e79bf 100644 --- a/packages/common/src/client/sync/bucket/CrudEntry.ts +++ b/packages/common/src/client/sync/bucket/CrudEntry.ts @@ -66,7 +66,7 @@ export class CrudEntry { opData?: Record; /** - * For tables where the `includeOld` option has been enabled, this tracks previous values for + * For tables where the `trackOld` option has been enabled, this tracks previous values for * `UPDATE` and `DELETE` statements. */ oldData?: Record; @@ -83,7 +83,7 @@ export class CrudEntry { /** * Client-side metadata attached with this write. * - * This field is only available when the `includeMetadata` option was set to `true` when creating a table + * This field is only available when the `trackMetadata` option was set to `true` when creating a table * and the insert or update statement set the `_metadata` column. */ metadata?: string; diff --git a/packages/common/src/db/schema/Table.ts b/packages/common/src/db/schema/Table.ts index 637094e2f..c56da93b4 100644 --- a/packages/common/src/db/schema/Table.ts +++ b/packages/common/src/db/schema/Table.ts @@ -14,9 +14,9 @@ interface SharedTableOptions { localOnly?: boolean; insertOnly?: boolean; viewName?: string; - includeOld?: boolean | IncludeOldOptions; - includeMetadata?: boolean; - ignoreEmptyUpdate?: boolean; + trackOld?: boolean | TrackOldOptions; + trackMetadata?: boolean; + ignoreEmptyUpdates?: boolean; } /** Whether to include old columns when PowerSync tracks local changes. @@ -24,7 +24,7 @@ interface SharedTableOptions { * Including old columns may be helpful for some backend connector implementations, which is * why it can be enabled on per-table or per-columm basis. */ -export interface IncludeOldOptions { +export interface TrackOldOptions { /** When defined, a list of column names for which old values should be tracked. */ columns?: string[]; /** When enabled, only include values that have actually been changed by an update. */ @@ -56,9 +56,9 @@ export const DEFAULT_TABLE_OPTIONS = { indexes: [], insertOnly: false, localOnly: false, - includeOld: false, - includeMetadata: false, - ignoreEmptyUpdate: false + trackOld: false, + trackMetadata: false, + ignoreEmptyUpdates: false }; export const InvalidSQLCharacters = /["'%,.#\s[\]]/; @@ -200,9 +200,9 @@ export class Table { viewName: options?.viewName, insertOnly: options?.insertOnly, localOnly: options?.localOnly, - includeOld: options?.includeOld, - includeMetadata: options?.includeMetadata, - ignoreEmptyUpdate: options?.ignoreEmptyUpdate + trackOld: options?.trackOld, + trackMetadata: options?.trackMetadata, + ignoreEmptyUpdates: options?.ignoreEmptyUpdates }; this.applyDefaultOptions(); @@ -212,9 +212,9 @@ export class Table { private applyDefaultOptions() { this.options.insertOnly ??= DEFAULT_TABLE_OPTIONS.insertOnly; this.options.localOnly ??= DEFAULT_TABLE_OPTIONS.localOnly; - this.options.includeOld ??= DEFAULT_TABLE_OPTIONS.includeOld; - this.options.includeMetadata ??= DEFAULT_TABLE_OPTIONS.includeMetadata; - this.options.ignoreEmptyUpdate ??= DEFAULT_TABLE_OPTIONS.ignoreEmptyUpdate; + this.options.trackOld ??= DEFAULT_TABLE_OPTIONS.trackOld; + this.options.trackMetadata ??= DEFAULT_TABLE_OPTIONS.trackMetadata; + this.options.ignoreEmptyUpdates ??= DEFAULT_TABLE_OPTIONS.ignoreEmptyUpdates; } get name() { @@ -255,16 +255,16 @@ export class Table { return this.options.insertOnly!; } - get includeOld() { - return this.options.includeOld!; + get trackOld() { + return this.options.trackOld!; } - get includeMetadata() { - return this.options.includeMetadata!; + get trackMetadata() { + return this.options.trackMetadata!; } - get ignoreEmptyUpdate() { - return this.options.ignoreEmptyUpdate!; + get ignoreEmptyUpdates() { + return this.options.ignoreEmptyUpdates!; } get internalName() { @@ -298,10 +298,10 @@ export class Table { throw new Error(`Table has too many columns. The maximum number of columns is ${MAX_AMOUNT_OF_COLUMNS}.`); } - if (this.includeMetadata && this.localOnly) { + if (this.trackMetadata && this.localOnly) { throw new Error(`Can't include metadata for local-only tables.`); } - if (this.includeOld != false && this.localOnly) { + if (this.trackOld != false && this.localOnly) { throw new Error(`Can't include old values for local-only tables.`); } @@ -341,17 +341,17 @@ export class Table { } toJSON() { - const includeOld = this.includeOld; + const trackOld = this.trackOld; return { name: this.name, view_name: this.viewName, local_only: this.localOnly, insert_only: this.insertOnly, - include_old: includeOld && ((includeOld as any).columns ?? true), - include_old_only_when_changed: typeof includeOld == 'object' && includeOld.onlyWhenChanged == true, - include_metadata: this.includeMetadata, - ignore_empty_update: this.ignoreEmptyUpdate, + include_old: trackOld && ((trackOld as any).columns ?? true), + include_old_only_when_changed: typeof trackOld == 'object' && trackOld.onlyWhenChanged == true, + include_metadata: this.trackMetadata, + ignore_empty_update: this.ignoreEmptyUpdates, columns: this.columns.map((c) => c.toJSON()), indexes: this.indexes.map((e) => e.toJSON(this)) }; diff --git a/packages/common/tests/db/schema/Table.test.ts b/packages/common/tests/db/schema/Table.test.ts index 3678719a1..cae0e9b70 100644 --- a/packages/common/tests/db/schema/Table.test.ts +++ b/packages/common/tests/db/schema/Table.test.ts @@ -136,19 +136,19 @@ describe('Table', () => { } expect(createTable({}).toJSON().include_metadata).toBe(false); - expect(createTable({ includeMetadata: true }).toJSON().include_metadata).toBe(true); + expect(createTable({ trackMetadata: true }).toJSON().include_metadata).toBe(true); - expect(createTable({ includeOld: true }).toJSON().include_old).toBe(true); - expect(createTable({ includeOld: true }).toJSON().include_old_only_when_changed).toBe(false); + expect(createTable({ trackOld: true }).toJSON().include_old).toBe(true); + expect(createTable({ trackOld: true }).toJSON().include_old_only_when_changed).toBe(false); - const complexIncldueOld = createTable({ includeOld: { + const complexIncldueOld = createTable({ trackOld: { columns: ['foo', 'bar'], onlyWhenChanged: true, } }); expect(complexIncldueOld.toJSON().include_old).toStrictEqual(['foo', 'bar']); expect(complexIncldueOld.toJSON().include_old_only_when_changed).toBe(true); - expect(createTable({ ignoreEmptyUpdate: true }).toJSON().ignore_empty_update).toBe(true); + expect(createTable({ ignoreEmptyUpdates: true }).toJSON().ignore_empty_update).toBe(true); }); describe('validate', () => { @@ -205,7 +205,7 @@ describe('Table', () => { { name: column.text }, - { localOnly: true, includeMetadata: true } + { localOnly: true, trackMetadata: true } ).validate() ).toThrowError("Can't include metadata for local-only tables."); }); @@ -216,7 +216,7 @@ describe('Table', () => { { name: column.text }, - { localOnly: true, includeOld: true } + { localOnly: true, trackOld: true } ).validate() ).toThrowError("Can't include old values for local-only tables."); @@ -225,7 +225,7 @@ describe('Table', () => { { name: column.text }, - { localOnly: true, includeOld: { onlyWhenChanged: false } } + { localOnly: true, trackOld: { onlyWhenChanged: false } } ).validate() ).toThrowError("Can't include old values for local-only tables."); }); diff --git a/packages/node/tests/crud.test.ts b/packages/node/tests/crud.test.ts index b2e50e066..f29c9baa5 100644 --- a/packages/node/tests/crud.test.ts +++ b/packages/node/tests/crud.test.ts @@ -9,7 +9,7 @@ databaseTest('include metadata', async ({ database }) => { { name: column.text }, - { includeMetadata: true } + { trackMetadata: true } ) }); await database.updateSchema(schema); @@ -26,7 +26,7 @@ databaseTest('include old values', async ({ database }) => { { name: column.text }, - { includeOld: true } + { trackOld: true } ) }); await database.updateSchema(schema); @@ -46,7 +46,7 @@ databaseTest('include old values with column filter', async ({ database }) => { name: column.text, content: column.text }, - { includeOld: { columns: ['name'] } } + { trackOld: { columns: ['name'] } } ) }); await database.updateSchema(schema); @@ -66,7 +66,7 @@ databaseTest('include old values when changed', async ({ database }) => { name: column.text, content: column.text }, - { includeOld: { onlyWhenChanged: true } } + { trackOld: { onlyWhenChanged: true } } ) }); await database.updateSchema(schema); @@ -85,7 +85,7 @@ databaseTest('ignore empty update', async ({ database }) => { { name: column.text }, - { ignoreEmptyUpdate: true } + { ignoreEmptyUpdates: true } ) }); await database.updateSchema(schema); From 230d11346ad880d75026a592ab89365ddc0fc76c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 6 May 2025 13:22:34 +0200 Subject: [PATCH 5/5] Rename trackOld --- .changeset/giant-ladybugs-dress.md | 2 +- .../src/client/sync/bucket/CrudEntry.ts | 8 +++--- packages/common/src/db/schema/Table.ts | 26 +++++++++---------- packages/common/tests/db/schema/Table.test.ts | 10 +++---- packages/node/tests/crud.test.ts | 12 ++++----- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.changeset/giant-ladybugs-dress.md b/.changeset/giant-ladybugs-dress.md index 92c919230..d6011c9b8 100644 --- a/.changeset/giant-ladybugs-dress.md +++ b/.changeset/giant-ladybugs-dress.md @@ -2,7 +2,7 @@ '@powersync/common': minor --- -- Add `trackOld` option on `Table` which sets `CrudEntry.oldData` to previous values on updates. +- Add `trackPreviousValues` option on `Table` which sets `CrudEntry.previousValues` to previous values on updates. - Add `trackMetadata` option on `Table` which adds a `_metadata` column that can be used for updates. The configured metadata is available through `CrudEntry.metadata`. - Add `ignoreEmptyUpdates` option which skips creating CRUD entries for updates that don't change any values. diff --git a/packages/common/src/client/sync/bucket/CrudEntry.ts b/packages/common/src/client/sync/bucket/CrudEntry.ts index 4007e79bf..1d0133486 100644 --- a/packages/common/src/client/sync/bucket/CrudEntry.ts +++ b/packages/common/src/client/sync/bucket/CrudEntry.ts @@ -66,10 +66,10 @@ export class CrudEntry { opData?: Record; /** - * For tables where the `trackOld` option has been enabled, this tracks previous values for + * For tables where the `trackPreviousValues` option has been enabled, this tracks previous values for * `UPDATE` and `DELETE` statements. */ - oldData?: Record; + previousValues?: Record; /** * Table that contained the change. @@ -109,7 +109,7 @@ export class CrudEntry { id: string, transactionId?: number, opData?: Record, - oldData?: Record, + previousValues?: Record, metadata?: string ) { this.clientId = clientId; @@ -118,7 +118,7 @@ export class CrudEntry { this.opData = opData; this.table = table; this.transactionId = transactionId; - this.oldData = oldData; + this.previousValues = previousValues; this.metadata = metadata; } diff --git a/packages/common/src/db/schema/Table.ts b/packages/common/src/db/schema/Table.ts index c56da93b4..ae729681d 100644 --- a/packages/common/src/db/schema/Table.ts +++ b/packages/common/src/db/schema/Table.ts @@ -14,17 +14,17 @@ interface SharedTableOptions { localOnly?: boolean; insertOnly?: boolean; viewName?: string; - trackOld?: boolean | TrackOldOptions; + trackPrevious?: boolean | TrackPreviousOptions; trackMetadata?: boolean; ignoreEmptyUpdates?: boolean; } -/** Whether to include old columns when PowerSync tracks local changes. +/** Whether to include previous column values when PowerSync tracks local changes. * - * Including old columns may be helpful for some backend connector implementations, which is + * Including old values may be helpful for some backend connector implementations, which is * why it can be enabled on per-table or per-columm basis. */ -export interface TrackOldOptions { +export interface TrackPreviousOptions { /** When defined, a list of column names for which old values should be tracked. */ columns?: string[]; /** When enabled, only include values that have actually been changed by an update. */ @@ -56,7 +56,7 @@ export const DEFAULT_TABLE_OPTIONS = { indexes: [], insertOnly: false, localOnly: false, - trackOld: false, + trackPrevious: false, trackMetadata: false, ignoreEmptyUpdates: false }; @@ -200,7 +200,7 @@ export class Table { viewName: options?.viewName, insertOnly: options?.insertOnly, localOnly: options?.localOnly, - trackOld: options?.trackOld, + trackPrevious: options?.trackPrevious, trackMetadata: options?.trackMetadata, ignoreEmptyUpdates: options?.ignoreEmptyUpdates }; @@ -212,7 +212,7 @@ export class Table { private applyDefaultOptions() { this.options.insertOnly ??= DEFAULT_TABLE_OPTIONS.insertOnly; this.options.localOnly ??= DEFAULT_TABLE_OPTIONS.localOnly; - this.options.trackOld ??= DEFAULT_TABLE_OPTIONS.trackOld; + this.options.trackPrevious ??= DEFAULT_TABLE_OPTIONS.trackPrevious; this.options.trackMetadata ??= DEFAULT_TABLE_OPTIONS.trackMetadata; this.options.ignoreEmptyUpdates ??= DEFAULT_TABLE_OPTIONS.ignoreEmptyUpdates; } @@ -255,8 +255,8 @@ export class Table { return this.options.insertOnly!; } - get trackOld() { - return this.options.trackOld!; + get trackPrevious() { + return this.options.trackPrevious!; } get trackMetadata() { @@ -301,7 +301,7 @@ export class Table { if (this.trackMetadata && this.localOnly) { throw new Error(`Can't include metadata for local-only tables.`); } - if (this.trackOld != false && this.localOnly) { + if (this.trackPrevious != false && this.localOnly) { throw new Error(`Can't include old values for local-only tables.`); } @@ -341,15 +341,15 @@ export class Table { } toJSON() { - const trackOld = this.trackOld; + const trackPrevious = this.trackPrevious; return { name: this.name, view_name: this.viewName, local_only: this.localOnly, insert_only: this.insertOnly, - include_old: trackOld && ((trackOld as any).columns ?? true), - include_old_only_when_changed: typeof trackOld == 'object' && trackOld.onlyWhenChanged == true, + include_old: trackPrevious && ((trackPrevious as any).columns ?? true), + include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true, include_metadata: this.trackMetadata, ignore_empty_update: this.ignoreEmptyUpdates, columns: this.columns.map((c) => c.toJSON()), diff --git a/packages/common/tests/db/schema/Table.test.ts b/packages/common/tests/db/schema/Table.test.ts index cae0e9b70..f5a51560e 100644 --- a/packages/common/tests/db/schema/Table.test.ts +++ b/packages/common/tests/db/schema/Table.test.ts @@ -138,10 +138,10 @@ describe('Table', () => { expect(createTable({}).toJSON().include_metadata).toBe(false); expect(createTable({ trackMetadata: true }).toJSON().include_metadata).toBe(true); - expect(createTable({ trackOld: true }).toJSON().include_old).toBe(true); - expect(createTable({ trackOld: true }).toJSON().include_old_only_when_changed).toBe(false); + expect(createTable({ trackPrevious: true }).toJSON().include_old).toBe(true); + expect(createTable({ trackPrevious: true }).toJSON().include_old_only_when_changed).toBe(false); - const complexIncldueOld = createTable({ trackOld: { + const complexIncldueOld = createTable({ trackPrevious: { columns: ['foo', 'bar'], onlyWhenChanged: true, } }); @@ -216,7 +216,7 @@ describe('Table', () => { { name: column.text }, - { localOnly: true, trackOld: true } + { localOnly: true, trackPrevious: true } ).validate() ).toThrowError("Can't include old values for local-only tables."); @@ -225,7 +225,7 @@ describe('Table', () => { { name: column.text }, - { localOnly: true, trackOld: { onlyWhenChanged: false } } + { localOnly: true, trackPrevious: { onlyWhenChanged: false } } ).validate() ).toThrowError("Can't include old values for local-only tables."); }); diff --git a/packages/node/tests/crud.test.ts b/packages/node/tests/crud.test.ts index f29c9baa5..d9f35c57d 100644 --- a/packages/node/tests/crud.test.ts +++ b/packages/node/tests/crud.test.ts @@ -26,7 +26,7 @@ databaseTest('include old values', async ({ database }) => { { name: column.text }, - { trackOld: true } + { trackPrevious: true } ) }); await database.updateSchema(schema); @@ -35,7 +35,7 @@ databaseTest('include old values', async ({ database }) => { await database.execute('UPDATE lists SET name = ?', ['new name']); const batch = await database.getNextCrudTransaction(); - expect(batch?.crud[0].oldData).toStrictEqual({name: 'entry'}); + expect(batch?.crud[0].previousValues).toStrictEqual({name: 'entry'}); }); databaseTest('include old values with column filter', async ({ database }) => { @@ -46,7 +46,7 @@ databaseTest('include old values with column filter', async ({ database }) => { name: column.text, content: column.text }, - { trackOld: { columns: ['name'] } } + { trackPrevious: { columns: ['name'] } } ) }); await database.updateSchema(schema); @@ -55,7 +55,7 @@ databaseTest('include old values with column filter', async ({ database }) => { await database.execute('UPDATE lists SET name = ?, content = ?', ['new name', 'new content']); const batch = await database.getNextCrudTransaction(); - expect(batch?.crud[0].oldData).toStrictEqual({name: 'name'}); + expect(batch?.crud[0].previousValues).toStrictEqual({name: 'name'}); }); databaseTest('include old values when changed', async ({ database }) => { @@ -66,7 +66,7 @@ databaseTest('include old values when changed', async ({ database }) => { name: column.text, content: column.text }, - { trackOld: { onlyWhenChanged: true } } + { trackPrevious: { onlyWhenChanged: true } } ) }); await database.updateSchema(schema); @@ -75,7 +75,7 @@ databaseTest('include old values when changed', async ({ database }) => { await database.execute('UPDATE lists SET name = ?', ['new name']); const batch = await database.getNextCrudTransaction(); - expect(batch?.crud[0].oldData).toStrictEqual({name: 'name'}); + expect(batch?.crud[0].previousValues).toStrictEqual({name: 'name'}); }); databaseTest('ignore empty update', async ({ database }) => {