From 033c4015432705710aef7a7f42dd990e0beaa30a Mon Sep 17 00:00:00 2001 From: ty walch Date: Mon, 17 Feb 2025 13:49:12 -0500 Subject: [PATCH 1/5] Adds execution options `seek` and `atleast` --- index.d.ts | 41 ++++--- src/entity.js | 59 +++++++--- test/pagination.spec.ts | 247 ++++++++++++++++++++++++++++++++++++++++ test/test-utils.ts | 204 +++++++++++++++++++++++++++++++++ 4 files changed, 521 insertions(+), 30 deletions(-) create mode 100644 test/pagination.spec.ts create mode 100644 test/test-utils.ts diff --git a/index.d.ts b/index.d.ts index f3ac8e97..b91c4874 100644 --- a/index.d.ts +++ b/index.d.ts @@ -19,21 +19,27 @@ type TransactGetCommandInput = { TransactItems: TransactGetItem[]; }; +export type DocumentClientV2 = { + get: DocumentClientMethod; + put: DocumentClientMethod; + delete: DocumentClientMethod; + update: DocumentClientMethod; + batchWrite: DocumentClientMethod; + batchGet: DocumentClientMethod; + scan: DocumentClientMethod; + transactGet: DocumentClientMethod; + transactWrite: DocumentClientMethod; + query: DocumentClientMethod; + createSet: (...params: any[]) => any; +}; + +export type DocumentClientV3 = { + send: (command: any) => Promise; +}; + export type DocumentClient = - | { - get: DocumentClientMethod; - put: DocumentClientMethod; - delete: DocumentClientMethod; - update: DocumentClientMethod; - batchWrite: DocumentClientMethod; - batchGet: DocumentClientMethod; - scan: DocumentClientMethod; - transactGet: DocumentClientMethod; - transactWrite: DocumentClientMethod; - } - | { - send: (command: any) => Promise; - }; + | DocumentClientV2 + | DocumentClientV3; export type AllCollectionNames< E extends { [name: string]: Entity }, @@ -2530,6 +2536,8 @@ export interface QueryOptions { table?: string; limit?: number; count?: number; + seek?: boolean; + atleast?: number; originalErr?: boolean; ignoreOwnership?: boolean; pages?: number | "all"; @@ -2537,7 +2545,6 @@ export interface QueryOptions { logger?: ElectroEventListener; data?: "raw" | "includeKeys" | "attributes"; order?: "asc" | "desc"; - consistent?: boolean; } @@ -2638,6 +2645,8 @@ type ServiceQueryGoTerminalOptions = { data?: "raw" | "includeKeys" | "attributes"; table?: string; limit?: number; + atleast?: number; + seek?: boolean; params?: object; originalErr?: boolean; ignoreOwnership?: boolean; @@ -2655,6 +2664,8 @@ type GoQueryTerminalOptions = { table?: string; limit?: number; count?: number; + atleast?: number; + seek?: boolean; params?: object; originalErr?: boolean; ignoreOwnership?: boolean; diff --git a/src/entity.js b/src/entity.js index 659cddf6..8ca3f180 100644 --- a/src/entity.js +++ b/src/entity.js @@ -375,9 +375,8 @@ class Entity { } upsert(attributes = {}) { - let index = TableIndex; return this._makeChain( - index, + TableIndex, this._clausesWithFilters, clauses.index, ).upsert(attributes); @@ -395,9 +394,8 @@ class Entity { } update(facets = {}) { - let index = TableIndex; return this._makeChain( - index, + TableIndex this._clausesWithFilters, clauses.index, ).update(facets); @@ -426,21 +424,19 @@ class Entity { } async transactWrite(parameters, config) { - let response = await this._exec( + return this._exec( MethodTypes.transactWrite, parameters, config, ); - return response; } async transactGet(parameters, config) { - let response = await this._exec( + return this._exec( MethodTypes.transactGet, parameters, config, ); - return response; } async go(method, parameters = {}, config = {}) { @@ -689,6 +685,7 @@ class Entity { let iterations = 0; let count = 0; let hydratedUnprocessed = []; + let morePaginationRequired = false; const shouldHydrate = config.hydrate && method === MethodTypes.query; do { let response = await this._exec( @@ -761,14 +758,29 @@ class Entity { } else { return response; } + iterations++; - } while ( - ExclusiveStartKey && - (pages === AllPages || - config.count !== undefined || - iterations < pages) && - (config.count === undefined || count < config.count) - ); + + const countOptionRequiresMorePagination = ( + config.count !== undefined && count < config.count + ); + + const pagesOptionRequiresMorePagination = + pages === AllPages || iterations < pages; + + const atleastOptionRequiresMorePagination = + config.atleast !== undefined && !config._isCollectionQuery && results.length < config.atleast; + + const seekOptionRequiresMorePagination = + config.seek && !config._isCollectionQuery && results.length === 0; + + morePaginationRequired = + atleastOptionRequiresMorePagination || + countOptionRequiresMorePagination || + pagesOptionRequiresMorePagination || + seekOptionRequiresMorePagination; + + } while (ExclusiveStartKey && morePaginationRequired); const cursor = this._formatReturnPager(config, ExclusiveStartKey); @@ -1641,6 +1653,8 @@ class Entity { _isPagination: false, _isCollectionQuery: false, pages: 1, + seek: false, + atleast: 0, count: undefined, listeners: [], preserveBatchOrder: false, @@ -1671,6 +1685,21 @@ class Entity { } } + if (option.atleast !== undefined) { + if (isNaN(option.atleast)) { + throw new e.ElectroError( + e.ErrorCodes.InvalidOptions, + `Invalid value for query option "atleast" provided. Unable to parse integer value.`, + ); + } + + config.atleast = parseInt(option.atleast); + } + + if (option.seek) { + config.seek = option.seek; + } + if (typeof option.compare === "string") { const type = ComparisonTypes[option.compare.toLowerCase()]; if (type) { diff --git a/test/pagination.spec.ts b/test/pagination.spec.ts new file mode 100644 index 00000000..b71e7cfe --- /dev/null +++ b/test/pagination.spec.ts @@ -0,0 +1,247 @@ +process.env.AWS_NODEJS_CONNECTION_REUSE_ENABLED = "1"; +import DynamoDB from "aws-sdk/clients/dynamodb"; +import { expect } from "chai"; +import { v4 as uuid } from "uuid"; +import { Entity, DocumentClient, EntityItem } from "../index"; + +const table = 'electro'; + +const client = new DynamoDB.DocumentClient({ + region: "us-east-1", + endpoint: process.env.LOCAL_DYNAMO_ENDPOINT || 'http://localhost:8000', +}); + +type DocumentClientMethod = keyof Extract; + +type DocumentClientSpyCall = { + type: DocumentClientMethod; + params: any; +} + +type DocumentClientSpy = DocumentClient & { + calls: DocumentClientSpyCall[]; +} + +function createDocumentClientSpy(client: DocumentClient): DocumentClientSpy { + if ('send' in client) { + throw new Error('v2 client only for now'); + } + + const calls: DocumentClientSpyCall[] = []; + return { + get calls() { + return calls; + }, + get: (params: any) => (calls.push({type: 'get', params}), client.get(params)), + put: (params: any) => (calls.push({type: 'put', params}), client.put(params)), + delete: (params: any) => (calls.push({type: 'delete', params}), client.delete(params)), + update: (params: any) => (calls.push({type: 'update', params}), client.update(params)), + batchWrite: (params: any) => (calls.push({type: 'batchWrite', params}), client.batchWrite(params)), + batchGet: (params: any) => (calls.push({type: 'batchGet', params}), client.batchGet(params)), + scan: (params: any) => (calls.push({type: 'scan', params}), client.scan(params)), + transactGet: (params: any) => (calls.push({type: 'transactGet', params}), client.transactGet(params)), + transactWrite: (params: any) => (calls.push({type: 'transactWrite', params}), client.transactWrite(params)), + query: (params: any) => (calls.push({type: 'query', params}), client.query(params)), + createSet: (params: any) => (calls.push({type: 'createSet', params}), client.createSet(params)), + } +} + +function createObjectOfSize(kb: number): object { + const sizeInBytes = kb * 1024; + const obj: { [key: string]: string } = {}; + let currentSize = 0; + + while (currentSize < sizeInBytes) { + const key = `key${currentSize}`; + const value = "a".repeat(1024); // 1 KB string + obj[key] = value; + currentSize += key.length + value.length; + } + + return obj; +} + +function createEntity(table: string, client: DocumentClient) { + return new Entity({ + model: { + entity: 'paginator', + version: '0', + service: 'pagination-test' + }, + attributes: { + id: { + type: 'string' + }, + index: { + type: 'number', + required: true, + }, + data: { + type: 'any' + }, + }, + indexes: { + record: { + pk: { + field: 'pk', + composite: ['id'], + }, + sk: { + field: 'sk', + composite: ['index'], + }, + } + }, + }, { client, table }); +} + +type PaginatorEntity = ReturnType; + +type PaginatorItem = EntityItem; + +function createItems(count: number, size: number, id: string = uuid()): PaginatorItem[] { + let index = 0; + return Array.from({ length: count }, () => ({ + id: id, + index: index++, + data: createObjectOfSize(size), + })); +} + +describe('entity pagination', () => { + const id = uuid(); + // 100 items of 100 KB each + const items = createItems(100, 100, id) + + before(async () => { + const entity = createEntity(table, client); + await entity.put(items).go(); + }); + + describe('when using seek', () => { + it('should only paginate once if there are no results', async () => { + const clientSpy = createDocumentClientSpy(client); + const entity = createEntity(table, clientSpy); + + // use an id that doesn't exist + const result = await entity.query + .record({ id: 'unknown' }) + .go({ seek: true }); + + // nothing should be returned + expect(result).to.deep.equal({ data: [], cursor: null }); + // only one call should have been made (didn't try to paginate) + expect(clientSpy.calls.length).to.equal(1); + }); + + it('should only paginate once if there is only one page of results', async () => { + const id = uuid(); + const clientSpy = createDocumentClientSpy(client); + const entity = createEntity(table, clientSpy); + // 20 items of 1 KB each + const items = createItems(20, 1, id); + await entity.put(items).go(); + + const result = await entity.query + .record({ id }) + .go({ seek: true }); + + // all items should be returned in one request + expect(result.data).to.have.length(items.length); + // only one "query" call should have been made (didn't try to paginate) + expect(clientSpy.calls.filter(({ type }) => type === 'query')).to.have.length(1); + }); + + it('should paginate multiple times if a cursor is returned but results are not', async () => { + const clientSpy = createDocumentClientSpy(client); + const entity = createEntity(table, clientSpy); + + // the items actually returned from DynamoDB + const returned: any[][] = []; + const result = await entity.query + .record({ id }) + .where(({ index }, { gt }) => gt(index, 60)) + .go({ + seek: true, + params: { Limit: 5 }, + logger: (event) => { + if (event.type === 'results') { + returned.push(event.results.Items); + } + } + }); + + // it should return items (the amount doesn't matter) + expect(result.data.length).to.be.greaterThan(0); + // it should have made more than one call to get results + expect(clientSpy.calls.length).to.be.greaterThan(1); + // the first call should have returned 0 items (the user need behind 'seek') + expect(returned[0]).to.have.length(0); + }); + }); + + describe('when using atleast', () => { + it('should throw if the atleast option cannot be parsed as a number', async () => { + const entity = createEntity(table, client); + const result = await entity.query + .record({ id: 'unknown', run: 'unknown' }) + // @ts-expect-error + .go({ atleast: 'abc' }) + .then(() => null) + .catch((err: any) => err); + + expect(result.message).to.equal('Invalid value for query option "atleast" provided. Unable to parse integer value. - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-options'); + }); + + it('should continue to paginate until at least the number of request items are returned', async () => { + const clientSpy = createDocumentClientSpy(client); + const entity = createEntity(table, clientSpy); + + const result = await entity.query + .record({ id }) + .go({ + atleast: 11, + params: { Limit: 5 }, + }); + + expect(result.data.length).to.be.greaterThan(10); + expect(clientSpy.calls.length).to.be.greaterThan(2); + }); + }); + + describe('when using count', () => { + it('should only return 10 items to the user', async () => { + const clientSpy = createDocumentClientSpy(client); + const entity = createEntity(table, clientSpy); + let returned: any = null; + const result = await entity.query + .record({ id }) + .go({ + count: 10, + logger: (event) => { + if (event.type === 'results') { + returned = event.results.Items; + } + } + }); + + expect(result.data).to.have.length(10); + expect(returned).to.not.be.null; + expect(returned.length).to.be.greaterThan(10); + }); + }); + + describe('when using pages', () => { + it('should query all items when using pages all', async () => { + const clientSpy = createDocumentClientSpy(client); + const entity = createEntity(table, clientSpy); + + const result = await entity.query + .record({ id }) + .go({ pages: 'all' }); + + expect(result.data.sort((a, z) => a.index - z.index)).to.deep.equal(items.sort((a, z) => a.index - z.index)); + expect(clientSpy.calls).to.have.length(10); + }); + }) +}); \ No newline at end of file diff --git a/test/test-utils.ts b/test/test-utils.ts new file mode 100644 index 00000000..cc267adb --- /dev/null +++ b/test/test-utils.ts @@ -0,0 +1,204 @@ +import { + DocumentClient, + DocumentClientMethod, + ElectroEventListener, + ElectroDBMethodTypes, + ElectroQueryEvent, + ElectroResultsEvent, + ElectroEvent, +} from "../index"; + +import { + ScanOutput, + QueryOutput, + PutItemOutput, + GetItemOutput, + UpdateItemOutput, + DeleteItemOutput, + BatchGetItemOutput, + BatchWriteItemOutput, + TransactGetItemsOutput, + TransactWriteItemsOutput, + DocumentClient as V2Client +} from "aws-sdk/clients/dynamodb"; + +import { + DynamoDBClient as V3Client +} from "@aws-sdk/client-dynamodb/dist-types/DynamoDBClient"; + +export const DYNAMODB_ENDPOINT = process.env.LOCAL_DYNAMO_ENDPOINT || 'http://loalhost:8000'; + +export const v2Client = new V2Client({ + region: "us-east-1", + endpoint: DYNAMODB_ENDPOINT, +}); + +export const v3Client = new V3Client({ + region: "us-east-1", + endpoint: DYNAMODB_ENDPOINT, +}); + +export type V2DocumentClient = Extract; + +export function createObjectOfSize(kb: number): object { + const sizeInBytes = kb * 1024; + const obj: { [key: string]: string } = {}; + let currentSize = 0; + + while (currentSize < sizeInBytes) { + const key = `key${currentSize}`; + const value = "a".repeat(1024); // 1 KB string + obj[key] = value; + currentSize += key.length + value.length; + } + + return obj; +} + +export type V2DocumentClientMethodName = keyof V2DocumentClient; + +export type V2DocumentClientSpyCall = { + type: V2DocumentClientMethodName; + result: any; + params: any; +} + +export type V2DocumentClientSpy = V2DocumentClient & { + calls: V2DocumentClientSpyCall[]; +} + +export function createV2DocumentClientSpy(client: V2DocumentClient): V2DocumentClientSpy { + if ('send' in client) { + throw new Error('v2 client only for now'); + } + + const calls: V2DocumentClientSpyCall[] = []; + + function createSpyFn(method: V2DocumentClientMethodName) : DocumentClientMethod { + return (params: any) => { + return { + promise: async () => { + if (method in client) { + const result = await client[method](params).promise(); + calls.push({ type: method, params, result }); + return result; + } + + throw new Error(`Method ${method} not found in client`); + } + } + } + } + + return { + get calls() { + return calls; + }, + get: createSpyFn('get'), + put: createSpyFn('put'), + delete: createSpyFn('delete'), + update: createSpyFn('update'), + batchWrite: createSpyFn('batchWrite'), + batchGet: createSpyFn('batchGet'), + scan: createSpyFn('scan'), + transactGet: createSpyFn('transactGet'), + transactWrite: createSpyFn('transactWrite'), + query: createSpyFn('query'), + createSet: (...options: any[]) => client.createSet(...options), + } +} + +export type CreateDataFnOptions = { + index: number; + data: object; +} + +export type CreateDataFn = (options: CreateDataFnOptions) => T; + +export function createData(count: number, size: number, fn: CreateDataFn): T[] { + return Array.from({ length: count }, () => createObjectOfSize(size)) + .map((data, index) => fn({ index, data })); +} + +export type CreateV2DocumentClientStubOptions = { + get?: GetItemOutput[]; + put?: PutItemOutput[]; + delete?: DeleteItemOutput[]; + update?: UpdateItemOutput[]; + batchWrite?: BatchWriteItemOutput[]; + batchGet?: BatchGetItemOutput[]; + scan?: ScanOutput[]; + transactGet?: TransactGetItemsOutput[]; + transactWrite?: TransactWriteItemsOutput[]; + query?: QueryOutput[]; + createSet?: any[]; +} + +export function createV2DocumentClientStub(options: CreateV2DocumentClientStubOptions): V2DocumentClient { + function createStubFn(method: V2DocumentClientMethodName, stubbed: any[] = []) : DocumentClientMethod { + return () => { + return { + promise: async () => { + if (stubbed.length === 0) { + throw new Error(`No stub found for method '${method}'`); + } + + return [...stubbed].shift(); + } + } + } + } + + return { + get: createStubFn('get', options.get), + put: createStubFn('put', options.put), + delete: createStubFn('delete', options.delete), + update: createStubFn('update', options.update), + batchWrite: createStubFn('batchWrite', options.batchWrite), + batchGet: createStubFn('batchGet', options.batchGet), + scan: createStubFn('scan', options.scan), + transactGet: createStubFn('transactGet', options.transactGet), + transactWrite: createStubFn('transactWrite', options.transactWrite), + query: createStubFn('query', options.query), + createSet: () => [], + } +} + +export function createDebugLogger(label: string, filters: ElectroDBMethodTypes[] = []): ElectroEventListener { + return (event) => { + if (filters.length > 0 && !filters.includes(event.method)) { + return; + } else if (event.type === 'query') { + console.log(label, JSON.stringify(event.params, null, 4)); + } else { + console.log(label, JSON.stringify(event.results, null, 4)); + } + } +} + +export type EventCollector = ElectroEventListener & { + queries: ElectroQueryEvent[], + results: ElectroResultsEvent[], + calls: ElectroEvent[]; +} + +export function createEventCollector(): EventCollector { + const calls: ElectroEvent[] = []; + const queries: ElectroQueryEvent[] = [] + const results: ElectroResultsEvent[] = []; + const collector = (event: ElectroEvent) => { + calls.push(event); + + if (event.type === 'query') { + queries.push(event); + } else { + results.push(event); + } + } + + collector.queries = queries; + collector.results = results; + collector.calls = calls; + + return collector; +} \ No newline at end of file From c72ff669f9d7e3e79efe8fb95d8e19e666f51eca Mon Sep 17 00:00:00 2001 From: ty walch Date: Tue, 18 Feb 2025 13:08:11 -0500 Subject: [PATCH 2/5] Renames `atleast` to `until` --- CHANGELOG.md | 8 ++++- index.d.ts | 9 +++--- package.json | 2 +- src/entity.js | 22 +++++++------- test/test-utils.ts | 17 +++++++---- ...pec.ts => ts_connected.pagination.spec.ts} | 30 +++++++++++++++---- www/src/pages/en/reference/events-logging.mdx | 27 +++++++++++++++-- www/src/partials/query-options.mdx | 10 +++++++ 8 files changed, 96 insertions(+), 29 deletions(-) rename test/{pagination.spec.ts => ts_connected.pagination.spec.ts} (88%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5a290be..bf767ae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -579,4 +579,10 @@ All notable changes to this project will be documented in this file. Breaking ch ## [3.4.1] ### Fixed -- [Issue #475](https://github.com/tywalch/electrodb/issues/475); Fixes issue where some users reported errors exporting entities and/or types when using the `CustomAttributeType` function. They would receive an error similar to `Exported variable '...' has or is using name 'OpaquePrimitiveSymbol' from external module "..." but cannot be named.`. \ No newline at end of file +- [Issue #475](https://github.com/tywalch/electrodb/issues/475); Fixes issue where some users reported errors exporting entities and/or types when using the `CustomAttributeType` function. They would receive an error similar to `Exported variable '...' has or is using name 'OpaquePrimitiveSymbol' from external module "..." but cannot be named.`. + +## [3.5.0] +### Added +- New `seek` execution option instructs ElectroDB to continue querying DynamoDB until at least one item is returned. This is useful when the circumstances of a table, model, and/or query request results in an empty return. +- New `until` execution option instructs ElectroDB to continue querying DynamoDB until at least the number of items specified is returned. Unlike `count`, this option will not guarantee the exact number of items returned, but will ensure at least the number specified is returned. This option has fewer post-processing steps than `count` and is more performant in cases where the exact number of items is not critical. +- Adds `params` parameter to `ElectroResultsEvent` event. This is useful for debugging, specifically correlating the request and response from DynamoDB. \ No newline at end of file diff --git a/index.d.ts b/index.d.ts index cfa2a183..49e7f29f 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1034,11 +1034,12 @@ export interface ElectroQueryEvent

{ params: P; } -export interface ElectroResultsEvent { +export interface ElectroResultsEvent { type: "results"; method: ElectroDBMethodTypes; config: any; results: R; + params: P; success: boolean; } @@ -2537,7 +2538,7 @@ export interface QueryOptions { limit?: number; count?: number; seek?: boolean; - atleast?: number; + until?: number; originalErr?: boolean; ignoreOwnership?: boolean; pages?: number | "all"; @@ -2645,7 +2646,7 @@ type ServiceQueryGoTerminalOptions = { data?: "raw" | "includeKeys" | "attributes"; table?: string; limit?: number; - atleast?: number; + until?: number; seek?: boolean; params?: object; originalErr?: boolean; @@ -2664,7 +2665,7 @@ type GoQueryTerminalOptions = { table?: string; limit?: number; count?: number; - atleast?: number; + until?: number; seek?: boolean; params?: object; originalErr?: boolean; diff --git a/package.json b/package.json index 07bb0d77..6b9aac96 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "electrodb", - "version": "3.4.1", + "version": "3.5.0", "description": "A library to more easily create and interact with multiple entities and heretical relationships in dynamodb", "main": "index.js", "scripts": { diff --git a/src/entity.js b/src/entity.js index 8ca3f180..efd428e8 100644 --- a/src/entity.js +++ b/src/entity.js @@ -395,17 +395,16 @@ class Entity { update(facets = {}) { return this._makeChain( - TableIndex + TableIndex, this._clausesWithFilters, clauses.index, ).update(facets); } patch(facets = {}) { - let index = TableIndex; let options = {}; return this._makeChain( - index, + TableIndex, this._clausesWithFilters, clauses.index, options, @@ -501,6 +500,7 @@ class Entity { config, success, results, + params, }, config.listeners, ); @@ -768,14 +768,14 @@ class Entity { const pagesOptionRequiresMorePagination = pages === AllPages || iterations < pages; - const atleastOptionRequiresMorePagination = - config.atleast !== undefined && !config._isCollectionQuery && results.length < config.atleast; + const untilOptionRequiresMorePagination = + config.until !== undefined && !config._isCollectionQuery && results.length < config.until; const seekOptionRequiresMorePagination = config.seek && !config._isCollectionQuery && results.length === 0; morePaginationRequired = - atleastOptionRequiresMorePagination || + untilOptionRequiresMorePagination || countOptionRequiresMorePagination || pagesOptionRequiresMorePagination || seekOptionRequiresMorePagination; @@ -1654,7 +1654,7 @@ class Entity { _isCollectionQuery: false, pages: 1, seek: false, - atleast: 0, + until: 0, count: undefined, listeners: [], preserveBatchOrder: false, @@ -1685,15 +1685,15 @@ class Entity { } } - if (option.atleast !== undefined) { - if (isNaN(option.atleast)) { + if (option.until !== undefined) { + if (isNaN(option.until)) { throw new e.ElectroError( e.ErrorCodes.InvalidOptions, - `Invalid value for query option "atleast" provided. Unable to parse integer value.`, + `Invalid value for query option "until" provided. Unable to parse integer value.`, ); } - config.atleast = parseInt(option.atleast); + config.until = parseInt(option.until); } if (option.seek) { diff --git a/test/test-utils.ts b/test/test-utils.ts index cc267adb..34a4197f 100644 --- a/test/test-utils.ts +++ b/test/test-utils.ts @@ -24,7 +24,7 @@ import { import { DynamoDBClient as V3Client -} from "@aws-sdk/client-dynamodb/dist-types/DynamoDBClient"; +} from "@aws-sdk/client-dynamodb"; export const DYNAMODB_ENDPOINT = process.env.LOCAL_DYNAMO_ENDPOINT || 'http://loalhost:8000'; @@ -176,23 +176,30 @@ export function createDebugLogger(label: string, filters: ElectroDBMethodTypes[] } } +export type EventCollectorCall = { + request: ElectroQueryEvent; + response: ElectroResultsEvent; +} + export type EventCollector = ElectroEventListener & { queries: ElectroQueryEvent[], results: ElectroResultsEvent[], - calls: ElectroEvent[]; + calls: EventCollectorCall[]; } export function createEventCollector(): EventCollector { - const calls: ElectroEvent[] = []; + const calls: EventCollectorCall[] = []; const queries: ElectroQueryEvent[] = [] const results: ElectroResultsEvent[] = []; + let current = {} as EventCollectorCall; const collector = (event: ElectroEvent) => { - calls.push(event); - if (event.type === 'query') { queries.push(event); + current.request = event; } else { + current.response = event; results.push(event); + calls.push({...current}); } } diff --git a/test/pagination.spec.ts b/test/ts_connected.pagination.spec.ts similarity index 88% rename from test/pagination.spec.ts rename to test/ts_connected.pagination.spec.ts index b71e7cfe..f29b2a72 100644 --- a/test/pagination.spec.ts +++ b/test/ts_connected.pagination.spec.ts @@ -3,6 +3,7 @@ import DynamoDB from "aws-sdk/clients/dynamodb"; import { expect } from "chai"; import { v4 as uuid } from "uuid"; import { Entity, DocumentClient, EntityItem } from "../index"; +import { createEventCollector } from './test-utils'; const table = 'electro'; @@ -180,17 +181,17 @@ describe('entity pagination', () => { }); }); - describe('when using atleast', () => { - it('should throw if the atleast option cannot be parsed as a number', async () => { + describe('when using until', () => { + it('should throw if the until option cannot be parsed as a number', async () => { const entity = createEntity(table, client); const result = await entity.query .record({ id: 'unknown', run: 'unknown' }) // @ts-expect-error - .go({ atleast: 'abc' }) + .go({ until: 'abc' }) .then(() => null) .catch((err: any) => err); - expect(result.message).to.equal('Invalid value for query option "atleast" provided. Unable to parse integer value. - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-options'); + expect(result.message).to.equal('Invalid value for query option "until" provided. Unable to parse integer value. - For more detail on this error reference: https://electrodb.dev/en/reference/errors/#invalid-options'); }); it('should continue to paginate until at least the number of request items are returned', async () => { @@ -200,13 +201,32 @@ describe('entity pagination', () => { const result = await entity.query .record({ id }) .go({ - atleast: 11, + until: 11, params: { Limit: 5 }, }); expect(result.data.length).to.be.greaterThan(10); expect(clientSpy.calls.length).to.be.greaterThan(2); }); + + it('should abandon until directive when there are no more results', async () => { + const clientSpy = createDocumentClientSpy(client); + const entity = createEntity(table, clientSpy); + + const collector = createEventCollector(); + + const result = await entity.query + .record({ id }) + .go({ + until: items.length + 100, + params: { Limit: 5 }, + logger: collector, + }); + + console.log(JSON.stringify(events, null, 2)); + expect(result.data).to.have.length(items.length); + expect(clientSpy.calls.length).to.be.equal(items.length / 5 + 1); + }); }); describe('when using count', () => { diff --git a/www/src/pages/en/reference/events-logging.mdx b/www/src/pages/en/reference/events-logging.mdx index 3f09c706..000f80eb 100644 --- a/www/src/pages/en/reference/events-logging.mdx +++ b/www/src/pages/en/reference/events-logging.mdx @@ -95,7 +95,7 @@ The `results` event occurs when results are returned from DynamoDB. The event in _Type::_ ```typescript -interface ElectroResultsEvent { +interface ElectroResultsEvent { type: "results"; method: | "put" @@ -112,6 +112,7 @@ interface ElectroResultsEvent { config: any; results: R; success: boolean; + params: P; } ``` @@ -142,7 +143,29 @@ _Example Output:_ "__edb_v__": "1", "pk": "$test_1#prop1_22874c81-27c4-4264-92c3-b280aa79aa30" } - } + }, + "params": { + "UpdateExpression": "SET #prop3 = :prop3_u0, #prop1 = :prop1_u0, #prop2 = :prop2_u0, #__edb_e__ = :__edb_e___u0, #__edb_v__ = :__edb_v___u0", + "ExpressionAttributeNames": { + "#prop3": "prop3", + "#prop1": "prop1", + "#prop2": "prop2", + "#__edb_e__": "__edb_e__", + "#__edb_v__": "__edb_v__" + }, + "ExpressionAttributeValues": { + ":prop3_u0": "3ec9ed0c-7497-4d05-bdb8-86c09a618047", + ":prop1_u0": "22874c81-27c4-4264-92c3-b280aa79aa30", + ":prop2_u0": "366aade8-a7c0-4328-8e14-0331b185de4e", + ":__edb_e___u0": "entity", + ":__edb_v___u0": "1" + }, + "TableName": "electro", + "Key": { + "pk": "$test_1#prop1_22874c81-27c4-4264-92c3-b280aa79aa30", + "sk": "$testcollection#entity_1#prop2_366aade8-a7c0-4328-8e14-0331b185de4e" + } + }, } ``` diff --git a/www/src/partials/query-options.mdx b/www/src/partials/query-options.mdx index 483dae48..40f4aede 100644 --- a/www/src/partials/query-options.mdx +++ b/www/src/partials/query-options.mdx @@ -13,6 +13,8 @@ By default, **ElectroDB** enables you to work with records as the names and prop ignoreOwnership?: boolean; limit?: number; count?: number; + seek?: boolean; + atleast?: number; pages?: number | 'all'; logger?: (event) => void; listeners Array<(event) => void>; @@ -60,6 +62,14 @@ Used a convenience wrapper for the DynamoDB parameter `Limit` [[read more](https **Default Value:** _none_ A target for the number of items to return from DynamoDB. If this option is passed, Queries on entities will paginate DynamoDB until the items found match the count is reached or all items for that query have been returned. It is important to understand that this option may result in ElectroDB making multiple calls to DynamoDB to satisfy the count. For this reason, you should also consider using the `pages` option to limit the number of requests (or "pages") made to DynamoDB. +### until +**Default Value:** _none_ +Instructs ElectroDB to continue querying DynamoDB until at least the number of items specified is returned. Unlike [count](#count), this option will not guarantee the exact number of items returned, but will ensure at least the number specified is returned. This option has fewer post-processing steps than `count` and is more performant in cases where the exact number of items is not critical. + +### seek +**Default Value:** `false` +Instructs ElectroDB to continue querying DynamoDB until at least one item is returned. This is useful when the circumstances of a table, model, and/or query request results in an empty return. + ### pages **Default Value:** 1 How many DynamoDB pages should a query iterate through before stopping. To have ElectroDB automatically paginate through all results, pass the string value `'all'`. From 3ad95aab4b5c8a21a1fea2c1783184d4378c3b5a Mon Sep 17 00:00:00 2001 From: ty walch Date: Tue, 18 Feb 2025 13:10:11 -0500 Subject: [PATCH 3/5] Fixes test --- test/test-utils.ts | 12 ------------ test/ts_connected.pagination.spec.ts | 1 - 2 files changed, 13 deletions(-) diff --git a/test/test-utils.ts b/test/test-utils.ts index 34a4197f..7f9daf82 100644 --- a/test/test-utils.ts +++ b/test/test-utils.ts @@ -176,36 +176,24 @@ export function createDebugLogger(label: string, filters: ElectroDBMethodTypes[] } } -export type EventCollectorCall = { - request: ElectroQueryEvent; - response: ElectroResultsEvent; -} - export type EventCollector = ElectroEventListener & { queries: ElectroQueryEvent[], results: ElectroResultsEvent[], - calls: EventCollectorCall[]; } export function createEventCollector(): EventCollector { - const calls: EventCollectorCall[] = []; const queries: ElectroQueryEvent[] = [] const results: ElectroResultsEvent[] = []; - let current = {} as EventCollectorCall; const collector = (event: ElectroEvent) => { if (event.type === 'query') { queries.push(event); - current.request = event; } else { - current.response = event; results.push(event); - calls.push({...current}); } } collector.queries = queries; collector.results = results; - collector.calls = calls; return collector; } \ No newline at end of file diff --git a/test/ts_connected.pagination.spec.ts b/test/ts_connected.pagination.spec.ts index f29b2a72..bdbb238b 100644 --- a/test/ts_connected.pagination.spec.ts +++ b/test/ts_connected.pagination.spec.ts @@ -223,7 +223,6 @@ describe('entity pagination', () => { logger: collector, }); - console.log(JSON.stringify(events, null, 2)); expect(result.data).to.have.length(items.length); expect(clientSpy.calls.length).to.be.equal(items.length / 5 + 1); }); From b3759287801f15dc9d578ef8aa53483132a6ae33 Mon Sep 17 00:00:00 2001 From: ty walch Date: Tue, 18 Feb 2025 13:35:11 -0500 Subject: [PATCH 4/5] Fixes test --- test/ts_connected.logger.spec.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/ts_connected.logger.spec.ts b/test/ts_connected.logger.spec.ts index e547593d..a4d014cc 100644 --- a/test/ts_connected.logger.spec.ts +++ b/test/ts_connected.logger.spec.ts @@ -177,7 +177,7 @@ type EventProperties = { const eventProperties: EventProperties = { query: ["config", "method", "params", "type"], - results: ["config", "method", "results", "success", "type"], + results: ["config", "method", "params", "results", "success", "type"], }; async function testListeners(fn: TestListenerCallback) { @@ -524,6 +524,13 @@ describe("listener functions", () => { method: "get", success: true, results: {}, + params: { + Key: { + pk: `$service#prop1_${prop1}`, + sk: `$entity1_version#prop2_${prop2}`, + }, + TableName: "electro", + }, }, ]); }); @@ -560,6 +567,13 @@ describe("listener functions", () => { method: "get", success: true, results: {}, + params: { + Key: { + pk: `$service#prop1_${prop1}`, + sk: `$entity1_version#prop2_${prop2}`, + }, + TableName: "electro", + }, }, ]); }); From ed6a456980f94ffba7dcd8153d8b36b05724cc32 Mon Sep 17 00:00:00 2001 From: ty walch Date: Wed, 19 Feb 2025 11:43:31 -0500 Subject: [PATCH 5/5] Add support for services --- src/entity.js | 9 ++- test/ts_connected.pagination.spec.ts | 95 +++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 20 deletions(-) diff --git a/src/entity.js b/src/entity.js index efd428e8..69bbf336 100644 --- a/src/entity.js +++ b/src/entity.js @@ -724,6 +724,9 @@ class Entity { } results[entity] = results[entity] || []; results[entity] = [...results[entity], ...items]; + if (config.count) { + count += items.length; + } } } else if (Array.isArray(response.data)) { let prevCount = count; @@ -762,17 +765,17 @@ class Entity { iterations++; const countOptionRequiresMorePagination = ( - config.count !== undefined && count < config.count + config.count !== undefined && !config._isCollectionQuery && count < config.count ); const pagesOptionRequiresMorePagination = pages === AllPages || iterations < pages; const untilOptionRequiresMorePagination = - config.until !== undefined && !config._isCollectionQuery && results.length < config.until; + config.until !== undefined && count < config.until; const seekOptionRequiresMorePagination = - config.seek && !config._isCollectionQuery && results.length === 0; + config.seek && count === 0; morePaginationRequired = untilOptionRequiresMorePagination || diff --git a/test/ts_connected.pagination.spec.ts b/test/ts_connected.pagination.spec.ts index bdbb238b..315c98cd 100644 --- a/test/ts_connected.pagination.spec.ts +++ b/test/ts_connected.pagination.spec.ts @@ -2,8 +2,8 @@ process.env.AWS_NODEJS_CONNECTION_REUSE_ENABLED = "1"; import DynamoDB from "aws-sdk/clients/dynamodb"; import { expect } from "chai"; import { v4 as uuid } from "uuid"; -import { Entity, DocumentClient, EntityItem } from "../index"; -import { createEventCollector } from './test-utils'; +import { Entity, Service, DocumentClient, EntityItem } from "../index"; +import { createEventCollector, createDebugLogger } from './test-utils'; const table = 'electro'; @@ -62,10 +62,10 @@ function createObjectOfSize(kb: number): object { return obj; } -function createEntity(table: string, client: DocumentClient) { +function createEntity(entityName: string, client: DocumentClient) { return new Entity({ model: { - entity: 'paginator', + entity: entityName, version: '0', service: 'pagination-test' }, @@ -83,6 +83,7 @@ function createEntity(table: string, client: DocumentClient) { }, indexes: { record: { + collection: 'pager', pk: { field: 'pk', composite: ['id'], @@ -109,20 +110,26 @@ function createItems(count: number, size: number, id: string = uuid()): Paginato })); } -describe('entity pagination', () => { +describe('entity and service pagination', () => { + const thing1Name = 'thing1'; + const thing2Name = 'thing2'; const id = uuid(); // 100 items of 100 KB each - const items = createItems(100, 100, id) + const items = createItems(100, 100, id); + const thing1 = createEntity(thing1Name, client); + const thing2 = createEntity(thing2Name, client); before(async () => { - const entity = createEntity(table, client); - await entity.put(items).go(); + await Promise.all([ + thing1.put(items).go(), + thing2.put(items).go(), + ]); }); describe('when using seek', () => { it('should only paginate once if there are no results', async () => { const clientSpy = createDocumentClientSpy(client); - const entity = createEntity(table, clientSpy); + const entity = createEntity(uuid(), clientSpy); // use an id that doesn't exist const result = await entity.query @@ -138,7 +145,7 @@ describe('entity pagination', () => { it('should only paginate once if there is only one page of results', async () => { const id = uuid(); const clientSpy = createDocumentClientSpy(client); - const entity = createEntity(table, clientSpy); + const entity = createEntity(uuid(), clientSpy); // 20 items of 1 KB each const items = createItems(20, 1, id); await entity.put(items).go(); @@ -155,7 +162,7 @@ describe('entity pagination', () => { it('should paginate multiple times if a cursor is returned but results are not', async () => { const clientSpy = createDocumentClientSpy(client); - const entity = createEntity(table, clientSpy); + const entity = createEntity(thing1Name, clientSpy); // the items actually returned from DynamoDB const returned: any[][] = []; @@ -183,8 +190,7 @@ describe('entity pagination', () => { describe('when using until', () => { it('should throw if the until option cannot be parsed as a number', async () => { - const entity = createEntity(table, client); - const result = await entity.query + const result = await thing1.query .record({ id: 'unknown', run: 'unknown' }) // @ts-expect-error .go({ until: 'abc' }) @@ -196,7 +202,7 @@ describe('entity pagination', () => { it('should continue to paginate until at least the number of request items are returned', async () => { const clientSpy = createDocumentClientSpy(client); - const entity = createEntity(table, clientSpy); + const entity = createEntity(thing1Name, clientSpy); const result = await entity.query .record({ id }) @@ -209,9 +215,26 @@ describe('entity pagination', () => { expect(clientSpy.calls.length).to.be.greaterThan(2); }); + it('should continue to paginate until at least the number of request items are returned on a service', async () => { + const clientSpy = createDocumentClientSpy(client); + const thing1 = createEntity(thing1Name, clientSpy); + const thing2 = createEntity(thing2Name, clientSpy); + const paginator = new Service({ thing1, thing2 }); + + const result = await paginator.collections + .pager({ id }) + .go({ + until: 11, + params: { Limit: 5 }, + }); + + expect(result.data.thing1.length + result.data.thing2.length).to.be.greaterThan(10); + expect(clientSpy.calls.length).to.be.greaterThan(2); + }); + it('should abandon until directive when there are no more results', async () => { const clientSpy = createDocumentClientSpy(client); - const entity = createEntity(table, clientSpy); + const entity = createEntity(thing1Name, clientSpy); const collector = createEventCollector(); @@ -226,12 +249,33 @@ describe('entity pagination', () => { expect(result.data).to.have.length(items.length); expect(clientSpy.calls.length).to.be.equal(items.length / 5 + 1); }); + + it('should abandon until directive when there are no more results with service', async () => { + const clientSpy = createDocumentClientSpy(client); + const thing1 = createEntity(thing1Name, clientSpy); + const thing2 = createEntity(thing2Name, clientSpy); + const paginator = new Service({ thing1, thing2 }); + + const collector = createEventCollector(); + + const result = await paginator.collections + .pager({ id }) + .go({ + until: items.length + 100, + params: { Limit: 5 }, + logger: collector, + }); + + expect(result.data.thing1).to.have.length(items.length); + expect(result.data.thing2).to.have.length(items.length); + expect(clientSpy.calls.length).to.be.equal((items.length * 2) / 5 + 1); + }); }); describe('when using count', () => { it('should only return 10 items to the user', async () => { const clientSpy = createDocumentClientSpy(client); - const entity = createEntity(table, clientSpy); + const entity = createEntity(thing1Name, clientSpy); let returned: any = null; const result = await entity.query .record({ id }) @@ -253,7 +297,7 @@ describe('entity pagination', () => { describe('when using pages', () => { it('should query all items when using pages all', async () => { const clientSpy = createDocumentClientSpy(client); - const entity = createEntity(table, clientSpy); + const entity = createEntity(thing1Name, clientSpy); const result = await entity.query .record({ id }) @@ -262,5 +306,22 @@ describe('entity pagination', () => { expect(result.data.sort((a, z) => a.index - z.index)).to.deep.equal(items.sort((a, z) => a.index - z.index)); expect(clientSpy.calls).to.have.length(10); }); + + it('should query all items when using pages all with a service', async () => { + const clientSpy = createDocumentClientSpy(client); + const thing1 = createEntity(thing1Name, clientSpy); + const thing2 = createEntity(thing2Name, clientSpy); + const paginator = new Service({ thing1, thing2 }); + + const result = await paginator.collections + .pager({ id }) + .go({ pages: 'all' }); + + const received = [...result.data.thing1, ...result.data.thing2].sort((a, z) => a.index - z.index); + const expected = [...items, ...items].sort((a, z) => a.index - z.index); + + expect(received).to.deep.equal(expected); + expect(clientSpy.calls).to.have.length(19); + }); }) }); \ No newline at end of file