From 9c158bcfbc2462c64192a3db4e219f3d224889b4 Mon Sep 17 00:00:00 2001 From: Alex Movsunov Date: Wed, 6 Sep 2023 11:42:17 +0400 Subject: [PATCH 01/21] avro streams pipelines clean code restore cr --- .vscode/launch.json | 20 ++++++++ package.json | 2 + src/app/app.module.ts | 1 + src/common/registry/storage/key.storage.ts | 9 ++++ .../registry/storage/operator.storage.ts | 9 ++++ src/common/streams/index.ts | 1 + src/common/streams/streamify.ts | 30 +++++++++++ src/http/keys/keys.controller.ts | 2 +- .../sr-modules-keys.controller.ts | 2 +- .../sr-module-operators-keys.response.ts | 29 +++++++++++ src/http/sr-modules-operators-keys/index.ts | 1 + .../sr-modules-operators-keys.controller.ts | 32 ++++++++++-- .../sr-modules-operators-keys.service.ts | 49 +++++++++++++++--- .../sr-modules-operators-keys.types.ts | 12 +++++ .../curated-module.service.ts | 50 +++++++++++++------ .../interfaces/staking-module.interface.ts | 2 + yarn.lock | 19 +++++++ 17 files changed, 245 insertions(+), 25 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 src/common/streams/index.ts create mode 100644 src/common/streams/streamify.ts create mode 100644 src/http/sr-modules-operators-keys/sr-modules-operators-keys.types.ts diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..488d2a6a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,20 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Debug Nest Framework", + "runtimeExecutable": "yarn", + "runtimeArgs": [ + "start:debug", + ], + "cwd": "${workspaceFolder}/src", + "autoAttachChildProcesses": true, + "restart": true, + "sourceMaps": true, + "stopOnEntry": false, + "console": "integratedTerminal", + } + ] +} diff --git a/package.json b/package.json index f5d56e40..2dffc3d1 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "ethers": "^5.5.4", "fastify-swagger": "^4.13.1", "jsonstream": "^1.0.3", + "pg-query-stream": "^4.5.3", "prom-client": "^14.0.1", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", @@ -81,6 +82,7 @@ "@nestjs/testing": "^8.2.5", "@types/cache-manager": "^3.4.2", "@types/jest": "^27.4.0", + "@types/jsonstream": "^0.8.31", "@types/node": "^17.0.9", "@types/supertest": "^2.0.11", "@typescript-eslint/eslint-plugin": "^5.10.0", diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 9efe2c5a..a6cd8a12 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -38,6 +38,7 @@ import { StakingRouterModule } from 'staking-router-modules'; autoLoadEntities: false, cache: { enabled: false }, debug: false, + safe: true, registerRequestContext: true, allowGlobalContext: false, }; diff --git a/src/common/registry/storage/key.storage.ts b/src/common/registry/storage/key.storage.ts index 09b785ab..dd3ffcba 100644 --- a/src/common/registry/storage/key.storage.ts +++ b/src/common/registry/storage/key.storage.ts @@ -16,6 +16,15 @@ export class RegistryKeyStorageService { return await this.repository.find(where, options); } + findStream(where: FilterQuery, fields?: string[]): AsyncIterable { + const knex = this.repository.getKnex(); + return knex + .select(fields || '*') + .from('registry_key') + .where(where) + .stream(); + } + /** find all keys */ async findAll(moduleAddress: string): Promise { return await this.repository.find( diff --git a/src/common/registry/storage/operator.storage.ts b/src/common/registry/storage/operator.storage.ts index 347a9c88..d71fad1e 100644 --- a/src/common/registry/storage/operator.storage.ts +++ b/src/common/registry/storage/operator.storage.ts @@ -16,6 +16,15 @@ export class RegistryOperatorStorageService { return await this.repository.find(where, options); } + findStream(where: FilterQuery, fields?: string[]): AsyncIterable { + const knex = this.repository.getKnex(); + return knex + .select(fields || '*') + .from('registry_operator') + .where(where) + .stream(); + } + /** find all operators */ async findAll(moduleAddress: string): Promise { return await this.repository.find( diff --git a/src/common/streams/index.ts b/src/common/streams/index.ts new file mode 100644 index 00000000..84d8fd0f --- /dev/null +++ b/src/common/streams/index.ts @@ -0,0 +1 @@ +export * from './streamify'; diff --git a/src/common/streams/streamify.ts b/src/common/streams/streamify.ts new file mode 100644 index 00000000..d4f02849 --- /dev/null +++ b/src/common/streams/streamify.ts @@ -0,0 +1,30 @@ +import { Readable, ReadableOptions } from 'stream'; + +class GeneratorToStream extends Readable { + constructor(options: ReadableOptions, protected readonly generator: AsyncGenerator) { + super(options); + } + + _read() { + try { + this.generator + .next() + .then((result) => { + if (!result.done) { + this.push(result.value); + } else { + this.push(null); + } + }) + .catch((e) => { + this.emit('error', e); + }); + } catch (e) { + this.emit('error', e); + } + } +} + +export function streamify(generator: AsyncGenerator) { + return new GeneratorToStream({ objectMode: true }, generator); +} diff --git a/src/http/keys/keys.controller.ts b/src/http/keys/keys.controller.ts index 45a950cb..d51bf89b 100644 --- a/src/http/keys/keys.controller.ts +++ b/src/http/keys/keys.controller.ts @@ -53,7 +53,7 @@ export class KeysController { try { for (const keysGenerator of keysGenerators) { for await (const keysBatch of keysGenerator) { - jsonStream.write(keysBatch); + jsonStream.write(JSON.stringify(keysBatch)); } } } finally { diff --git a/src/http/sr-modules-keys/sr-modules-keys.controller.ts b/src/http/sr-modules-keys/sr-modules-keys.controller.ts index d916319c..8f51cc45 100644 --- a/src/http/sr-modules-keys/sr-modules-keys.controller.ts +++ b/src/http/sr-modules-keys/sr-modules-keys.controller.ts @@ -71,7 +71,7 @@ export class SRModulesKeysController { reply.type('application/json').send(jsonStream); for await (const keysBatch of keysGenerator) { - jsonStream.write(keysBatch); + jsonStream.write(JSON.stringify(keysBatch)); } jsonStream.end(); diff --git a/src/http/sr-modules-operators-keys/entities/sr-module-operators-keys.response.ts b/src/http/sr-modules-operators-keys/entities/sr-module-operators-keys.response.ts index 05c679eb..27ef46db 100644 --- a/src/http/sr-modules-operators-keys/entities/sr-module-operators-keys.response.ts +++ b/src/http/sr-modules-operators-keys/entities/sr-module-operators-keys.response.ts @@ -45,3 +45,32 @@ export class SRModuleOperatorsKeysResponse { }) meta!: ELMeta; } + +export class SRModulesOperatorsKeysStreamResponse { + @ApiProperty({ + type: 'array', + items: { oneOf: [{ $ref: getSchemaPath(CuratedOperator) }] }, + description: 'Operators of staking router module', + }) + operators?: SRModuleOperator[]; + + @ApiProperty({ + type: 'array', + items: { oneOf: [{ $ref: getSchemaPath(CuratedKey) }] }, + description: 'Keys of staking router module', + }) + keys?: SRModuleKey[]; + + @ApiProperty({ + type: 'array', + items: { oneOf: [{ $ref: getSchemaPath(SRModule) }] }, + description: 'List of Staking Router', + }) + modules?: SRModule[]; + + @ApiProperty({ + nullable: true, + type: () => ELMeta, + }) + meta?: ELMeta; +} diff --git a/src/http/sr-modules-operators-keys/index.ts b/src/http/sr-modules-operators-keys/index.ts index 9ef30a0a..dbf0531b 100644 --- a/src/http/sr-modules-operators-keys/index.ts +++ b/src/http/sr-modules-operators-keys/index.ts @@ -1,3 +1,4 @@ export * from './sr-modules-operators-keys.module'; export * from './sr-modules-operators-keys.controller'; export * from './sr-modules-operators-keys.service'; +export * from './sr-modules-operators-keys.types'; diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts index c02f5c14..b2e0d622 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts @@ -1,13 +1,15 @@ +import { pipeline } from 'node:stream/promises'; +import { IsolationLevel } from '@mikro-orm/core'; import { Controller, Get, Version, Param, Query, NotFoundException, HttpStatus, Res } from '@nestjs/common'; import { ApiOperation, ApiResponse, ApiTags, ApiParam, ApiNotFoundResponse } from '@nestjs/swagger'; -import { SRModuleOperatorsKeysResponse } from './entities'; +import { SRModuleOperatorsKeysResponse, SRModulesOperatorsKeysStreamResponse } from './entities'; import { ModuleId, KeyQuery } from 'http/common/entities/'; import { SRModulesOperatorsKeysService } from './sr-modules-operators-keys.service'; import { TooEarlyResponse } from 'http/common/entities/http-exceptions'; import { EntityManager } from '@mikro-orm/knex'; import * as JSONStream from 'jsonstream'; import type { FastifyReply } from 'fastify'; -import { IsolationLevel } from '@mikro-orm/core'; +import { streamify } from 'common/streams'; @Controller('/modules') @ApiTags('operators-keys') @@ -63,7 +65,7 @@ export class SRModulesOperatorsKeysController { reply.type('application/json').send(jsonStream); for await (const keysBatch of keysGenerator) { - jsonStream.write(keysBatch); + jsonStream.write(JSON.stringify(keysBatch)); } jsonStream.end(); @@ -71,4 +73,28 @@ export class SRModulesOperatorsKeysController { { isolationLevel: IsolationLevel.REPEATABLE_READ }, ); } + + @Version('2') + @ApiOperation({ summary: 'Comprehensive stream for staking router modules, operators and their keys' }) + @ApiResponse({ + status: 200, + description: 'Stream of all SR modules, operators and keys', + type: SRModulesOperatorsKeysStreamResponse, + }) + @ApiResponse({ + status: 425, + description: 'Meta has not exist yet, maybe data was not written in db yet', + type: TooEarlyResponse, + }) + @Get('operators/keys') + async getModulesOperatorsKeysStream(@Res() reply: FastifyReply) { + const jsonStream = JSONStream.stringify(); + + reply.type('application/json').send(jsonStream); + + await this.entityManager.transactional( + () => pipeline([streamify(this.srModulesOperatorsKeys.getModulesOperatorsKeysGenerator()), jsonStream]), + { isolationLevel: IsolationLevel.REPEATABLE_READ }, + ); + } } diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.service.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.service.ts index cab680ea..f2930471 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.service.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.service.ts @@ -6,14 +6,15 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { StakingRouterService } from 'staking-router-modules/staking-router.service'; import { KeyEntity, OperatorEntity } from 'staking-router-modules/interfaces/staking-module.interface'; import { EntityManager } from '@mikro-orm/knex'; +import { MetaStreamRecord, ModulesOperatorsKeysRecord } from './sr-modules-operators-keys.types'; @Injectable() export class SRModulesOperatorsKeysService { constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, protected readonly configService: ConfigService, - protected stakingRouterService: StakingRouterService, - protected readonly entityManager: EntityManager, + protected readonly stakingRouterService: StakingRouterService, + readonly entityManager: EntityManager, ) {} public async get( @@ -29,13 +30,49 @@ export class SRModulesOperatorsKeysService { const moduleInstance = this.stakingRouterService.getStakingRouterModuleImpl(module.type); - const keysGenerator: AsyncGenerator = await moduleInstance.getKeysStream( - module.stakingModuleAddress, - filters, - ); + const keysGenerator: AsyncGenerator = moduleInstance.getKeysStream(module.stakingModuleAddress, filters); const operatorsFilter = filters.operatorIndex ? { index: filters.operatorIndex } : {}; const operators: OperatorEntity[] = await moduleInstance.getOperators(module.stakingModuleAddress, operatorsFilter); return { operators, keysGenerator, module, meta: { elBlockSnapshot } }; } + + public async *getModulesOperatorsKeysGenerator(): AsyncGenerator { + const { stakingModules, elBlockSnapshot } = await this.stakingRouterService.getStakingModulesAndMeta(); + + const meta: MetaStreamRecord = { elBlockSnapshot }; + for (const stakingModule of stakingModules) { + const moduleInstance = this.stakingRouterService.getStakingRouterModuleImpl(stakingModule.type); + + const keysGenerator = moduleInstance.getKeysStream(stakingModule.stakingModuleAddress, {}); + const operatorsGenerator = moduleInstance.getOperatorsStream(stakingModule.stakingModuleAddress, {}); + + let nextKey = await keysGenerator.next(); + let nextOperator = await operatorsGenerator.next(); + + yield { + stakingModule, + meta, + key: nextKey.value || null, + operator: nextOperator.value || null, + }; + + do { + if (!nextKey.done) { + nextKey = await keysGenerator.next(); + } + + if (!nextOperator.done) { + nextOperator = await operatorsGenerator.next(); + } + + yield { + stakingModule: null, + meta: null, + key: nextKey.value || null, + operator: nextOperator.value || null, + }; + } while (!nextKey.done || !nextOperator.done); + } + } } diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.types.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.types.ts new file mode 100644 index 00000000..b229bac3 --- /dev/null +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.types.ts @@ -0,0 +1,12 @@ +import { ELBlockSnapshot } from 'http/common/entities'; +import { KeyEntity, OperatorEntity } from 'staking-router-modules/interfaces/staking-module.interface'; +import { SrModuleEntity } from 'storage/sr-module.entity'; + +export type MetaStreamRecord = { elBlockSnapshot: ELBlockSnapshot } | null; + +export type ModulesOperatorsKeysRecord = { + stakingModule: SrModuleEntity | null; + key: KeyEntity | null; + operator: OperatorEntity | null; + meta: MetaStreamRecord; +}; diff --git a/src/staking-router-modules/curated-module.service.ts b/src/staking-router-modules/curated-module.service.ts index a3726fca..fb73e29a 100644 --- a/src/staking-router-modules/curated-module.service.ts +++ b/src/staking-router-modules/curated-module.service.ts @@ -56,29 +56,51 @@ export class CuratedModuleService implements StakingModuleInterface { public async *getKeysStream(contractAddress: string, filters: KeysFilter): AsyncGenerator { const where = {}; if (filters.operatorIndex != undefined) { - where['operatorIndex'] = filters.operatorIndex; + where['operator_index'] = filters.operatorIndex; } if (filters.used != undefined) { where['used'] = filters.used; } - where['moduleAddress'] = contractAddress; - - const batchSize = 10000; - let offset = 0; + where['module_address'] = contractAddress; - while (true) { - const chunk = await this.keyStorageService.find(where, { limit: batchSize, offset }); - if (chunk.length === 0) { - break; - } + const keyStream = this.keyStorageService.findStream(where, [ + 'index', + 'operator_index as operatorIndex', + 'key', + 'deposit_signature as depositSignature', + 'used', + 'module_address as moduleAddress', + ]); - offset += batchSize; + for await (const record of keyStream) { + yield record; + } + } - for (const record of chunk) { - yield record; - } + public async *getOperatorsStream(moduleAddress: string, filters?: OperatorsFilter): AsyncGenerator { + const where = {}; + if (filters?.index != undefined) { + where['index'] = filters.index; + } + // we store operators of modules with the same impl at the same table + where['module_address'] = moduleAddress; + + const operatorStream = this.operatorStorageService.findStream(where, [ + 'index', + 'active', + 'name', + 'reward_address as rewardAddress', + 'staking_limit as stakingLimit', + 'stopped_validators as stoppedValidators', + 'total_signing_keys as totalSigningKeys', + 'used_signing_keys as usedSigningKeys', + 'module_address as moduleAddress', + ]); + + for await (const record of operatorStream) { + yield record; } } diff --git a/src/staking-router-modules/interfaces/staking-module.interface.ts b/src/staking-router-modules/interfaces/staking-module.interface.ts index 377e1c75..1cc22012 100644 --- a/src/staking-router-modules/interfaces/staking-module.interface.ts +++ b/src/staking-router-modules/interfaces/staking-module.interface.ts @@ -47,6 +47,8 @@ export interface StakingModuleInterface { getOperators(moduleAddress: string, filters?: OperatorsFilter): Promise; + getOperatorsStream(moduleAddress: string, filters?: OperatorsFilter): AsyncGenerator; + getOperator(moduleAddress: string, index: number): Promise; getCurrentNonce(moduleAddress: string, blockHash: string): Promise; diff --git a/yarn.lock b/yarn.lock index 53277f6e..ca8dacf7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1612,6 +1612,13 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/jsonstream@^0.8.31": + version "0.8.31" + resolved "https://registry.yarnpkg.com/@types/jsonstream/-/jsonstream-0.8.31.tgz#8df1c36c80dbb36e40794217d782022455d1e89a" + integrity sha512-32nJ7wl+q0lebxHo8iXqpmgLmkj6lYNHKp97+h8qteQ6O6IKnY+GyD/Eitqi4ul2Bbw3MScfRKtaHxt8gpC98w== + dependencies: + "@types/node" "*" + "@types/node-fetch@^2.5.12": version "2.6.2" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.2.tgz#d1a9c5fd049d9415dce61571557104dec3ec81da" @@ -5528,6 +5535,11 @@ pg-connection-string@2.5.0, pg-connection-string@^2.5.0: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== +pg-cursor@^2.10.3: + version "2.10.3" + resolved "https://registry.yarnpkg.com/pg-cursor/-/pg-cursor-2.10.3.tgz#4b44fbaede168a4785def56b8ac195e7df354472" + integrity sha512-rDyBVoqPVnx/PTmnwQAYgusSeAKlTL++gmpf5klVK+mYMFEqsOc6VHHZnPKc/4lOvr4r6fiMuoxSFuBF1dx4FQ== + pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" @@ -5543,6 +5555,13 @@ pg-protocol@^1.5.0: resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.5.0.tgz#b5dd452257314565e2d54ab3c132adc46565a6a0" integrity sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ== +pg-query-stream@^4.5.3: + version "4.5.3" + resolved "https://registry.yarnpkg.com/pg-query-stream/-/pg-query-stream-4.5.3.tgz#841ce414064d7b14bd2540d2267bdf40779d26f2" + integrity sha512-ufa94r/lHJdjAm3+zPZEO0gXAmCb4tZPaOt7O76mjcxdL/HxwTuryy76km+u0odBBgtfdKFYq/9XGfiYeQF0yA== + dependencies: + pg-cursor "^2.10.3" + pg-types@^2.1.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" From 0f1970456fd17f4b76ad6ff0a23d705be3fe0a0c Mon Sep 17 00:00:00 2001 From: Alex Movsunov Date: Mon, 18 Sep 2023 14:15:25 +0400 Subject: [PATCH 02/21] fix e2e --- .vscode/launch.json | 35 +++++++++++++++++++ package.json | 1 - src/http/keys/keys.controller.ts | 2 +- src/http/keys/keys.e2e-spec.ts | 14 +++++++- .../sr-modules-operators-keys.controller.ts | 2 +- yarn.lock | 7 ---- 6 files changed, 50 insertions(+), 11 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 488d2a6a..5a07de47 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -15,6 +15,41 @@ "sourceMaps": true, "stopOnEntry": false, "console": "integratedTerminal", + }, + { + "type": "node", + "request": "launch", + "name": "Debug Jest e2e", + "runtimeExecutable": "yarn", + "runtimeArgs": [ + "test:e2e", + ], + "cwd": "${workspaceFolder}/src", + "autoAttachChildProcesses": true, + "restart": true, + "sourceMaps": true, + "stopOnEntry": false, + "console": "integratedTerminal", + }, + { + "name": "Jest file", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/jest", + "args": [ + "${fileBasenameNoExtension}", + "--runInBand", + "--watch", + "--coverage=false", + "--no-cache" + ], + "cwd": "${workspaceRoot}", + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "sourceMaps": true, + "windows": { + "program": "${workspaceFolder}/node_modules/jest/bin/jest" + } } ] } diff --git a/package.json b/package.json index 2dffc3d1..5e855efc 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,6 @@ "@nestjs/testing": "^8.2.5", "@types/cache-manager": "^3.4.2", "@types/jest": "^27.4.0", - "@types/jsonstream": "^0.8.31", "@types/node": "^17.0.9", "@types/supertest": "^2.0.11", "@typescript-eslint/eslint-plugin": "^5.10.0", diff --git a/src/http/keys/keys.controller.ts b/src/http/keys/keys.controller.ts index d51bf89b..45a950cb 100644 --- a/src/http/keys/keys.controller.ts +++ b/src/http/keys/keys.controller.ts @@ -53,7 +53,7 @@ export class KeysController { try { for (const keysGenerator of keysGenerators) { for await (const keysBatch of keysGenerator) { - jsonStream.write(JSON.stringify(keysBatch)); + jsonStream.write(keysBatch); } } } finally { diff --git a/src/http/keys/keys.e2e-spec.ts b/src/http/keys/keys.e2e-spec.ts index 9cc733fe..7334b881 100644 --- a/src/http/keys/keys.e2e-spec.ts +++ b/src/http/keys/keys.e2e-spec.ts @@ -3,11 +3,12 @@ import { Test } from '@nestjs/testing'; import { Global, INestApplication, Module, ValidationPipe, VersioningType } from '@nestjs/common'; import { KeyRegistryService, + RegistryKey, RegistryKeyStorageService, RegistryStorageModule, RegistryStorageService, } from '../../common/registry'; -import { MikroORM } from '@mikro-orm/core'; +import { FilterQuery, MikroORM } from '@mikro-orm/core'; import { MikroOrmModule } from '@mikro-orm/nestjs'; import { KeysController } from './keys.controller'; import { StakingRouterModule } from '../../staking-router-modules/staking-router.module'; @@ -131,6 +132,15 @@ describe('KeyController (e2e)', () => { } } + class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { + async *findStream(where: FilterQuery, fields?: string[] | undefined): AsyncIterable { + const result = await this.find(where); + for (const key of result) { + yield key; + } + } + } + beforeAll(async () => { const imports = [ // sqlite3 only supports serializable transactions, ignoring the isolation level param @@ -151,6 +161,8 @@ describe('KeyController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) + .overrideProvider(RegistryKeyStorageService) + .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts index b2e0d622..03d71717 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts @@ -65,7 +65,7 @@ export class SRModulesOperatorsKeysController { reply.type('application/json').send(jsonStream); for await (const keysBatch of keysGenerator) { - jsonStream.write(JSON.stringify(keysBatch)); + jsonStream.write(keysBatch); } jsonStream.end(); diff --git a/yarn.lock b/yarn.lock index ca8dacf7..75a8f25b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1612,13 +1612,6 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== -"@types/jsonstream@^0.8.31": - version "0.8.31" - resolved "https://registry.yarnpkg.com/@types/jsonstream/-/jsonstream-0.8.31.tgz#8df1c36c80dbb36e40794217d782022455d1e89a" - integrity sha512-32nJ7wl+q0lebxHo8iXqpmgLmkj6lYNHKp97+h8qteQ6O6IKnY+GyD/Eitqi4ul2Bbw3MScfRKtaHxt8gpC98w== - dependencies: - "@types/node" "*" - "@types/node-fetch@^2.5.12": version "2.6.2" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.2.tgz#d1a9c5fd049d9415dce61571557104dec3ec81da" From e017147ed489b630d5ab4f9a5b069566ce106027 Mon Sep 17 00:00:00 2001 From: Alex Movsunov Date: Mon, 25 Sep 2023 19:54:58 +0400 Subject: [PATCH 03/21] pull new code --- .../entities/sr-module-operators-keys.response.ts | 14 +++++++------- .../sr-modules-operators-keys.types.ts | 7 +++---- yarn.lock | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/http/sr-modules-operators-keys/entities/sr-module-operators-keys.response.ts b/src/http/sr-modules-operators-keys/entities/sr-module-operators-keys.response.ts index 583d3096..65ab6f80 100644 --- a/src/http/sr-modules-operators-keys/entities/sr-module-operators-keys.response.ts +++ b/src/http/sr-modules-operators-keys/entities/sr-module-operators-keys.response.ts @@ -1,4 +1,4 @@ -import { ApiProperty, ApiExtraModels } from '@nestjs/swagger'; +import { ApiProperty, ApiExtraModels, getSchemaPath } from '@nestjs/swagger'; import { Key, Operator, StakingModuleResponse, ELMeta } from '../../common/entities/'; @ApiExtraModels(Operator) @@ -41,24 +41,24 @@ export class SRModuleOperatorsKeysResponse { export class SRModulesOperatorsKeysStreamResponse { @ApiProperty({ type: 'array', - items: { oneOf: [{ $ref: getSchemaPath(CuratedOperator) }] }, + items: { oneOf: [{ $ref: getSchemaPath(Operator) }] }, description: 'Operators of staking router module', }) - operators?: SRModuleOperator[]; + operators?: Operator[]; @ApiProperty({ type: 'array', - items: { oneOf: [{ $ref: getSchemaPath(CuratedKey) }] }, + items: { oneOf: [{ $ref: getSchemaPath(Key) }] }, description: 'Keys of staking router module', }) - keys?: SRModuleKey[]; + keys?: Key[]; @ApiProperty({ type: 'array', - items: { oneOf: [{ $ref: getSchemaPath(SRModule) }] }, + items: { oneOf: [{ $ref: getSchemaPath(StakingModuleResponse) }] }, description: 'List of Staking Router', }) - modules?: SRModule[]; + modules?: StakingModuleResponse[]; @ApiProperty({ nullable: true, diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.types.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.types.ts index b229bac3..2cc6ea5a 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.types.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.types.ts @@ -1,12 +1,11 @@ -import { ELBlockSnapshot } from 'http/common/entities'; -import { KeyEntity, OperatorEntity } from 'staking-router-modules/interfaces/staking-module.interface'; +import { ELBlockSnapshot, Key, Operator } from 'http/common/entities'; import { SrModuleEntity } from 'storage/sr-module.entity'; export type MetaStreamRecord = { elBlockSnapshot: ELBlockSnapshot } | null; export type ModulesOperatorsKeysRecord = { stakingModule: SrModuleEntity | null; - key: KeyEntity | null; - operator: OperatorEntity | null; + key: Key | null; + operator: Operator | null; meta: MetaStreamRecord; }; diff --git a/yarn.lock b/yarn.lock index a1e8baba..bf91dfd7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -85,7 +85,7 @@ "@aragon/apps-vault" "^4.0.0" "@aragon/os" "4.2.0" -"@aragon/apps-lido@github:lidofinance/aragon-apps#master": +"@aragon/apps-lido@lidofinance/aragon-apps#master": version "1.0.0" resolved "https://codeload.github.com/lidofinance/aragon-apps/tar.gz/b09834d29c0db211ddd50f50905cbeff257fc8e0" From 24cc07d3a32a372a212ceeacbf7c07812f951628 Mon Sep 17 00:00:00 2001 From: Alex Movsunov Date: Tue, 26 Sep 2023 10:39:28 +0400 Subject: [PATCH 04/21] fix e2e tests --- src/http/keys/keys.e2e-spec.ts | 2 +- .../sr-modules-keys.controller.ts | 2 +- .../sr-modules-keys.e2e-spec.ts | 11 +++++++++++ .../sr-modules-operators-keys.e2e-spec.ts | 11 +++++++++++ .../sr-modules-operators.e2e-spec.ts | 12 ++++++++++++ .../sr-modules-validators.e2e-spec.ts | 11 +++++++++++ src/http/sr-modules/sr-modules.e2e-spec.ts | 18 +++++++++++++++++- 7 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/http/keys/keys.e2e-spec.ts b/src/http/keys/keys.e2e-spec.ts index e9344bf1..25030106 100644 --- a/src/http/keys/keys.e2e-spec.ts +++ b/src/http/keys/keys.e2e-spec.ts @@ -54,7 +54,7 @@ describe('KeyController (e2e)', () => { } class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { - async *findStream(where: FilterQuery, fields?: string[] | undefined): AsyncIterable { + async *findStream(where: FilterQuery, fields): AsyncIterable { const result = await this.find(where); for (const key of result) { yield key; diff --git a/src/http/sr-modules-keys/sr-modules-keys.controller.ts b/src/http/sr-modules-keys/sr-modules-keys.controller.ts index 6c5b9747..e2d9a806 100644 --- a/src/http/sr-modules-keys/sr-modules-keys.controller.ts +++ b/src/http/sr-modules-keys/sr-modules-keys.controller.ts @@ -87,7 +87,7 @@ export class SRModulesKeysController { reply.type('application/json').send(jsonStream); for await (const keysBatch of keysGenerator) { - jsonStream.write(JSON.stringify(keysBatch)); + jsonStream.write(keysBatch); } jsonStream.end(); diff --git a/src/http/sr-modules-keys/sr-modules-keys.e2e-spec.ts b/src/http/sr-modules-keys/sr-modules-keys.e2e-spec.ts index cecdc37c..80b298f6 100644 --- a/src/http/sr-modules-keys/sr-modules-keys.e2e-spec.ts +++ b/src/http/sr-modules-keys/sr-modules-keys.e2e-spec.ts @@ -63,6 +63,15 @@ describe('SRModulesKeysController (e2e)', () => { } } + class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { + async *findStream(where, fields): AsyncIterable { + const result = await this.find(where); + for (const key of result) { + yield key; + } + } + } + beforeAll(async () => { const imports = [ // sqlite3 only supports serializable transactions, ignoring the isolation level param @@ -83,6 +92,8 @@ describe('SRModulesKeysController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) + .overrideProvider(RegistryKeyStorageService) + .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts index 44833846..99a2fbf1 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts @@ -55,6 +55,15 @@ describe('SRModulesOperatorsKeysController (e2e)', () => { } } + class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { + async *findStream(where, fields): AsyncIterable { + const result = await this.find(where); + for (const key of result) { + yield key; + } + } + } + beforeAll(async () => { const imports = [ // sqlite3 only supports serializable transactions, ignoring the isolation level param @@ -75,6 +84,8 @@ describe('SRModulesOperatorsKeysController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) + .overrideProvider(RegistryKeyStorageService) + .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); diff --git a/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts b/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts index d20d9c07..af290b7f 100644 --- a/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts +++ b/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts @@ -3,6 +3,7 @@ import { Test } from '@nestjs/testing'; import { Global, INestApplication, Module, ValidationPipe, VersioningType } from '@nestjs/common'; import { KeyRegistryService, + RegistryKeyStorageService, RegistryOperatorStorageService, RegistryStorageModule, RegistryStorageService, @@ -69,6 +70,15 @@ describe('SRModuleOperatorsController (e2e)', () => { } } + class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { + async *findStream(where, fields): AsyncIterable { + const result = await this.find(where); + for (const key of result) { + yield key; + } + } + } + beforeAll(async () => { const imports = [ // sqlite3 only supports serializable transactions, ignoring the isolation level param @@ -89,6 +99,8 @@ describe('SRModuleOperatorsController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) + .overrideProvider(RegistryKeyStorageService) + .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); diff --git a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts index c6a3e86a..87719cd9 100644 --- a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts +++ b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts @@ -82,6 +82,15 @@ describe('SRModulesValidatorsController (e2e)', () => { } } + class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { + async *findStream(where, fields): AsyncIterable { + const result = await this.find(where); + for (const key of result) { + yield key; + } + } + } + const consensusServiceMock = { getBlockV2: (args: { blockId: string | number }) => { return block; @@ -135,6 +144,8 @@ describe('SRModulesValidatorsController (e2e)', () => { }) .overrideProvider(ConsensusService) .useValue(consensusServiceMock) + .overrideProvider(RegistryKeyStorageService) + .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); diff --git a/src/http/sr-modules/sr-modules.e2e-spec.ts b/src/http/sr-modules/sr-modules.e2e-spec.ts index 16d4794f..f0c48da7 100644 --- a/src/http/sr-modules/sr-modules.e2e-spec.ts +++ b/src/http/sr-modules/sr-modules.e2e-spec.ts @@ -4,7 +4,12 @@ import { Global, INestApplication, Module, ValidationPipe, VersioningType } from import { MikroORM } from '@mikro-orm/core'; import { MikroOrmModule } from '@mikro-orm/nestjs'; -import { KeyRegistryService, RegistryStorageModule, RegistryStorageService } from '../../common/registry'; +import { + KeyRegistryService, + RegistryKeyStorageService, + RegistryStorageModule, + RegistryStorageService, +} from '../../common/registry'; import { StakingRouterModule } from '../../staking-router-modules/staking-router.module'; import { dvtModule, curatedModule, dvtModuleResp, curatedModuleResp, dvtModuleInUpperCase } from '../module.fixture'; import { SRModuleStorageService } from '../../storage/sr-module.storage'; @@ -45,6 +50,15 @@ describe('SRModulesController (e2e)', () => { } } + class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { + async *findStream(where, fields): AsyncIterable { + const result = await this.find(where); + for (const key of result) { + yield key; + } + } + } + beforeAll(async () => { const imports = [ // sqlite3 only supports serializable transactions, ignoring the isolation level param @@ -66,6 +80,8 @@ describe('SRModulesController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) + .overrideProvider(RegistryKeyStorageService) + .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); From 9f02fe4b88a392687f30856425a4ced669f228d2 Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Fri, 20 Oct 2023 09:18:23 +0400 Subject: [PATCH 05/21] VAL-369: Use postgres in e2e --- .dockerignore | 3 - .vscode/launch.json | 2 +- Dockerfile.e2e | 15 +++++ docker-compose.e2e.yml | 43 +++++++++++++ package.json | 4 +- src/app/app-testing.module.ts | 25 ++++++-- src/app/database-testing.module.ts | 29 +++++++++ src/app/index.ts | 1 + src/http/keys/keys.e2e-spec.ts | 28 ++------- .../sr-modules-keys.e2e-spec.ts | 25 ++------ .../sr-modules-operators-keys.e2e-spec.ts | 14 +---- .../sr-modules-operators.e2e-spec.ts | 25 ++------ .../sr-modules-validators.e2e-spec.ts | 62 +++++-------------- src/http/sr-modules/sr-modules.e2e-spec.ts | 34 ++-------- 14 files changed, 147 insertions(+), 163 deletions(-) create mode 100644 Dockerfile.e2e create mode 100644 docker-compose.e2e.yml create mode 100644 src/app/database-testing.module.ts diff --git a/.dockerignore b/.dockerignore index fc1211d6..4230b796 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,9 +2,6 @@ /dist /node_modules -# Env -.env - # Logs logs *.log diff --git a/.vscode/launch.json b/.vscode/launch.json index 5a07de47..2a6f524b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -22,7 +22,7 @@ "name": "Debug Jest e2e", "runtimeExecutable": "yarn", "runtimeArgs": [ - "test:e2e", + "test:e2e:docker:debug", ], "cwd": "${workspaceFolder}/src", "autoAttachChildProcesses": true, diff --git a/Dockerfile.e2e b/Dockerfile.e2e new file mode 100644 index 00000000..4d871920 --- /dev/null +++ b/Dockerfile.e2e @@ -0,0 +1,15 @@ +FROM node:18.14.2-alpine3.16 as building + +RUN apk add --no-cache git=2.36.6-r0 + +WORKDIR /app + +COPY package.json yarn.lock chronix.config.ts .env ./ +COPY jest* ./ +COPY ./tsconfig*.json ./ + +RUN yarn install --frozen-lockfile --non-interactive && yarn cache clean +COPY ./src ./src +RUN yarn typechain + +CMD ["yarn", "test:e2e:docker"] diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 00000000..0e84c19f --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,43 @@ +version: '3.7' + +services: + e2e_pgdb: + container_name: e2e_pgdb + image: postgres:14-alpine + restart: unless-stopped + environment: + - POSTGRES_DB=${DB_NAME} + - POSTGRES_USER=${DB_USER} + - POSTGRES_PASSWORD=${DB_PASSWORD} + expose: + - 5432:5432 + volumes: + - ./.volumes/pgdata-${CHAIN_ID}/:/var/lib/postgresql/data + + e2e_keys_api: + container_name: e2e_keys_api + build: + context: ./ + dockerfile: Dockerfile.e2e + environment: + - NODE_ENV=production + - PORT=${PORT} + - CORS_WHITELIST_REGEXP=${CORS_WHITELIST_REGEXP} + - GLOBAL_THROTTLE_TTL=${GLOBAL_THROTTLE_TTL} + - GLOBAL_THROTTLE_LIMIT=${GLOBAL_THROTTLE_LIMIT} + - GLOBAL_CACHE_TTL=${GLOBAL_CACHE_TTL} + - LOG_LEVEL=${LOG_LEVEL} + - LOG_FORMAT=${LOG_FORMAT} + - PROVIDERS_URLS=${PROVIDERS_URLS} + - CL_API_URLS=${CL_API_URLS} + - CHAIN_ID=${CHAIN_ID} + - DB_NAME=${DB_NAME} + - DB_PORT=5432 + - DB_HOST=e2e_pgdb + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} + - JOB_INTERVAL_REGISTRY=${JOB_INTERVAL_REGISTRY} + - VALIDATOR_REGISTRY_ENABLE=${VALIDATOR_REGISTRY_ENABLE} + - JOB_INTERVAL_VALIDATORS_REGISTRY=${JOB_INTERVAL_VALIDATORS_REGISTRY} + depends_on: + - e2e_pgdb diff --git a/package.json b/package.json index ac9ae448..ed990a82 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,9 @@ "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", - "test:e2e": "jest --config jest-e2e.json && chronix test", + "test:e2e": "docker-compose --env-file=./.env -f docker-compose.e2e.yml up --build --abort-on-container-exit", + "test:e2e:docker": "mikro-orm schema:drop -r && mikro-orm migration:up && jest -i --config jest-e2e.json && chronix test", + "test:e2e:docker:debug": "mikro-orm migration:up && node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest -i --config jest-e2e.json && chronix test", "typechain": "typechain --target=ethers-v5 --out-dir ./src/generated ./src/staking-router-modules/contracts/abi/*.json", "chronix:compile": "chronix compile", "chronix:test": "chronix test" diff --git a/src/app/app-testing.module.ts b/src/app/app-testing.module.ts index 8de87552..f9e150fd 100644 --- a/src/app/app-testing.module.ts +++ b/src/app/app-testing.module.ts @@ -1,7 +1,8 @@ import { APP_INTERCEPTOR } from '@nestjs/core'; import { Module } from '@nestjs/common'; import { PrometheusModule } from '../common/prometheus'; -import { ConfigModule } from '../common/config'; +import { ConfigModule, ConfigService } from '../common/config'; +import config from 'mikro-orm.config'; import { SentryInterceptor } from '../common/sentry'; import { HealthModule } from '../common/health'; import { AppService } from './app.service'; @@ -22,11 +23,23 @@ import { KeysUpdateModule } from 'jobs/keys-update'; ConfigModule, ExecutionProviderModule, ConsensusProviderModule, - MikroOrmModule.forRoot({ - dbName: ':memory:', - type: 'sqlite', - allowGlobalContext: true, - entities: ['./**/*.entity.ts'], + MikroOrmModule.forRootAsync({ + async useFactory(configService: ConfigService) { + return { + ...config, + dbName: configService.get('DB_NAME'), + host: configService.get('DB_HOST'), + port: configService.get('DB_PORT'), + user: configService.get('DB_USER'), + password: configService.get('DB_PASSWORD'), + autoLoadEntities: false, + cache: { enabled: false }, + debug: false, + registerRequestContext: true, + allowGlobalContext: true, + }; + }, + inject: [ConfigService], }), LoggerModule.forRoot({ transports: [nullTransport()] }), ScheduleModule.forRoot(), diff --git a/src/app/database-testing.module.ts b/src/app/database-testing.module.ts new file mode 100644 index 00000000..4e9d4bc8 --- /dev/null +++ b/src/app/database-testing.module.ts @@ -0,0 +1,29 @@ +import { Module } from '@nestjs/common'; +import { MikroOrmModule } from '@mikro-orm/nestjs'; +import config from 'mikro-orm.config'; +import { ConfigModule, ConfigService } from 'common/config'; + +@Module({ + imports: [ + ConfigModule, + MikroOrmModule.forRootAsync({ + async useFactory(configService: ConfigService) { + return { + ...config, + dbName: configService.get('DB_NAME'), + host: configService.get('DB_HOST'), + port: configService.get('DB_PORT'), + user: configService.get('DB_USER'), + password: configService.get('DB_PASSWORD'), + autoLoadEntities: false, + cache: { enabled: false }, + debug: false, + registerRequestContext: true, + allowGlobalContext: true, + }; + }, + inject: [ConfigService], + }), + ], +}) +export class DatabaseTestingModule {} diff --git a/src/app/index.ts b/src/app/index.ts index 006ae467..e8af5d49 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -1,3 +1,4 @@ export * from './app.constants'; export * from './app.module'; export * from './app.service'; +export * from './database-testing.module'; diff --git a/src/http/keys/keys.e2e-spec.ts b/src/http/keys/keys.e2e-spec.ts index ac2b5627..3001b5b4 100644 --- a/src/http/keys/keys.e2e-spec.ts +++ b/src/http/keys/keys.e2e-spec.ts @@ -3,13 +3,11 @@ import { Test } from '@nestjs/testing'; import { Global, INestApplication, Module, ValidationPipe, VersioningType } from '@nestjs/common'; import { KeyRegistryService, - RegistryKey, RegistryKeyStorageService, RegistryStorageModule, RegistryStorageService, } from '../../common/registry'; -import { FilterQuery, MikroORM } from '@mikro-orm/core'; -import { MikroOrmModule } from '@mikro-orm/nestjs'; +import { MikroORM } from '@mikro-orm/core'; import { KeysController } from './keys.controller'; import { StakingRouterModule } from '../../staking-router-modules/staking-router.module'; @@ -22,6 +20,7 @@ import { FastifyAdapter, NestFastifyApplication } from '@nestjs/platform-fastify import { elMeta } from '../el-meta.fixture'; import { curatedKeyWithDuplicate, curatedModule, curatedModuleKeys, dvtModule, keys } from '../db.fixtures'; import { curatedModuleKeysResponse, dvtModuleKeysResponse } from 'http/keys.fixtures'; +import { DatabaseTestingModule } from 'app'; describe('KeyController (e2e)', () => { let app: INestApplication; @@ -51,25 +50,9 @@ describe('KeyController (e2e)', () => { } } - class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { - async *findStream(where: FilterQuery, fields): AsyncIterable { - const result = await this.find(where); - for (const key of result) { - yield key; - } - } - } - beforeAll(async () => { const imports = [ - // sqlite3 only supports serializable transactions, ignoring the isolation level param - // TODO: use postgres - MikroOrmModule.forRoot({ - dbName: ':memory:', - type: 'sqlite', - allowGlobalContext: true, - entities: ['./**/*.entity.ts'], - }), + DatabaseTestingModule, LoggerModule.forRoot({ transports: [nullTransport()] }), KeyRegistryModule, StakingRouterModule, @@ -80,8 +63,6 @@ describe('KeyController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) - .overrideProvider(RegistryKeyStorageService) - .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); @@ -90,7 +71,8 @@ describe('KeyController (e2e)', () => { registryStorage = moduleRef.get(RegistryStorageService); const generator = moduleRef.get(MikroORM).getSchemaGenerator(); - await generator.updateSchema(); + await generator.refreshDatabase(); + await generator.clearDatabase(); app = moduleRef.createNestApplication(new FastifyAdapter()); app.enableVersioning({ type: VersioningType.URI }); diff --git a/src/http/sr-modules-keys/sr-modules-keys.e2e-spec.ts b/src/http/sr-modules-keys/sr-modules-keys.e2e-spec.ts index 4cf2a523..5b505373 100644 --- a/src/http/sr-modules-keys/sr-modules-keys.e2e-spec.ts +++ b/src/http/sr-modules-keys/sr-modules-keys.e2e-spec.ts @@ -8,7 +8,6 @@ import { RegistryStorageService, } from '../../common/registry'; import { MikroORM } from '@mikro-orm/core'; -import { MikroOrmModule } from '@mikro-orm/nestjs'; import { StakingRouterModule } from '../../staking-router-modules/staking-router.module'; import { SRModuleStorageService } from '../../storage/sr-module.storage'; @@ -22,6 +21,7 @@ import { curatedModule, dvtModule, keys } from '../db.fixtures'; import { dvtModuleResp, curatedModuleResp } from '../module.fixture'; import { curatedModuleKeysResponse, dvtModuleKeysResponse } from '../keys.fixtures'; import { elMeta } from '../el-meta.fixture'; +import { DatabaseTestingModule } from 'app'; describe('SRModulesKeysController (e2e)', () => { let app: INestApplication; @@ -62,25 +62,9 @@ describe('SRModulesKeysController (e2e)', () => { } } - class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { - async *findStream(where, fields): AsyncIterable { - const result = await this.find(where); - for (const key of result) { - yield key; - } - } - } - beforeAll(async () => { const imports = [ - // sqlite3 only supports serializable transactions, ignoring the isolation level param - // TODO: use postgres - MikroOrmModule.forRoot({ - dbName: ':memory:', - type: 'sqlite', - allowGlobalContext: true, - entities: ['./**/*.entity.ts'], - }), + DatabaseTestingModule, LoggerModule.forRoot({ transports: [nullTransport()] }), KeyRegistryModule, StakingRouterModule, @@ -91,8 +75,6 @@ describe('SRModulesKeysController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) - .overrideProvider(RegistryKeyStorageService) - .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); @@ -101,7 +83,8 @@ describe('SRModulesKeysController (e2e)', () => { registryStorage = moduleRef.get(RegistryStorageService); const generator = moduleRef.get(MikroORM).getSchemaGenerator(); - await generator.updateSchema(); + await generator.refreshDatabase(); + await generator.clearDatabase(); app = moduleRef.createNestApplication(new FastifyAdapter()); app.enableVersioning({ type: VersioningType.URI }); diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts index a1532bb4..13f05bca 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts @@ -56,15 +56,6 @@ describe('SRModulesOperatorsKeysController (e2e)', () => { } } - class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { - async *findStream(where, fields): AsyncIterable { - const result = await this.find(where); - for (const key of result) { - yield key; - } - } - } - beforeAll(async () => { const imports = [ // sqlite3 only supports serializable transactions, ignoring the isolation level param @@ -85,8 +76,6 @@ describe('SRModulesOperatorsKeysController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) - .overrideProvider(RegistryKeyStorageService) - .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); @@ -96,7 +85,8 @@ describe('SRModulesOperatorsKeysController (e2e)', () => { operatorsStorageService = moduleRef.get(RegistryOperatorStorageService); const generator = moduleRef.get(MikroORM).getSchemaGenerator(); - await generator.updateSchema(); + await generator.refreshDatabase(); + await generator.clearDatabase(); app = moduleRef.createNestApplication(new FastifyAdapter()); app.enableVersioning({ type: VersioningType.URI }); diff --git a/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts b/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts index ebb1fb09..2d27abcc 100644 --- a/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts +++ b/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts @@ -3,7 +3,6 @@ import { Test } from '@nestjs/testing'; import { Global, INestApplication, Module, ValidationPipe, VersioningType } from '@nestjs/common'; import { KeyRegistryService, - RegistryKeyStorageService, RegistryOperatorStorageService, RegistryStorageModule, RegistryStorageService, @@ -24,6 +23,7 @@ import { elMeta } from '../el-meta.fixture'; import { operators, dvtModule, curatedModule, srModules } from '../db.fixtures'; import { dvtModuleResp, curatedModuleResp } from '../module.fixture'; import { dvtOperatorsResp, curatedOperatorsResp } from '../operator.fixtures'; +import { DatabaseTestingModule } from 'app'; describe('SRModuleOperatorsController (e2e)', () => { let app: INestApplication; @@ -64,25 +64,9 @@ describe('SRModuleOperatorsController (e2e)', () => { } } - class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { - async *findStream(where, fields): AsyncIterable { - const result = await this.find(where); - for (const key of result) { - yield key; - } - } - } - beforeAll(async () => { const imports = [ - // sqlite3 only supports serializable transactions, ignoring the isolation level param - // TODO: use postgres - MikroOrmModule.forRoot({ - dbName: ':memory:', - type: 'sqlite', - allowGlobalContext: true, - entities: ['./**/*.entity.ts'], - }), + DatabaseTestingModule, LoggerModule.forRoot({ transports: [nullTransport()] }), KeyRegistryModule, StakingRouterModule, @@ -93,8 +77,6 @@ describe('SRModuleOperatorsController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) - .overrideProvider(RegistryKeyStorageService) - .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); @@ -103,7 +85,8 @@ describe('SRModuleOperatorsController (e2e)', () => { registryStorage = moduleRef.get(RegistryStorageService); const generator = moduleRef.get(MikroORM).getSchemaGenerator(); - await generator.updateSchema(); + await generator.refreshDatabase(); + await generator.clearDatabase(); app = moduleRef.createNestApplication(new FastifyAdapter()); app.enableVersioning({ type: VersioningType.URI }); diff --git a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts index 56b2b1c1..58aae8aa 100644 --- a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts +++ b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts @@ -1,16 +1,12 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { Test } from '@nestjs/testing'; import { Global, INestApplication, Module, ValidationPipe, VersioningType } from '@nestjs/common'; import { KeyRegistryService, - RegistryKey, RegistryKeyStorageService, - RegistryOperator, RegistryStorageModule, RegistryStorageService, } from '../../common/registry'; import { MikroORM } from '@mikro-orm/core'; -import { MikroOrmModule } from '@mikro-orm/nestjs'; import { SRModulesValidatorsController } from './sr-modules-validators.controller'; import { StakingRouterModule } from '../../staking-router-modules/staking-router.module'; @@ -25,17 +21,10 @@ import { FastifyAdapter, NestFastifyApplication } from '@nestjs/platform-fastify import { elMeta } from '../el-meta.fixture'; import { keys, dvtModule, curatedModule } from '../db.fixtures'; import { ConfigService } from '../../common/config'; -import { - ConsensusMetaEntity, - ConsensusValidatorEntity, - ValidatorsRegistryInterface, -} from '@lido-nestjs/validators-registry'; +import { ValidatorsRegistryInterface } from '@lido-nestjs/validators-registry'; import { ConsensusModule, ConsensusService } from '@lido-nestjs/consensus'; import { FetchModule } from '@lido-nestjs/fetch'; -import { ConfigModule } from '../../common/config'; import { ValidatorsModule } from '../../validators'; -import { SrModuleEntity } from '../../storage/sr-module.entity'; -import { ElMetaEntity } from '../../storage/el-meta.entity'; import { block, header, @@ -51,6 +40,7 @@ import { dvtOpOneRespExitMessages20percent, dvtOpOneRespExitMessages5maxAmount, } from '../consensus.fixtures'; +import { DatabaseTestingModule } from 'app'; describe('SRModulesValidatorsController (e2e)', () => { let app: INestApplication; @@ -60,6 +50,7 @@ describe('SRModulesValidatorsController (e2e)', () => { let elMetaStorageService: ElMetaStorageService; let registryStorage: RegistryStorageService; let validatorsRegistry: ValidatorsRegistryInterface; + let configService: ConfigService; async function cleanDB() { await keysStorageService.removeAll(); @@ -81,15 +72,6 @@ describe('SRModulesValidatorsController (e2e)', () => { } } - class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { - async *findStream(where, fields): AsyncIterable { - const result = await this.find(where); - for (const key of result) { - yield key; - } - } - } - const consensusServiceMock = { getBlockV2: (args: { blockId: string | number }) => { return block; @@ -104,25 +86,10 @@ describe('SRModulesValidatorsController (e2e)', () => { beforeAll(async () => { const imports = [ - // sqlite3 only supports serializable transactions, ignoring the isolation level param - // TODO: use postgres - MikroOrmModule.forRoot({ - dbName: ':memory:', - type: 'sqlite', - allowGlobalContext: true, - entities: [ - RegistryKey, - RegistryOperator, - ConsensusValidatorEntity, - ConsensusMetaEntity, - SrModuleEntity, - ElMetaEntity, - ], - }), + DatabaseTestingModule, LoggerModule.forRoot({ transports: [nullTransport()] }), KeyRegistryModule, StakingRouterModule, - ConfigModule, ConsensusModule.forRoot({ imports: [FetchModule], }), @@ -134,28 +101,29 @@ describe('SRModulesValidatorsController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) - .overrideProvider(ConfigService) - .useValue({ - get(path) { - const conf = { VALIDATOR_REGISTRY_ENABLE: true }; - return conf[path]; - }, - }) .overrideProvider(ConsensusService) .useValue(consensusServiceMock) - .overrideProvider(RegistryKeyStorageService) - .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); + configService = moduleRef.get(ConfigService); keysStorageService = moduleRef.get(RegistryKeyStorageService); moduleStorageService = moduleRef.get(SRModuleStorageService); registryStorage = moduleRef.get(RegistryStorageService); // validatorsStorage = moduleRef.get(StorageService); validatorsRegistry = moduleRef.get(ValidatorsRegistryInterface); + jest.spyOn(configService, 'get').mockImplementation((path) => { + if (path === 'VALIDATOR_REGISTRY_ENABLE') { + return true; + } + + return configService.get(path); + }); + const generator = moduleRef.get(MikroORM).getSchemaGenerator(); - await generator.updateSchema(); + await generator.refreshDatabase(); + await generator.clearDatabase(); app = moduleRef.createNestApplication(new FastifyAdapter()); app.enableVersioning({ type: VersioningType.URI }); diff --git a/src/http/sr-modules/sr-modules.e2e-spec.ts b/src/http/sr-modules/sr-modules.e2e-spec.ts index 816d1b1f..47b3de5c 100644 --- a/src/http/sr-modules/sr-modules.e2e-spec.ts +++ b/src/http/sr-modules/sr-modules.e2e-spec.ts @@ -2,14 +2,8 @@ import { Test } from '@nestjs/testing'; import { Global, INestApplication, Module, ValidationPipe, VersioningType } from '@nestjs/common'; import { MikroORM } from '@mikro-orm/core'; -import { MikroOrmModule } from '@mikro-orm/nestjs'; - -import { - KeyRegistryService, - RegistryKeyStorageService, - RegistryStorageModule, - RegistryStorageService, -} from '../../common/registry'; + +import { KeyRegistryService, RegistryStorageModule, RegistryStorageService } from '../../common/registry'; import { StakingRouterModule } from '../../staking-router-modules/staking-router.module'; import { dvtModuleResp, curatedModuleResp, dvtModuleAddressWithChecksum } from '../module.fixture'; import { SRModuleStorageService } from '../../storage/sr-module.storage'; @@ -24,6 +18,7 @@ import { SRModulesService } from './sr-modules.service'; import { elMeta } from '../el-meta.fixture'; import { curatedModule, dvtModule } from '../db.fixtures'; +import { DatabaseTestingModule } from 'app'; describe('SRModulesController (e2e)', () => { let app: INestApplication; @@ -51,25 +46,9 @@ describe('SRModulesController (e2e)', () => { } } - class RegistryKeyStorageServiceMock extends RegistryKeyStorageService { - async *findStream(where, fields): AsyncIterable { - const result = await this.find(where); - for (const key of result) { - yield key; - } - } - } - beforeAll(async () => { const imports = [ - // sqlite3 only supports serializable transactions, ignoring the isolation level param - // TODO: use postgres - MikroOrmModule.forRoot({ - dbName: ':memory:', - type: 'sqlite', - allowGlobalContext: true, - entities: ['./**/*.entity.ts'], - }), + DatabaseTestingModule, LoggerModule.forRoot({ transports: [nullTransport()] }), KeyRegistryModule, StakingRouterModule, @@ -81,8 +60,6 @@ describe('SRModulesController (e2e)', () => { const moduleRef = await Test.createTestingModule({ imports, controllers, providers }) .overrideProvider(KeyRegistryService) .useClass(KeysRegistryServiceMock) - .overrideProvider(RegistryKeyStorageService) - .useClass(RegistryKeyStorageServiceMock) .compile(); elMetaStorageService = moduleRef.get(ElMetaStorageService); @@ -90,7 +67,8 @@ describe('SRModulesController (e2e)', () => { registryStorage = moduleRef.get(RegistryStorageService); const generator = moduleRef.get(MikroORM).getSchemaGenerator(); - await generator.updateSchema(); + await generator.refreshDatabase(); + await generator.clearDatabase(); app = moduleRef.createNestApplication(new FastifyAdapter()); app.enableVersioning({ type: VersioningType.URI }); From 7e729eb160f4c7a652e6e71336a11935a44e2737 Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Mon, 23 Oct 2023 08:52:44 +0300 Subject: [PATCH 06/21] feat: github service-container for e2e tests --- .github/workflows/test.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 837f30b3..4767aaf3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,20 @@ on: pull_request jobs: test: runs-on: ubuntu-latest + + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DB: node_operator_keys_service_db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - name: Checkout repo uses: actions/checkout@v2.3.5 @@ -20,8 +34,13 @@ jobs: - name: Run tests run: yarn test - name: Run tests - run: yarn test:e2e + run: yarn test:e2e:docker env: PROVIDERS_URLS: ${{ secrets.PROVIDERS_URLS }} CHAIN_ID: ${{ secrets.CHAIN_ID }} CHRONIX_PROVIDER_MAINNET_URL: ${{ secrets.CHRONIX_PROVIDER_MAINNET_URL }} + DB_NAME: node_operator_keys_service_db + DB_PORT: 5432 + DB_HOST: postgres + DB_USER: postgres + DB_PASSWORD: postgres \ No newline at end of file From da5001bb09b9b5c5df4347c4b7144251ad00753f Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Mon, 23 Oct 2023 11:00:43 +0300 Subject: [PATCH 07/21] fix: ci --- .github/workflows/test.yml | 8 +++++--- .../sr-modules-operators/sr-modules-operators.e2e-spec.ts | 1 - .../sr-modules-validators.e2e-spec.ts | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4767aaf3..4fd40fff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + ports: + - 5432:5432 steps: - name: Checkout repo @@ -33,7 +35,7 @@ jobs: run: yarn lint - name: Run tests run: yarn test - - name: Run tests + - name: Run E2E tests run: yarn test:e2e:docker env: PROVIDERS_URLS: ${{ secrets.PROVIDERS_URLS }} @@ -41,6 +43,6 @@ jobs: CHRONIX_PROVIDER_MAINNET_URL: ${{ secrets.CHRONIX_PROVIDER_MAINNET_URL }} DB_NAME: node_operator_keys_service_db DB_PORT: 5432 - DB_HOST: postgres + DB_HOST: localhost DB_USER: postgres - DB_PASSWORD: postgres \ No newline at end of file + DB_PASSWORD: postgres diff --git a/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts b/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts index 2d27abcc..11a3c76a 100644 --- a/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts +++ b/src/http/sr-modules-operators/sr-modules-operators.e2e-spec.ts @@ -8,7 +8,6 @@ import { RegistryStorageService, } from '../../common/registry'; import { MikroORM } from '@mikro-orm/core'; -import { MikroOrmModule } from '@mikro-orm/nestjs'; import { StakingRouterModule } from '../../staking-router-modules/staking-router.module'; import { SRModuleStorageService } from '../../storage/sr-module.storage'; diff --git a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts index 58aae8aa..fc743328 100644 --- a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts +++ b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { Test } from '@nestjs/testing'; import { Global, INestApplication, Module, ValidationPipe, VersioningType } from '@nestjs/common'; import { From 044369207d2afd702fc40ed2271678bcd285db79 Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Mon, 23 Oct 2023 18:20:10 +0300 Subject: [PATCH 08/21] fix tests --- .github/workflows/test.yml | 1 + docker-compose.e2e.yml | 2 +- package.json | 2 +- .../sr-modules-operators-keys.e2e-spec.ts | 10 ++-------- .../sr-modules-validators.e2e-spec.ts | 8 -------- 5 files changed, 5 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4fd40fff..6d13a9fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,3 +46,4 @@ jobs: DB_HOST: localhost DB_USER: postgres DB_PASSWORD: postgres + VALIDATOR_REGISTRY_ENABLE: true diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index 0e84c19f..4818af83 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -37,7 +37,7 @@ services: - DB_USER=${DB_USER} - DB_PASSWORD=${DB_PASSWORD} - JOB_INTERVAL_REGISTRY=${JOB_INTERVAL_REGISTRY} - - VALIDATOR_REGISTRY_ENABLE=${VALIDATOR_REGISTRY_ENABLE} + - VALIDATOR_REGISTRY_ENABLE=true - JOB_INTERVAL_VALIDATORS_REGISTRY=${JOB_INTERVAL_VALIDATORS_REGISTRY} depends_on: - e2e_pgdb diff --git a/package.json b/package.json index ed990a82..e9819b3a 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "docker-compose --env-file=./.env -f docker-compose.e2e.yml up --build --abort-on-container-exit", "test:e2e:docker": "mikro-orm schema:drop -r && mikro-orm migration:up && jest -i --config jest-e2e.json && chronix test", - "test:e2e:docker:debug": "mikro-orm migration:up && node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest -i --config jest-e2e.json && chronix test", + "test:e2e:docker:debug": "mikro-orm schema:drop -r && mikro-orm migration:up && node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest -i --config jest-e2e.json && chronix test", "typechain": "typechain --target=ethers-v5 --out-dir ./src/generated ./src/staking-router-modules/contracts/abi/*.json", "chronix:compile": "chronix compile", "chronix:test": "chronix test" diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts index 13f05bca..4953d20c 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts @@ -26,6 +26,7 @@ import { elMeta } from '../el-meta.fixture'; import { keys, operators, dvtModule, curatedModule } from '../db.fixtures'; import { dvtModuleKeysResponse } from '../keys.fixtures'; import { dvtOperatorsResp } from '../operator.fixtures'; +import { DatabaseTestingModule } from 'app'; describe('SRModulesOperatorsKeysController (e2e)', () => { let app: INestApplication; @@ -58,14 +59,7 @@ describe('SRModulesOperatorsKeysController (e2e)', () => { beforeAll(async () => { const imports = [ - // sqlite3 only supports serializable transactions, ignoring the isolation level param - // TODO: use postgres - MikroOrmModule.forRoot({ - dbName: ':memory:', - type: 'sqlite', - allowGlobalContext: true, - entities: ['./**/*.entity.ts'], - }), + DatabaseTestingModule, LoggerModule.forRoot({ transports: [nullTransport()] }), KeyRegistryModule, StakingRouterModule, diff --git a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts index fc743328..b781a725 100644 --- a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts +++ b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts @@ -114,14 +114,6 @@ describe('SRModulesValidatorsController (e2e)', () => { // validatorsStorage = moduleRef.get(StorageService); validatorsRegistry = moduleRef.get(ValidatorsRegistryInterface); - jest.spyOn(configService, 'get').mockImplementation((path) => { - if (path === 'VALIDATOR_REGISTRY_ENABLE') { - return true; - } - - return configService.get(path); - }); - const generator = moduleRef.get(MikroORM).getSchemaGenerator(); await generator.refreshDatabase(); await generator.clearDatabase(); From f14ee771692bc0c1cbd4c6c1406ca65f49dd613d Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Mon, 23 Oct 2023 19:11:48 +0300 Subject: [PATCH 09/21] revert changes --- .github/workflows/test.yml | 2 +- src/app/app-testing.module.ts | 25 +++++-------------- .../sr-modules-operators-keys.e2e-spec.ts | 1 - .../sr-modules-validators.e2e-spec.ts | 8 ++++++ 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d13a9fe..5ffaae05 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,7 @@ jobs: run: yarn test:e2e:docker env: PROVIDERS_URLS: ${{ secrets.PROVIDERS_URLS }} + CL_API_URLS: "https://e2e-test.lido.fi," CHAIN_ID: ${{ secrets.CHAIN_ID }} CHRONIX_PROVIDER_MAINNET_URL: ${{ secrets.CHRONIX_PROVIDER_MAINNET_URL }} DB_NAME: node_operator_keys_service_db @@ -46,4 +47,3 @@ jobs: DB_HOST: localhost DB_USER: postgres DB_PASSWORD: postgres - VALIDATOR_REGISTRY_ENABLE: true diff --git a/src/app/app-testing.module.ts b/src/app/app-testing.module.ts index f9e150fd..8de87552 100644 --- a/src/app/app-testing.module.ts +++ b/src/app/app-testing.module.ts @@ -1,8 +1,7 @@ import { APP_INTERCEPTOR } from '@nestjs/core'; import { Module } from '@nestjs/common'; import { PrometheusModule } from '../common/prometheus'; -import { ConfigModule, ConfigService } from '../common/config'; -import config from 'mikro-orm.config'; +import { ConfigModule } from '../common/config'; import { SentryInterceptor } from '../common/sentry'; import { HealthModule } from '../common/health'; import { AppService } from './app.service'; @@ -23,23 +22,11 @@ import { KeysUpdateModule } from 'jobs/keys-update'; ConfigModule, ExecutionProviderModule, ConsensusProviderModule, - MikroOrmModule.forRootAsync({ - async useFactory(configService: ConfigService) { - return { - ...config, - dbName: configService.get('DB_NAME'), - host: configService.get('DB_HOST'), - port: configService.get('DB_PORT'), - user: configService.get('DB_USER'), - password: configService.get('DB_PASSWORD'), - autoLoadEntities: false, - cache: { enabled: false }, - debug: false, - registerRequestContext: true, - allowGlobalContext: true, - }; - }, - inject: [ConfigService], + MikroOrmModule.forRoot({ + dbName: ':memory:', + type: 'sqlite', + allowGlobalContext: true, + entities: ['./**/*.entity.ts'], }), LoggerModule.forRoot({ transports: [nullTransport()] }), ScheduleModule.forRoot(), diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts index 4953d20c..80d2f281 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.e2e-spec.ts @@ -9,7 +9,6 @@ import { RegistryStorageService, } from '../../common/registry'; import { MikroORM } from '@mikro-orm/core'; -import { MikroOrmModule } from '@mikro-orm/nestjs'; import { SRModulesOperatorsKeysController } from './sr-modules-operators-keys.controller'; import { StakingRouterModule } from '../../staking-router-modules/staking-router.module'; diff --git a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts index b781a725..fc743328 100644 --- a/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts +++ b/src/http/sr-modules-validators/sr-modules-validators.e2e-spec.ts @@ -114,6 +114,14 @@ describe('SRModulesValidatorsController (e2e)', () => { // validatorsStorage = moduleRef.get(StorageService); validatorsRegistry = moduleRef.get(ValidatorsRegistryInterface); + jest.spyOn(configService, 'get').mockImplementation((path) => { + if (path === 'VALIDATOR_REGISTRY_ENABLE') { + return true; + } + + return configService.get(path); + }); + const generator = moduleRef.get(MikroORM).getSchemaGenerator(); await generator.refreshDatabase(); await generator.clearDatabase(); From 01634ac71212cc6a49d678969790c35cc35e2e56 Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Mon, 23 Oct 2023 19:36:19 +0300 Subject: [PATCH 10/21] fix: get rid of console.log --- docker-compose.e2e.yml | 2 +- src/common/registry/test/fetch/operator.fetch.spec.ts | 1 - .../sr-modules-operators/sr-modules-operators.controller.ts | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index 4818af83..0e84c19f 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -37,7 +37,7 @@ services: - DB_USER=${DB_USER} - DB_PASSWORD=${DB_PASSWORD} - JOB_INTERVAL_REGISTRY=${JOB_INTERVAL_REGISTRY} - - VALIDATOR_REGISTRY_ENABLE=true + - VALIDATOR_REGISTRY_ENABLE=${VALIDATOR_REGISTRY_ENABLE} - JOB_INTERVAL_VALIDATORS_REGISTRY=${JOB_INTERVAL_VALIDATORS_REGISTRY} depends_on: - e2e_pgdb diff --git a/src/common/registry/test/fetch/operator.fetch.spec.ts b/src/common/registry/test/fetch/operator.fetch.spec.ts index 1f8b3d25..5ff8a939 100644 --- a/src/common/registry/test/fetch/operator.fetch.spec.ts +++ b/src/common/registry/test/fetch/operator.fetch.spec.ts @@ -79,7 +79,6 @@ describe('Operators', () => { // operatorFields(operator); return iface.encodeFunctionResult('getNodeOperator', operatorFields(operator)); }); - console.log('aaaaa', expected); const result = await fetchService.fetch(address); expect(result).toEqual([expected]); diff --git a/src/http/sr-modules-operators/sr-modules-operators.controller.ts b/src/http/sr-modules-operators/sr-modules-operators.controller.ts index 95eed771..cc67ad11 100644 --- a/src/http/sr-modules-operators/sr-modules-operators.controller.ts +++ b/src/http/sr-modules-operators/sr-modules-operators.controller.ts @@ -83,7 +83,6 @@ export class SRModulesOperatorsController { }) @Get('modules/:module_id/operators/:operator_id') getModuleOperator(@Param('module_id', ModuleIdPipe) module_id: string | number, @Param() operator: OperatorId) { - console.log(module_id, typeof module_id); return this.srModulesOperators.getModuleOperator(module_id, operator.operator_id); } } From a197e9c2d2671f7b9bc698820a9328fe057699f0 Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Tue, 7 Nov 2023 13:25:55 +0400 Subject: [PATCH 11/21] cr --- src/app/app.module.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 0a2c479d..657a4890 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -38,7 +38,6 @@ import { StakingRouterModule } from '../staking-router-modules'; autoLoadEntities: false, cache: { enabled: false }, debug: false, - safe: true, registerRequestContext: true, allowGlobalContext: false, }; From 9d0897a2b111ec469018b418ce80f55f7a65060d Mon Sep 17 00:00:00 2001 From: Eddort Date: Tue, 7 Nov 2023 10:59:05 +0100 Subject: [PATCH 12/21] feat: stream json error handling --- src/http/keys/keys.controller.ts | 9 +++++++-- src/http/sr-modules-keys/sr-modules-keys.controller.ts | 7 ++++++- .../sr-modules-operators-keys.controller.ts | 7 ++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/http/keys/keys.controller.ts b/src/http/keys/keys.controller.ts index 0defea68..a29d6486 100644 --- a/src/http/keys/keys.controller.ts +++ b/src/http/keys/keys.controller.ts @@ -48,7 +48,7 @@ export class KeysController { const jsonStream = JSONStream.stringify('{ "meta": ' + JSON.stringify(meta) + ', "data": [', ',', ']}'); reply.type('application/json').send(jsonStream); - // TODO: is it necessary to check the error? or 'finally' is ok? + try { for (const keysGenerator of keysGenerators) { for await (const key of keysGenerator) { @@ -56,8 +56,13 @@ export class KeysController { jsonStream.write(keyReponse); } } - } finally { jsonStream.end(); + } catch (streamError) { + // Handle the error during streaming. + console.error('Error during streaming:', streamError); + // destroy method closes the stream without ']' and corrupt the result + // https://github.com/dominictarr/through/blob/master/index.js#L78 + jsonStream.destroy(); } }, { isolationLevel: IsolationLevel.REPEATABLE_READ }, diff --git a/src/http/sr-modules-keys/sr-modules-keys.controller.ts b/src/http/sr-modules-keys/sr-modules-keys.controller.ts index ed70003f..972200dc 100644 --- a/src/http/sr-modules-keys/sr-modules-keys.controller.ts +++ b/src/http/sr-modules-keys/sr-modules-keys.controller.ts @@ -98,8 +98,13 @@ export class SRModulesKeysController { jsonStream.write(keyReponse); } - } finally { jsonStream.end(); + } catch (streamError) { + // Handle the error during streaming. + console.error('Error during streaming:', streamError); + // destroy method closes the stream without ']' and corrupt the result + // https://github.com/dominictarr/through/blob/master/index.js#L78 + jsonStream.destroy(); } }, { isolationLevel: IsolationLevel.REPEATABLE_READ }, diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts index 64b4c309..9519b7d9 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts @@ -76,8 +76,13 @@ export class SRModulesOperatorsKeysController { const keyResponse = new Key(key); jsonStream.write(keyResponse); } - } finally { jsonStream.end(); + } catch (streamError) { + // Handle the error during streaming. + console.error('Error during streaming:', streamError); + // destroy method closes the stream without ']' and corrupt the result + // https://github.com/dominictarr/through/blob/master/index.js#L78 + jsonStream.destroy(); } }, { isolationLevel: IsolationLevel.REPEATABLE_READ }, From 3a05f6101b5af5fc46858fc535dfc90d56a6719d Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Wed, 8 Nov 2023 08:00:37 +0400 Subject: [PATCH 13/21] cr 1 --- docker-compose.e2e.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index 0e84c19f..265db991 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -21,23 +21,7 @@ services: dockerfile: Dockerfile.e2e environment: - NODE_ENV=production - - PORT=${PORT} - - CORS_WHITELIST_REGEXP=${CORS_WHITELIST_REGEXP} - - GLOBAL_THROTTLE_TTL=${GLOBAL_THROTTLE_TTL} - - GLOBAL_THROTTLE_LIMIT=${GLOBAL_THROTTLE_LIMIT} - - GLOBAL_CACHE_TTL=${GLOBAL_CACHE_TTL} - - LOG_LEVEL=${LOG_LEVEL} - - LOG_FORMAT=${LOG_FORMAT} - - PROVIDERS_URLS=${PROVIDERS_URLS} - - CL_API_URLS=${CL_API_URLS} - - CHAIN_ID=${CHAIN_ID} - - DB_NAME=${DB_NAME} - DB_PORT=5432 - DB_HOST=e2e_pgdb - - DB_USER=${DB_USER} - - DB_PASSWORD=${DB_PASSWORD} - - JOB_INTERVAL_REGISTRY=${JOB_INTERVAL_REGISTRY} - - VALIDATOR_REGISTRY_ENABLE=${VALIDATOR_REGISTRY_ENABLE} - - JOB_INTERVAL_VALIDATORS_REGISTRY=${JOB_INTERVAL_VALIDATORS_REGISTRY} depends_on: - e2e_pgdb From 17ed1ee55c5dbf091f74c4ebdd23df861fea15c0 Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Wed, 8 Nov 2023 13:08:40 +0400 Subject: [PATCH 14/21] experiment --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ffaae05..4bbab83a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: run: yarn test:e2e:docker env: PROVIDERS_URLS: ${{ secrets.PROVIDERS_URLS }} - CL_API_URLS: "https://e2e-test.lido.fi," + # CL_API_URLS: "https://e2e-test.lido.fi," CHAIN_ID: ${{ secrets.CHAIN_ID }} CHRONIX_PROVIDER_MAINNET_URL: ${{ secrets.CHRONIX_PROVIDER_MAINNET_URL }} DB_NAME: node_operator_keys_service_db From 1c36f23c3e1698c57a8f1be8ee9586b6bbfcdf61 Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Wed, 8 Nov 2023 13:19:19 +0400 Subject: [PATCH 15/21] VALIDATOR_REGISTRY_ENABLE=false --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4bbab83a..76093cff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: run: yarn test:e2e:docker env: PROVIDERS_URLS: ${{ secrets.PROVIDERS_URLS }} - # CL_API_URLS: "https://e2e-test.lido.fi," + VALIDATOR_REGISTRY_ENABLE: false CHAIN_ID: ${{ secrets.CHAIN_ID }} CHRONIX_PROVIDER_MAINNET_URL: ${{ secrets.CHRONIX_PROVIDER_MAINNET_URL }} DB_NAME: node_operator_keys_service_db From be43c98993a68e17efa6237d88654a6a944da031 Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Wed, 8 Nov 2023 13:25:39 +0400 Subject: [PATCH 16/21] rollback --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76093cff..5ffaae05 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: run: yarn test:e2e:docker env: PROVIDERS_URLS: ${{ secrets.PROVIDERS_URLS }} - VALIDATOR_REGISTRY_ENABLE: false + CL_API_URLS: "https://e2e-test.lido.fi," CHAIN_ID: ${{ secrets.CHAIN_ID }} CHRONIX_PROVIDER_MAINNET_URL: ${{ secrets.CHRONIX_PROVIDER_MAINNET_URL }} DB_NAME: node_operator_keys_service_db From cf06652f97fe060455739d1f606a8d8f7ab94abe Mon Sep 17 00:00:00 2001 From: Eddort Date: Wed, 8 Nov 2023 12:41:47 +0100 Subject: [PATCH 17/21] feat: add timeout to keys and operators stream --- src/common/registry/storage/key.storage.ts | 7 ++++++- .../registry/storage/operator.storage.ts | 7 ++++++- src/common/registry/utils/stream.utils.ts | 21 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 src/common/registry/utils/stream.utils.ts diff --git a/src/common/registry/storage/key.storage.ts b/src/common/registry/storage/key.storage.ts index dd3ffcba..441a190f 100644 --- a/src/common/registry/storage/key.storage.ts +++ b/src/common/registry/storage/key.storage.ts @@ -1,6 +1,7 @@ import { QueryOrder } from '@mikro-orm/core'; import { FilterQuery, FindOptions } from '@mikro-orm/core'; import { Injectable } from '@nestjs/common'; +import { addTimeoutToStream } from '../utils/stream.utils'; import { RegistryKey } from './key.entity'; import { RegistryKeyRepository } from './key.repository'; @@ -18,11 +19,15 @@ export class RegistryKeyStorageService { findStream(where: FilterQuery, fields?: string[]): AsyncIterable { const knex = this.repository.getKnex(); - return knex + const stream = knex .select(fields || '*') .from('registry_key') .where(where) .stream(); + + addTimeoutToStream(stream, 60_000); + + return stream; } /** find all keys */ diff --git a/src/common/registry/storage/operator.storage.ts b/src/common/registry/storage/operator.storage.ts index d71fad1e..2230416e 100644 --- a/src/common/registry/storage/operator.storage.ts +++ b/src/common/registry/storage/operator.storage.ts @@ -1,6 +1,7 @@ import { QueryOrder } from '@mikro-orm/core'; import { FilterQuery, FindOptions } from '@mikro-orm/core'; import { Injectable } from '@nestjs/common'; +import { addTimeoutToStream } from '../utils/stream.utils'; import { RegistryOperator } from './operator.entity'; import { RegistryOperatorRepository } from './operator.repository'; @@ -18,11 +19,15 @@ export class RegistryOperatorStorageService { findStream(where: FilterQuery, fields?: string[]): AsyncIterable { const knex = this.repository.getKnex(); - return knex + const stream = knex .select(fields || '*') .from('registry_operator') .where(where) .stream(); + + addTimeoutToStream(stream, 60_000); + + return stream; } /** find all operators */ diff --git a/src/common/registry/utils/stream.utils.ts b/src/common/registry/utils/stream.utils.ts new file mode 100644 index 00000000..24eb1c0f --- /dev/null +++ b/src/common/registry/utils/stream.utils.ts @@ -0,0 +1,21 @@ +import { PassThrough } from 'stream'; + +export const addTimeoutToStream = (stream: PassThrough, ms: number) => { + let timeout = setTimeout(() => { + stream.destroy(); + }, ms); + + const debounce = () => { + clearTimeout(timeout); + timeout = setTimeout(() => { + stream.destroy(); + }, ms); + }; + + const debounceTimeoutHandler = stream.on('data', debounce); + + stream.once('close', () => { + clearTimeout(timeout); + debounceTimeoutHandler.off('data', debounce); + }); +}; From 4dd16ab1f229c10186b8e9c377debdda4f71c82b Mon Sep 17 00:00:00 2001 From: Eddort Date: Thu, 9 Nov 2023 11:15:02 +0100 Subject: [PATCH 18/21] fix: replace console logs by error --- src/http/keys/keys.controller.ts | 2 +- src/http/sr-modules-keys/sr-modules-keys.controller.ts | 2 +- .../sr-modules-operators-keys.controller.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http/keys/keys.controller.ts b/src/http/keys/keys.controller.ts index a29d6486..c795d86b 100644 --- a/src/http/keys/keys.controller.ts +++ b/src/http/keys/keys.controller.ts @@ -59,10 +59,10 @@ export class KeysController { jsonStream.end(); } catch (streamError) { // Handle the error during streaming. - console.error('Error during streaming:', streamError); // destroy method closes the stream without ']' and corrupt the result // https://github.com/dominictarr/through/blob/master/index.js#L78 jsonStream.destroy(); + throw streamError; } }, { isolationLevel: IsolationLevel.REPEATABLE_READ }, diff --git a/src/http/sr-modules-keys/sr-modules-keys.controller.ts b/src/http/sr-modules-keys/sr-modules-keys.controller.ts index 972200dc..9e7b20e1 100644 --- a/src/http/sr-modules-keys/sr-modules-keys.controller.ts +++ b/src/http/sr-modules-keys/sr-modules-keys.controller.ts @@ -101,10 +101,10 @@ export class SRModulesKeysController { jsonStream.end(); } catch (streamError) { // Handle the error during streaming. - console.error('Error during streaming:', streamError); // destroy method closes the stream without ']' and corrupt the result // https://github.com/dominictarr/through/blob/master/index.js#L78 jsonStream.destroy(); + throw streamError; } }, { isolationLevel: IsolationLevel.REPEATABLE_READ }, diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts index 9519b7d9..e99e1ba0 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts @@ -79,10 +79,10 @@ export class SRModulesOperatorsKeysController { jsonStream.end(); } catch (streamError) { // Handle the error during streaming. - console.error('Error during streaming:', streamError); // destroy method closes the stream without ']' and corrupt the result // https://github.com/dominictarr/through/blob/master/index.js#L78 jsonStream.destroy(); + throw streamError; } }, { isolationLevel: IsolationLevel.REPEATABLE_READ }, From bf16d5b040f281207f51b6639f84aa5c225817e9 Mon Sep 17 00:00:00 2001 From: Eddort Date: Sun, 12 Nov 2023 19:27:14 +0100 Subject: [PATCH 19/21] feat: better error handling --- src/common/registry/storage/key.storage.ts | 2 +- .../registry/storage/operator.storage.ts | 2 +- src/common/registry/utils/stream.utils.ts | 15 ++++++++++++--- src/http/keys/keys.controller.ts | 12 ++++++++++-- .../sr-modules-keys.controller.ts | 8 ++++++-- .../sr-modules-operators-keys.controller.ts | 18 ++++++++++++++++-- 6 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/common/registry/storage/key.storage.ts b/src/common/registry/storage/key.storage.ts index 441a190f..bb3a2f28 100644 --- a/src/common/registry/storage/key.storage.ts +++ b/src/common/registry/storage/key.storage.ts @@ -25,7 +25,7 @@ export class RegistryKeyStorageService { .where(where) .stream(); - addTimeoutToStream(stream, 60_000); + addTimeoutToStream(stream, 60_000, 'A timeout occurred loading keys from the database'); return stream; } diff --git a/src/common/registry/storage/operator.storage.ts b/src/common/registry/storage/operator.storage.ts index 2230416e..a642fd01 100644 --- a/src/common/registry/storage/operator.storage.ts +++ b/src/common/registry/storage/operator.storage.ts @@ -25,7 +25,7 @@ export class RegistryOperatorStorageService { .where(where) .stream(); - addTimeoutToStream(stream, 60_000); + addTimeoutToStream(stream, 60_000, 'A timeout occurred loading operators from the database'); return stream; } diff --git a/src/common/registry/utils/stream.utils.ts b/src/common/registry/utils/stream.utils.ts index 24eb1c0f..dc98a156 100644 --- a/src/common/registry/utils/stream.utils.ts +++ b/src/common/registry/utils/stream.utils.ts @@ -1,17 +1,26 @@ import { PassThrough } from 'stream'; -export const addTimeoutToStream = (stream: PassThrough, ms: number) => { +/** + * This util adds a timeout for streaming, + * preventing errors related to stream hangs + * this can happen when working with knex + * @param stream nodejs stream + * @param ms timeout in ms + * @param errorMessage error text to be displayed when the stream is closed + */ +export const addTimeoutToStream = (stream: PassThrough, ms: number, errorMessage: string) => { let timeout = setTimeout(() => { - stream.destroy(); + stream.destroy(new Error(errorMessage)); }, ms); const debounce = () => { clearTimeout(timeout); timeout = setTimeout(() => { - stream.destroy(); + stream.destroy(new Error(errorMessage)); }, ms); }; + // we should stop streaming if more than "ms" has elapsed since the last receipt of data. const debounceTimeoutHandler = stream.on('data', debounce); stream.once('close', () => { diff --git a/src/http/keys/keys.controller.ts b/src/http/keys/keys.controller.ts index c795d86b..77526c01 100644 --- a/src/http/keys/keys.controller.ts +++ b/src/http/keys/keys.controller.ts @@ -10,7 +10,10 @@ import { HttpStatus, NotFoundException, Res, + LoggerService, + Inject, } from '@nestjs/common'; +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import type { FastifyReply } from 'fastify'; import { ApiNotFoundResponse, ApiOperation, ApiParam, ApiResponse, ApiTags } from '@nestjs/swagger'; import { KeysService } from './keys.service'; @@ -25,7 +28,11 @@ import { IsolationLevel } from '@mikro-orm/core'; @Controller('keys') @ApiTags('keys') export class KeysController { - constructor(protected readonly keysService: KeysService, protected readonly entityManager: EntityManager) {} + constructor( + @Inject(LOGGER_PROVIDER) protected logger: LoggerService, + protected readonly keysService: KeysService, + protected readonly entityManager: EntityManager, + ) {} @Version('1') @Get() @@ -56,13 +63,14 @@ export class KeysController { jsonStream.write(keyReponse); } } + jsonStream.end(); } catch (streamError) { // Handle the error during streaming. + this.logger.log('keys streaming error', streamError); // destroy method closes the stream without ']' and corrupt the result // https://github.com/dominictarr/through/blob/master/index.js#L78 jsonStream.destroy(); - throw streamError; } }, { isolationLevel: IsolationLevel.REPEATABLE_READ }, diff --git a/src/http/sr-modules-keys/sr-modules-keys.controller.ts b/src/http/sr-modules-keys/sr-modules-keys.controller.ts index 9e7b20e1..2970d631 100644 --- a/src/http/sr-modules-keys/sr-modules-keys.controller.ts +++ b/src/http/sr-modules-keys/sr-modules-keys.controller.ts @@ -10,7 +10,10 @@ import { HttpStatus, Res, HttpCode, + LoggerService, + Inject, } from '@nestjs/common'; +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { ApiNotFoundResponse, ApiOperation, ApiParam, ApiResponse, ApiTags } from '@nestjs/swagger'; import { SRModuleKeyListResponse, GroupedByModuleKeyListResponse } from './entities'; import { SRModulesKeysService } from './sr-modules-keys.service'; @@ -27,6 +30,7 @@ import { ModuleIdPipe } from '../common/pipeline/module-id-pipe'; @ApiTags('sr-module-keys') export class SRModulesKeysController { constructor( + @Inject(LOGGER_PROVIDER) protected logger: LoggerService, protected readonly srModulesKeysService: SRModulesKeysService, protected readonly entityManager: EntityManager, ) {} @@ -95,16 +99,16 @@ export class SRModulesKeysController { try { for await (const key of keysGenerator) { const keyReponse = new Key(key); - jsonStream.write(keyReponse); } + jsonStream.end(); } catch (streamError) { // Handle the error during streaming. + this.logger.error('module-keys streaming error', streamError); // destroy method closes the stream without ']' and corrupt the result // https://github.com/dominictarr/through/blob/master/index.js#L78 jsonStream.destroy(); - throw streamError; } }, { isolationLevel: IsolationLevel.REPEATABLE_READ }, diff --git a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts index e99e1ba0..db426359 100644 --- a/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts +++ b/src/http/sr-modules-operators-keys/sr-modules-operators-keys.controller.ts @@ -1,6 +1,17 @@ import { pipeline } from 'node:stream/promises'; import { IsolationLevel } from '@mikro-orm/core'; -import { Controller, Get, Version, Param, Query, NotFoundException, HttpStatus, Res } from '@nestjs/common'; +import { + Controller, + Get, + Version, + Param, + Query, + NotFoundException, + HttpStatus, + Res, + LoggerService, + Inject, +} from '@nestjs/common'; import { ApiOperation, ApiResponse, ApiTags, ApiParam, ApiNotFoundResponse } from '@nestjs/swagger'; import { SRModuleOperatorsKeysResponse, SRModulesOperatorsKeysStreamResponse } from './entities'; import { KeyQuery, Key } from 'http/common/entities/'; @@ -11,11 +22,13 @@ import * as JSONStream from 'jsonstream'; import type { FastifyReply } from 'fastify'; import { streamify } from 'common/streams'; import { ModuleIdPipe } from 'http/common/pipeline/module-id-pipe'; +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; @Controller('/modules') @ApiTags('operators-keys') export class SRModulesOperatorsKeysController { constructor( + @Inject(LOGGER_PROVIDER) protected logger: LoggerService, protected readonly srModulesOperatorsKeys: SRModulesOperatorsKeysService, protected readonly entityManager: EntityManager, ) {} @@ -76,13 +89,14 @@ export class SRModulesOperatorsKeysController { const keyResponse = new Key(key); jsonStream.write(keyResponse); } + jsonStream.end(); } catch (streamError) { // Handle the error during streaming. + this.logger.error('operators-keys streaming error', streamError); // destroy method closes the stream without ']' and corrupt the result // https://github.com/dominictarr/through/blob/master/index.js#L78 jsonStream.destroy(); - throw streamError; } }, { isolationLevel: IsolationLevel.REPEATABLE_READ }, From 789948ca42de69af573bbdde89367b6fd77bab0a Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Mon, 20 Nov 2023 20:47:06 +0400 Subject: [PATCH 20/21] chore: code review --- .github/workflows/test.yml | 2 +- src/common/registry/storage/key.storage.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ffaae05..9d85514e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ jobs: services: postgres: - image: postgres + image: postgres:16-alpine env: POSTGRES_PASSWORD: postgres POSTGRES_USER: postgres diff --git a/src/common/registry/storage/key.storage.ts b/src/common/registry/storage/key.storage.ts index bb3a2f28..5a81cb25 100644 --- a/src/common/registry/storage/key.storage.ts +++ b/src/common/registry/storage/key.storage.ts @@ -23,6 +23,10 @@ export class RegistryKeyStorageService { .select(fields || '*') .from('registry_key') .where(where) + .orderBy([ + { column: 'operatorIndex', order: 'asc' }, + { column: 'index', order: 'asc' }, + ]) .stream(); addTimeoutToStream(stream, 60_000, 'A timeout occurred loading keys from the database'); From dd6d2d33680bf30e9c216ef8b90a023cca86c86d Mon Sep 17 00:00:00 2001 From: Alexander Movsunov Date: Mon, 20 Nov 2023 20:50:51 +0400 Subject: [PATCH 21/21] chore: code review 2 --- src/common/registry/storage/operator.storage.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/registry/storage/operator.storage.ts b/src/common/registry/storage/operator.storage.ts index a642fd01..90229d52 100644 --- a/src/common/registry/storage/operator.storage.ts +++ b/src/common/registry/storage/operator.storage.ts @@ -23,6 +23,10 @@ export class RegistryOperatorStorageService { .select(fields || '*') .from('registry_operator') .where(where) + .orderBy([ + { column: 'operatorIndex', order: 'asc' }, + { column: 'index', order: 'asc' }, + ]) .stream(); addTimeoutToStream(stream, 60_000, 'A timeout occurred loading operators from the database');