Merge pull request #255 from lidofinance/develop

develop -> main

chronix.config.ts

@@ -3,7 +3,7 @@ export default {
     port: 8001,
   },
   commands: {
-    test: 'jest --config  jest-e2e-onchain.json',
+    test: 'jest -i --config  jest-e2e-onchain.json',
   },
   plugins: ['simple-dvt-v1'],
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "lido-keys-api",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Lido Node Operators keys service",
   "author": "Lido team",
   "private": true,
@@ -52,13 +52,13 @@
     "@lido-nestjs/fetch": "^1.4.0",
     "@lido-nestjs/logger": "^1.3.2",
     "@lido-nestjs/middleware": "^1.2.0",
-    "@lido-nestjs/validators-registry": "^1.3.0",
-    "@mikro-orm/cli": "^5.5.3",
-    "@mikro-orm/core": "^5.5.3",
-    "@mikro-orm/knex": "^5.5.3",
-    "@mikro-orm/migrations": "^5.5.3",
+    "@lido-nestjs/validators-registry": "^1.3.1",
+    "@mikro-orm/cli": "^5.7.13",
+    "@mikro-orm/core": "^5.7.13",
+    "@mikro-orm/knex": "^5.7.13",
+    "@mikro-orm/migrations": "^5.7.13",
     "@mikro-orm/nestjs": "^5.1.2",
-    "@mikro-orm/postgresql": "^5.5.3",
+    "@mikro-orm/postgresql": "^5.7.13",
     "@nestjs/common": "^8.2.5",
     "@nestjs/config": "^1.1.6",
     "@nestjs/core": "^8.2.5",
@@ -85,7 +85,6 @@
     "@types/k6": "~0.44.2"
   },
   "devDependencies": {
-    "@mikro-orm/sqlite": "5.5.3",
     "@nestjs/cli": "^8.2.5",
     "@nestjs/schematics": "^8.0.5",
     "@nestjs/testing": "^8.2.5",

sample.env

@@ -47,3 +47,7 @@ CL_API_URLS=https://quiknode.pro/<token>
 # for some applications we dont need validators registry library
 # value below is default
 VALIDATOR_REGISTRY_ENABLE=true
+
+# When streaming lasts more than STREAM_TIMEOUT seconds, the stream will be destroyed
+# This prevents the retrieval of outdated data.
+STREAM_TIMEOUT=60000

src/app/app-testing.module.ts

@@ -7,13 +7,13 @@ import { HealthModule } from '../common/health';
 import { AppService } from './app.service';
 import { ExecutionProviderModule } from '../common/execution-provider';
 import { ConsensusProviderModule } from '../common/consensus-provider';
-import { MikroOrmModule } from '@mikro-orm/nestjs';
 import { ScheduleModule } from '@nestjs/schedule';
 import { LoggerModule, nullTransport } from '@lido-nestjs/logger';
 import { SimpleFallbackJsonRpcBatchProvider } from '@lido-nestjs/execution';
 import { KeyRegistryModule } from 'common/registry';
 import { StakingRouterModule } from 'staking-router-modules';
 import { KeysUpdateModule } from 'jobs/keys-update';
+import { DatabaseE2ETestingModule } from './database-e2e-testing.module';
 
 @Module({
   imports: [
@@ -22,12 +22,7 @@ import { KeysUpdateModule } from 'jobs/keys-update';
     ConfigModule,
     ExecutionProviderModule,
     ConsensusProviderModule,
-    MikroOrmModule.forRoot({
-      dbName: ':memory:',
-      type: 'sqlite',
-      allowGlobalContext: true,
-      entities: ['./**/*.entity.ts'],
-    }),
+    DatabaseE2ETestingModule.forRoot(),
     LoggerModule.forRoot({ transports: [nullTransport()] }),
     ScheduleModule.forRoot(),
     KeyRegistryModule.forRootAsync({

src/app/database-testing.module.ts → src/app/database-e2e-testing.module.ts

@@ -4,10 +4,10 @@ import config from 'mikro-orm.config';
 import { ConfigModule, ConfigService } from 'common/config';
 
 @Module({})
-export class DatabaseTestingModule {
+export class DatabaseE2ETestingModule {
   static forRoot(mikroOrmConfigOverrides: Partial<typeof config> = {}): DynamicModule {
     return {
-      module: DatabaseTestingModule,
+      module: DatabaseE2ETestingModule,
       imports: [
         ConfigModule,
         MikroOrmModule.forRootAsync({

src/app/index.ts

@@ -1,4 +1,4 @@
 export * from './app.constants';
 export * from './app.module';
 export * from './app.service';
-export * from './database-testing.module';
+export * from './database-e2e-testing.module';

src/app/simple-dvt-deploy.e2e-chain.ts

@@ -40,7 +40,7 @@ describe('Simple DVT deploy', () => {
   let dvtNodeOperator2WithoutKeys: chronix.StoryResult<'simple-dvt/set-node-operator-name'>;
 
   let moduleRef: TestingModule;
-
+  let configService: ConfigService;
   let keysStorageService: RegistryKeyStorageService;
   let moduleStorageService: SRModuleStorageService;
   let elMetaStorageService: ElMetaStorageService;
@@ -61,6 +61,7 @@ describe('Simple DVT deploy', () => {
     session = await sdk.env.hardhat({
       fork: forkUrl,
       chainId: 1,
+      forkBlockNumber: 19282216
     });
 
     moduleRef = await Test.createTestingModule({ imports: [AppModule] })
@@ -84,26 +85,27 @@ describe('Simple DVT deploy', () => {
         validatorsRegistryLastSlot: jest.fn(),
         validatorsEnabled: jest.fn(),
       })
-      .overrideProvider(ConfigService)
-      .useValue({
-        get(path) {
-          const conf = {
-            LIDO_LOCATOR_ADDRESS: '0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb',
-          };
-          return conf[path];
-        },
-      })
       .compile();
 
     const generator = moduleRef.get(MikroORM).getSchemaGenerator();
-    await generator.updateSchema();
+    await generator.refreshDatabase();
+    await generator.clearDatabase();
 
+    configService = moduleRef.get(ConfigService);
     elMetaStorageService = moduleRef.get(ElMetaStorageService);
     keysStorageService = moduleRef.get(RegistryKeyStorageService);
     moduleStorageService = moduleRef.get(SRModuleStorageService);
 
     keysUpdateService = moduleRef.get(KeysUpdateService);
     stakingRouterService = moduleRef.get(StakingRouterService);
+
+    jest.spyOn(configService, 'get').mockImplementation((path: any) => {
+      if (path === 'LIDO_LOCATOR_ADDRESS') {
+        return '0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb';
+      }
+
+      return configService.get(path);
+    });
   });
 
   afterAll(async () => {

src/common/config/env.validation.ts

@@ -171,6 +171,11 @@ export class EnvironmentVariables {
   @IsPositive()
   @Transform(({ value }) => parseInt(value, 10))
   KEYS_FETCH_BATCH_SIZE = 1100;
+
+  @IsOptional()
+  @IsPositive()
+  @Transform(({ value }) => parseInt(value, 10))
+  STREAM_TIMEOUT = 60_000;
 }
 
 export function validate(config: Record<string, unknown>) {

src/common/registry/storage/constants.ts

@@ -1,6 +1,6 @@
 export const KEY_LEN = 98;
 export const DEPOSIT_SIGNATURE_LEN = 194;
 export const ADDRESS_LEN = 42;
-export const STREAM_TIMEOUT = 60_000;
+export const DEFAULT_STREAM_TIMEOUT = 60_000;
 export const STREAM_KEYS_TIMEOUT_MESSAGE = 'A timeout occurred loading keys from the database';
 export const STREAM_OPERATORS_TIMEOUT_MESSAGE = 'A timeout occurred loading operators from the database';

src/common/registry/storage/key.storage.ts

@@ -1,14 +1,15 @@
 import { QueryOrder } from '@mikro-orm/core';
 import { FilterQuery, FindOptions } from '@mikro-orm/core';
 import { Injectable } from '@nestjs/common';
+import { ConfigService } from 'common/config';
 import { addTimeoutToStream } from '../utils/stream.utils';
 import { RegistryKey } from './key.entity';
 import { RegistryKeyRepository } from './key.repository';
-import { STREAM_KEYS_TIMEOUT_MESSAGE, STREAM_TIMEOUT } from './constants';
+import { DEFAULT_STREAM_TIMEOUT, STREAM_KEYS_TIMEOUT_MESSAGE } from './constants';
 
 @Injectable()
 export class RegistryKeyStorageService {
-  constructor(private readonly repository: RegistryKeyRepository) {}
+  constructor(private readonly repository: RegistryKeyRepository, private readonly configService: ConfigService) {}
 
   /** find keys */
   async find<P extends string = never>(
@@ -27,7 +28,8 @@ export class RegistryKeyStorageService {
     const knex = qb.getKnexQuery();
     const stream = knex.stream();
 
-    addTimeoutToStream(stream, STREAM_TIMEOUT, STREAM_KEYS_TIMEOUT_MESSAGE);
+    const streamTimeout = this.configService.get('STREAM_TIMEOUT') ?? DEFAULT_STREAM_TIMEOUT;
+    addTimeoutToStream(stream, streamTimeout, STREAM_KEYS_TIMEOUT_MESSAGE);
 
     return stream;
   }

src/common/registry/storage/operator.storage.ts

@@ -1,14 +1,15 @@
 import { QueryOrder } from '@mikro-orm/core';
 import { FilterQuery, FindOptions } from '@mikro-orm/core';
 import { Injectable } from '@nestjs/common';
+import { ConfigService } from 'common/config';
 import { addTimeoutToStream } from '../utils/stream.utils';
 import { RegistryOperator } from './operator.entity';
 import { RegistryOperatorRepository } from './operator.repository';
-import { STREAM_OPERATORS_TIMEOUT_MESSAGE, STREAM_TIMEOUT } from './constants';
+import { DEFAULT_STREAM_TIMEOUT, STREAM_OPERATORS_TIMEOUT_MESSAGE } from './constants';
 
 @Injectable()
 export class RegistryOperatorStorageService {
-  constructor(private readonly repository: RegistryOperatorRepository) {}
+  constructor(private readonly repository: RegistryOperatorRepository, private readonly configService: ConfigService) {}
 
   /** find operators */
   async find<P extends string = never>(
@@ -28,7 +29,8 @@ export class RegistryOperatorStorageService {
     const knex = qb.getKnexQuery();
     const stream = knex.stream();
 
-    addTimeoutToStream(stream, STREAM_TIMEOUT, STREAM_OPERATORS_TIMEOUT_MESSAGE);
+    const streamTimeout = this.configService.get('STREAM_TIMEOUT') ?? DEFAULT_STREAM_TIMEOUT;
+    addTimeoutToStream(stream, streamTimeout, STREAM_OPERATORS_TIMEOUT_MESSAGE);
 
     return stream;
   }

src/common/registry/storage/registry-storage.module.ts

@@ -6,12 +6,14 @@ import { RegistryKeyStorageService } from './key.storage';
 import { MikroOrmModule } from '@mikro-orm/nestjs';
 import { RegistryKey } from './key.entity';
 import { RegistryOperator } from './operator.entity';
+import { ConfigModule } from 'common/config';
 
 @Module({
   imports: [
     MikroOrmModule.forFeature({
       entities: [RegistryKey, RegistryOperator],
     }),
+    ConfigModule,
   ],
   providers: [RegistryStorageService, RegistryOperatorStorageService, RegistryKeyStorageService],
   exports: [RegistryStorageService, RegistryOperatorStorageService, RegistryKeyStorageService],

src/common/registry/test/fetch/key-batch.fetch.e2e-spec.ts

@@ -27,7 +27,8 @@ describe('Fetch keys in batch', () => {
   });
 
   test('fetch one key', async () => {
-    const keys = await fetchService.fetch(address, 17, 0, 3, { blockTag: 9641262 });
+    const overrides = { blockTag: 10573030 };
+    const keys = await fetchService.fetch(address, 17, 0, 3, overrides);
 
     expect(keys).toBeInstanceOf(Array);
 

src/common/registry/test/fetch/key.fetch.e2e-spec.ts

@@ -28,7 +28,7 @@ describe('Keys', () => {
   });
 
   test('fetch one key', async () => {
-    const key = await fetchService.fetchOne(address, 21, 0, { blockTag: 6912872 });
+    const key = await fetchService.fetchOne(address, 17, 0, { blockTag: 10573030 });
 
     expect(key).toBeInstanceOf(Object);
 
@@ -40,23 +40,23 @@ describe('Keys', () => {
   });
 
   test('fetch operator keys', async () => {
-    const keys = await fetchService.fetch(address, 21, 0, -1, {
-      blockTag: 6912872,
+    const keys = await fetchService.fetch(address, 17, 0, 3, {
+      blockTag: 10573030,
     });
     expect(keys).toBeInstanceOf(Array);
     expect(keys.length).toBe(3);
   }, 15_000);
 
   test('fetch several keys', async () => {
-    const keys = await fetchService.fetch(address, 21, 0, 2, {
-      blockTag: 6912872,
+    const keys = await fetchService.fetch(address, 17, 0, 2, {
+      blockTag: 10573030,
     });
 
     expect(keys).toBeInstanceOf(Array);
     expect(keys.length).toBe(2);
 
-    expect(keys[0].operatorIndex).toBe(21);
-    expect(keys[1].operatorIndex).toBe(21);
+    expect(keys[0].operatorIndex).toBe(17);
+    expect(keys[1].operatorIndex).toBe(17);
 
     expect(keys[0].index).toBe(0);
     expect(keys[1].index).toBe(1);

src/common/registry/test/fetch/operator.fetch.e2e-spec.ts

@@ -52,15 +52,17 @@ describe('Operators', () => {
 
   test('fetch all operators', async () => {
     const operators = await fetchService.fetch(address, 0, -1, {
-      blockTag: 9641262,
+      blockTag: 10573030,
     });
 
     expect(operators).toBeInstanceOf(Array);
     expect(operators.length).toBeGreaterThan(0);
   }, 30_000);
 
   test('fetch multiply operators', async () => {
-    const operators = await fetchService.fetch(address, 1, 3);
+    const operators = await fetchService.fetch(address, 1, 3, {
+      blockTag: 10573030,
+    });
 
     expect(operators).toBeInstanceOf(Array);
     expect(operators.length).toBe(2);

src/common/registry/test/key-registry/async.spec.ts → src/common/registry/test/key-registry/async.e2e-spec.ts

@@ -1,12 +1,11 @@
-import { MikroOrmModule } from '@mikro-orm/nestjs';
 import { ModuleMetadata } from '@nestjs/common';
 import { Test } from '@nestjs/testing';
 import { nullTransport, LoggerModule } from '@lido-nestjs/logger';
 import { getNetwork } from '@ethersproject/networks';
 import { getDefaultProvider } from '@ethersproject/providers';
-import { KeyRegistryModule, KeyRegistryService, RegistryStorageService } from '../../';
+import { KeyRegistryModule, KeyRegistryService, RegistryStorageService } from '../..';
 import { MikroORM } from '@mikro-orm/core';
-import { mikroORMConfig } from '../testing.utils';
+import { DatabaseE2ETestingModule } from 'app';
 
 describe('Async module initializing', () => {
   const provider = getDefaultProvider('mainnet');
@@ -19,15 +18,16 @@ describe('Async module initializing', () => {
     const storageService = moduleRef.get(RegistryStorageService);
 
     const generator = moduleRef.get(MikroORM).getSchemaGenerator();
-    await generator.updateSchema();
+    await generator.refreshDatabase();
+    await generator.clearDatabase();
 
     expect(registryService).toBeDefined();
     await storageService.onModuleDestroy();
   };
 
   test('forRootAsync', async () => {
     await testModules([
-      MikroOrmModule.forRoot(mikroORMConfig),
+      DatabaseE2ETestingModule.forRoot(),
       LoggerModule.forRoot({ transports: [nullTransport()] }),
       KeyRegistryModule.forRootAsync({
         async useFactory() {
@@ -39,12 +39,7 @@ describe('Async module initializing', () => {
 
   test('forFeatureAsync', async () => {
     await testModules([
-      MikroOrmModule.forRoot({
-        dbName: ':memory:',
-        type: 'sqlite',
-        allowGlobalContext: true,
-        entities: ['./**/*.entity.ts'],
-      }),
+      DatabaseE2ETestingModule.forRoot(),
       LoggerModule.forRoot({ transports: [nullTransport()] }),
       KeyRegistryModule.forFeatureAsync({
         async useFactory() {