Merge pull request #254 from lidofinance/fix/val-668-update-knex

Update mikro orm to version using knex with stream end fix

package.json

@@ -52,13 +52,13 @@
     "@lido-nestjs/fetch": "^1.4.0",
     "@lido-nestjs/logger": "^1.3.2",
     "@lido-nestjs/middleware": "^1.2.0",
-    "@lido-nestjs/validators-registry": "^1.3.0",
-    "@mikro-orm/cli": "^5.5.3",
-    "@mikro-orm/core": "^5.5.3",
-    "@mikro-orm/knex": "^5.5.3",
-    "@mikro-orm/migrations": "^5.5.3",
+    "@lido-nestjs/validators-registry": "^1.3.1",
+    "@mikro-orm/cli": "^5.7.13",
+    "@mikro-orm/core": "^5.7.13",
+    "@mikro-orm/knex": "^5.7.13",
+    "@mikro-orm/migrations": "^5.7.13",
     "@mikro-orm/nestjs": "^5.1.2",
-    "@mikro-orm/postgresql": "^5.5.3",
+    "@mikro-orm/postgresql": "^5.7.13",
     "@nestjs/common": "^8.2.5",
     "@nestjs/config": "^1.1.6",
     "@nestjs/core": "^8.2.5",

src/common/registry/test/fetch/key-batch.fetch.e2e-spec.ts

@@ -27,7 +27,8 @@ describe('Fetch keys in batch', () => {
   });
 
   test('fetch one key', async () => {
-    const keys = await fetchService.fetch(address, 17, 0, 3, { blockTag: 9641262 });
+    const overrides = { blockTag: 10573030 };
+    const keys = await fetchService.fetch(address, 17, 0, 3, overrides);
 
     expect(keys).toBeInstanceOf(Array);
 

src/common/registry/test/fetch/key.fetch.e2e-spec.ts

@@ -28,7 +28,7 @@ describe('Keys', () => {
   });
 
   test('fetch one key', async () => {
-    const key = await fetchService.fetchOne(address, 21, 0, { blockTag: 6912872 });
+    const key = await fetchService.fetchOne(address, 17, 0, { blockTag: 10573030 });
 
     expect(key).toBeInstanceOf(Object);
 
@@ -40,23 +40,23 @@ describe('Keys', () => {
   });
 
   test('fetch operator keys', async () => {
-    const keys = await fetchService.fetch(address, 21, 0, -1, {
-      blockTag: 6912872,
+    const keys = await fetchService.fetch(address, 17, 0, 3, {
+      blockTag: 10573030,
     });
     expect(keys).toBeInstanceOf(Array);
     expect(keys.length).toBe(3);
   }, 15_000);
 
   test('fetch several keys', async () => {
-    const keys = await fetchService.fetch(address, 21, 0, 2, {
-      blockTag: 6912872,
+    const keys = await fetchService.fetch(address, 17, 0, 2, {
+      blockTag: 10573030,
     });
 
     expect(keys).toBeInstanceOf(Array);
     expect(keys.length).toBe(2);
 
-    expect(keys[0].operatorIndex).toBe(21);
-    expect(keys[1].operatorIndex).toBe(21);
+    expect(keys[0].operatorIndex).toBe(17);
+    expect(keys[1].operatorIndex).toBe(17);
 
     expect(keys[0].index).toBe(0);
     expect(keys[1].index).toBe(1);

src/common/registry/test/fetch/operator.fetch.e2e-spec.ts

@@ -52,15 +52,17 @@ describe('Operators', () => {
 
   test('fetch all operators', async () => {
     const operators = await fetchService.fetch(address, 0, -1, {
-      blockTag: 9641262,
+      blockTag: 10573030,
     });
 
     expect(operators).toBeInstanceOf(Array);
     expect(operators.length).toBeGreaterThan(0);
   }, 30_000);
 
   test('fetch multiply operators', async () => {
-    const operators = await fetchService.fetch(address, 1, 3);
+    const operators = await fetchService.fetch(address, 1, 3, {
+      blockTag: 10573030,
+    });
 
     expect(operators).toBeInstanceOf(Array);
     expect(operators.length).toBe(2);

src/common/registry/test/key-registry/connect.e2e-spec.ts

@@ -2,8 +2,7 @@ import { Test } from '@nestjs/testing';
 import { nullTransport, LoggerModule } from '@lido-nestjs/logger';
 import { BatchProviderModule, ExtendedJsonRpcBatchProvider } from '@lido-nestjs/execution';
 import { KeyRegistryModule, KeyRegistryService, RegistryOperatorFetchService, RegistryStorageService } from '../../';
-import { clearDb, compareTestOperators } from '../testing.utils';
-import { operators } from '../fixtures/connect.fixture';
+import { clearDb } from '../testing.utils';
 import { MikroORM } from '@mikro-orm/core';
 import { REGISTRY_CONTRACT_ADDRESSES } from '@lido-nestjs/contracts';
 import * as dotenv from 'dotenv';
@@ -21,11 +20,11 @@ describe('Registry', () => {
     process.exit(1);
   }
   const address = REGISTRY_CONTRACT_ADDRESSES[process.env.CHAIN_ID];
-  const operatorsWithModuleAddress = operators.map((key) => {
-    return { ...key, moduleAddress: address };
-  });
+  // const operatorsWithModuleAddress = operators.map((key) => {
+  //   return { ...key, moduleAddress: address };
+  // });
 
-  const blockHash = '0x4ef0f15a8a04a97f60a9f76ba83d27bcf98dac9635685cd05fe1d78bd6e93418';
+  const blockHash = '0x42e6d3fe6df4bc4bdfda27595a015ac9fd5af65cf9bd9d8ad0f2ac802dd73749';
 
   beforeEach(async () => {
     const imports = [
@@ -51,7 +50,6 @@ describe('Registry', () => {
     storageService = moduleRef.get(RegistryStorageService);
     registryOperatorFetchService = moduleRef.get(RegistryOperatorFetchService);
     mikroOrm = moduleRef.get(MikroORM);
-
     jest.spyOn(registryOperatorFetchService, 'getFinalizedBlockTag').mockImplementation(() => ({ blockHash } as any));
 
     const generator = mikroOrm.getSchemaGenerator();
@@ -67,11 +65,14 @@ describe('Registry', () => {
   test('Update', async () => {
     await registryService.update(address, blockHash);
 
-    await compareTestOperators(address, registryService, {
-      operators: operatorsWithModuleAddress,
-    });
+    // TODO: consider uncommit on holesky
+    // await compareTestOperators(address, registryService, {
+    //   operators: operatorsWithModuleAddress,
+    // });
 
+    const operators = await registryService.getOperatorsFromStorage(address);
+    expect(operators).toHaveLength(89);
     const keys = await registryService.getOperatorsKeysFromStorage(address);
-    expect(keys).toHaveLength(20527);
+    expect(keys).toHaveLength(57816);
   }, 400_000);
 });

src/common/registry/test/validator-registry/connect.e2e-spec.ts

@@ -10,8 +10,7 @@ import {
   RegistryOperatorFetchService,
 } from '../../';
 
-import { clearDb, compareTestOperators } from '../testing.utils';
-import { operators } from '../fixtures/connect.fixture';
+import { clearDb } from '../testing.utils';
 import { MikroORM } from '@mikro-orm/core';
 import { REGISTRY_CONTRACT_ADDRESSES } from '@lido-nestjs/contracts';
 import * as dotenv from 'dotenv';
@@ -30,11 +29,11 @@ describe('Registry', () => {
   }
   const address = REGISTRY_CONTRACT_ADDRESSES[process.env.CHAIN_ID];
 
-  const operatorsWithModuleAddress = operators.map((key) => {
-    return { ...key, moduleAddress: address };
-  });
+  // const operatorsWithModuleAddress = operators.map((key) => {
+  //   return { ...key, moduleAddress: address };
+  // });
 
-  const blockHash = '0x4ef0f15a8a04a97f60a9f76ba83d27bcf98dac9635685cd05fe1d78bd6e93418';
+  const blockHash = '0x42e6d3fe6df4bc4bdfda27595a015ac9fd5af65cf9bd9d8ad0f2ac802dd73749';
 
   beforeEach(async () => {
     const imports = [
@@ -77,10 +76,13 @@ describe('Registry', () => {
   test('Update', async () => {
     await registryService.update(address, blockHash);
 
-    await compareTestOperators(address, registryService, {
-      operators: operatorsWithModuleAddress,
-    });
+    // await compareTestOperators(address, registryService, {
+    //   operators: operatorsWithModuleAddress,
+    // });
+
+    const operators = await registryService.getOperatorsFromStorage(address);
+    expect(operators).toHaveLength(89);
     const keys = await registryService.getOperatorsKeysFromStorage(address);
-    expect(keys).toHaveLength(15283);
+    expect(keys).toHaveLength(55131);
   }, 400_000);
 });

src/jobs/keys-update/keys-update.service.ts

@@ -117,6 +117,7 @@ export class KeysUpdateService {
   public async update(): Promise<{ number: number; hash: string; timestamp: number } | undefined> {
     // reading latest block from blockchain
     const currElMeta = await this.executionProvider.getBlock('latest');
+    this.logger.log('Fetched latest block');
     // read from database last execution layer data
     const prevElMeta = await this.elMetaStorage.get();
 

src/validators/validators.service.ts

@@ -4,14 +4,15 @@ import {
   ConsensusValidatorsAndMetadata,
   Validator,
   ConsensusMeta,
+  ValidatorStatus,
 } from '@lido-nestjs/validators-registry';
 import { LOGGER_PROVIDER, LoggerService } from '../common/logger';
 import { ConfigService } from '../common/config';
 import { QueryOrder } from '@mikro-orm/core';
 
 export interface ValidatorsFilter {
   pubkeys: string[];
-  statuses: string[];
+  statuses: ValidatorStatus[];
   max_amount: number | undefined;
   percent: number | undefined;
 }