From dabea500a556ec0bc75b428469552dbaf6bd2d01 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Mon, 5 Feb 2024 12:49:03 +0400 Subject: [PATCH 01/15] feat: common --- .env | 0 .env.example | 13 ++ .eslintrc.js | 7 + .gitignore | 4 + .nvmrc | 2 +- .prettierrc | 5 +- package.json | 10 ++ src/cli/cli.service.ts | 11 +- src/common/config/config.service.ts | 10 +- src/common/config/env.validation.ts | 15 +++ src/common/handlers/handlers.module.ts | 1 + src/common/handlers/handlers.service.ts | 126 +++++++++++++++++- src/common/logger/logger.module.ts | 10 +- src/common/providers/base/rest-provider.ts | 92 +++++++++++++ src/common/providers/consensus/consensus.ts | 123 ++++++++++++++++- .../providers/consensus/response.interface.ts | 124 +++++++++++++++++ src/common/providers/keysapi/keysapi.spec.ts | 1 + src/common/providers/keysapi/keysapi.ts | 73 +++++++++- src/common/providers/providers.module.ts | 6 +- .../download-progress.spec.ts | 19 +++ .../download-progress/download-progress.ts | 67 ++++++++++ src/common/utils/utils.module.ts | 9 ++ src/main.ts | 4 +- tsconfig.json | 2 +- yarn.lock | 103 +++++++++++++- 25 files changed, 794 insertions(+), 43 deletions(-) delete mode 100644 .env create mode 100644 src/common/providers/base/rest-provider.ts create mode 100644 src/common/providers/consensus/response.interface.ts create mode 100644 src/common/utils/download-progress/download-progress.spec.ts create mode 100644 src/common/utils/download-progress/download-progress.ts create mode 100644 src/common/utils/utils.module.ts diff --git a/.env b/.env deleted file mode 100644 index e69de29..0000000 diff --git a/.env.example b/.env.example index e69de29..0184643 100644 --- a/.env.example +++ b/.env.example @@ -0,0 +1,13 @@ +# CLI working mode +#EL_RPC_URLS=https://mainnet.infura.io/v3/... +#CL_API_URLS=https://quiknode.pro/... +#TX_SENDER_PRIVATE_KEY=... + +# Daemon working mode +#ETH_NETWORK=1 +#EL_RPC_URLS=https://mainnet.infura.io/v3/... +#CL_API_URLS=https://quiknode.pro/... +#KEYSAPI_API_URLS=https://keys-api.lido.fi/ +#LIDO_STAKING_MODULE_ADDRESS="0x9D4AF1Ee19Dad8857db3a45B0374c81c8A1C6320" +#TX_SENDER_PRIVATE_KEY=... + diff --git a/.eslintrc.js b/.eslintrc.js index 8f9e52c..67d7c6a 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -18,6 +18,13 @@ module.exports = { jest: true, }, ignorePatterns: ['.eslintrc.js'], + settings: { + 'import/resolver': { + typescript: { + project: './tsconfig.json', + }, + }, + }, rules: { '@typescript-eslint/interface-name-prefix': 'off', '@typescript-eslint/explicit-function-return-type': 'off', diff --git a/.gitignore b/.gitignore index aa0e6d0..7834f27 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,10 @@ # ENV /.env +# Storage +/.keys-indexer-* +/.roots-stack-* + # Logs logs *.log diff --git a/.nvmrc b/.nvmrc index 860cc50..7ea6a59 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v18.17.1 +v20.11.0 diff --git a/.prettierrc b/.prettierrc index dcb7279..9db28a2 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,4 +1,5 @@ { "singleQuote": true, - "trailingComma": "all" -} \ No newline at end of file + "trailingComma": "all", + "printWidth": 120 +} diff --git a/package.json b/package.json index 297bc63..84ea395 100644 --- a/package.json +++ b/package.json @@ -22,15 +22,21 @@ "test:e2e": "jest --config ./test/jest-e2e.json" }, "dependencies": { + "@huanshiwushuang/lowdb": "^6.0.2", "@lido-nestjs/logger": "^1.3.2", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.1.1", "@nestjs/core": "^10.0.0", "@nestjs/platform-express": "^10.0.0", + "@types/cli-progress": "^3.11.5", + "cli-progress": "^3.12.0", "nest-commander": "^3.12.5", "nest-winston": "^1.9.4", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.1", + "stream-chain": "^2.2.5", + "stream-json": "^1.8.0", + "undici": "^6.4.0", "winston": "^3.11.0" }, "devDependencies": { @@ -42,12 +48,16 @@ "@swc/jest": "^0.2.30", "@types/express": "^4.17.17", "@types/jest": "^29.5.2", + "@types/lowdb": "1.0.0", "@types/node": "^20.3.1", + "@types/stream-chain": "^2.0.4", + "@types/stream-json": "^1.7.7", "@types/supertest": "^6.0.0", "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^6.0.0", "eslint": "^8.42.0", "eslint-config-prettier": "^9.0.0", + "eslint-import-resolver-typescript": "^3.6.1", "eslint-plugin-import": "^2.29.1", "eslint-plugin-prettier": "^5.0.0", "jest": "^29.5.0", diff --git a/src/cli/cli.service.ts b/src/cli/cli.service.ts index 0d62947..cb18225 100644 --- a/src/cli/cli.service.ts +++ b/src/cli/cli.service.ts @@ -1,16 +1,9 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; -import { - Inject, - Injectable, - LoggerService, - OnApplicationBootstrap, -} from '@nestjs/common'; +import { Inject, Injectable, LoggerService, OnApplicationBootstrap } from '@nestjs/common'; @Injectable() export class CliService implements OnApplicationBootstrap { - constructor( - @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, - ) {} + constructor(@Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService) {} async onApplicationBootstrap() { this.logger.log('Working mode: CLI'); } diff --git a/src/common/config/config.service.ts b/src/common/config/config.service.ts index 39b64a1..270747b 100644 --- a/src/common/config/config.service.ts +++ b/src/common/config/config.service.ts @@ -7,16 +7,10 @@ export class ConfigService extends ConfigServiceSource { * List of env variables that should be hidden */ public get secrets(): string[] { - return [ - ...this.get('EL_RPC_URLS'), - ...this.get('CL_API_URLS'), - ...this.get('KEYSAPI_API_URLS'), - ]; + return [...this.get('EL_RPC_URLS'), ...this.get('CL_API_URLS'), ...this.get('KEYSAPI_API_URLS')]; } - public get( - key: T, - ): EnvironmentVariables[T] { + public get(key: T): EnvironmentVariables[T] { return super.get(key, { infer: true }) as EnvironmentVariables[T]; } } diff --git a/src/common/config/env.validation.ts b/src/common/config/env.validation.ts index 5ede437..b134f9e 100644 --- a/src/common/config/env.validation.ts +++ b/src/common/config/env.validation.ts @@ -32,6 +32,21 @@ export class EnvironmentVariables { @IsEnum(WorkingMode) public WORKING_MODE = WorkingMode.Daemon; + public START_ROOT?: string; + + @IsNotEmpty() + public LIDO_STAKING_MODULE_ADDRESS: string; + + @IsNumber() + @Min(30 * 60 * 1000) + @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) + public KEYS_INDEXER_RUNNING_PERIOD: number = 3 * 60 * 60 * 1000; + + @IsNumber() + @Min(384000) // epoch time in ms + @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) + public KEYS_INDEXER_KEYAPI_FRESHNESS_PERIOD: number = 8 * 60 * 60 * 1000; + @IsNumber() @Min(1025) @Max(65535) diff --git a/src/common/handlers/handlers.module.ts b/src/common/handlers/handlers.module.ts index 3e9aa7d..fed3190 100644 --- a/src/common/handlers/handlers.module.ts +++ b/src/common/handlers/handlers.module.ts @@ -4,5 +4,6 @@ import { HandlersService } from './handlers.service'; @Module({ providers: [HandlersService], + exports: [HandlersService], }) export class HandlersModule {} diff --git a/src/common/handlers/handlers.service.ts b/src/common/handlers/handlers.service.ts index a1f5eb2..5eed365 100644 --- a/src/common/handlers/handlers.service.ts +++ b/src/common/handlers/handlers.service.ts @@ -1,4 +1,126 @@ -import { Injectable } from '@nestjs/common'; +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; +import { Inject, Injectable, LoggerService } from '@nestjs/common'; + +import { BlockInfoResponse, RootHex } from '../providers/consensus/response.interface'; + +export interface KeyInfo { + operatorId: number; + keyIndex: number; + pubKey: string; + withdrawableEpoch: number; +} + +type KeyInfoFn = (valIndex: number) => KeyInfo | undefined; @Injectable() -export class HandlersService {} +export class HandlersService { + constructor(@Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService) {} + + public async prove(blockRoot: RootHex, blockInfo: BlockInfoResponse, keyInfoFn: KeyInfoFn): Promise { + const slashings = await this.getUnprovenSlashings(blockRoot, blockInfo, keyInfoFn); + const withdrawals = await this.getUnprovenWithdrawals(blockRoot, blockInfo, keyInfoFn); + if (!slashings.length && !withdrawals.length) return; + const payload = await this.buildProvePayload(slashings, withdrawals); + // TODO: ask before sending if CLI or daemon in watch mode + await this.sendProves(payload); + } + + private async buildProvePayload(slashings: string[], withdrawals: string[]): Promise { + // TODO: implement + // this.consensus.getState(...) + return {}; + } + private async sendProves(payload: any): Promise { + // TODO: implement + } + + private async getUnprovenSlashings( + blockRoot: RootHex, + blockInfo: BlockInfoResponse, + keyInfoFn: KeyInfoFn, + ): Promise { + const slashings = [ + ...this.getSlashedProposers(blockInfo, keyInfoFn), + ...this.getSlashedAttesters(blockInfo, keyInfoFn), + ]; + if (!slashings.length) return []; + const unproven = []; + for (const slashing of slashings) { + // TODO: implement + // const proved = await this.execution.isSlashingProved(slashing); + const proved = false; + if (!proved) unproven.push(slashing); + } + if (!unproven.length) { + this.logger.log(`No slashings to prove. Root [${blockRoot}]`); + return []; + } + this.logger.warn(`๐Ÿ” Unproven slashings: ${unproven}`); + return unproven; + } + + private async getUnprovenWithdrawals( + blockRoot: RootHex, + blockInfo: BlockInfoResponse, + keyInfoFn: KeyInfoFn, + ): Promise { + const withdrawals = this.getFullWithdrawals(blockInfo, keyInfoFn); + if (!withdrawals.length) return []; + const unproven = []; + for (const withdrawal of withdrawals) { + // TODO: implement + // const proved = await this.execution.isSlashingProved(slashing); + const proved = false; + if (!proved) unproven.push(withdrawal); + } + if (!unproven.length) { + this.logger.log(`No full withdrawals to prove. Root [${blockRoot}]`); + return []; + } + this.logger.warn(`๐Ÿ” Unproven full withdrawals: ${unproven}`); + return unproven; + } + + private getSlashedAttesters( + blockInfo: BlockInfoResponse, + keyInfoFn: (valIndex: number) => KeyInfo | undefined, + ): string[] { + const slashed = []; + for (const att of blockInfo.message.body.attester_slashings) { + const accused = att.attestation_1.attesting_indices.filter((x) => + att.attestation_2.attesting_indices.includes(x), + ); + slashed.push(...accused.filter((item) => keyInfoFn(Number(item)))); + } + return slashed; + } + + private getSlashedProposers( + blockInfo: BlockInfoResponse, + keyInfoFn: (valIndex: number) => KeyInfo | undefined, + ): string[] { + const slashed = []; + for (const prop of blockInfo.message.body.proposer_slashings) { + if (keyInfoFn(Number(prop.signed_header_1.proposer_index))) { + slashed.push(prop.signed_header_1.proposer_index); + } + } + return slashed; + } + + private getFullWithdrawals( + blockInfo: BlockInfoResponse, + keyInfoFn: (valIndex: number) => KeyInfo | undefined, + ): string[] { + const fullWithdrawals = []; + const blockEpoch = Number(blockInfo.message.slot) / 32; + const withdrawals = blockInfo.message.body.execution_payload.withdrawals; + for (const withdrawal of withdrawals) { + const keyInfo = keyInfoFn(Number(withdrawal.validator_index)); + if (keyInfo && blockEpoch >= keyInfo.withdrawableEpoch) { + fullWithdrawals.push(withdrawal.validator_index); + } + } + return fullWithdrawals; + } +} diff --git a/src/common/logger/logger.module.ts b/src/common/logger/logger.module.ts index 9dd1e6b..b5e6ff6 100644 --- a/src/common/logger/logger.module.ts +++ b/src/common/logger/logger.module.ts @@ -1,8 +1,4 @@ -import { - LoggerModule as Logger, - jsonTransport, - simpleTransport, -} from '@lido-nestjs/logger'; +import { LoggerModule as Logger, jsonTransport, simpleTransport } from '@lido-nestjs/logger'; import { Module } from '@nestjs/common'; import { ConfigModule } from '../config/config.module'; @@ -20,9 +16,7 @@ import { LogFormat } from '../config/interfaces'; const format = configService.get('LOG_FORMAT'); const isJSON = format === LogFormat.JSON; - const transports = isJSON - ? jsonTransport({ secrets }) - : simpleTransport({ secrets }); + const transports = isJSON ? jsonTransport({ secrets }) : simpleTransport({ secrets }); return { level, transports }; }, diff --git a/src/common/providers/base/rest-provider.ts b/src/common/providers/base/rest-provider.ts new file mode 100644 index 0000000..5bd4ee2 --- /dev/null +++ b/src/common/providers/base/rest-provider.ts @@ -0,0 +1,92 @@ +import { LoggerService } from '@nestjs/common'; +import { request } from 'undici'; +import { IncomingHttpHeaders } from 'undici/types/header'; +import BodyReadable from 'undici/types/readable'; + +import { PrometheusService } from '../../prometheus/prometheus.service'; + +export interface RequestPolicy { + timeout: number; + maxRetries: number; + fallbacks: Array; +} + +export interface RequestOptions { + streamed?: boolean; + requestPolicy?: RequestPolicy; + signal?: AbortSignal; +} + +export abstract class BaseRestProvider { + protected readonly mainUrl: string; + protected readonly requestPolicy: RequestPolicy; + + protected constructor( + urls: Array, + responseTimeout: number, + maxRetries: number, + protected readonly logger: LoggerService, + protected readonly prometheus?: PrometheusService, + ) { + this.mainUrl = urls[0]; + this.requestPolicy = { + timeout: responseTimeout, + maxRetries: maxRetries, + fallbacks: urls.slice(1), + }; + } + + // TODO: Request should have: + // 1. metrics (if it is daemon mode) + // 2. retries + // 3. fallbacks + + protected async baseGet( + base: string, + endpoint: string, + options?: RequestOptions, + ): Promise { + options = { + streamed: false, + requestPolicy: this.requestPolicy, + ...options, + } as RequestOptions; + const { body, headers, statusCode } = await request(new URL(endpoint, base), { + method: 'GET', + headersTimeout: (options.requestPolicy as RequestPolicy).timeout, + signal: options.signal, + }); + if (statusCode !== 200) { + const hostname = new URL(base).hostname; + throw new Error(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`); + } + return options.streamed ? { body: body, headers: headers } : ((await body.json()) as T); + } + + protected async basePost( + base: string, + endpoint: string, + requestBody: any, + options?: RequestOptions, + ): Promise { + options = { + streamed: false, + requestPolicy: this.requestPolicy, + ...options, + } as RequestOptions; + const { body, headers, statusCode } = await request(new URL(endpoint, base), { + method: 'POST', + headersTimeout: (options.requestPolicy as RequestPolicy).timeout, + signal: options.signal, + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + }); + if (statusCode !== 200) { + const hostname = new URL(base).hostname; + throw new Error(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`); + } + return options.streamed ? { body: body, headers: headers } : ((await body.json()) as T); + } +} diff --git a/src/common/providers/consensus/consensus.ts b/src/common/providers/consensus/consensus.ts index 0c755d2..849c179 100644 --- a/src/common/providers/consensus/consensus.ts +++ b/src/common/providers/consensus/consensus.ts @@ -1,4 +1,123 @@ -import { Injectable } from '@nestjs/common'; +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; +import { Inject, Injectable, LoggerService, OnApplicationBootstrap, Optional } from '@nestjs/common'; +import { chain } from 'stream-chain'; +import { parser } from 'stream-json'; +import { connectTo } from 'stream-json/Assembler'; +import { IncomingHttpHeaders } from 'undici/types/header'; +import BodyReadable from 'undici/types/readable'; + +import { + BlockHeaderResponse, + BlockId, + BlockInfoResponse, + GenesisResponse, + RootHex, + StateId, + StateValidatorResponse, +} from './response.interface'; +import { ConfigService } from '../../config/config.service'; +import { PrometheusService } from '../../prometheus/prometheus.service'; +import { DownloadProgress } from '../../utils/download-progress/download-progress'; +import { BaseRestProvider } from '../base/rest-provider'; @Injectable() -export class Consensus {} +export class Consensus extends BaseRestProvider implements OnApplicationBootstrap { + private readonly endpoints = { + version: 'eth/v1/node/version', + genesis: 'eth/v1/beacon/genesis', + blockInfo: (blockId: BlockId): string => `eth/v2/beacon/blocks/${blockId}`, + beaconHeader: (blockId: BlockId): string => `eth/v1/beacon/headers/${blockId}`, + beaconHeadersByParentRoot: (parentRoot: RootHex): string => `eth/v1/beacon/headers?parent_root=${parentRoot}`, + validators: (stateId: StateId): string => `eth/v1/beacon/states/${stateId}/validators`, + state: (stateId: StateId): string => `eth/v2/debug/beacon/states/${stateId}`, + }; + + public genesisTimestamp: number; + // TODO: configurable + public SLOTS_PER_EPOCH: number = 32; + public SECONDS_PER_SLOT: number = 12; + + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + @Optional() protected readonly prometheus: PrometheusService, + protected readonly config: ConfigService, + protected readonly progress: DownloadProgress, + ) { + super( + config.get('CL_API_URLS') as Array, + config.get('CL_API_RESPONSE_TIMEOUT'), + config.get('CL_API_MAX_RETRIES'), + logger, + prometheus, + ); + } + + public async onApplicationBootstrap(): Promise { + this.logger.log(`Getting genesis timestamp`); + const resp = await this.getGenesis(); + this.genesisTimestamp = Number(resp.genesis_time); + } + + public slotToTimestamp(slot: number): number { + return this.genesisTimestamp + slot * this.SECONDS_PER_SLOT; + } + + public async getGenesis(): Promise { + return (await this.baseGet(this.mainUrl, this.endpoints.genesis)).data as GenesisResponse; + } + + public async getBlockInfo(blockId: BlockId): Promise { + return (await this.baseGet(this.mainUrl, this.endpoints.blockInfo(blockId))).data as BlockInfoResponse; + } + + public async getBeaconHeader(blockId: BlockId): Promise { + return (await this.baseGet(this.mainUrl, this.endpoints.beaconHeader(blockId))).data as BlockHeaderResponse; + } + + public async getBeaconHeadersByParentRoot( + parentRoot: RootHex, + ): Promise<{ finalized: boolean; data: BlockHeaderResponse[] }> { + return (await this.baseGet(this.mainUrl, this.endpoints.beaconHeadersByParentRoot(parentRoot))) as { + finalized: boolean; + data: BlockHeaderResponse[]; + }; + } + + public async getValidators(stateId: StateId, signal?: AbortSignal): Promise { + const resp: { body: BodyReadable; headers: IncomingHttpHeaders } = await this.baseGet( + this.mainUrl, + this.endpoints.validators(stateId), + { + streamed: true, + signal, + }, + ); + // Progress bar + // TODO: Enable for CLI only + //this.progress.show('Validators from state', resp); + // Data processing + const pipeline = chain([resp.body, parser()]); + return await new Promise((resolve) => { + connectTo(pipeline).on('done', (asm) => resolve(asm.current.data)); + }); + } + + public async getState(stateId: StateId, signal?: AbortSignal): Promise { + const resp: { body: BodyReadable; headers: IncomingHttpHeaders } = await this.baseGet( + this.mainUrl, + this.endpoints.state(stateId), + { + streamed: true, + signal, + }, + ); + // Progress bar + // TODO: Enable for CLI only + //this.progress.show(`State [${stateId}]`, resp); + // Data processing + const pipeline = chain([resp.body, parser()]); + return await new Promise((resolve) => { + connectTo(pipeline).on('done', (asm) => resolve(asm.current)); + }); + } +} diff --git a/src/common/providers/consensus/response.interface.ts b/src/common/providers/consensus/response.interface.ts new file mode 100644 index 0000000..31e7667 --- /dev/null +++ b/src/common/providers/consensus/response.interface.ts @@ -0,0 +1,124 @@ +export type BLSSignature = string; +export type ValidatorIndex = string; +export type RootHex = string; +export type Slot = number; +export type Epoch = number; +export type BlockId = RootHex | Slot | 'head' | 'genesis' | 'finalized'; +export type StateId = RootHex | Slot | 'head' | 'genesis' | 'finalized' | 'justified'; + +export enum ValStatus { + ActiveOngoing = 'active_ongoing', + ActiveExiting = 'active_exiting', + PendingQueued = 'pending_queued', + PendingInitialized = 'pending_initialized', + ActiveSlashed = 'active_slashed', + ExitedSlashed = 'exited_slashed', + ExitedUnslashed = 'exited_unslashed', + WithdrawalPossible = 'withdrawal_possible', + WithdrawalDone = 'withdrawal_done', +} + +export interface BlockHeaderResponse { + root: RootHex; + canonical: boolean; + header: { + message: { + slot: Slot; + proposer_index: ValidatorIndex; + parent_root: RootHex; + state_root: RootHex; + body_root: RootHex; + }; + signature: BLSSignature; + }; +} + +export interface BlockInfoResponse { + message: { + slot: string; + proposer_index: ValidatorIndex; + body: { + attestations: BeaconBlockAttestation[]; + proposer_slashings: { + signed_header_1: { + proposer_index: string; + }; + signed_header_2: { + proposer_index: string; + }; + }[]; + attester_slashings: { + attestation_1: { + attesting_indices: string[]; + }; + attestation_2: { + attesting_indices: string[]; + }; + }[]; + execution_payload: { + withdrawals: Withdrawal[]; + }; + }; + }; +} + +export interface Withdrawal { + index: string; + validator_index: ValidatorIndex; + address: string; + amount: string; +} + +export interface GenesisResponse { + /** + * example: 1590832934 + * The genesis_time configured for the beacon node, which is the unix time in seconds at which the Eth2.0 chain began. + */ + genesis_time: string; + + /** + * example: 0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2 + * pattern: ^0x[a-fA-F0-9]{64}$ + */ + genesis_validators_root: string; + + /** + * example: 0x00000000 + * pattern: ^0x[a-fA-F0-9]{8}$ + * a fork version number + */ + genesis_fork_version: string; +} + +export interface BeaconBlockAttestation { + aggregation_bits: string; + data: { + slot: string; + index: string; + beacon_block_root: RootHex; + source: { + epoch: string; + root: RootHex; + }; + target: { + epoch: string; + root: RootHex; + }; + }; +} + +export interface StateValidatorResponse { + index: string; + balance: string; + status: (typeof ValStatus)[keyof typeof ValStatus]; + validator: { + pubkey: string; + withdrawal_credentials: string; + effective_balance: string; + slashed: boolean; + activation_eligibility_epoch: string; + activation_epoch: string; + exit_epoch: string; + withdrawable_epoch: string; + }; +} diff --git a/src/common/providers/keysapi/keysapi.spec.ts b/src/common/providers/keysapi/keysapi.spec.ts index 59e018c..3f745b3 100644 --- a/src/common/providers/keysapi/keysapi.spec.ts +++ b/src/common/providers/keysapi/keysapi.spec.ts @@ -1,4 +1,5 @@ import { Test, TestingModule } from '@nestjs/testing'; + import { Keysapi } from './keysapi'; describe('Keysapi', () => { diff --git a/src/common/providers/keysapi/keysapi.ts b/src/common/providers/keysapi/keysapi.ts index f81d585..6529b11 100644 --- a/src/common/providers/keysapi/keysapi.ts +++ b/src/common/providers/keysapi/keysapi.ts @@ -1,4 +1,73 @@ -import { Injectable } from '@nestjs/common'; +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; +import { Inject, Injectable, LoggerService, Optional } from '@nestjs/common'; +import { chain } from 'stream-chain'; +import { parser } from 'stream-json'; +import { connectTo } from 'stream-json/Assembler'; +import { IncomingHttpHeaders } from 'undici/types/header'; +import BodyReadable from 'undici/types/readable'; + +import { ConfigService } from '../../config/config.service'; +import { PrometheusService } from '../../prometheus/prometheus.service'; +import { BaseRestProvider } from '../base/rest-provider'; @Injectable() -export class Keysapi {} +export class Keysapi extends BaseRestProvider { + private readonly endpoints = { + status: 'v1/status', + modules: 'v1/modules', + moduleKeys: (module_id: string | number): string => `v1/modules/${module_id}/keys`, + findModuleKeys: (module_id: string | number): string => `v1/modules/${module_id}/keys/find`, + }; + + // TODO: types + + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + @Optional() protected readonly prometheus: PrometheusService, + protected readonly config: ConfigService, + ) { + super( + config.get('KEYSAPI_API_URLS') as Array, + config.get('KEYSAPI_API_RESPONSE_TIMEOUT'), + config.get('KEYSAPI_API_MAX_RETRIES'), + logger, + prometheus, + ); + } + + public healthCheck(finalizedTimestamp: number, keysApiMetadata: any): void { + if ( + finalizedTimestamp - keysApiMetadata.elBlockSnapshot.timestamp > + this.config.get('KEYS_INDEXER_KEYAPI_FRESHNESS_PERIOD') + ) { + throw new Error('KeysApi is outdated'); + } + } + + public async getStatus(): Promise { + return await this.baseGet(this.mainUrl, this.endpoints.status); + } + + public async getModules(): Promise { + return await this.baseGet(this.mainUrl, this.endpoints.modules); + } + + public async getModuleKeys(module_id: string | number, signal?: AbortSignal): Promise { + const resp: { body: BodyReadable; headers: IncomingHttpHeaders } = await this.baseGet( + this.mainUrl, + this.endpoints.moduleKeys(module_id), + { + streamed: true, + signal, + }, + ); + const pipeline = chain([resp.body, parser()]); + return await new Promise((resolve) => { + connectTo(pipeline).on('done', (asm) => resolve(asm.current)); + }); + } + + public async findModuleKeys(module_id: string | number, keysToFind: string[], signal?: AbortSignal): Promise { + return await this.basePost(this.mainUrl, this.endpoints.findModuleKeys(module_id), { pubkeys: keysToFind, signal }); + } +} diff --git a/src/common/providers/providers.module.ts b/src/common/providers/providers.module.ts index e401e93..77cdaf4 100644 --- a/src/common/providers/providers.module.ts +++ b/src/common/providers/providers.module.ts @@ -1,17 +1,19 @@ import { Module } from '@nestjs/common'; +import { ConditionalModule } from '@nestjs/config'; import { Consensus } from './consensus/consensus'; import { Execution } from './execution/execution'; import { Keysapi } from './keysapi/keysapi'; -import { ConditionalModule } from '@nestjs/config'; -import { PrometheusModule } from '../prometheus/prometheus.module'; import { WorkingMode } from '../config/env.validation'; +import { PrometheusModule } from '../prometheus/prometheus.module'; +import { UtilsModule } from '../utils/utils.module'; @Module({ imports: [ ConditionalModule.registerWhen(PrometheusModule, (env: NodeJS.ProcessEnv) => { return env['WORKING_MODE'] === WorkingMode.Daemon; }), + UtilsModule, ], providers: [Execution, Consensus, Keysapi], exports: [Execution, Consensus, Keysapi], diff --git a/src/common/utils/download-progress/download-progress.spec.ts b/src/common/utils/download-progress/download-progress.spec.ts new file mode 100644 index 0000000..b30bfa5 --- /dev/null +++ b/src/common/utils/download-progress/download-progress.spec.ts @@ -0,0 +1,19 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { DownloadProgress } from './download-progress'; + +describe('DownloadProgress', () => { + let provider: DownloadProgress; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [DownloadProgress], + }).compile(); + + provider = module.get(DownloadProgress); + }); + + it('should be defined', () => { + expect(provider).toBeDefined(); + }); +}); diff --git a/src/common/utils/download-progress/download-progress.ts b/src/common/utils/download-progress/download-progress.ts new file mode 100644 index 0000000..57d2d2f --- /dev/null +++ b/src/common/utils/download-progress/download-progress.ts @@ -0,0 +1,67 @@ +import { Injectable } from '@nestjs/common'; +import { MultiBar, Presets, SingleBar } from 'cli-progress'; +import { IncomingHttpHeaders } from 'undici/types/header'; +import BodyReadable from 'undici/types/readable'; + +@Injectable() +export class DownloadProgress { + private multibar: MultiBar; + + constructor() {} + + // TODO: how it works when error occurs from other promises in the same time ? + + public show(name: string, resp: { body: BodyReadable; headers: IncomingHttpHeaders }): void { + const totalContentLength = Number(resp.headers['content-length']); + let downloaded = 0; + let speed = '0.00'; + let downloadedMb = 0; + let ratio = 0; + const start = Date.now(); + const bar = this.add(name, Number((totalContentLength / 1024 / 1024).toFixed(2))); + + resp.body.on('data', (chunk) => { + downloaded += chunk.length; + ratio = downloaded / totalContentLength; + downloadedMb = Number(downloaded / 1024 / 1024); + speed = (downloadedMb / ((Date.now() - start) / 1000)).toFixed(2); + bar.update(ratio, { + speed, + downloaded: downloadedMb.toFixed(2), + status: 'โคต๏ธ ', + }); + }); + resp.body.on('end', () => renderFinish('โœ… Downloaded\n')); + resp.body.on('error', () => renderFinish('โŒ Failed\n')); + + const renderFinish = (status: string) => { + bar.update(ratio, { + speed, + downloaded: downloadedMb.toFixed(2), + status, + }); + bar.render(); + bar.stop(); + }; + } + + private add(name: string, size: number): SingleBar { + if (!this.multibar) { + this.initMultibar(); + } + return this.multibar.create(1, 0, { name, size }); + } + + private initMultibar(): void { + this.multibar = new MultiBar( + { + fps: 1, + hideCursor: true, + noTTYOutput: true, + emptyOnZero: true, + format: ` | {name} |{bar}| {percentage}% || {downloaded} of {size} Mb | Speed: {speed} Mb/s | {status}`, + }, + Presets.shades_grey, + ); + } +} diff --git a/src/common/utils/utils.module.ts b/src/common/utils/utils.module.ts new file mode 100644 index 0000000..ad7be84 --- /dev/null +++ b/src/common/utils/utils.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; + +import { DownloadProgress } from './download-progress/download-progress'; + +@Module({ + providers: [DownloadProgress], + exports: [DownloadProgress], +}) +export class UtilsModule {} diff --git a/src/main.ts b/src/main.ts index 1fcd41c..99551d9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,7 +7,7 @@ import { ConfigService } from './common/config/config.service'; import { WorkingMode } from './common/config/env.validation'; import { DaemonModule } from './daemon/daemon.module'; -async function bootstrapCli() { +async function bootstrapCLI() { const cliApp = await CommandFactory.createWithoutRunning(CliModule, { bufferLogs: true, }); @@ -28,7 +28,7 @@ async function bootstrapDaemon() { async function bootstrap() { switch (process.env.WORKING_MODE) { case WorkingMode.CLI: - await bootstrapCli(); + await bootstrapCLI(); break; case WorkingMode.Daemon: await bootstrapDaemon(); diff --git a/tsconfig.json b/tsconfig.json index a1c778d..a86f36d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,6 +16,6 @@ "noImplicitAny": true, "strictBindCallApply": true, "forceConsistentCasingInFileNames": true, - "noFallthroughCasesInSwitch": true + "noFallthroughCasesInSwitch": true, } } diff --git a/yarn.lock b/yarn.lock index 4168039..1e57076 100644 --- a/yarn.lock +++ b/yarn.lock @@ -395,6 +395,11 @@ resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.56.0.tgz#ef20350fec605a7f7035a01764731b2de0f3782b" integrity sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A== +"@fastify/busboy@^2.0.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.0.tgz#0709e9f4cb252351c609c6e6d8d6779a8d25edff" + integrity sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA== + "@fig/complete-commander@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@fig/complete-commander/-/complete-commander-3.0.0.tgz#02826f8604adfb694e576b39ad53acd735aa9d18" @@ -409,6 +414,11 @@ dependencies: lodash "^4.17.21" +"@huanshiwushuang/lowdb@^6.0.2": + version "6.0.2" + resolved "https://registry.yarnpkg.com/@huanshiwushuang/lowdb/-/lowdb-6.0.2.tgz#983d4258e8f77ce9d4b51bae984e64e6230a9ee9" + integrity sha512-Icn1EUg2F9NjQ6h0l6QRnbLaoEHsofOubrCGSK76LbjOpAb6MUNW+B6Bs0OcA4iyoLayToLFqNYANF2u6m/WXQ== + "@humanwhocodes/config-array@^0.11.13": version "0.11.14" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" @@ -1072,6 +1082,13 @@ "@types/node" "*" "@types/responselike" "^1.0.0" +"@types/cli-progress@^3.11.5": + version "3.11.5" + resolved "https://registry.yarnpkg.com/@types/cli-progress/-/cli-progress-3.11.5.tgz#9518c745e78557efda057e3f96a5990c717268c3" + integrity sha512-D4PbNRbviKyppS5ivBGyFO29POlySLmA2HyUFE4p5QGazAMM3CwkKWcvTl8gvElSuxRh6FPKL8XmidX873ou4g== + dependencies: + "@types/node" "*" + "@types/connect@*": version "3.4.38" resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.38.tgz#5ba7f3bc4fbbdeaff8dded952e5ff2cc53f8d858" @@ -1186,6 +1203,18 @@ dependencies: "@types/node" "*" +"@types/lodash@*": + version "4.14.202" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.202.tgz#f09dbd2fb082d507178b2f2a5c7e74bd72ff98f8" + integrity sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ== + +"@types/lowdb@1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@types/lowdb/-/lowdb-1.0.0.tgz#603f73895660537f57cb248c1169311b451151bd" + integrity sha512-gYmB2gACsmtIOnmH9fIrxclLBUfvSS62pIfvKwXNTSkOt6wTH3OKjXa/TJyBfJYgyIQlJlCkK+9oJ9YvwdYwLg== + dependencies: + "@types/lodash" "*" + "@types/methods@^1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@types/methods/-/methods-1.1.4.tgz#d3b7ac30ac47c91054ea951ce9eed07b1051e547" @@ -1252,6 +1281,21 @@ resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.3.tgz#6209321eb2c1712a7e7466422b8cb1fc0d9dd5d8" integrity sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw== +"@types/stream-chain@*", "@types/stream-chain@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@types/stream-chain/-/stream-chain-2.0.4.tgz#065f0a75dc18db9f2747a8283ea450c37248a919" + integrity sha512-V7TsWLHrx79KumkHqSD7F8eR6POpEuWb6PuXJ7s/dRHAf3uVst3Jkp1yZ5XqIfECZLQ4a28vBVstTErmsMBvaQ== + dependencies: + "@types/node" "*" + +"@types/stream-json@^1.7.7": + version "1.7.7" + resolved "https://registry.yarnpkg.com/@types/stream-json/-/stream-json-1.7.7.tgz#8660101e15ee52e9a2370727334269ad7ec6a759" + integrity sha512-hHG7cLQ09H/m9i0jzL6UJAeLLxIWej90ECn0svO4T8J0nGcl89xZDQ2ujT4WKlvg0GWkcxJbjIDzW/v7BYUM6Q== + dependencies: + "@types/node" "*" + "@types/stream-chain" "*" + "@types/superagent@^8.1.0": version "8.1.1" resolved "https://registry.yarnpkg.com/@types/superagent/-/superagent-8.1.1.tgz#dbc620c5df3770b0c3092f947d6d5e808adae2bc" @@ -2079,6 +2123,13 @@ cli-cursor@^3.1.0: dependencies: restore-cursor "^3.1.0" +cli-progress@^3.12.0: + version "3.12.0" + resolved "https://registry.yarnpkg.com/cli-progress/-/cli-progress-3.12.0.tgz#807ee14b66bcc086258e444ad0f19e7d42577942" + integrity sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A== + dependencies: + string-width "^4.2.3" + cli-spinners@^2.5.0: version "2.9.2" resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-2.9.2.tgz#1773a8f4b9c4d6ac31563df53b3fc1d79462fe41" @@ -2525,7 +2576,7 @@ end-of-stream@^1.1.0: dependencies: once "^1.4.0" -enhanced-resolve@^5.0.0, enhanced-resolve@^5.15.0, enhanced-resolve@^5.7.0: +enhanced-resolve@^5.0.0, enhanced-resolve@^5.12.0, enhanced-resolve@^5.15.0, enhanced-resolve@^5.7.0: version "5.15.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz#1af946c7d93603eb88e9896cee4904dc012e9c35" integrity sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg== @@ -2659,7 +2710,20 @@ eslint-import-resolver-node@^0.3.9: is-core-module "^2.13.0" resolve "^1.22.4" -eslint-module-utils@^2.8.0: +eslint-import-resolver-typescript@^3.6.1: + version "3.6.1" + resolved "https://registry.yarnpkg.com/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-3.6.1.tgz#7b983680edd3f1c5bce1a5829ae0bc2d57fe9efa" + integrity sha512-xgdptdoi5W3niYeuQxKmzVDTATvLYqhpwmykwsh7f6HIOStGWEIL9iqZgQDF9u9OEzrRwR8no5q2VT+bjAujTg== + dependencies: + debug "^4.3.4" + enhanced-resolve "^5.12.0" + eslint-module-utils "^2.7.4" + fast-glob "^3.3.1" + get-tsconfig "^4.5.0" + is-core-module "^2.11.0" + is-glob "^4.0.3" + +eslint-module-utils@^2.7.4, eslint-module-utils@^2.8.0: version "2.8.0" resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.8.0.tgz#e439fee65fc33f6bba630ff621efc38ec0375c49" integrity sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw== @@ -2937,7 +3001,7 @@ fast-diff@^1.1.2: resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.3.0.tgz#ece407fa550a64d638536cd727e129c61616e0f0" integrity sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw== -fast-glob@^3.2.5, fast-glob@^3.2.9: +fast-glob@^3.2.5, fast-glob@^3.2.9, fast-glob@^3.3.1: version "3.3.2" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129" integrity sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow== @@ -3245,6 +3309,13 @@ get-symbol-description@^1.0.0: call-bind "^1.0.2" get-intrinsic "^1.1.1" +get-tsconfig@^4.5.0: + version "4.7.2" + resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.7.2.tgz#0dcd6fb330391d46332f4c6c1bf89a6514c2ddce" + integrity sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A== + dependencies: + resolve-pkg-maps "^1.0.0" + glob-parent@^5.1.2, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -3610,7 +3681,7 @@ is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7: resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== -is-core-module@^2.13.0, is-core-module@^2.13.1: +is-core-module@^2.11.0, is-core-module@^2.13.0, is-core-module@^2.13.1: version "2.13.1" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384" integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw== @@ -5157,6 +5228,11 @@ resolve-from@^5.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== +resolve-pkg-maps@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f" + integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== + resolve.exports@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-2.0.2.tgz#f8c934b8e6a13f539e38b7098e2e36134f01e800" @@ -5493,6 +5569,18 @@ statuses@2.0.1: resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== +stream-chain@^2.2.5: + version "2.2.5" + resolved "https://registry.yarnpkg.com/stream-chain/-/stream-chain-2.2.5.tgz#b30967e8f14ee033c5b9a19bbe8a2cba90ba0d09" + integrity sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA== + +stream-json@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/stream-json/-/stream-json-1.8.0.tgz#53f486b2e3b4496c506131f8d7260ba42def151c" + integrity sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw== + dependencies: + stream-chain "^2.2.5" + streamsearch@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764" @@ -5977,6 +6065,13 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici@^6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.4.0.tgz#7ca0c3f73e1034f3c79e566183b61bb55b1410ea" + integrity sha512-wYaKgftNqf6Je7JQ51YzkEkEevzOgM7at5JytKO7BjaURQpERW8edQSMrr2xb+Yv4U8Yg47J24+lc9+NbeXMFA== + dependencies: + "@fastify/busboy" "^2.0.0" + universalify@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d" From 666e40d43c9e2cfb40630455215abefa8737cf26 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Mon, 5 Feb 2024 12:49:37 +0400 Subject: [PATCH 02/15] feat: daemon --- src/common/handlers/handlers.service.ts | 9 +- src/daemon/daemon.module.ts | 15 +- src/daemon/daemon.service.ts | 46 +++++- src/daemon/services/keys-indexer.ts | 210 ++++++++++++++++++++++++ src/daemon/services/roots-processor.ts | 33 ++++ src/daemon/services/roots-provider.ts | 54 ++++++ src/daemon/services/roots-stack.ts | 62 +++++++ src/daemon/utils/sleep.ts | 1 + 8 files changed, 414 insertions(+), 16 deletions(-) create mode 100644 src/daemon/services/keys-indexer.ts create mode 100644 src/daemon/services/roots-processor.ts create mode 100644 src/daemon/services/roots-provider.ts create mode 100644 src/daemon/services/roots-stack.ts create mode 100644 src/daemon/utils/sleep.ts diff --git a/src/common/handlers/handlers.service.ts b/src/common/handlers/handlers.service.ts index 5eed365..6340bfb 100644 --- a/src/common/handlers/handlers.service.ts +++ b/src/common/handlers/handlers.service.ts @@ -23,15 +23,22 @@ export class HandlersService { const payload = await this.buildProvePayload(slashings, withdrawals); // TODO: ask before sending if CLI or daemon in watch mode await this.sendProves(payload); + this.logger.log(`๐Ÿ Proves sent. Root [${blockRoot}]`); } private async buildProvePayload(slashings: string[], withdrawals: string[]): Promise { // TODO: implement // this.consensus.getState(...) + if (slashings.length || withdrawals.length) { + this.logger.warn(`๐Ÿ“ฆ Prove payload: slashings [${slashings}], withdrawals [${withdrawals}]`); + } return {}; } private async sendProves(payload: any): Promise { // TODO: implement + if (payload) { + this.logger.warn(`๐Ÿ“ก Sending proves`); + } } private async getUnprovenSlashings( @@ -114,7 +121,7 @@ export class HandlersService { ): string[] { const fullWithdrawals = []; const blockEpoch = Number(blockInfo.message.slot) / 32; - const withdrawals = blockInfo.message.body.execution_payload.withdrawals; + const withdrawals = blockInfo.message.body.execution_payload?.withdrawals ?? []; for (const withdrawal of withdrawals) { const keyInfo = keyInfoFn(Number(withdrawal.validator_index)); if (keyInfo && blockEpoch >= keyInfo.withdrawableEpoch) { diff --git a/src/daemon/daemon.module.ts b/src/daemon/daemon.module.ts index 1773959..eddb16a 100644 --- a/src/daemon/daemon.module.ts +++ b/src/daemon/daemon.module.ts @@ -1,6 +1,10 @@ import { Module } from '@nestjs/common'; import { DaemonService } from './daemon.service'; +import { KeysIndexer } from './services/keys-indexer'; +import { RootsProcessor } from './services/roots-processor'; +import { RootsProvider } from './services/roots-provider'; +import { RootsStack } from './services/roots-stack'; import { ConfigModule } from '../common/config/config.module'; import { HandlersModule } from '../common/handlers/handlers.module'; import { LoggerModule } from '../common/logger/logger.module'; @@ -8,13 +12,8 @@ import { PrometheusModule } from '../common/prometheus/prometheus.module'; import { ProvidersModule } from '../common/providers/providers.module'; @Module({ - imports: [ - LoggerModule, - ConfigModule, - PrometheusModule, - ProvidersModule, - HandlersModule, - ], - providers: [DaemonService], + imports: [LoggerModule, ConfigModule, PrometheusModule, ProvidersModule, HandlersModule], + providers: [DaemonService, KeysIndexer, RootsProvider, RootsProcessor, RootsStack], + exports: [DaemonService], }) export class DaemonModule {} diff --git a/src/daemon/daemon.service.ts b/src/daemon/daemon.service.ts index 59494d5..c999dc2 100644 --- a/src/daemon/daemon.service.ts +++ b/src/daemon/daemon.service.ts @@ -1,18 +1,50 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; -import { - Inject, - Injectable, - LoggerService, - OnApplicationBootstrap, -} from '@nestjs/common'; +import { Inject, Injectable, LoggerService, OnApplicationBootstrap } from '@nestjs/common'; + +import { KeysIndexer } from './services/keys-indexer'; +import { RootsProcessor } from './services/roots-processor'; +import { RootsProvider } from './services/roots-provider'; +import sleep from './utils/sleep'; +import { ConfigService } from '../common/config/config.service'; +import { Consensus } from '../common/providers/consensus/consensus'; @Injectable() export class DaemonService implements OnApplicationBootstrap { constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly config: ConfigService, + protected readonly consensus: Consensus, + protected readonly keysIndexer: KeysIndexer, + protected readonly rootsProvider: RootsProvider, + protected readonly rootsProcessor: RootsProcessor, ) {} async onApplicationBootstrap() { - this.logger.log('Working mode: DAEMON'); + this.loop().then(); + } + + private async loop() { + while (true) { + try { + await this.baseRun(); + } catch (e) { + this.logger.error(e); + await sleep(1000); + } + } + } + + private async baseRun() { + this.logger.log('๐Ÿ—ฟ Get finalized header'); + const header = await this.consensus.getBeaconHeader('finalized'); + this.logger.log(`๐Ÿ’Ž Finalized slot [${header.header.message.slot}]. Root [${header.root}]`); + await this.keysIndexer.run(header); + const nextRoot = await this.rootsProvider.getNext(header); + if (nextRoot) { + await this.rootsProcessor.process(nextRoot); + } else { + this.logger.log(`๐Ÿ’ค Wait for the next finalized root`); + await sleep(12000); + } } } diff --git a/src/daemon/services/keys-indexer.ts b/src/daemon/services/keys-indexer.ts new file mode 100644 index 0000000..aea34b3 --- /dev/null +++ b/src/daemon/services/keys-indexer.ts @@ -0,0 +1,210 @@ +import { Low } from '@huanshiwushuang/lowdb'; +import { JSONFile } from '@huanshiwushuang/lowdb/node'; +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; +import { Inject, Injectable, LoggerService, OnApplicationBootstrap } from '@nestjs/common'; + +import { ConfigService } from '../../common/config/config.service'; +import { KeyInfo } from '../../common/handlers/handlers.service'; +import { Consensus } from '../../common/providers/consensus/consensus'; +import { + BlockHeaderResponse, + RootHex, + Slot, + StateValidatorResponse, +} from '../../common/providers/consensus/response.interface'; +import { Keysapi } from '../../common/providers/keysapi/keysapi'; + +type Info = { + moduleAddress: string; + moduleId: number; + storageStateSlot: number; + lastValidatorsCount: number; +}; + +type Storage = { + [valIndex: number]: KeyInfo; +}; + +@Injectable() +export class KeysIndexer implements OnApplicationBootstrap { + private startedAt: number = 0; + + private info: Low; + private storage: Low; + + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly config: ConfigService, + protected readonly consensus: Consensus, + protected readonly keysapi: Keysapi, + ) {} + + public async onApplicationBootstrap(): Promise { + await this.initOrReadServiceData(); + } + + public getKey = (valIndex: number): KeyInfo | undefined => { + return this.storage.data[valIndex]; + }; + + public async run(finalizedHeader: BlockHeaderResponse): Promise { + // At one time only one task should be running + if (this.startedAt > 0) { + this.logger.warn(`๐Ÿ”‘ Keys indexer has been running for ${Date.now() - this.startedAt}ms`); + return; + } + const slot = Number(finalizedHeader.header.message.slot); + if (this.isNotTimeToRun(slot)) { + this.logger.log('No need to run keys indexer'); + return; + } + this.logger.log(`๐Ÿ”‘ Keys indexer is running`); + const stateRoot = finalizedHeader.header.message.state_root; + if (this.info.data.storageStateSlot == 0) { + await this.baseRun(stateRoot, slot); + return; + } + // We shouldn't wait for task to finish + // to avoid block processing if indexing fails or stuck + this.startedAt = Date.now(); + this.baseRun(stateRoot, slot) + .catch((e) => this.logger.error(e)) + .finally(() => (this.startedAt = 0)); + } + + private async baseRun(stateRoot: RootHex, finalizedSlot: Slot): Promise { + this.logger.log(`Get validators. State root [${stateRoot}]`); + const validators = await this.consensus.getValidators(stateRoot); + this.logger.log(`Total validators count: ${validators.length}`); + // TODO: do we need to store already full withdrawn keys ? + this.info.data.lastValidatorsCount == 0 + ? await this.initStorage(validators, finalizedSlot) + : await this.updateStorage(validators, finalizedSlot); + this.logger.log(`CSM validators count: ${Object.keys(this.storage.data).length}`); + this.info.data.storageStateSlot = finalizedSlot; + this.info.data.lastValidatorsCount = validators.length; + await this.info.write(); + } + + private async initStorage(validators: StateValidatorResponse[], finalizedSlot: Slot): Promise { + this.logger.log(`Init keys data`); + const csmKeys = await this.keysapi.getModuleKeys(this.info.data.moduleId); + this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); + const keysMap = new Map(); + csmKeys.data.keys.forEach((k: any) => keysMap.set(k.key, { ...k })); + for (const v of validators) { + const keyInfo = keysMap.get(v.validator.pubkey); + if (!keyInfo) continue; + this.storage.data[Number(v.index)] = { + operatorId: keyInfo.operatorIndex, + keyIndex: keyInfo.index, + pubKey: v.validator.pubkey, + // TODO: bigint? + withdrawableEpoch: Number(v.validator.withdrawable_epoch), + }; + } + await this.storage.write(); + } + + private async updateStorage(vals: StateValidatorResponse[], finalizedSlot: Slot): Promise { + // TODO: should we think about re-using validator indexes? + // TODO: should we think about changing WC for existing old vaidators ? + if (vals.length - this.info.data.lastValidatorsCount == 0) { + this.logger.log(`No new validators in the state`); + return; + } + vals = vals.slice(this.info.data.lastValidatorsCount); + const valKeys = vals.map((v: StateValidatorResponse) => v.validator.pubkey); + this.logger.log(`New appeared validators count: ${vals.length}`); + const csmKeys = await this.keysapi.findModuleKeys(this.info.data.moduleId, valKeys); + this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); + this.logger.log(`New appeared CSM validators count: ${csmKeys.data.keys.length}`); + for (const csmKey of csmKeys.data.keys) { + for (const newVal of vals) { + if (newVal.validator.pubkey != csmKey.key) continue; + this.storage.data[Number(newVal.index)] = { + operatorId: csmKey.operatorIndex, + keyIndex: csmKey.index, + pubKey: csmKey.key, + // TODO: bigint? + withdrawableEpoch: Number(newVal.validator.withdrawable_epoch), + }; + } + } + await this.storage.write(); + } + + public isNotTimeToRun(finalizedSlot: Slot): boolean { + const storageTimestamp = this.consensus.slotToTimestamp(this.info.data.storageStateSlot) * 1000; + return ( + this.info.data.storageStateSlot == finalizedSlot || + this.config.get('KEYS_INDEXER_RUNNING_PERIOD') >= Date.now() - storageTimestamp + ); + } + + public eligibleForAnyDuty(slotNumber: Slot): boolean { + return this.eligibleForSlashings(slotNumber) || this.eligibleForFullWithdrawals(slotNumber); + } + + public eligibleForEveryDuty(slotNumber: Slot): boolean { + const eligibleForSlashings = this.eligibleForSlashings(slotNumber); + const eligibleForFullWithdrawals = this.eligibleForFullWithdrawals(slotNumber); + if (!eligibleForSlashings) + this.logger.warn( + '๐Ÿšจ Current keys indexer data might not be ready to detect slashing. ' + + 'The root will be processed later again', + ); + if (!eligibleForFullWithdrawals) + this.logger.warn( + 'โš ๏ธ Current keys indexer data might not be ready to detect full withdrawal. ' + + 'The root will be processed later again', + ); + return eligibleForSlashings && eligibleForFullWithdrawals; + } + + private eligibleForSlashings(slotNumber: Slot): boolean { + // We are ok with oudated indexer for detection slasing + // because of a bunch of delays between deposit and validator appearing + // TODO: get constants from node + const ETH1_FOLLOW_DISTANCE = 2048; // ~8 hours + const EPOCHS_PER_ETH1_VOTING_PERIOD = 64; // ~6.8 hours + const safeDelay = ETH1_FOLLOW_DISTANCE + EPOCHS_PER_ETH1_VOTING_PERIOD * 32; + if (this.info.data.storageStateSlot >= slotNumber) return true; + return slotNumber - this.info.data.storageStateSlot <= safeDelay; // ~14.8 hours + } + + private eligibleForFullWithdrawals(slotNumber: Slot): boolean { + // We are ok with oudated indexer for detection withdrawal + // because of MIN_VALIDATOR_WITHDRAWABILITY_DELAY + // TODO: get constants from node + const MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 256; + const safeDelay = MIN_VALIDATOR_WITHDRAWABILITY_DELAY * 32; + if (this.info.data.storageStateSlot >= slotNumber) return true; + return slotNumber - this.info.data.storageStateSlot <= safeDelay; // ~27 hours + } + + private async initOrReadServiceData() { + const defaultInfo: Info = { + moduleAddress: this.config.get('LIDO_STAKING_MODULE_ADDRESS'), + moduleId: 0, + storageStateSlot: 0, + lastValidatorsCount: 0, + }; + this.info = new Low(new JSONFile('.keys-indexer-info.json'), defaultInfo); + this.storage = new Low(new JSONFile('.keys-indexer-storage.json'), {}); + await this.info.read(); + await this.storage.read(); + + if (this.info.data.moduleId == 0) { + const modules = (await this.keysapi.getModules()).data; + const module = modules.find( + (m: any) => m.stakingModuleAddress.toLowerCase() === this.info.data.moduleAddress.toLowerCase(), + ); + if (!module) { + throw new Error(`Module with address ${this.info.data.moduleAddress} not found`); + } + this.info.data.moduleId = module.id; + await this.info.write(); + } + } +} diff --git a/src/daemon/services/roots-processor.ts b/src/daemon/services/roots-processor.ts new file mode 100644 index 0000000..5ca18fd --- /dev/null +++ b/src/daemon/services/roots-processor.ts @@ -0,0 +1,33 @@ +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; +import { Inject, Injectable, LoggerService } from '@nestjs/common'; + +import { KeysIndexer } from './keys-indexer'; +import { RootsStack } from './roots-stack'; +import { HandlersService } from '../../common/handlers/handlers.service'; +import { Consensus } from '../../common/providers/consensus/consensus'; +import { RootHex } from '../../common/providers/consensus/response.interface'; + +@Injectable() +export class RootsProcessor { + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly consensus: Consensus, + protected readonly keysIndexer: KeysIndexer, + protected readonly rootsStack: RootsStack, + protected readonly handlers: HandlersService, + ) {} + + public async process(blockRoot: RootHex): Promise { + this.logger.log(`๐Ÿ›ƒ Root in processing [${blockRoot}]`); + const blockInfo = await this.consensus.getBlockInfo(blockRoot); + const rootSlot = { + blockRoot, + slotNumber: Number(blockInfo.message.slot), + }; + const indexerIsOK = this.keysIndexer.eligibleForEveryDuty(rootSlot.slotNumber); + if (!indexerIsOK) await this.rootsStack.push(rootSlot); // only new will be pushed + await this.handlers.prove(blockRoot, blockInfo, this.keysIndexer.getKey); + if (indexerIsOK) await this.rootsStack.purge(blockRoot); + await this.rootsStack.setLastProcessed(rootSlot); + } +} diff --git a/src/daemon/services/roots-provider.ts b/src/daemon/services/roots-provider.ts new file mode 100644 index 0000000..965b182 --- /dev/null +++ b/src/daemon/services/roots-provider.ts @@ -0,0 +1,54 @@ +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; +import { Inject, Injectable, LoggerService } from '@nestjs/common'; + +import { RootSlot, RootsStack } from './roots-stack'; +import { ConfigService } from '../../common/config/config.service'; +import { Consensus } from '../../common/providers/consensus/consensus'; +import { BlockHeaderResponse, RootHex } from '../../common/providers/consensus/response.interface'; + +@Injectable() +export class RootsProvider { + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly config: ConfigService, + protected readonly consensus: Consensus, + protected readonly rootsStack: RootsStack, + ) {} + + public async getNext(finalizedHeader: BlockHeaderResponse): Promise { + const stacked = this.getStacked(); + if (stacked) return stacked; + const lastProcessed = this.rootsStack.getLastProcessed(); + if (!lastProcessed) return this.getKnown(finalizedHeader); + return await this.getChild(lastProcessed, finalizedHeader); + } + + private getStacked(): RootHex | undefined { + const stacked = this.rootsStack.getNextEligible(); + if (!stacked) return; + this.logger.warn(`โญ๏ธ Next root to process [${stacked.blockRoot}]. Taken from ๐Ÿ“š stack of unprocessed roots`); + return stacked.blockRoot; + } + + private getKnown(finalizedHeader: BlockHeaderResponse): RootHex | undefined { + const configured = this.config.get('START_ROOT'); + if (configured) { + this.logger.log(`No processed roots. Start from โš™๏ธ configured root [${configured}]`); + return configured; + } + this.logger.log(`No processed roots. Start from ๐Ÿ’Ž last finalized root [${finalizedHeader.root}]`); + return finalizedHeader.root; + } + + private async getChild(lastProcessed: RootSlot, finalizedHeader: BlockHeaderResponse): Promise { + this.logger.log(`โฎ๏ธ Last processed root [${lastProcessed.blockRoot}]`); + if (lastProcessed.blockRoot == finalizedHeader.root) return; + const diff = Number(finalizedHeader.header.message.slot) - lastProcessed.slotNumber; + this.logger.warn(`Diff between last processed and finalized is ${diff} slots`); + const childHeader = await this.consensus.getBeaconHeadersByParentRoot(lastProcessed.blockRoot); + if (!childHeader || !childHeader.finalized) return; + const child = childHeader.data[0].root; + this.logger.log(`โญ๏ธ Next root to process [${child}]. Child of last processed`); + return child; + } +} diff --git a/src/daemon/services/roots-stack.ts b/src/daemon/services/roots-stack.ts new file mode 100644 index 0000000..d430ba6 --- /dev/null +++ b/src/daemon/services/roots-stack.ts @@ -0,0 +1,62 @@ +import { Low } from '@huanshiwushuang/lowdb'; +import { JSONFile } from '@huanshiwushuang/lowdb/node'; +import { Injectable, OnApplicationBootstrap } from '@nestjs/common'; + +import { KeysIndexer } from './keys-indexer'; +import { RootHex } from '../../common/providers/consensus/response.interface'; + +export type RootSlot = { blockRoot: string; slotNumber: number }; + +type Info = { + lastProcessedRootSlot: RootSlot | undefined; +}; + +type Storage = RootSlot[]; + +@Injectable() +export class RootsStack implements OnApplicationBootstrap { + private info: Low; + private storage: Low; + + constructor(protected readonly keysIndexer: KeysIndexer) {} + + async onApplicationBootstrap(): Promise { + await this.initOrReadServiceData(); + } + + public getNextEligible(): RootSlot | undefined { + return this.storage.data.find((s) => this.keysIndexer.eligibleForAnyDuty(s.slotNumber)); + } + + public async push(rs: RootSlot): Promise { + const idx = this.storage.data.findIndex((i) => rs.blockRoot == i.blockRoot); + if (idx !== -1) return; + this.storage.data.push(rs); + await this.storage.write(); + } + + public async purge(blockRoot: RootHex): Promise { + const idx = this.storage.data.findIndex((i) => blockRoot == i.blockRoot); + if (idx == -1) return; + this.storage.data.splice(idx, 1); + await this.storage.write(); + } + + public getLastProcessed(): RootSlot | undefined { + return this.info.data.lastProcessedRootSlot; + } + + public async setLastProcessed(item: RootSlot): Promise { + this.info.data.lastProcessedRootSlot = item; + await this.info.write(); + } + + private async initOrReadServiceData() { + this.info = new Low(new JSONFile('.roots-stack-info.json'), { + lastProcessedRootSlot: undefined, + }); + this.storage = new Low(new JSONFile('.roots-stack-storage.json'), []); + await this.info.read(); + await this.storage.read(); + } +} diff --git a/src/daemon/utils/sleep.ts b/src/daemon/utils/sleep.ts new file mode 100644 index 0000000..fc4f3a2 --- /dev/null +++ b/src/daemon/utils/sleep.ts @@ -0,0 +1 @@ +export default async (ms: number): Promise => await new Promise((resolve) => setTimeout(resolve, ms)); From 8098db053ac317c843ed8c70e33e6e74b38f9ebf Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Mon, 5 Feb 2024 14:32:02 +0400 Subject: [PATCH 03/15] chore: add missing libs --- package.json | 4 +- yarn.lock | 704 ++++++++++++++++++++++++++++----------------------- 2 files changed, 396 insertions(+), 312 deletions(-) diff --git a/package.json b/package.json index 84ea395..6330729 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,8 @@ "@nestjs/core": "^10.0.0", "@nestjs/platform-express": "^10.0.0", "@types/cli-progress": "^3.11.5", + "class-transformer": "^0.5.1", + "class-validator": "^0.14.1", "cli-progress": "^3.12.0", "nest-commander": "^3.12.5", "nest-winston": "^1.9.4", @@ -68,7 +70,7 @@ "ts-loader": "^9.4.3", "ts-node": "^10.9.1", "tsconfig-paths": "^4.2.0", - "typescript": "^5.1.3" + "typescript": "^5.3.3" }, "jest": { "moduleFileExtensions": [ diff --git a/yarn.lock b/yarn.lock index 1e57076..7dfcfde 100644 --- a/yarn.lock +++ b/yarn.lock @@ -27,15 +27,27 @@ rxjs "7.8.1" source-map "0.7.4" -"@angular-devkit/schematics-cli@17.0.9": - version "17.0.9" - resolved "https://registry.yarnpkg.com/@angular-devkit/schematics-cli/-/schematics-cli-17.0.9.tgz#6ee0ac81b37cab50e60b49a0541508552a75ab7f" - integrity sha512-tznzzB26sy8jVUlV9HhXcbFYZcIIFMAiDMOuyLko2LZFjfoqW+OPvwa1mwAQwvVVSQZVAKvdndFhzwyl/axwFQ== +"@angular-devkit/core@17.1.2": + version "17.1.2" + resolved "https://registry.yarnpkg.com/@angular-devkit/core/-/core-17.1.2.tgz#bf2c3475e9ff853dc53d8dc8ce9bbf8b2f1193f8" + integrity sha512-ku+/W/HMCBacSWFppenr9y6Lx8mDuTuQvn1IkTyBLiJOpWnzgVbx9kHDeaDchGa1PwLlJUBBrv27t3qgJOIDPw== dependencies: - "@angular-devkit/core" "17.0.9" - "@angular-devkit/schematics" "17.0.9" + ajv "8.12.0" + ajv-formats "2.1.1" + jsonc-parser "3.2.0" + picomatch "3.0.1" + rxjs "7.8.1" + source-map "0.7.4" + +"@angular-devkit/schematics-cli@17.1.2": + version "17.1.2" + resolved "https://registry.yarnpkg.com/@angular-devkit/schematics-cli/-/schematics-cli-17.1.2.tgz#7a77e8294071e5ba569e2ffb567b3301d1db3f07" + integrity sha512-bvXykYzSST05qFdlgIzUguNOb3z0hCa8HaTwtqdmQo9aFPf+P+/AC56I64t1iTchMjQtf3JrBQhYM25gUdcGbg== + dependencies: + "@angular-devkit/core" "17.1.2" + "@angular-devkit/schematics" "17.1.2" ansi-colors "4.1.3" - inquirer "9.2.11" + inquirer "9.2.12" symbol-observable "4.0.0" yargs-parser "21.1.1" @@ -50,7 +62,18 @@ ora "5.4.1" rxjs "7.8.1" -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.16.7", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.23.5": +"@angular-devkit/schematics@17.1.2": + version "17.1.2" + resolved "https://registry.yarnpkg.com/@angular-devkit/schematics/-/schematics-17.1.2.tgz#ca77a86ed44ab227614aff6e1f7ce4f3cd0c6ded" + integrity sha512-8S9RuM8olFN/gwN+mjbuF1CwHX61f0i59EGXz9tXLnKRUTjsRR+8vVMTAmX0dvVAT5fJTG/T69X+HX7FeumdqA== + dependencies: + "@angular-devkit/core" "17.1.2" + jsonc-parser "3.2.0" + magic-string "0.30.5" + ora "5.4.1" + rxjs "7.8.1" + +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.16.7", "@babel/code-frame@^7.23.5": version "7.23.5" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.23.5.tgz#9009b69a8c602293476ad598ff53e4562e15c244" integrity sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA== @@ -64,20 +87,20 @@ integrity sha512-uU27kfDRlhfKl+w1U6vp16IuvSLtjAxdArVXPa9BvLkrr7CYIsxH5adpHObeAGY/41+syctUWOZ140a2Rvkgjw== "@babel/core@^7.11.6", "@babel/core@^7.12.3": - version "7.23.7" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.7.tgz#4d8016e06a14b5f92530a13ed0561730b5c6483f" - integrity sha512-+UpDgowcmqe36d4NwqvKsyPMlOLNGMsfMmQ5WGCu+siCe3t3dfe9njrzGfdN4qq+bcNUt0+Vw6haRxBOycs4dw== + version "7.23.9" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.9.tgz#b028820718000f267870822fec434820e9b1e4d1" + integrity sha512-5q0175NOjddqpvvzU+kDiSOAk4PfdO6FvwCWoQ6RO7rTzEe8vlo+4HVfcnAREhD4npMs0e9uZypjTwzZPCf/cw== dependencies: "@ampproject/remapping" "^2.2.0" "@babel/code-frame" "^7.23.5" "@babel/generator" "^7.23.6" "@babel/helper-compilation-targets" "^7.23.6" "@babel/helper-module-transforms" "^7.23.3" - "@babel/helpers" "^7.23.7" - "@babel/parser" "^7.23.6" - "@babel/template" "^7.22.15" - "@babel/traverse" "^7.23.7" - "@babel/types" "^7.23.6" + "@babel/helpers" "^7.23.9" + "@babel/parser" "^7.23.9" + "@babel/template" "^7.23.9" + "@babel/traverse" "^7.23.9" + "@babel/types" "^7.23.9" convert-source-map "^2.0.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -177,14 +200,14 @@ resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.23.5.tgz#907a3fbd4523426285365d1206c423c4c5520307" integrity sha512-85ttAOMLsr53VgXkTbkx8oA6YTfT4q7/HzXSLEYmjcSTJPMPQtvq1BD79Byep5xMUYbGRzEpDsjUf3dyp54IKw== -"@babel/helpers@^7.23.7": - version "7.23.8" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.23.8.tgz#fc6b2d65b16847fd50adddbd4232c76378959e34" - integrity sha512-KDqYz4PiOWvDFrdHLPhKtCThtIcKVy6avWD2oG4GEvyQ+XDZwHD4YQd+H2vNMnq2rkdxsDkU82T+Vk8U/WXHRQ== +"@babel/helpers@^7.23.9": + version "7.23.9" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.23.9.tgz#c3e20bbe7f7a7e10cb9b178384b4affdf5995c7d" + integrity sha512-87ICKgU5t5SzOT7sBMfCOZQ2rHjRU+Pcb9BoILMYz600W6DkVRLFBPwQ18gwUVvggqXivaUakpnxWQGbpywbBQ== dependencies: - "@babel/template" "^7.22.15" - "@babel/traverse" "^7.23.7" - "@babel/types" "^7.23.6" + "@babel/template" "^7.23.9" + "@babel/traverse" "^7.23.9" + "@babel/types" "^7.23.9" "@babel/highlight@^7.23.4": version "7.23.4" @@ -195,10 +218,10 @@ chalk "^2.4.2" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.23.6": - version "7.23.6" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.6.tgz#ba1c9e512bda72a47e285ae42aff9d2a635a9e3b" - integrity sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ== +"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.23.9": + version "7.23.9" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.9.tgz#7b903b6149b0f8fa7ad564af646c4c38a77fc44b" + integrity sha512-9tcKgqKbs3xGJ+NtKF2ndOBBLVwPjl1SHxPQkd36r3Dlirw3xWUeGaTbqr7uGZcTaxkVNwc+03SVP7aCdWrTlA== "@babel/plugin-syntax-async-generators@^7.8.4": version "7.8.4" @@ -298,19 +321,19 @@ dependencies: "@babel/helper-plugin-utils" "^7.22.5" -"@babel/template@^7.22.15", "@babel/template@^7.3.3": - version "7.22.15" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" - integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== +"@babel/template@^7.22.15", "@babel/template@^7.23.9", "@babel/template@^7.3.3": + version "7.23.9" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.23.9.tgz#f881d0487cba2828d3259dcb9ef5005a9731011a" + integrity sha512-+xrD2BWLpvHKNmX2QbpdpsBaWnRxahMwJjO+KZk2JOElj5nSmKezyS1B4u+QbHMTX69t4ukm6hh9lsYQ7GHCKA== dependencies: - "@babel/code-frame" "^7.22.13" - "@babel/parser" "^7.22.15" - "@babel/types" "^7.22.15" + "@babel/code-frame" "^7.23.5" + "@babel/parser" "^7.23.9" + "@babel/types" "^7.23.9" -"@babel/traverse@^7.23.7": - version "7.23.7" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.7.tgz#9a7bf285c928cb99b5ead19c3b1ce5b310c9c305" - integrity sha512-tY3mM8rH9jM0YHFGyfC0/xf+SB5eKUu7HPj7/k3fpi9dAlsMc5YbQvDi0Sh2QTPXqMhyaAtzAr807TIyfQrmyg== +"@babel/traverse@^7.23.9": + version "7.23.9" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.9.tgz#2f9d6aead6b564669394c5ce0f9302bb65b9d950" + integrity sha512-I/4UJ9vs90OkBtY6iiiTORVMyIhJ4kAVmsKo9KFc8UOxMeUfi2hvtIBsET5u9GizXE6/GFSuKCTNfgCswuEjRg== dependencies: "@babel/code-frame" "^7.23.5" "@babel/generator" "^7.23.6" @@ -318,15 +341,15 @@ "@babel/helper-function-name" "^7.23.0" "@babel/helper-hoist-variables" "^7.22.5" "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.23.6" - "@babel/types" "^7.23.6" + "@babel/parser" "^7.23.9" + "@babel/types" "^7.23.9" debug "^4.3.1" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.6", "@babel/types@^7.3.3": - version "7.23.6" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.6.tgz#be33fdb151e1f5a56877d704492c240fc71c7ccd" - integrity sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg== +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.6", "@babel/types@^7.23.9", "@babel/types@^7.3.3": + version "7.23.9" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.9.tgz#1dd7b59a9a2b5c87f8b41e52770b5ecbf492e002" + integrity sha512-dQjSq/7HaSjRM43FFGnv5keM2HsxpmyV1PfaSVm0nzzjwwTmjOe6J4bC8e3+pTEIgHaHj+1ZlLThRJ2auc/w1Q== dependencies: "@babel/helper-string-parser" "^7.23.4" "@babel/helper-validator-identifier" "^7.22.20" @@ -722,12 +745,12 @@ traverse "^0.6.7" winston "^3.4.0" -"@ljharb/through@^2.3.9": - version "2.3.11" - resolved "https://registry.yarnpkg.com/@ljharb/through/-/through-2.3.11.tgz#783600ff12c06f21a76cc26e33abd0b1595092f9" - integrity sha512-ccfcIDlogiXNq5KcbAwbaO7lMh3Tm1i3khMPYpxlK8hH/W53zN81KM9coerRLOnTGu3nfXIniAmQbRI9OxbC0w== +"@ljharb/through@^2.3.11": + version "2.3.12" + resolved "https://registry.yarnpkg.com/@ljharb/through/-/through-2.3.12.tgz#c418c43060eee193adce48b15c2206096a28e9ea" + integrity sha512-ajo/heTlG3QgC8EGP6APIejksVAYt4ayz4tqoP3MolFELzcH1x1fzwEYRJTPO0IELutZ5HQ0c26/GqAYy79u3g== dependencies: - call-bind "^1.0.2" + call-bind "^1.0.5" "@lukeed/csprng@^1.0.0": version "1.1.0" @@ -749,13 +772,13 @@ os-filter-obj "^2.0.0" "@nestjs/cli@^10.0.0": - version "10.3.0" - resolved "https://registry.yarnpkg.com/@nestjs/cli/-/cli-10.3.0.tgz#5f9ef49a60baf4b39cb87e4b74240f7c9339e923" - integrity sha512-37h+wSDItY0NE/x3a/M9yb2cXzfsD4qoE26rHgFn592XXLelDN12wdnfn7dTIaiRZT7WOCdQ+BYP9mQikR4AsA== + version "10.3.1" + resolved "https://registry.yarnpkg.com/@nestjs/cli/-/cli-10.3.1.tgz#f13af238fd17ce22f5c4a3439828248938aca8e8" + integrity sha512-xzabUxTdZ7IcNXLzAq1YZgGJkAt6JNeeLVORj8MfMV0io2edgAn5ASn4tIOHvnsmKh6yX1kBaVEhTHiuENlplA== dependencies: - "@angular-devkit/core" "17.0.9" - "@angular-devkit/schematics" "17.0.9" - "@angular-devkit/schematics-cli" "17.0.9" + "@angular-devkit/core" "17.1.2" + "@angular-devkit/schematics" "17.1.2" + "@angular-devkit/schematics-cli" "17.1.2" "@nestjs/schematics" "^10.0.1" chalk "4.1.2" chokidar "3.5.3" @@ -773,13 +796,13 @@ tsconfig-paths "4.2.0" tsconfig-paths-webpack-plugin "4.1.0" typescript "5.3.3" - webpack "5.89.0" + webpack "5.90.1" webpack-node-externals "3.0.0" "@nestjs/common@^10.0.0": - version "10.3.0" - resolved "https://registry.yarnpkg.com/@nestjs/common/-/common-10.3.0.tgz#d78f0ff2062d1d53c79c170a79c12a1548e2e598" - integrity sha512-DGv34UHsZBxCM3H5QGE2XE/+oLJzz5+714JQjBhjD9VccFlQs3LRxo/epso4l7nJIiNlZkPyIUC8WzfU/5RTsQ== + version "10.3.1" + resolved "https://registry.yarnpkg.com/@nestjs/common/-/common-10.3.1.tgz#7aa5a0ffbd0123533adc1cfee8fd184b74bc2fc1" + integrity sha512-YuxeIlVemVQCuXMkNbBpNlmwZgp/Cu6dwCOjki63mhyYHEFX48GNNA4zZn5MFRjF4h7VSceABsScROuzsxs9LA== dependencies: uid "2.0.2" iterare "1.2.1" @@ -796,9 +819,9 @@ uuid "9.0.0" "@nestjs/core@^10.0.0": - version "10.3.0" - resolved "https://registry.yarnpkg.com/@nestjs/core/-/core-10.3.0.tgz#d5c6b26d6d9280664910d5481153d25c5da4ec00" - integrity sha512-N06P5ncknW/Pm8bj964WvLIZn2gNhHliCBoAO1LeBvNImYkecqKcrmLbY49Fa1rmMfEM3MuBHeDys3edeuYAOA== + version "10.3.1" + resolved "https://registry.yarnpkg.com/@nestjs/core/-/core-10.3.1.tgz#96f34ad7e6efab080ac7c3f07d5454c9705eccc4" + integrity sha512-mh6FwTKh2R3CmLRuB50BF5q/lzc+Mz+7qAlEvpgCiTSIfSXzbQ47vWpfgLirwkL3SlCvtFS8onxOeI69RpxvXA== dependencies: uid "2.0.2" "@nuxtjs/opencollective" "0.3.2" @@ -808,9 +831,9 @@ tslib "2.6.2" "@nestjs/platform-express@^10.0.0": - version "10.3.0" - resolved "https://registry.yarnpkg.com/@nestjs/platform-express/-/platform-express-10.3.0.tgz#ea69b048ef90b78b1001eb1c6b02d9d798f5f3af" - integrity sha512-E4hUW48bYv8OHbP9XQg6deefmXb0pDSSuE38SdhA0mJ37zGY7C5EqqBUdlQk4ttfD+OdnbIgJ1zOokT6dd2d7A== + version "10.3.1" + resolved "https://registry.yarnpkg.com/@nestjs/platform-express/-/platform-express-10.3.1.tgz#f72f337ddd96895e56284b13971bdbd3479e24b3" + integrity sha512-Rj21quI5h4Lry7q9an+nO4ADQiQUy9A6XK74o5aTUHo3Ysm25ujqh2NgU4XbT3M2oXU9qzhE59OfhkQ7ZUvTAg== dependencies: body-parser "1.20.2" cors "2.8.5" @@ -830,9 +853,9 @@ pluralize "8.0.0" "@nestjs/testing@^10.0.0": - version "10.3.0" - resolved "https://registry.yarnpkg.com/@nestjs/testing/-/testing-10.3.0.tgz#a4de362de88f855ddee5ed6f5cc25bd6aaf4c4c3" - integrity sha512-8DM+bw1qASCvaEnoHUQhypCOf54+G5R21MeFBMvnSk5DtKaWVZuzDP2GjLeYCpTH19WeP6LrrjHv3rX2LKU02A== + version "10.3.1" + resolved "https://registry.yarnpkg.com/@nestjs/testing/-/testing-10.3.1.tgz#ea28a7d29122dd3a2df1542842e741a52dd7c474" + integrity sha512-74aSAugWT31jSPnStyRWDXgjHXWO3GYaUfAZ2T7Dml88UGkGy95iwaWgYy7aYM8/xVFKcDYkfL5FAYqZYce/yg== dependencies: tslib "2.6.2" @@ -887,9 +910,9 @@ integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== "@sinonjs/commons@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.0.tgz#beb434fe875d965265e04722ccfc21df7f755d72" - integrity sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA== + version "3.0.1" + resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.1.tgz#1029357e44ca901a615585f6d27738dbc89084cd" + integrity sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ== dependencies: type-detect "4.0.8" @@ -901,97 +924,99 @@ "@sinonjs/commons" "^3.0.0" "@swc/cli@^0.1.63": - version "0.1.63" - resolved "https://registry.yarnpkg.com/@swc/cli/-/cli-0.1.63.tgz#b4ab850f8c285d06d593428b14ffa3df782adcbb" - integrity sha512-EM9oxxHzmmsprYRbGqsS2M4M/Gr5Gkcl0ROYYIdlUyTkhOiX822EQiRCpPCwdutdnzH2GyaTN7wc6i0Y+CKd3A== + version "0.1.65" + resolved "https://registry.yarnpkg.com/@swc/cli/-/cli-0.1.65.tgz#bb51ce6f088a78ac99a07507c15a8d74c9336ecb" + integrity sha512-4NcgsvJVHhA7trDnMmkGLLvWMHu2kSy+qHx6QwRhhJhdiYdNUrhdp+ERxen73sYtaeEOYeLJcWrQ60nzKi6rpg== dependencies: "@mole-inc/bin-wrapper" "^8.0.1" commander "^7.1.0" fast-glob "^3.2.5" + minimatch "^9.0.3" semver "^7.3.8" slash "3.0.0" source-map "^0.7.3" -"@swc/core-darwin-arm64@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-darwin-arm64/-/core-darwin-arm64-1.3.104.tgz#ad8fcd333c09634279d6cf46c5dd2c00b47ef809" - integrity sha512-rCnVj8x3kn6s914Adddu+zROHUn6mUEMkNKUckofs3W9OthNlZXJA3C5bS2MMTRFXCWamJ0Zmh6INFpz+f4Tfg== - -"@swc/core-darwin-x64@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-darwin-x64/-/core-darwin-x64-1.3.104.tgz#be2f270fb1f9d0aa2f27836f9ccb28ea4da26a7e" - integrity sha512-LBCWGTYkn1UjyxrmcLS3vZgtCDVhwxsQMV7jz5duc7Gas8SRWh6ZYqvUkjlXMDX1yx0uvzHrkaRw445+zDRj7Q== - -"@swc/core-linux-arm-gnueabihf@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.3.104.tgz#52c1425fbd4aa189d47a40eaebb335cbda96f917" - integrity sha512-iFbsWcx0TKHWnFBNCuUstYqRtfkyBx7FKv5To1Hx14EMuvvoCD/qUoJEiNfDQN5n/xU9g5xq4RdbjEWCFLhAbA== - -"@swc/core-linux-arm64-gnu@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.3.104.tgz#30da51b22f36887317fa5f49b8eb2ebe17d936de" - integrity sha512-1BIIp+nUPrRHHaJ35YJqrwXPwYSITp5robqqjyTwoKGw2kq0x+A964kpWul6v0d7A9Ial8fyH4m13eSWBodD2A== - -"@swc/core-linux-arm64-musl@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.3.104.tgz#c9a281ad655ba5a4217466c7e0ca6457202b2997" - integrity sha512-IyDNkzpKwvLqmRwTW+s8f8OsOSSj1N6juZKbvNHpZRfWZkz3T70q3vJlDBWQwy8z8cm7ckd7YUT3eKcSBPPowg== - -"@swc/core-linux-x64-gnu@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.3.104.tgz#2bd0cd4e92fbedb83aeb6526299a792579b624f2" - integrity sha512-MfX/wiRdTjE5uXHTDnaX69xI4UBfxIhcxbVlMj//N+7AX/G2pl2UFityfVMU2HpM12BRckrCxVI8F/Zy3DZkYQ== - -"@swc/core-linux-x64-musl@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.3.104.tgz#a3bb9b5eb9c524f87c586f43019fc544e2ef8bcf" - integrity sha512-5yeILaxA31gGEmquErO8yxlq1xu0XVt+fz5mbbKXKZMRRILxYxNzAGb5mzV41r0oHz6Vhv4AXX/WMCmeWl+HkQ== - -"@swc/core-win32-arm64-msvc@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.3.104.tgz#ec3b63321bbed1283c7873b7c3ecaaf03f8a42ee" - integrity sha512-rwcImsYnWDWGmeESG0XdGGOql5s3cG5wA8C4hHHKdH76zamPfDKKQFBsjmoNi0f1IsxaI9AJPeOmD4bAhT1ZoQ== - -"@swc/core-win32-ia32-msvc@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.3.104.tgz#47ef6d3dfb7093ff7da4848a59645672c0f25bef" - integrity sha512-ICDA+CJLYC7NkePnrbh/MvXwDQfy3rZSFgrVdrqRosv9DKHdFjYDnA9++7ozjrIdFdBrFW2NR7pyUcidlwhNzA== - -"@swc/core-win32-x64-msvc@1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.3.104.tgz#661de1921e869b0a6762e85c5e3232c007554ad8" - integrity sha512-fZJ1Ju62U4lMZVU+nHxLkFNcu0hG5Y0Yj/5zjrlbuX5N8J5eDndWAFsVnQhxRTZqKhZB53pvWRQs5FItSDqgXg== +"@swc/core-darwin-arm64@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-darwin-arm64/-/core-darwin-arm64-1.4.0.tgz#11abf23b884929a467ba270cf6789b9c50c4248b" + integrity sha512-UTJ/Vz+s7Pagef6HmufWt6Rs0aUu+EJF4Pzuwvr7JQQ5b1DZeAAUeUtkUTFx/PvCbM8Xfw4XdKBUZfrIKCfW8A== + +"@swc/core-darwin-x64@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-darwin-x64/-/core-darwin-x64-1.4.0.tgz#f044ddaca60c5081e907b148721ad7461f6f6dfe" + integrity sha512-f8v58u2GsGak8EtZFN9guXqE0Ep10Suny6xriaW2d8FGqESPyNrnBzli3aqkSeQk5gGqu2zJ7WiiKp3XoUOidA== + +"@swc/core-linux-arm-gnueabihf@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.4.0.tgz#52ceea673fc76692c0bd6d58e1863125c3e6173b" + integrity sha512-q2KAkBzmPcTnRij/Y1fgHCKAGevUX/H4uUESrw1J5gmUg9Qip6onKV80lTumA1/aooGJ18LOsB31qdbwmZk9OA== + +"@swc/core-linux-arm64-gnu@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.4.0.tgz#7f3ff1ab824ec48acdb39d231cbcb4096a4f9dd0" + integrity sha512-SknGu96W0mzHtLHWm+62fk5+Omp9fMPFO7AWyGFmz2tr8EgRRXtTSrBUnWhAbgcalnhen48GsvtMdxf1KNputg== + +"@swc/core-linux-arm64-musl@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.4.0.tgz#26c3b1f7947c19ef725997af716f230957d586f8" + integrity sha512-/k3TDvpBRMDNskHooNN1KqwUhcwkfBlIYxRTnJvsfT2C7My4pffR+4KXmt0IKynlTTbCdlU/4jgX4801FSuliw== + +"@swc/core-linux-x64-gnu@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.4.0.tgz#2c7d03a04a7d045394cfed7d46419ff8816ec22e" + integrity sha512-GYsTMvNt5+WTVlwwQzOOWsPMw6P/F41u5PGHWmfev8Nd4QJ1h3rWPySKk4mV42IJwH9MgQCVSl3ygwNqwl6kFg== + +"@swc/core-linux-x64-musl@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.4.0.tgz#0e76442dfb6d5026d8d6e7db6b2f4922b7692d0f" + integrity sha512-jGVPdM/VwF7kK/uYRW5N6FwzKf/FnDjGIR3RPvQokjYJy7Auk+3Oj21C0Jev7sIT9RYnO/TrFEoEozKeD/z2Qw== + +"@swc/core-win32-arm64-msvc@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.4.0.tgz#0177bebf312eb251d6749ab76259c0e08088e837" + integrity sha512-biHYm1AronEKlt47O/H8sSOBM2BKXMmWT+ApvlxUw50m1RGNnVnE0bgY7tylFuuSiWyXsQPJbmUV708JqORXVg== + +"@swc/core-win32-ia32-msvc@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.4.0.tgz#27fa650280e5651aa42129eaf03e02787b866417" + integrity sha512-TL5L2tFQb19kJwv6+elToGBj74QXCn9j+hZfwQatvZEJRA5rDK16eH6oAE751dGUArhnWlW3Vj65hViPvTuycw== + +"@swc/core-win32-x64-msvc@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.4.0.tgz#bd575c599bd6847bddc4863a3babd85e3db5e11e" + integrity sha512-e2xVezU7XZ2Stzn4i7TOQe2Kn84oYdG0M3A7XI7oTdcpsKCcKwgiMoroiAhqCv+iN20KNqhnWwJiUiTj/qN5AA== "@swc/core@^1.3.104": - version "1.3.104" - resolved "https://registry.yarnpkg.com/@swc/core/-/core-1.3.104.tgz#4346c4548ddff85ebc4a1acd2ce54ce6f36f5e34" - integrity sha512-9LWH/qzR/Pmyco+XwPiPfz59T1sryI7o5dmqb593MfCkaX5Fzl9KhwQTI47i21/bXYuCdfa9ySZuVkzXMirYxA== + version "1.4.0" + resolved "https://registry.yarnpkg.com/@swc/core/-/core-1.4.0.tgz#3a0ceeea5b889173f4592955fe1da4d071d86a76" + integrity sha512-wc5DMI5BJftnK0Fyx9SNJKkA0+BZSJQx8430yutWmsILkHMBD3Yd9GhlMaxasab9RhgKqZp7Ht30hUYO5ZDvQg== dependencies: "@swc/counter" "^0.1.1" "@swc/types" "^0.1.5" optionalDependencies: - "@swc/core-darwin-arm64" "1.3.104" - "@swc/core-darwin-x64" "1.3.104" - "@swc/core-linux-arm-gnueabihf" "1.3.104" - "@swc/core-linux-arm64-gnu" "1.3.104" - "@swc/core-linux-arm64-musl" "1.3.104" - "@swc/core-linux-x64-gnu" "1.3.104" - "@swc/core-linux-x64-musl" "1.3.104" - "@swc/core-win32-arm64-msvc" "1.3.104" - "@swc/core-win32-ia32-msvc" "1.3.104" - "@swc/core-win32-x64-msvc" "1.3.104" - -"@swc/counter@^0.1.1": - version "0.1.2" - resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.2.tgz#bf06d0770e47c6f1102270b744e17b934586985e" - integrity sha512-9F4ys4C74eSTEUNndnER3VJ15oru2NumfQxS8geE+f3eB5xvfxpWyqE5XlVnxb/R14uoXi6SLbBwwiDSkv+XEw== + "@swc/core-darwin-arm64" "1.4.0" + "@swc/core-darwin-x64" "1.4.0" + "@swc/core-linux-arm-gnueabihf" "1.4.0" + "@swc/core-linux-arm64-gnu" "1.4.0" + "@swc/core-linux-arm64-musl" "1.4.0" + "@swc/core-linux-x64-gnu" "1.4.0" + "@swc/core-linux-x64-musl" "1.4.0" + "@swc/core-win32-arm64-msvc" "1.4.0" + "@swc/core-win32-ia32-msvc" "1.4.0" + "@swc/core-win32-x64-msvc" "1.4.0" + +"@swc/counter@^0.1.1", "@swc/counter@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9" + integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ== "@swc/jest@^0.2.30": - version "0.2.30" - resolved "https://registry.yarnpkg.com/@swc/jest/-/jest-0.2.30.tgz#ad561bf90d1090ec0b71d54878f85a82d3af781f" - integrity sha512-80KKC6GYvgrpX1/7yKsRbu88V6OAJIcMGzOLCt0pPSg1nEwJg/lLAodVy2hCD8OcYApmY5gSwD4SnwgA5Y7Q7A== + version "0.2.36" + resolved "https://registry.yarnpkg.com/@swc/jest/-/jest-0.2.36.tgz#2797450a30d28b471997a17e901ccad946fe693e" + integrity sha512-8X80dp81ugxs4a11z1ka43FPhP+/e+mJNXJSxiNYk8gIX/jPBtY4gQTrKu/KIoco8bzKuPI5lUxjfLiGsfvnlw== dependencies: "@jest/create-cache-key-function" "^29.7.0" + "@swc/counter" "^0.1.3" jsonc-parser "^3.2.0" "@swc/types@^0.1.5": @@ -1117,15 +1142,15 @@ "@types/estree" "*" "@types/json-schema" "*" -"@types/estree@*", "@types/estree@^1.0.0": +"@types/estree@*", "@types/estree@^1.0.5": version "1.0.5" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== "@types/express-serve-static-core@^4.17.33": - version "4.17.41" - resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.41.tgz#5077defa630c2e8d28aa9ffc2c01c157c305bef6" - integrity sha512-OaJ7XLaelTgrvlZD8/aa0vvvxZdUmlCn6MtWeB7TkiKW70BQLc9XEPpDLPdbo52ZhXUCrznlWdCHWxJWtdyajA== + version "4.17.43" + resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.43.tgz#10d8444be560cb789c4735aea5eac6e5af45df54" + integrity sha512-oaYtiBirUOPQGSWNGPWnzyAFJ0BP3cwvN4oWZQY+zUBwpVIGsKUkpBpSztp74drYcjavs7SKFZ4DX1V2QeN8rg== dependencies: "@types/node" "*" "@types/qs" "*" @@ -1179,9 +1204,9 @@ "@types/istanbul-lib-report" "*" "@types/jest@^29.5.2": - version "29.5.11" - resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.5.11.tgz#0c13aa0da7d0929f078ab080ae5d4ced80fa2f2c" - integrity sha512-S2mHmYIVe13vrm6q4kN6fLYYAka15ALQki/vgDC3mIukEOx8WJlv0kQPM+d4w8Gp6u0uSdKND04IlTXBv0rwnQ== + version "29.5.12" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.5.12.tgz#7f7dc6eb4cf246d2474ed78744b05d06ce025544" + integrity sha512-eDC8bTvT/QhYdxJAulQikueigY5AsdBRH2yDKW3yveW7svY3+DzN84/2NUgkw10RTiJbWqZrTtoGVdYlvFJdLw== dependencies: expect "^29.0.0" pretty-format "^29.0.0" @@ -1231,9 +1256,9 @@ integrity sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w== "@types/node@*", "@types/node@^20.3.1": - version "20.11.5" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.5.tgz#be10c622ca7fcaa3cf226cf80166abc31389d86e" - integrity sha512-g557vgQjUUfN76MZAN/dt1z3dzcUsimuysco0KeluHgrPdJXkP/XdAURgyO2W9fZWHRtRBiVKzKn8vyOAwlG+w== + version "20.11.16" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.16.tgz#4411f79411514eb8e2926f036c86c9f0e4ec6708" + integrity sha512-gKb0enTmRCzXSSUJDq6/sPcqrfCv2mkkG6Jt/clpn5eiCbKTY+SgZUxo+p8ZKMof5dCp9vHQUAB7wOUTod22wQ== dependencies: undici-types "~5.26.4" @@ -1297,9 +1322,9 @@ "@types/stream-chain" "*" "@types/superagent@^8.1.0": - version "8.1.1" - resolved "https://registry.yarnpkg.com/@types/superagent/-/superagent-8.1.1.tgz#dbc620c5df3770b0c3092f947d6d5e808adae2bc" - integrity sha512-YQyEXA4PgCl7EVOoSAS3o0fyPFU6erv5mMixztQYe1bqbWmmn8c+IrqoxjQeZe4MgwXikgcaZPiI/DsbmOVlzA== + version "8.1.3" + resolved "https://registry.yarnpkg.com/@types/superagent/-/superagent-8.1.3.tgz#6222a466e89eac9c84ad8de11870d92097e6554a" + integrity sha512-R/CfN6w2XsixLb1Ii8INfn+BT9sGPvw74OavfkW4SwY+jeUcAwLZv2+bXLJkndnimxjEBm0RPHgcjW9pLCa8cw== dependencies: "@types/cookiejar" "^2.1.5" "@types/methods" "^1.1.4" @@ -1318,6 +1343,11 @@ resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c" integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw== +"@types/validator@^13.11.8": + version "13.11.8" + resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.11.8.tgz#bb1162ec0fe6f87c95ca812f15b996fcc5e1e2dc" + integrity sha512-c/hzNDBh7eRF+KbCf+OoZxKbnkpaK/cKp9iLQWqB7muXtM+MtL9SUUH8vCFcLn6dH1Qm05jiexK0ofWY7TfOhQ== + "@types/yargs-parser@*": version "21.0.3" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.3.tgz#815e30b786d2e8f0dcd85fd5bcf5e1a04d008f15" @@ -1331,15 +1361,15 @@ "@types/yargs-parser" "*" "@typescript-eslint/eslint-plugin@^6.0.0": - version "6.19.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.19.0.tgz#db03f3313b57a30fbbdad2e6929e88fc7feaf9ba" - integrity sha512-DUCUkQNklCQYnrBSSikjVChdc84/vMPDQSgJTHBZ64G9bA9w0Crc0rd2diujKbTdp6w2J47qkeHQLoi0rpLCdg== + version "6.20.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.20.0.tgz#9cf31546d2d5e884602626d89b0e0d2168ac25ed" + integrity sha512-fTwGQUnjhoYHeSF6m5pWNkzmDDdsKELYrOBxhjMrofPqCkoC2k3B2wvGHFxa1CTIqkEn88nlW1HVMztjo2K8Hg== dependencies: "@eslint-community/regexpp" "^4.5.1" - "@typescript-eslint/scope-manager" "6.19.0" - "@typescript-eslint/type-utils" "6.19.0" - "@typescript-eslint/utils" "6.19.0" - "@typescript-eslint/visitor-keys" "6.19.0" + "@typescript-eslint/scope-manager" "6.20.0" + "@typescript-eslint/type-utils" "6.20.0" + "@typescript-eslint/utils" "6.20.0" + "@typescript-eslint/visitor-keys" "6.20.0" debug "^4.3.4" graphemer "^1.4.0" ignore "^5.2.4" @@ -1348,46 +1378,46 @@ ts-api-utils "^1.0.1" "@typescript-eslint/parser@^6.0.0": - version "6.19.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.19.0.tgz#80344086f362181890ade7e94fc35fe0480bfdf5" - integrity sha512-1DyBLG5SH7PYCd00QlroiW60YJ4rWMuUGa/JBV0iZuqi4l4IK3twKPq5ZkEebmGqRjXWVgsUzfd3+nZveewgow== - dependencies: - "@typescript-eslint/scope-manager" "6.19.0" - "@typescript-eslint/types" "6.19.0" - "@typescript-eslint/typescript-estree" "6.19.0" - "@typescript-eslint/visitor-keys" "6.19.0" + version "6.20.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.20.0.tgz#17e314177304bdf498527e3c4b112e41287b7416" + integrity sha512-bYerPDF/H5v6V76MdMYhjwmwgMA+jlPVqjSDq2cRqMi8bP5sR3Z+RLOiOMad3nsnmDVmn2gAFCyNgh/dIrfP/w== + dependencies: + "@typescript-eslint/scope-manager" "6.20.0" + "@typescript-eslint/types" "6.20.0" + "@typescript-eslint/typescript-estree" "6.20.0" + "@typescript-eslint/visitor-keys" "6.20.0" debug "^4.3.4" -"@typescript-eslint/scope-manager@6.19.0": - version "6.19.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.19.0.tgz#b6d2abb825b29ab70cb542d220e40c61c1678116" - integrity sha512-dO1XMhV2ehBI6QN8Ufi7I10wmUovmLU0Oru3n5LVlM2JuzB4M+dVphCPLkVpKvGij2j/pHBWuJ9piuXx+BhzxQ== +"@typescript-eslint/scope-manager@6.20.0": + version "6.20.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.20.0.tgz#8a926e60f6c47feb5bab878246dc2ae465730151" + integrity sha512-p4rvHQRDTI1tGGMDFQm+GtxP1ZHyAh64WANVoyEcNMpaTFn3ox/3CcgtIlELnRfKzSs/DwYlDccJEtr3O6qBvA== dependencies: - "@typescript-eslint/types" "6.19.0" - "@typescript-eslint/visitor-keys" "6.19.0" + "@typescript-eslint/types" "6.20.0" + "@typescript-eslint/visitor-keys" "6.20.0" -"@typescript-eslint/type-utils@6.19.0": - version "6.19.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.19.0.tgz#522a494ef0d3e9fdc5e23a7c22c9331bbade0101" - integrity sha512-mcvS6WSWbjiSxKCwBcXtOM5pRkPQ6kcDds/juxcy/727IQr3xMEcwr/YLHW2A2+Fp5ql6khjbKBzOyjuPqGi/w== +"@typescript-eslint/type-utils@6.20.0": + version "6.20.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.20.0.tgz#d395475cd0f3610dd80c7d8716fa0db767da3831" + integrity sha512-qnSobiJQb1F5JjN0YDRPHruQTrX7ICsmltXhkV536mp4idGAYrIyr47zF/JmkJtEcAVnIz4gUYJ7gOZa6SmN4g== dependencies: - "@typescript-eslint/typescript-estree" "6.19.0" - "@typescript-eslint/utils" "6.19.0" + "@typescript-eslint/typescript-estree" "6.20.0" + "@typescript-eslint/utils" "6.20.0" debug "^4.3.4" ts-api-utils "^1.0.1" -"@typescript-eslint/types@6.19.0": - version "6.19.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.19.0.tgz#689b0498c436272a6a2059b09f44bcbd90de294a" - integrity sha512-lFviGV/vYhOy3m8BJ/nAKoAyNhInTdXpftonhWle66XHAtT1ouBlkjL496b5H5hb8dWXHwtypTqgtb/DEa+j5A== +"@typescript-eslint/types@6.20.0": + version "6.20.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.20.0.tgz#5ccd74c29011ae7714ae6973e4ec0c634708b448" + integrity sha512-MM9mfZMAhiN4cOEcUOEx+0HmuaW3WBfukBZPCfwSqFnQy0grXYtngKCqpQN339X3RrwtzspWJrpbrupKYUSBXQ== -"@typescript-eslint/typescript-estree@6.19.0": - version "6.19.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.19.0.tgz#0813ba364a409afb4d62348aec0202600cb468fa" - integrity sha512-o/zefXIbbLBZ8YJ51NlkSAt2BamrK6XOmuxSR3hynMIzzyMY33KuJ9vuMdFSXW+H0tVvdF9qBPTHA91HDb4BIQ== +"@typescript-eslint/typescript-estree@6.20.0": + version "6.20.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.20.0.tgz#5b2d0975949e6bdd8d45ee1471461ef5fadc5542" + integrity sha512-RnRya9q5m6YYSpBN7IzKu9FmLcYtErkDkc8/dKv81I9QiLLtVBHrjz+Ev/crAqgMNW2FCsoZF4g2QUylMnJz+g== dependencies: - "@typescript-eslint/types" "6.19.0" - "@typescript-eslint/visitor-keys" "6.19.0" + "@typescript-eslint/types" "6.20.0" + "@typescript-eslint/visitor-keys" "6.20.0" debug "^4.3.4" globby "^11.1.0" is-glob "^4.0.3" @@ -1395,25 +1425,25 @@ semver "^7.5.4" ts-api-utils "^1.0.1" -"@typescript-eslint/utils@6.19.0": - version "6.19.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.19.0.tgz#557b72c3eeb4f73bef8037c85dae57b21beb1a4b" - integrity sha512-QR41YXySiuN++/dC9UArYOg4X86OAYP83OWTewpVx5ct1IZhjjgTLocj7QNxGhWoTqknsgpl7L+hGygCO+sdYw== +"@typescript-eslint/utils@6.20.0": + version "6.20.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.20.0.tgz#0e52afcfaa51af5656490ba4b7437cc3aa28633d" + integrity sha512-/EKuw+kRu2vAqCoDwDCBtDRU6CTKbUmwwI7SH7AashZ+W+7o8eiyy6V2cdOqN49KsTcASWsC5QeghYuRDTyOOg== dependencies: "@eslint-community/eslint-utils" "^4.4.0" "@types/json-schema" "^7.0.12" "@types/semver" "^7.5.0" - "@typescript-eslint/scope-manager" "6.19.0" - "@typescript-eslint/types" "6.19.0" - "@typescript-eslint/typescript-estree" "6.19.0" + "@typescript-eslint/scope-manager" "6.20.0" + "@typescript-eslint/types" "6.20.0" + "@typescript-eslint/typescript-estree" "6.20.0" semver "^7.5.4" -"@typescript-eslint/visitor-keys@6.19.0": - version "6.19.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.19.0.tgz#4565e0ecd63ca1f81b96f1dd76e49f746c6b2b49" - integrity sha512-hZaUCORLgubBvtGpp1JEFEazcuEdfxta9j4iUwdSAr7mEsYYAp3EAUyCZk3VEEqGj6W+AV4uWyrDGtrlawAsgQ== +"@typescript-eslint/visitor-keys@6.20.0": + version "6.20.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.20.0.tgz#f7ada27f2803de89df0edd9fd7be22c05ce6a498" + integrity sha512-E8Cp98kRe4gKHjJD4NExXKz/zOJ1A2hhZc+IMVD6i7w4yjIvh6VyuRI0gRtxAsXtoC35uGMaQ9rjI2zJaXDEAw== dependencies: - "@typescript-eslint/types" "6.19.0" + "@typescript-eslint/types" "6.20.0" eslint-visitor-keys "^3.4.1" "@ungap/structured-clone@^1.2.0": @@ -1693,13 +1723,13 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -array-buffer-byte-length@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz#fabe8bc193fea865f317fe7807085ee0dee5aead" - integrity sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A== +array-buffer-byte-length@^1.0.0, array-buffer-byte-length@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz#1e5583ec16763540a27ae52eed99ff899223568f" + integrity sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg== dependencies: - call-bind "^1.0.2" - is-array-buffer "^3.0.1" + call-bind "^1.0.5" + is-array-buffer "^3.0.4" array-flatten@1.1.1: version "1.1.1" @@ -1727,6 +1757,17 @@ array-union@^2.1.0: resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== +array.prototype.filter@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/array.prototype.filter/-/array.prototype.filter-1.0.3.tgz#423771edeb417ff5914111fff4277ea0624c0d0e" + integrity sha512-VizNcj/RGJiUyQBgzwxzE5oHdeuXY5hSbbmKMlphj1cy1Vl7Pn2asCGbSrru6hSQjmCzqTBPVWAF/whmEOVHbw== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + es-array-method-boxes-properly "^1.0.0" + is-string "^1.0.7" + array.prototype.findlastindex@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.3.tgz#b37598438f97b579166940814e2c0493a4f50207" @@ -1759,16 +1800,17 @@ array.prototype.flatmap@^1.3.2: es-shim-unscopables "^1.0.0" arraybuffer.prototype.slice@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.2.tgz#98bd561953e3e74bb34938e77647179dfe6e9f12" - integrity sha512-yMBKppFur/fbHu9/6USUe03bZ4knMYiwFBcyiaXB8Go0qNehwX6inYPzK9U0NeQvGxKthcmHcaR8P5MStSRBAw== + version "1.0.3" + resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.3.tgz#097972f4255e41bc3425e37dc3f6421cf9aefde6" + integrity sha512-bMxMKAjg13EBSVscxTaYA4mRc5t1UAXa2kXiGTNfZ079HIWXEkKmkgFrh/nJqamaLSrXO5H4WFFkPEaLJWbs3A== dependencies: - array-buffer-byte-length "^1.0.0" - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - get-intrinsic "^1.2.1" - is-array-buffer "^3.0.2" + array-buffer-byte-length "^1.0.1" + call-bind "^1.0.5" + define-properties "^1.2.1" + es-abstract "^1.22.3" + es-errors "^1.2.1" + get-intrinsic "^1.2.3" + is-array-buffer "^3.0.4" is-shared-array-buffer "^1.0.2" asap@^2.0.0: @@ -1786,10 +1828,10 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== -available-typed-arrays@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" - integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== +available-typed-arrays@^1.0.5, available-typed-arrays@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.6.tgz#ac812d8ce5a6b976d738e1c45f08d0b00bc7d725" + integrity sha512-j1QzY8iPNPG4o4xmO3ptzpRxTciqD3MgEHtifP/YnJpIo58Xu+ne4BejlbkuaLfXn/nz6HFiw29bLpj2PNMdGg== babel-jest@^29.7.0: version "29.7.0" @@ -1958,13 +2000,13 @@ braces@^3.0.2, braces@~3.0.2: dependencies: fill-range "^7.0.1" -browserslist@^4.14.5, browserslist@^4.22.2: - version "4.22.2" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.22.2.tgz#704c4943072bd81ea18997f3bd2180e89c77874b" - integrity sha512-0UgcrvQmBDvZHFGdYUehrCNIazki7/lUP3kkoi/r3YB2amZbFM9J43ZRkJTXBUZK4gmx56+Sqk9+Vs9mwZx9+A== +browserslist@^4.21.10, browserslist@^4.22.2: + version "4.22.3" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.22.3.tgz#299d11b7e947a6b843981392721169e27d60c5a6" + integrity sha512-UAp55yfwNv0klWNapjs/ktHoguxuQNGnOzxYmfnXIS+8AsRDZkSDxg7R1AX3GKzn078SBI5dzwzj/Yx0Or0e3A== dependencies: - caniuse-lite "^1.0.30001565" - electron-to-chromium "^1.4.601" + caniuse-lite "^1.0.30001580" + electron-to-chromium "^1.4.648" node-releases "^2.0.14" update-browserslist-db "^1.0.13" @@ -2025,7 +2067,7 @@ cacheable-request@^7.0.2: normalize-url "^6.0.1" responselike "^2.0.0" -call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.4, call-bind@^1.0.5: +call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.5.tgz#6fa2b7845ce0ea49bf4d8b9ef64727a2c2e2e513" integrity sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ== @@ -2049,10 +2091,10 @@ camelcase@^6.2.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -caniuse-lite@^1.0.30001565: - version "1.0.30001579" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001579.tgz#45c065216110f46d6274311a4b3fcf6278e0852a" - integrity sha512-u5AUVkixruKHJjw/pj9wISlcMpgFWzSrczLZbrqBSxukQixmg0SJ5sZTpvaFvxU0HoQKd4yoyAogyrAz9pzJnA== +caniuse-lite@^1.0.30001580: + version "1.0.30001584" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001584.tgz#5e3ea0625d048d5467670051687655b1f7bf7dfd" + integrity sha512-LOz7CCQ9M1G7OjJOF9/mzmqmj3jE/7VOmrfw6Mgs0E8cjOsbRXQJHsPBfmBOXDskXKrHLyyW3n7kpDW/4BsfpQ== chalk@4.1.2, chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.1, chalk@^4.1.2: version "4.1.2" @@ -2116,6 +2158,20 @@ cjs-module-lexer@^1.0.0: resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz#6c370ab19f8a3394e318fe682686ec0ac684d107" integrity sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ== +class-transformer@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/class-transformer/-/class-transformer-0.5.1.tgz#24147d5dffd2a6cea930a3250a677addf96ab336" + integrity sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw== + +class-validator@^0.14.1: + version "0.14.1" + resolved "https://registry.yarnpkg.com/class-validator/-/class-validator-0.14.1.tgz#ff2411ed8134e9d76acfeb14872884448be98110" + integrity sha512-2VEG9JICxIqTpoK1eMzZqaV+u/EiwEJkMGzTrZf6sU/fwsnOITVgYJ8yojSy6CaXtO9V0Cc6ZQZ8h8m4UBuLwQ== + dependencies: + "@types/validator" "^13.11.8" + libphonenumber-js "^1.10.53" + validator "^13.9.0" + cli-cursor@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" @@ -2539,10 +2595,10 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== -electron-to-chromium@^1.4.601: - version "1.4.640" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.640.tgz#76290a36fa4b5f1f4cadaf1fc582478ebb3ac246" - integrity sha512-z/6oZ/Muqk4BaE7P69bXhUhpJbUM9ZJeka43ZwxsDshKtePns4mhBlh8bU5+yrnOnz3fhG82XLzGUXazOmsWnA== +electron-to-chromium@^1.4.648: + version "1.4.656" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.656.tgz#b374fb7cab9b782a5bc967c0ce0e19826186b9c9" + integrity sha512-9AQB5eFTHyR3Gvt2t/NwR0le2jBSUNwCnMbUCejFWHD+so4tH40/dRLgoE+jxlPeWS43XJewyvCv+I8LPMl49Q== emittery@^0.13.1: version "0.13.1" @@ -2591,7 +2647,7 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -es-abstract@^1.22.1: +es-abstract@^1.22.1, es-abstract@^1.22.3: version "1.22.3" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.22.3.tgz#48e79f5573198de6dee3589195727f4f74bc4f32" integrity sha512-eiiY8HQeYfYH2Con2berK+To6GrK2RxbPawDkGq4UiCQQfZHb6wX9qQqkbpPqaxQFcl8d9QzZqo0tGE0VcrdwA== @@ -2636,6 +2692,16 @@ es-abstract@^1.22.1: unbox-primitive "^1.0.2" which-typed-array "^1.1.13" +es-array-method-boxes-properly@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-array-method-boxes-properly/-/es-array-method-boxes-properly-1.0.0.tgz#873f3e84418de4ee19c5be752990b2e44718d09e" + integrity sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA== + +es-errors@^1.0.0, es-errors@^1.2.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" + integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== + es-module-lexer@^1.2.1: version "1.4.1" resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-1.4.1.tgz#41ea21b43908fe6a287ffcbe4300f790555331f5" @@ -3028,9 +3094,9 @@ fast-safe-stringify@2.1.1, fast-safe-stringify@^2.1.1: integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== fastq@^1.6.0: - version "1.16.0" - resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.16.0.tgz#83b9a9375692db77a822df081edb6a9cf6839320" - integrity sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA== + version "1.17.0" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.17.0.tgz#ca5e1a90b5e68f97fc8b61330d5819b82f5fab03" + integrity sha512-zGygtijUMT7jnk3h26kUms3BkSDp4IfIKjmnqI2tvx6nuBfiF1UqOxbnLfzdv+apBy+53oaImsKtMw/xYbW+1w== dependencies: reusify "^1.0.4" @@ -3269,11 +3335,12 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@^1.2.0, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.2.tgz#281b7622971123e1ef4b3c90fd7539306da93f3b" - integrity sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA== +get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2, get-intrinsic@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.3.tgz#9d2d284a238e62672f556361e7d4e1a4686ae50e" + integrity sha512-JIcZczvcMVE7AUOP+X72bh8HqHBRxFdz5PDHYtNG/lE3yk9b3KZBJlwFcTyPYjg3L4RLLmZJzvjxhaZVapxFrQ== dependencies: + es-errors "^1.0.0" function-bind "^1.1.2" has-proto "^1.0.1" has-symbols "^1.0.3" @@ -3470,12 +3537,12 @@ has-symbols@^1.0.2, has-symbols@^1.0.3: resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== -has-tostringtag@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz#7e133818a7d394734f941e73c3d3f9291e658b25" - integrity sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ== +has-tostringtag@^1.0.0, has-tostringtag@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc" + integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw== dependencies: - has-symbols "^1.0.2" + has-symbols "^1.0.3" hasown@^2.0.0: version "2.0.0" @@ -3536,9 +3603,9 @@ ieee754@^1.1.13, ieee754@^1.2.1: integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== ignore@^5.2.0, ignore@^5.2.4: - version "5.3.0" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.0.tgz#67418ae40d34d6999c95ff56016759c718c82f78" - integrity sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg== + version "5.3.1" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef" + integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw== import-fresh@^3.2.1, import-fresh@^3.3.0: version "3.3.0" @@ -3595,12 +3662,12 @@ inquirer@8.2.6: through "^2.3.6" wrap-ansi "^6.0.1" -inquirer@9.2.11: - version "9.2.11" - resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-9.2.11.tgz#e9003755c233a414fceda1891c23bd622cad4a95" - integrity sha512-B2LafrnnhbRzCWfAdOXisUzL89Kg8cVJlYmhqoi3flSiV/TveO+nsXwgKr9h9PIo+J1hz7nBSk6gegRIMBBf7g== +inquirer@9.2.12: + version "9.2.12" + resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-9.2.12.tgz#0348e9311765b7c93fce143bb1c0ef1ae879b1d7" + integrity sha512-mg3Fh9g2zfuVWJn6lhST0O7x4n03k7G8Tx5nvikJkbq8/CK47WDVm+UznF0G6s5Zi0KcyUisr6DU8T67N5U+1Q== dependencies: - "@ljharb/through" "^2.3.9" + "@ljharb/through" "^2.3.11" ansi-escapes "^4.3.2" chalk "^5.3.0" cli-cursor "^3.1.0" @@ -3635,14 +3702,13 @@ ipaddr.js@1.9.1: resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== -is-array-buffer@^3.0.1, is-array-buffer@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.2.tgz#f2653ced8412081638ecb0ebbd0c41c6e0aecbbe" - integrity sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w== +is-array-buffer@^3.0.2, is-array-buffer@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.4.tgz#7a1f92b3d61edd2bc65d24f130530ea93d7fae98" + integrity sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw== dependencies: call-bind "^1.0.2" - get-intrinsic "^1.2.0" - is-typed-array "^1.1.10" + get-intrinsic "^1.2.1" is-arrayish@^0.2.1: version "0.2.1" @@ -3789,11 +3855,11 @@ is-symbol@^1.0.2, is-symbol@^1.0.3: has-symbols "^1.0.2" is-typed-array@^1.1.10, is-typed-array@^1.1.12, is-typed-array@^1.1.9: - version "1.1.12" - resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.12.tgz#d0bab5686ef4a76f7a73097b95470ab199c57d4a" - integrity sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg== + version "1.1.13" + resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.13.tgz#d6c5ca56df62334959322d7d7dd1cca50debe229" + integrity sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw== dependencies: - which-typed-array "^1.1.11" + which-typed-array "^1.1.14" is-unicode-supported@^0.1.0: version "0.1.0" @@ -4323,11 +4389,16 @@ json5@^2.2.2, json5@^2.2.3: resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== -jsonc-parser@3.2.0, jsonc-parser@^3.2.0: +jsonc-parser@3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/jsonc-parser/-/jsonc-parser-3.2.0.tgz#31ff3f4c2b9793f89c67212627c51c6394f88e76" integrity sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w== +jsonc-parser@^3.2.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/jsonc-parser/-/jsonc-parser-3.2.1.tgz#031904571ccf929d7670ee8c547545081cb37f1a" + integrity sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA== + jsonfile@^6.0.1: version "6.1.0" resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" @@ -4367,6 +4438,11 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" +libphonenumber-js@^1.10.53: + version "1.10.55" + resolved "https://registry.yarnpkg.com/libphonenumber-js/-/libphonenumber-js-1.10.55.tgz#ec864e369bf7babde02021d06b5f2433d7e9c78e" + integrity sha512-MrTg2JFLscgmTY6/oT9vopYETlgUls/FU6OaeeamGwk4LFxjIgOUML/ZSZICgR0LPYXaonVJo40lzMvaaTJlQA== + lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" @@ -4454,9 +4530,9 @@ lru-cache@^6.0.0: yallist "^4.0.0" "lru-cache@^9.1.1 || ^10.0.0": - version "10.1.0" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.1.0.tgz#2098d41c2dc56500e6c88584aa656c84de7d0484" - integrity sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag== + version "10.2.0" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.2.0.tgz#0bd445ca57363465900f4d1f9bd8db343a4d95c3" + integrity sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q== magic-string@0.30.5: version "0.30.5" @@ -4561,7 +4637,7 @@ mimic-response@^3.1.0: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== -minimatch@9.0.3, minimatch@^9.0.1: +minimatch@9.0.3, minimatch@^9.0.1, minimatch@^9.0.3: version "9.0.3" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.3.tgz#a6e00c3de44c3a542bfaae70abfc22420a6da825" integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== @@ -4763,14 +4839,15 @@ object.fromentries@^2.0.7: es-abstract "^1.22.1" object.groupby@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/object.groupby/-/object.groupby-1.0.1.tgz#d41d9f3c8d6c778d9cbac86b4ee9f5af103152ee" - integrity sha512-HqaQtqLnp/8Bn4GL16cj+CUYbnpe1bh0TtEaWvybszDG4tgxCJuRpV8VGuvNaI1fAnI4lUJzDG55MXcOH4JZcQ== + version "1.0.2" + resolved "https://registry.yarnpkg.com/object.groupby/-/object.groupby-1.0.2.tgz#494800ff5bab78fd0eff2835ec859066e00192ec" + integrity sha512-bzBq58S+x+uo0VjurFT0UktpKHOZmv4/xePiOA1nbB9pMqpGK7rUPNgf+1YC+7mE+0HzhTMqNUuCqvKhj6FnBw== dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - get-intrinsic "^1.2.1" + array.prototype.filter "^1.0.3" + call-bind "^1.0.5" + define-properties "^1.2.1" + es-abstract "^1.22.3" + es-errors "^1.0.0" object.values@^1.1.7: version "1.1.7" @@ -5016,9 +5093,9 @@ prettier-linter-helpers@^1.0.0: fast-diff "^1.1.2" prettier@^3.0.0, prettier@^3.1.0: - version "3.2.4" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.4.tgz#4723cadeac2ce7c9227de758e5ff9b14e075f283" - integrity sha512-FWu1oLHKCrtpO1ypU6J0SbK2d9Ckwysq6bHj/uaCP26DxrPpppCLQRGVuqAxSTvhF00AcvDRyYrLNW7ocBhFFQ== + version "3.2.5" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.5.tgz#e52bc3090586e824964a8813b09aba6233b28368" + integrity sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A== pretty-format@^29.0.0, pretty-format@^29.7.0: version "29.7.0" @@ -5773,7 +5850,7 @@ tapable@^2.1.1, tapable@^2.2.0, tapable@^2.2.1: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -terser-webpack-plugin@^5.3.7: +terser-webpack-plugin@^5.3.10: version "5.3.10" resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz#904f4c9193c6fd2a03f693a2150c62a92f40d199" integrity sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w== @@ -5888,9 +5965,9 @@ ts-api-utils@^1.0.1: integrity sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg== ts-jest@^29.1.0: - version "29.1.1" - resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-29.1.1.tgz#f58fe62c63caf7bfcc5cc6472082f79180f0815b" - integrity sha512-D6xjnnbP17cC85nliwGiL+tpoKN0StpgE0TeOjXQTU6MVCfsB4v7aW05CgQ/1OywGb0x/oy9hHFnN+sczTiRaA== + version "29.1.2" + resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-29.1.2.tgz#7613d8c81c43c8cb312c6904027257e814c40e09" + integrity sha512-br6GJoH/WUX4pu7FbZXuWGKGNDuU7b8Uj77g/Sp7puZV6EXzuByl6JrECvm0MzVzSTkSHWTihsXt+5XYER5b+g== dependencies: bs-logger "0.x" fast-json-stable-stringify "2.x" @@ -6038,7 +6115,7 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA== -typescript@5.3.3, typescript@^5.1.3: +typescript@5.3.3, typescript@^5.3.3: version "5.3.3" resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.3.3.tgz#b3ce6ba258e72e6305ba66f5c9b452aaee3ffe37" integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw== @@ -6066,9 +6143,9 @@ undici-types@~5.26.4: integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== undici@^6.4.0: - version "6.4.0" - resolved "https://registry.yarnpkg.com/undici/-/undici-6.4.0.tgz#7ca0c3f73e1034f3c79e566183b61bb55b1410ea" - integrity sha512-wYaKgftNqf6Je7JQ51YzkEkEevzOgM7at5JytKO7BjaURQpERW8edQSMrr2xb+Yv4U8Yg47J24+lc9+NbeXMFA== + version "6.6.0" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.6.0.tgz#a1d618347f4aef6dd23bec9cd2ca9f71209dabd2" + integrity sha512-p8VvLAgnx6g9pydV0GG/kciSx3ZCq5PLeEU4yefjoZCc1HSeiMxbrFzYIZlgSMrX3l0CoTJ37C6edu13acE40A== dependencies: "@fastify/busboy" "^2.0.0" @@ -6126,6 +6203,11 @@ v8-to-istanbul@^9.0.1: "@types/istanbul-lib-coverage" "^2.0.1" convert-source-map "^2.0.0" +validator@^13.9.0: + version "13.11.0" + resolved "https://registry.yarnpkg.com/validator/-/validator-13.11.0.tgz#23ab3fd59290c61248364eabf4067f04955fbb1b" + integrity sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ== + vary@^1, vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" @@ -6168,19 +6250,19 @@ webpack-sources@^3.2.3: resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde" integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== -webpack@5.89.0: - version "5.89.0" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.89.0.tgz#56b8bf9a34356e93a6625770006490bf3a7f32dc" - integrity sha512-qyfIC10pOr70V+jkmud8tMfajraGCZMBWJtrmuBymQKCrLTRejBI8STDp1MCyZu/QTdZSeacCQYpYNQVOzX5kw== +webpack@5.90.1: + version "5.90.1" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.90.1.tgz#62ab0c097d7cbe83d32523dbfbb645cdb7c3c01c" + integrity sha512-SstPdlAC5IvgFnhiRok8hqJo/+ArAbNv7rhU4fnWGHNVfN59HSQFaxZDSAL3IFG2YmqxuRs+IU33milSxbPlog== dependencies: "@types/eslint-scope" "^3.7.3" - "@types/estree" "^1.0.0" + "@types/estree" "^1.0.5" "@webassemblyjs/ast" "^1.11.5" "@webassemblyjs/wasm-edit" "^1.11.5" "@webassemblyjs/wasm-parser" "^1.11.5" acorn "^8.7.1" acorn-import-assertions "^1.9.0" - browserslist "^4.14.5" + browserslist "^4.21.10" chrome-trace-event "^1.0.2" enhanced-resolve "^5.15.0" es-module-lexer "^1.2.1" @@ -6194,7 +6276,7 @@ webpack@5.89.0: neo-async "^2.6.2" schema-utils "^3.2.0" tapable "^2.1.1" - terser-webpack-plugin "^5.3.7" + terser-webpack-plugin "^5.3.10" watchpack "^2.4.0" webpack-sources "^3.2.3" @@ -6217,16 +6299,16 @@ which-boxed-primitive@^1.0.2: is-string "^1.0.5" is-symbol "^1.0.3" -which-typed-array@^1.1.11, which-typed-array@^1.1.13: - version "1.1.13" - resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.13.tgz#870cd5be06ddb616f504e7b039c4c24898184d36" - integrity sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow== +which-typed-array@^1.1.13, which-typed-array@^1.1.14: + version "1.1.14" + resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.14.tgz#1f78a111aee1e131ca66164d8bdc3ab062c95a06" + integrity sha512-VnXFiIW8yNn9kIHN88xvZ4yOWchftKDsRJ8fEPacX/wl1lOvBrhsJ/OeJCXq7B0AaijRuqgzSKalJoPk+D8MPg== dependencies: - available-typed-arrays "^1.0.5" - call-bind "^1.0.4" + available-typed-arrays "^1.0.6" + call-bind "^1.0.5" for-each "^0.3.3" gopd "^1.0.1" - has-tostringtag "^1.0.0" + has-tostringtag "^1.0.1" which@^1.2.9: version "1.3.1" @@ -6243,9 +6325,9 @@ which@^2.0.1: isexe "^2.0.0" winston-transport@^4.5.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.6.0.tgz#f1c1a665ad1b366df72199e27892721832a19e1b" - integrity sha512-wbBA9PbPAHxKiygo7ub7BYRiKxms0tpfU2ljtWzb3SjRjv5yl6Ozuy/TkXf00HTAt+Uylo3gSkNwzc4ME0wiIg== + version "4.7.0" + resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.7.0.tgz#e302e6889e6ccb7f383b926df6936a5b781bd1f0" + integrity sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg== dependencies: logform "^2.3.2" readable-stream "^3.6.0" From ea57fd8ea53f169e23441cc143260b74ee1f9f8c Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Mon, 5 Feb 2024 15:18:01 +0400 Subject: [PATCH 04/15] chore: add desc to README --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 39989a5..48f676c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,27 @@ Tool for reporting Slashings and Withdrawals for Lido Community Staking Module +### Daemon working mode + +The tool is a daemon that listens to the CL and reports any slashings and withdrawals to the Lido Community Staking Module. + +The algorithm is as follows: +0. Get the current CL finalized head. +1. Run `KeysIndexer` service to get the current validator set of the CS Module. + > It is necessary at the first startup. All subsequent runs of the indexer will be performed when necessary and independently of the main processing* +2. Choose the next block service to process from `RootsProvider`. + > The provider chooses the next root with the following priority: + > - Return the root from `RootsStack` service if exists and `KeyIndexer` is helthy enougth to be trusted completely to process this root + > - *When no any processed roots yet* Return a configured root (from `.env` file) or the last finalized root + > - Return a finalized child root of the last processed root + > - Sleep 12s if nothing to process and **return to the step 0** +3. Run `RootsProcessor` service to process the root. + > The processor does the following: + > - Get the block info from CL by the root + > - If the current state of `KeysIndexer` is outdated (~15-27h behind from the block) to be trusted completely, add the block root to `RootsStack` + > - If the block has a slashing or withdrawal, report it to the CS Module + > - If the current state of `KeysIndexer` is helthy enougth to be trusted completely, remove the root from `RootsStack` + ## Installation ```bash From fb8d12fd41ab287c799e2d041f4f4d64b354b4b7 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Mon, 5 Feb 2024 16:14:17 +0400 Subject: [PATCH 05/15] chore: add statements --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 48f676c..b486231 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,12 @@ The algorithm is as follows: > - If the block has a slashing or withdrawal, report it to the CS Module > - If the current state of `KeysIndexer` is helthy enougth to be trusted completely, remove the root from `RootsStack` +So, according to the algorithm, there are these statements: +1. We always go sequentially by the finalized routs of blocks, taking the next one by the root of the previous one. In this way we avoid missing some blocks. +2. If for some reason the daemon crashes, it will start from the last root running before the crash when it is launched +3. If for some reason KeysAPI crashed or CL node stopped giving validators, we can use the previously successfully received data to guarantee that our slashings will report for another ~15h and withdrawals for ~27h (because of the new validators appearing time and `MIN_VALIDATOR_WITHDRAWABILITY_DELAY`) +If any of these time thresholds are breached, we can't be sure that if there was a slashing or a full withdrawal there was definitely not our validator there. That's why we put the root block in the stack just in case, to process it again later when KeysAPI and CL node are well. + ## Installation ```bash From 7805685d6badcba0bbf66f14c3bfd30be410d3ae Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Thu, 8 Feb 2024 20:33:02 +0400 Subject: [PATCH 06/15] feat: use SSZ state representation from node --- package.json | 5 +- src/common/handlers/handlers.module.ts | 2 + src/common/handlers/handlers.service.ts | 18 ++++- src/common/providers/base/rest-provider.ts | 2 + src/common/providers/consensus/consensus.ts | 21 +++++- src/daemon/services/keys-indexer.ts | 67 +++++++++++------- yarn.lock | 76 +++++++++++++++++++-- 7 files changed, 154 insertions(+), 37 deletions(-) diff --git a/package.json b/package.json index 6330729..30f485e 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "dependencies": { "@huanshiwushuang/lowdb": "^6.0.2", "@lido-nestjs/logger": "^1.3.2", + "@lodestar/types": "^1.15.0", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.1.1", "@nestjs/core": "^10.0.0", @@ -45,8 +46,8 @@ "@nestjs/cli": "^10.0.0", "@nestjs/schematics": "^10.0.0", "@nestjs/testing": "^10.0.0", - "@swc/cli": "^0.1.63", - "@swc/core": "^1.3.104", + "@swc/cli": "^0.3.9", + "@swc/core": "^1.4.0", "@swc/jest": "^0.2.30", "@types/express": "^4.17.17", "@types/jest": "^29.5.2", diff --git a/src/common/handlers/handlers.module.ts b/src/common/handlers/handlers.module.ts index fed3190..4c4ca80 100644 --- a/src/common/handlers/handlers.module.ts +++ b/src/common/handlers/handlers.module.ts @@ -1,8 +1,10 @@ import { Module } from '@nestjs/common'; import { HandlersService } from './handlers.service'; +import { ProvidersModule } from '../providers/providers.module'; @Module({ + imports: [ProvidersModule], providers: [HandlersService], exports: [HandlersService], }) diff --git a/src/common/handlers/handlers.service.ts b/src/common/handlers/handlers.service.ts index 6340bfb..955ef7d 100644 --- a/src/common/handlers/handlers.service.ts +++ b/src/common/handlers/handlers.service.ts @@ -1,6 +1,7 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { Inject, Injectable, LoggerService } from '@nestjs/common'; +import { Consensus } from '../providers/consensus/consensus'; import { BlockInfoResponse, RootHex } from '../providers/consensus/response.interface'; export interface KeyInfo { @@ -14,24 +15,35 @@ type KeyInfoFn = (valIndex: number) => KeyInfo | undefined; @Injectable() export class HandlersService { - constructor(@Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService) {} + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly consensus: Consensus, + ) {} public async prove(blockRoot: RootHex, blockInfo: BlockInfoResponse, keyInfoFn: KeyInfoFn): Promise { const slashings = await this.getUnprovenSlashings(blockRoot, blockInfo, keyInfoFn); const withdrawals = await this.getUnprovenWithdrawals(blockRoot, blockInfo, keyInfoFn); if (!slashings.length && !withdrawals.length) return; - const payload = await this.buildProvePayload(slashings, withdrawals); + const payload = await this.buildProvePayload(blockInfo, slashings, withdrawals); // TODO: ask before sending if CLI or daemon in watch mode await this.sendProves(payload); this.logger.log(`๐Ÿ Proves sent. Root [${blockRoot}]`); } - private async buildProvePayload(slashings: string[], withdrawals: string[]): Promise { + private async buildProvePayload( + blockInfo: BlockInfoResponse, + slashings: string[], + withdrawals: string[], + ): Promise { // TODO: implement // this.consensus.getState(...) if (slashings.length || withdrawals.length) { this.logger.warn(`๐Ÿ“ฆ Prove payload: slashings [${slashings}], withdrawals [${withdrawals}]`); } + // const { ssz } = await eval('import("@lodestar/types")'); + // const stateSSZ = await this.consensus.getStateSSZ(header.header.message.slot); + // const stateView = ssz.deneb.BeaconState.deserializeToView(stateSSZ); + // const validatorsInfo = stateView.validators.type.elementType.toJson(stateView.validators.get(1337)); return {}; } private async sendProves(payload: any): Promise { diff --git a/src/common/providers/base/rest-provider.ts b/src/common/providers/base/rest-provider.ts index 5bd4ee2..412d399 100644 --- a/src/common/providers/base/rest-provider.ts +++ b/src/common/providers/base/rest-provider.ts @@ -15,6 +15,7 @@ export interface RequestOptions { streamed?: boolean; requestPolicy?: RequestPolicy; signal?: AbortSignal; + headers?: Record; } export abstract class BaseRestProvider { @@ -55,6 +56,7 @@ export abstract class BaseRestProvider { method: 'GET', headersTimeout: (options.requestPolicy as RequestPolicy).timeout, signal: options.signal, + headers: options.headers, }); if (statusCode !== 200) { const hostname = new URL(base).hostname; diff --git a/src/common/providers/consensus/consensus.ts b/src/common/providers/consensus/consensus.ts index 849c179..cc12e68 100644 --- a/src/common/providers/consensus/consensus.ts +++ b/src/common/providers/consensus/consensus.ts @@ -103,7 +103,7 @@ export class Consensus extends BaseRestProvider implements OnApplicationBootstra } public async getState(stateId: StateId, signal?: AbortSignal): Promise { - const resp: { body: BodyReadable; headers: IncomingHttpHeaders } = await this.baseGet( + const { body } = await this.baseGet<{ body: BodyReadable; headers: IncomingHttpHeaders }>( this.mainUrl, this.endpoints.state(stateId), { @@ -115,9 +115,26 @@ export class Consensus extends BaseRestProvider implements OnApplicationBootstra // TODO: Enable for CLI only //this.progress.show(`State [${stateId}]`, resp); // Data processing - const pipeline = chain([resp.body, parser()]); + const pipeline = chain([body, parser()]); return await new Promise((resolve) => { connectTo(pipeline).on('done', (asm) => resolve(asm.current)); }); } + + public async getStateSSZ(stateId: StateId, signal?: AbortSignal): Promise { + const { body } = await this.baseGet<{ body: BodyReadable; headers: IncomingHttpHeaders }>( + this.mainUrl, + this.endpoints.state(stateId), + { + streamed: true, + signal, + headers: { accept: 'application/octet-stream' }, + }, + ); + // Progress bar + // TODO: Enable for CLI only + //this.progress.show(`State [${stateId}]`, resp); + // Data processing + return new Uint8Array(await body.arrayBuffer()); + } } diff --git a/src/daemon/services/keys-indexer.ts b/src/daemon/services/keys-indexer.ts index aea34b3..d6ab62b 100644 --- a/src/daemon/services/keys-indexer.ts +++ b/src/daemon/services/keys-indexer.ts @@ -1,3 +1,5 @@ +import { BooleanType, ByteVectorType, ContainerNodeStructType, UintNumberType } from '@chainsafe/ssz'; +import { ListCompositeTreeView } from '@chainsafe/ssz/lib/view/listComposite'; import { Low } from '@huanshiwushuang/lowdb'; import { JSONFile } from '@huanshiwushuang/lowdb/node'; import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; @@ -6,12 +8,7 @@ import { Inject, Injectable, LoggerService, OnApplicationBootstrap } from '@nest import { ConfigService } from '../../common/config/config.service'; import { KeyInfo } from '../../common/handlers/handlers.service'; import { Consensus } from '../../common/providers/consensus/consensus'; -import { - BlockHeaderResponse, - RootHex, - Slot, - StateValidatorResponse, -} from '../../common/providers/consensus/response.interface'; +import { BlockHeaderResponse, RootHex, Slot } from '../../common/providers/consensus/response.interface'; import { Keysapi } from '../../common/providers/keysapi/keysapi'; type Info = { @@ -25,6 +22,21 @@ type Storage = { [valIndex: number]: KeyInfo; }; +let types: typeof import('@lodestar/types'); + +type Validators = ListCompositeTreeView< + ContainerNodeStructType<{ + pubkey: ByteVectorType; + withdrawalCredentials: ByteVectorType; + effectiveBalance: UintNumberType; + slashed: BooleanType; + activationEligibilityEpoch: UintNumberType; + activationEpoch: UintNumberType; + exitEpoch: UintNumberType; + withdrawableEpoch: UintNumberType; + }> +>; + @Injectable() export class KeysIndexer implements OnApplicationBootstrap { private startedAt: number = 0; @@ -40,6 +52,8 @@ export class KeysIndexer implements OnApplicationBootstrap { ) {} public async onApplicationBootstrap(): Promise { + // ugly hack to import ESModule to CommonJS project + types = await eval(`import('@lodestar/types')`); await this.initOrReadServiceData(); } @@ -74,60 +88,63 @@ export class KeysIndexer implements OnApplicationBootstrap { private async baseRun(stateRoot: RootHex, finalizedSlot: Slot): Promise { this.logger.log(`Get validators. State root [${stateRoot}]`); - const validators = await this.consensus.getValidators(stateRoot); - this.logger.log(`Total validators count: ${validators.length}`); + const stateSSZ = await this.consensus.getStateSSZ(stateRoot); + const stateView = types.ssz.deneb.BeaconState.deserializeToView(stateSSZ); + this.logger.log(`Total validators count: ${stateView.validators.length}`); // TODO: do we need to store already full withdrawn keys ? this.info.data.lastValidatorsCount == 0 - ? await this.initStorage(validators, finalizedSlot) - : await this.updateStorage(validators, finalizedSlot); + ? await this.initStorage(stateView.validators, finalizedSlot) + : await this.updateStorage(stateView.validators, finalizedSlot); this.logger.log(`CSM validators count: ${Object.keys(this.storage.data).length}`); this.info.data.storageStateSlot = finalizedSlot; - this.info.data.lastValidatorsCount = validators.length; + this.info.data.lastValidatorsCount = stateView.validators.length; await this.info.write(); } - private async initStorage(validators: StateValidatorResponse[], finalizedSlot: Slot): Promise { + private async initStorage(validators: Validators, finalizedSlot: Slot): Promise { this.logger.log(`Init keys data`); const csmKeys = await this.keysapi.getModuleKeys(this.info.data.moduleId); this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); const keysMap = new Map(); csmKeys.data.keys.forEach((k: any) => keysMap.set(k.key, { ...k })); - for (const v of validators) { - const keyInfo = keysMap.get(v.validator.pubkey); + for (const [i, v] of validators.getAllReadonlyValues().entries()) { + const keyInfo = keysMap.get('0x'.concat(Buffer.from(v.pubkey).toString('hex'))); if (!keyInfo) continue; - this.storage.data[Number(v.index)] = { + this.storage.data[i] = { operatorId: keyInfo.operatorIndex, keyIndex: keyInfo.index, - pubKey: v.validator.pubkey, + pubKey: v.pubkey.toString(), // TODO: bigint? - withdrawableEpoch: Number(v.validator.withdrawable_epoch), + withdrawableEpoch: v.withdrawableEpoch, }; } await this.storage.write(); } - private async updateStorage(vals: StateValidatorResponse[], finalizedSlot: Slot): Promise { + private async updateStorage(validators: Validators, finalizedSlot: Slot): Promise { // TODO: should we think about re-using validator indexes? // TODO: should we think about changing WC for existing old vaidators ? - if (vals.length - this.info.data.lastValidatorsCount == 0) { + if (validators.length - this.info.data.lastValidatorsCount == 0) { this.logger.log(`No new validators in the state`); return; } - vals = vals.slice(this.info.data.lastValidatorsCount); - const valKeys = vals.map((v: StateValidatorResponse) => v.validator.pubkey); + // TODO: can be better + const vals = validators.getAllReadonlyValues().slice(this.info.data.lastValidatorsCount); + const valKeys = vals.map((v) => '0x'.concat(Buffer.from(v.pubkey).toString('hex'))); this.logger.log(`New appeared validators count: ${vals.length}`); const csmKeys = await this.keysapi.findModuleKeys(this.info.data.moduleId, valKeys); this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); this.logger.log(`New appeared CSM validators count: ${csmKeys.data.keys.length}`); for (const csmKey of csmKeys.data.keys) { - for (const newVal of vals) { - if (newVal.validator.pubkey != csmKey.key) continue; - this.storage.data[Number(newVal.index)] = { + for (const [i, v] of vals.entries()) { + if (valKeys[i] != csmKey.key) continue; + const index = i + this.info.data.lastValidatorsCount; + this.storage.data[index] = { operatorId: csmKey.operatorIndex, keyIndex: csmKey.index, pubKey: csmKey.key, // TODO: bigint? - withdrawableEpoch: Number(newVal.validator.withdrawable_epoch), + withdrawableEpoch: v.withdrawableEpoch, }; } } diff --git a/yarn.lock b/yarn.lock index 7dfcfde..e2493ac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -360,6 +360,27 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@chainsafe/as-sha256@^0.4.1": + version "0.4.1" + resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.4.1.tgz#cfc0737e25f8c206767bdb6703e7943e5d44513e" + integrity sha512-IqeeGwQihK6Y2EYLFofqs2eY2ep1I2MvQXHzOAI+5iQN51OZlUkrLgyAugu2x86xZewDk5xas7lNczkzFzF62w== + +"@chainsafe/persistent-merkle-tree@^0.6.1": + version "0.6.1" + resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.6.1.tgz#37bde25cf6cbe1660ad84311aa73157dc86ec7f2" + integrity sha512-gcENLemRR13+1MED2NeZBMA7FRS0xQPM7L2vhMqvKkjqtFT4YfjSVADq5U0iLuQLhFUJEMVuA8fbv5v+TN6O9A== + dependencies: + "@chainsafe/as-sha256" "^0.4.1" + "@noble/hashes" "^1.3.0" + +"@chainsafe/ssz@^0.14.0": + version "0.14.0" + resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.14.0.tgz#fe9e4fd3cf673013bd57f77c3ab0fdc5ebc5d916" + integrity sha512-KTc33pWu7ItXlzMAz5/1osOHsvhx25kpM3j7Ez+PNZLyyhIoNzAhhozvxy+ul0fCDfHbvaCRp3lJQnzsb5Iv0A== + dependencies: + "@chainsafe/as-sha256" "^0.4.1" + "@chainsafe/persistent-merkle-tree" "^0.6.1" + "@colors/colors@1.5.0": version "1.5.0" resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" @@ -752,6 +773,19 @@ dependencies: call-bind "^1.0.5" +"@lodestar/params@^1.15.0": + version "1.15.0" + resolved "https://registry.yarnpkg.com/@lodestar/params/-/params-1.15.0.tgz#07c75e32bbf0644ecc7b0354fe57cfb69a830c8c" + integrity sha512-E0cNJtSR3WVLs3rSGdL/uxPaUetr2jonLbckqAV4859nKHuF53Hm7awmMyoqSRKUnz012zsb96zLaoxe6kUqow== + +"@lodestar/types@^1.15.0": + version "1.15.0" + resolved "https://registry.yarnpkg.com/@lodestar/types/-/types-1.15.0.tgz#ed8d4896b427e577a98fed5f0301209cb9a91ab1" + integrity sha512-dpU0B2te3WdfuTBah/qUdL3+OCYRFPsmtVFL+x94g0HZx914/LZjPS+G6CCoSjxtonYPK9iFU/hhfdQB0ioIQw== + dependencies: + "@chainsafe/ssz" "^0.14.0" + "@lodestar/params" "^1.15.0" + "@lukeed/csprng@^1.0.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@lukeed/csprng/-/csprng-1.1.0.tgz#1e3e4bd05c1cc7a0b2ddbd8a03f39f6e4b5e6cfe" @@ -859,6 +893,11 @@ dependencies: tslib "2.6.2" +"@noble/hashes@^1.3.0": + version "1.3.3" + resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.3.3.tgz#39908da56a4adc270147bb07968bf3b16cfe1699" + integrity sha512-V7/fPHgl+jsVPXqqeOzT8egNj2iBIVt+ECeMMG8TdcnTikP3oaBtUVqpT/gYCR68aEBJSF+XbYUxStjbFMqIIA== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" @@ -923,15 +962,17 @@ dependencies: "@sinonjs/commons" "^3.0.0" -"@swc/cli@^0.1.63": - version "0.1.65" - resolved "https://registry.yarnpkg.com/@swc/cli/-/cli-0.1.65.tgz#bb51ce6f088a78ac99a07507c15a8d74c9336ecb" - integrity sha512-4NcgsvJVHhA7trDnMmkGLLvWMHu2kSy+qHx6QwRhhJhdiYdNUrhdp+ERxen73sYtaeEOYeLJcWrQ60nzKi6rpg== +"@swc/cli@^0.3.9": + version "0.3.9" + resolved "https://registry.yarnpkg.com/@swc/cli/-/cli-0.3.9.tgz#f9b331e9ec8f1f2154b1a77a8617c6dfbfc44fec" + integrity sha512-e5grxGEyNT0fYZEFmhSrRYL1kFAZAXlv+WjfQ35J6J9Hl0EtrMVymAEbGabetg2Q/2FX6HiRcjgc9LrdUCBk4A== dependencies: "@mole-inc/bin-wrapper" "^8.0.1" + "@swc/counter" "^0.1.3" commander "^7.1.0" fast-glob "^3.2.5" minimatch "^9.0.3" + piscina "^4.3.0" semver "^7.3.8" slash "3.0.0" source-map "^0.7.3" @@ -986,7 +1027,7 @@ resolved "https://registry.yarnpkg.com/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.4.0.tgz#bd575c599bd6847bddc4863a3babd85e3db5e11e" integrity sha512-e2xVezU7XZ2Stzn4i7TOQe2Kn84oYdG0M3A7XI7oTdcpsKCcKwgiMoroiAhqCv+iN20KNqhnWwJiUiTj/qN5AA== -"@swc/core@^1.3.104": +"@swc/core@^1.4.0": version "1.4.0" resolved "https://registry.yarnpkg.com/@swc/core/-/core-1.4.0.tgz#3a0ceeea5b889173f4592955fe1da4d071d86a76" integrity sha512-wc5DMI5BJftnK0Fyx9SNJKkA0+BZSJQx8430yutWmsILkHMBD3Yd9GhlMaxasab9RhgKqZp7Ht30hUYO5ZDvQg== @@ -4751,11 +4792,24 @@ nest-winston@^1.6.2, nest-winston@^1.9.4: dependencies: fast-safe-stringify "^2.1.1" +nice-napi@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/nice-napi/-/nice-napi-1.0.2.tgz#dc0ab5a1eac20ce548802fc5686eaa6bc654927b" + integrity sha512-px/KnJAJZf5RuBGcfD+Sp2pAKq0ytz8j+1NehvgIGFkvtvFrDM3T8E4x/JJODXK9WZow8RRGrbA9QQ3hs+pDhA== + dependencies: + node-addon-api "^3.0.0" + node-gyp-build "^4.2.2" + node-abort-controller@^3.0.1: version "3.1.1" resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== +node-addon-api@^3.0.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" + integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A== + node-emoji@1.11.0: version "1.11.0" resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c" @@ -4770,6 +4824,11 @@ node-fetch@^2.6.1: dependencies: whatwg-url "^5.0.0" +node-gyp-build@^4.2.2: + version "4.8.0" + resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.8.0.tgz#3fee9c1731df4581a3f9ead74664369ff00d26dd" + integrity sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og== + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" @@ -5068,6 +5127,13 @@ pirates@^4.0.4: resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9" integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg== +piscina@^4.3.0: + version "4.3.1" + resolved "https://registry.yarnpkg.com/piscina/-/piscina-4.3.1.tgz#eaa59461caa27f07c637e667b14c36a0bd7e7daf" + integrity sha512-MBj0QYm3hJQ/C/wIXTN1OCYC8uQ4BBJ4LVele2P4ZwVQAH04vkk8E1SpDbuemLAL1dZorbuOob9rYqJeWCcCRg== + optionalDependencies: + nice-napi "^1.0.2" + pkg-dir@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" From efb9d795ff292a4678c0969be946d8fb06940c4c Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 21 Feb 2024 18:13:40 +0400 Subject: [PATCH 07/15] fix: review and handlers --- src/common/handlers/handlers.service.ts | 78 ++++++++++++++------- src/common/handlers/types.ts | 5 ++ src/common/providers/consensus/consensus.ts | 10 ++- src/daemon/services/keys-indexer.ts | 26 ++++--- src/daemon/services/roots-processor.ts | 2 +- 5 files changed, 82 insertions(+), 39 deletions(-) create mode 100644 src/common/handlers/types.ts diff --git a/src/common/handlers/handlers.service.ts b/src/common/handlers/handlers.service.ts index 955ef7d..e55627b 100644 --- a/src/common/handlers/handlers.service.ts +++ b/src/common/handlers/handlers.service.ts @@ -1,8 +1,9 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { Inject, Injectable, LoggerService } from '@nestjs/common'; +import { WithdrawalsProvePayload } from './types'; import { Consensus } from '../providers/consensus/consensus'; -import { BlockInfoResponse, RootHex } from '../providers/consensus/response.interface'; +import { BlockHeaderResponse, BlockInfoResponse, RootHex, Withdrawal } from '../providers/consensus/response.interface'; export interface KeyInfo { operatorId: number; @@ -20,39 +21,65 @@ export class HandlersService { protected readonly consensus: Consensus, ) {} - public async prove(blockRoot: RootHex, blockInfo: BlockInfoResponse, keyInfoFn: KeyInfoFn): Promise { + public async proveIfNeeded(blockRoot: RootHex, blockInfo: BlockInfoResponse, keyInfoFn: KeyInfoFn): Promise { const slashings = await this.getUnprovenSlashings(blockRoot, blockInfo, keyInfoFn); const withdrawals = await this.getUnprovenWithdrawals(blockRoot, blockInfo, keyInfoFn); if (!slashings.length && !withdrawals.length) return; - const payload = await this.buildProvePayload(blockInfo, slashings, withdrawals); - // TODO: ask before sending if CLI or daemon in watch mode - await this.sendProves(payload); - this.logger.log(`๐Ÿ Proves sent. Root [${blockRoot}]`); + const header = await this.consensus.getBeaconHeader(blockRoot); + // TODO: wait until appears next block if doesn't exist + const nextHeader = await this.consensus.getBeaconHeadersByParentRoot(blockRoot); + const stateView = await this.consensus.getStateView(header.header.message.state_root); + if (slashings.length) { + for (const payload of this.buildSlashingsProvePayloads(blockInfo, nextHeader.data[0], stateView, slashings)) { + // TODO: ask before sending if CLI or daemon in watch mode + await this.sendSlashingsProve(payload); + } + } + if (withdrawals.length) { + for (const payload of this.buildWithdrawalsProvePayloads(blockInfo, nextHeader.data[0], stateView, withdrawals)) { + // TODO: ask before sending if CLI or daemon in watch mode + await this.sendWithdrawalsProve(payload); + } + } + if (!slashings.length || !withdrawals.length) this.logger.log(`๐Ÿ Proves sent. Root [${blockRoot}]`); } - private async buildProvePayload( + private *buildSlashingsProvePayloads( blockInfo: BlockInfoResponse, + nextHeader: BlockHeaderResponse, + stateView: any, // TODO: type slashings: string[], - withdrawals: string[], - ): Promise { - // TODO: implement - // this.consensus.getState(...) - if (slashings.length || withdrawals.length) { - this.logger.warn(`๐Ÿ“ฆ Prove payload: slashings [${slashings}], withdrawals [${withdrawals}]`); + ): Generator { + this.logger.warn(`๐Ÿ“ฆ Building prove payloads | Slashings: [${slashings}]`); + for (const slashing of slashings) { + // const validatorsInfo = stateView.validators.type.elementType.toJson(stateView.validators.get(1337)); + yield slashing; } - // const { ssz } = await eval('import("@lodestar/types")'); - // const stateSSZ = await this.consensus.getStateSSZ(header.header.message.slot); - // const stateView = ssz.deneb.BeaconState.deserializeToView(stateSSZ); - // const validatorsInfo = stateView.validators.type.elementType.toJson(stateView.validators.get(1337)); - return {}; } - private async sendProves(payload: any): Promise { - // TODO: implement - if (payload) { - this.logger.warn(`๐Ÿ“ก Sending proves`); + + private *buildWithdrawalsProvePayloads( + blockInfo: BlockInfoResponse, + nextHeader: BlockHeaderResponse, + stateView: any, // TODO: type + withdrawals: Withdrawal[], + ): Generator { + this.logger.warn(`๐Ÿ“ฆ Building prove payloads | Withdrawals: [${withdrawals}]`); + for (const withdrawal of withdrawals) { + // const validatorsInfo = stateView.validators.type.elementType.toJson(stateView.validators.get(1337)); + yield withdrawal as WithdrawalsProvePayload; } } + private async sendSlashingsProve(payload: any): Promise { + // TODO: implement + this.logger.warn(`๐Ÿ“ก Sending slashings prove`); + } + + private async sendWithdrawalsProve(payload: any): Promise { + // TODO: implement + this.logger.warn(`๐Ÿ“ก Sending withdrawals prove`); + } + private async getUnprovenSlashings( blockRoot: RootHex, blockInfo: BlockInfoResponse, @@ -82,7 +109,7 @@ export class HandlersService { blockRoot: RootHex, blockInfo: BlockInfoResponse, keyInfoFn: KeyInfoFn, - ): Promise { + ): Promise { const withdrawals = this.getFullWithdrawals(blockInfo, keyInfoFn); if (!withdrawals.length) return []; const unproven = []; @@ -130,14 +157,15 @@ export class HandlersService { private getFullWithdrawals( blockInfo: BlockInfoResponse, keyInfoFn: (valIndex: number) => KeyInfo | undefined, - ): string[] { + ): Withdrawal[] { const fullWithdrawals = []; const blockEpoch = Number(blockInfo.message.slot) / 32; const withdrawals = blockInfo.message.body.execution_payload?.withdrawals ?? []; for (const withdrawal of withdrawals) { const keyInfo = keyInfoFn(Number(withdrawal.validator_index)); if (keyInfo && blockEpoch >= keyInfo.withdrawableEpoch) { - fullWithdrawals.push(withdrawal.validator_index); + // TODO: think about sync committee case (balance > 0 after full withdrawal) + fullWithdrawals.push(withdrawal); } } return fullWithdrawals; diff --git a/src/common/handlers/types.ts b/src/common/handlers/types.ts new file mode 100644 index 0000000..a0680e3 --- /dev/null +++ b/src/common/handlers/types.ts @@ -0,0 +1,5 @@ +export type WithdrawalsProvePayload = WithdrawalsGeneralProvePayload | WithdrawalsHistoricalProvePayload; + +export type WithdrawalsGeneralProvePayload = any; + +export type WithdrawalsHistoricalProvePayload = any; diff --git a/src/common/providers/consensus/consensus.ts b/src/common/providers/consensus/consensus.ts index cc12e68..c80811b 100644 --- a/src/common/providers/consensus/consensus.ts +++ b/src/common/providers/consensus/consensus.ts @@ -20,6 +20,8 @@ import { PrometheusService } from '../../prometheus/prometheus.service'; import { DownloadProgress } from '../../utils/download-progress/download-progress'; import { BaseRestProvider } from '../base/rest-provider'; +let types: typeof import('@lodestar/types'); + @Injectable() export class Consensus extends BaseRestProvider implements OnApplicationBootstrap { private readonly endpoints = { @@ -53,6 +55,8 @@ export class Consensus extends BaseRestProvider implements OnApplicationBootstra } public async onApplicationBootstrap(): Promise { + // ugly hack to import ESModule to CommonJS project + types = await eval(`import('@lodestar/types')`); this.logger.log(`Getting genesis timestamp`); const resp = await this.getGenesis(); this.genesisTimestamp = Number(resp.genesis_time); @@ -121,7 +125,7 @@ export class Consensus extends BaseRestProvider implements OnApplicationBootstra }); } - public async getStateSSZ(stateId: StateId, signal?: AbortSignal): Promise { + public async getStateView(stateId: StateId, signal?: AbortSignal) { const { body } = await this.baseGet<{ body: BodyReadable; headers: IncomingHttpHeaders }>( this.mainUrl, this.endpoints.state(stateId), @@ -135,6 +139,8 @@ export class Consensus extends BaseRestProvider implements OnApplicationBootstra // TODO: Enable for CLI only //this.progress.show(`State [${stateId}]`, resp); // Data processing - return new Uint8Array(await body.arrayBuffer()); + const bodyBites = new Uint8Array(await body.arrayBuffer()); + // TODO: select fork + return types.ssz.deneb.BeaconState.deserializeToView(bodyBites); } } diff --git a/src/daemon/services/keys-indexer.ts b/src/daemon/services/keys-indexer.ts index d6ab62b..33084ce 100644 --- a/src/daemon/services/keys-indexer.ts +++ b/src/daemon/services/keys-indexer.ts @@ -22,8 +22,6 @@ type Storage = { [valIndex: number]: KeyInfo; }; -let types: typeof import('@lodestar/types'); - type Validators = ListCompositeTreeView< ContainerNodeStructType<{ pubkey: ByteVectorType; @@ -37,6 +35,19 @@ type Validators = ListCompositeTreeView< }> >; +// At one time only one task should be running +function Single(target: any, propertyKey: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; + descriptor.value = function (...args: any[]) { + if (this.startedAt > 0) { + this.logger.warn(`๐Ÿ”‘ Keys indexer has been running for ${Date.now() - this.startedAt}ms`); + return; + } + originalMethod.apply(this, args); + }; + return descriptor; +} + @Injectable() export class KeysIndexer implements OnApplicationBootstrap { private startedAt: number = 0; @@ -52,8 +63,6 @@ export class KeysIndexer implements OnApplicationBootstrap { ) {} public async onApplicationBootstrap(): Promise { - // ugly hack to import ESModule to CommonJS project - types = await eval(`import('@lodestar/types')`); await this.initOrReadServiceData(); } @@ -61,12 +70,8 @@ export class KeysIndexer implements OnApplicationBootstrap { return this.storage.data[valIndex]; }; + @Single public async run(finalizedHeader: BlockHeaderResponse): Promise { - // At one time only one task should be running - if (this.startedAt > 0) { - this.logger.warn(`๐Ÿ”‘ Keys indexer has been running for ${Date.now() - this.startedAt}ms`); - return; - } const slot = Number(finalizedHeader.header.message.slot); if (this.isNotTimeToRun(slot)) { this.logger.log('No need to run keys indexer'); @@ -88,8 +93,7 @@ export class KeysIndexer implements OnApplicationBootstrap { private async baseRun(stateRoot: RootHex, finalizedSlot: Slot): Promise { this.logger.log(`Get validators. State root [${stateRoot}]`); - const stateSSZ = await this.consensus.getStateSSZ(stateRoot); - const stateView = types.ssz.deneb.BeaconState.deserializeToView(stateSSZ); + const stateView = await this.consensus.getStateView(stateRoot); this.logger.log(`Total validators count: ${stateView.validators.length}`); // TODO: do we need to store already full withdrawn keys ? this.info.data.lastValidatorsCount == 0 diff --git a/src/daemon/services/roots-processor.ts b/src/daemon/services/roots-processor.ts index 5ca18fd..cf48df3 100644 --- a/src/daemon/services/roots-processor.ts +++ b/src/daemon/services/roots-processor.ts @@ -26,7 +26,7 @@ export class RootsProcessor { }; const indexerIsOK = this.keysIndexer.eligibleForEveryDuty(rootSlot.slotNumber); if (!indexerIsOK) await this.rootsStack.push(rootSlot); // only new will be pushed - await this.handlers.prove(blockRoot, blockInfo, this.keysIndexer.getKey); + await this.handlers.proveIfNeeded(blockRoot, blockInfo, this.keysIndexer.getKey); if (indexerIsOK) await this.rootsStack.purge(blockRoot); await this.rootsStack.setLastProcessed(rootSlot); } From 8f147009c3297b442c5a8795115b1bc8620291ce Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Tue, 27 Feb 2024 15:58:53 +0400 Subject: [PATCH 08/15] fix: add fork selector for state --- package.json | 1 + src/common/providers/consensus/consensus.ts | 18 ++++++++++-------- yarn.lock | 5 +++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 30f485e..4bf859f 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "dependencies": { "@huanshiwushuang/lowdb": "^6.0.2", "@lido-nestjs/logger": "^1.3.2", + "@lodestar/params": "^1.16.0", "@lodestar/types": "^1.15.0", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.1.1", diff --git a/src/common/providers/consensus/consensus.ts b/src/common/providers/consensus/consensus.ts index c80811b..a17d10d 100644 --- a/src/common/providers/consensus/consensus.ts +++ b/src/common/providers/consensus/consensus.ts @@ -1,5 +1,5 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; -import { Inject, Injectable, LoggerService, OnApplicationBootstrap, Optional } from '@nestjs/common'; +import { Inject, Injectable, LoggerService, OnModuleInit, Optional } from '@nestjs/common'; import { chain } from 'stream-chain'; import { parser } from 'stream-json'; import { connectTo } from 'stream-json/Assembler'; @@ -20,10 +20,11 @@ import { PrometheusService } from '../../prometheus/prometheus.service'; import { DownloadProgress } from '../../utils/download-progress/download-progress'; import { BaseRestProvider } from '../base/rest-provider'; -let types: typeof import('@lodestar/types'); +let ssz: typeof import('@lodestar/types').ssz; +let ForkName: typeof import('@lodestar/params').ForkName; @Injectable() -export class Consensus extends BaseRestProvider implements OnApplicationBootstrap { +export class Consensus extends BaseRestProvider implements OnModuleInit { private readonly endpoints = { version: 'eth/v1/node/version', genesis: 'eth/v1/beacon/genesis', @@ -54,9 +55,9 @@ export class Consensus extends BaseRestProvider implements OnApplicationBootstra ); } - public async onApplicationBootstrap(): Promise { + public async onModuleInit(): Promise { // ugly hack to import ESModule to CommonJS project - types = await eval(`import('@lodestar/types')`); + ssz = await eval(`import('@lodestar/types').then((m) => m.ssz)`); this.logger.log(`Getting genesis timestamp`); const resp = await this.getGenesis(); this.genesisTimestamp = Number(resp.genesis_time); @@ -126,7 +127,7 @@ export class Consensus extends BaseRestProvider implements OnApplicationBootstra } public async getStateView(stateId: StateId, signal?: AbortSignal) { - const { body } = await this.baseGet<{ body: BodyReadable; headers: IncomingHttpHeaders }>( + const { body, headers } = await this.baseGet<{ body: BodyReadable; headers: IncomingHttpHeaders }>( this.mainUrl, this.endpoints.state(stateId), { @@ -135,12 +136,13 @@ export class Consensus extends BaseRestProvider implements OnApplicationBootstra headers: { accept: 'application/octet-stream' }, }, ); + const version = headers['eth-consensus-version'] as keyof typeof ForkName; // Progress bar // TODO: Enable for CLI only //this.progress.show(`State [${stateId}]`, resp); // Data processing const bodyBites = new Uint8Array(await body.arrayBuffer()); - // TODO: select fork - return types.ssz.deneb.BeaconState.deserializeToView(bodyBites); + // TODO: high memory usage + return ssz.allForks[version].BeaconState.deserializeToView(bodyBites); } } diff --git a/yarn.lock b/yarn.lock index e2493ac..0333a6c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -778,6 +778,11 @@ resolved "https://registry.yarnpkg.com/@lodestar/params/-/params-1.15.0.tgz#07c75e32bbf0644ecc7b0354fe57cfb69a830c8c" integrity sha512-E0cNJtSR3WVLs3rSGdL/uxPaUetr2jonLbckqAV4859nKHuF53Hm7awmMyoqSRKUnz012zsb96zLaoxe6kUqow== +"@lodestar/params@^1.16.0": + version "1.16.0" + resolved "https://registry.yarnpkg.com/@lodestar/params/-/params-1.16.0.tgz#417ffb1ee9e179b2402ad624dac4c2c6d7d08e51" + integrity sha512-Fx3mO5mhAqo9VTIt6ZfgF/Wiw6qAYxfete6ZG9AWs2I56lDPEeXRDOCkggItkPeSYXesX8o9vQP6Dkiwfrm2yg== + "@lodestar/types@^1.15.0": version "1.15.0" resolved "https://registry.yarnpkg.com/@lodestar/types/-/types-1.15.0.tgz#ed8d4896b427e577a98fed5f0301209cb9a91ab1" From 37e341041aa3e7541fb872f0d1a8160885769d42 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Tue, 27 Feb 2024 15:59:52 +0400 Subject: [PATCH 09/15] fix: handlers. a bit --- src/common/handlers/handlers.service.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/common/handlers/handlers.service.ts b/src/common/handlers/handlers.service.ts index e55627b..2d4baef 100644 --- a/src/common/handlers/handlers.service.ts +++ b/src/common/handlers/handlers.service.ts @@ -27,16 +27,17 @@ export class HandlersService { if (!slashings.length && !withdrawals.length) return; const header = await this.consensus.getBeaconHeader(blockRoot); // TODO: wait until appears next block if doesn't exist - const nextHeader = await this.consensus.getBeaconHeadersByParentRoot(blockRoot); + const nextHeaders = await this.consensus.getBeaconHeadersByParentRoot(blockRoot); + const nextHeader = nextHeaders.data[0]; const stateView = await this.consensus.getStateView(header.header.message.state_root); if (slashings.length) { - for (const payload of this.buildSlashingsProvePayloads(blockInfo, nextHeader.data[0], stateView, slashings)) { + for (const payload of this.buildSlashingsProvePayloads(blockInfo, nextHeader, stateView, slashings)) { // TODO: ask before sending if CLI or daemon in watch mode await this.sendSlashingsProve(payload); } } if (withdrawals.length) { - for (const payload of this.buildWithdrawalsProvePayloads(blockInfo, nextHeader.data[0], stateView, withdrawals)) { + for (const payload of this.buildWithdrawalsProvePayloads(blockInfo, nextHeader, stateView, withdrawals)) { // TODO: ask before sending if CLI or daemon in watch mode await this.sendWithdrawalsProve(payload); } @@ -72,11 +73,13 @@ export class HandlersService { private async sendSlashingsProve(payload: any): Promise { // TODO: implement + this.logger.log(payload); this.logger.warn(`๐Ÿ“ก Sending slashings prove`); } private async sendWithdrawalsProve(payload: any): Promise { // TODO: implement + this.logger.log(payload); this.logger.warn(`๐Ÿ“ก Sending withdrawals prove`); } @@ -123,7 +126,7 @@ export class HandlersService { this.logger.log(`No full withdrawals to prove. Root [${blockRoot}]`); return []; } - this.logger.warn(`๐Ÿ” Unproven full withdrawals: ${unproven}`); + this.logger.warn(`๐Ÿ” Unproven full withdrawals: ${unproven.length}`); return unproven; } @@ -159,11 +162,11 @@ export class HandlersService { keyInfoFn: (valIndex: number) => KeyInfo | undefined, ): Withdrawal[] { const fullWithdrawals = []; - const blockEpoch = Number(blockInfo.message.slot) / 32; + const blockEpoch = Number((Number(blockInfo.message.slot) / 32).toFixed()); const withdrawals = blockInfo.message.body.execution_payload?.withdrawals ?? []; for (const withdrawal of withdrawals) { const keyInfo = keyInfoFn(Number(withdrawal.validator_index)); - if (keyInfo && blockEpoch >= keyInfo.withdrawableEpoch) { + if (keyInfo?.withdrawableEpoch != null && keyInfo && blockEpoch >= keyInfo.withdrawableEpoch) { // TODO: think about sync committee case (balance > 0 after full withdrawal) fullWithdrawals.push(withdrawal); } From c5766f335bf90f04a2311d9e58a64ef7aaf3e4f4 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Tue, 27 Feb 2024 16:01:01 +0400 Subject: [PATCH 10/15] fix: keys-indexer init\update data --- src/common/providers/keysapi/keysapi.ts | 1 + src/daemon/daemon.service.ts | 2 +- src/daemon/services/keys-indexer.ts | 154 ++++++++++++++---------- 3 files changed, 93 insertions(+), 64 deletions(-) diff --git a/src/common/providers/keysapi/keysapi.ts b/src/common/providers/keysapi/keysapi.ts index 6529b11..c17bb77 100644 --- a/src/common/providers/keysapi/keysapi.ts +++ b/src/common/providers/keysapi/keysapi.ts @@ -61,6 +61,7 @@ export class Keysapi extends BaseRestProvider { signal, }, ); + // TODO: ignore depositSignature ? const pipeline = chain([resp.body, parser()]); return await new Promise((resolve) => { connectTo(pipeline).on('done', (asm) => resolve(asm.current)); diff --git a/src/daemon/daemon.service.ts b/src/daemon/daemon.service.ts index c999dc2..360e6f3 100644 --- a/src/daemon/daemon.service.ts +++ b/src/daemon/daemon.service.ts @@ -38,7 +38,7 @@ export class DaemonService implements OnApplicationBootstrap { this.logger.log('๐Ÿ—ฟ Get finalized header'); const header = await this.consensus.getBeaconHeader('finalized'); this.logger.log(`๐Ÿ’Ž Finalized slot [${header.header.message.slot}]. Root [${header.root}]`); - await this.keysIndexer.run(header); + await this.keysIndexer.update(header); const nextRoot = await this.rootsProvider.getNext(header); if (nextRoot) { await this.rootsProcessor.process(nextRoot); diff --git a/src/daemon/services/keys-indexer.ts b/src/daemon/services/keys-indexer.ts index 33084ce..2fa48da 100644 --- a/src/daemon/services/keys-indexer.ts +++ b/src/daemon/services/keys-indexer.ts @@ -1,9 +1,10 @@ +import { iterateNodesAtDepth } from '@chainsafe/persistent-merkle-tree'; import { BooleanType, ByteVectorType, ContainerNodeStructType, UintNumberType } from '@chainsafe/ssz'; import { ListCompositeTreeView } from '@chainsafe/ssz/lib/view/listComposite'; import { Low } from '@huanshiwushuang/lowdb'; import { JSONFile } from '@huanshiwushuang/lowdb/node'; import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; -import { Inject, Injectable, LoggerService, OnApplicationBootstrap } from '@nestjs/common'; +import { Inject, Injectable, LoggerService, OnModuleInit } from '@nestjs/common'; import { ConfigService } from '../../common/config/config.service'; import { KeyInfo } from '../../common/handlers/handlers.service'; @@ -49,7 +50,7 @@ function Single(target: any, propertyKey: string, descriptor: PropertyDescriptor } @Injectable() -export class KeysIndexer implements OnApplicationBootstrap { +export class KeysIndexer implements OnModuleInit { private startedAt: number = 0; private info: Low; @@ -62,7 +63,7 @@ export class KeysIndexer implements OnApplicationBootstrap { protected readonly keysapi: Keysapi, ) {} - public async onApplicationBootstrap(): Promise { + public async onModuleInit(): Promise { await this.initOrReadServiceData(); } @@ -71,87 +72,37 @@ export class KeysIndexer implements OnApplicationBootstrap { }; @Single - public async run(finalizedHeader: BlockHeaderResponse): Promise { + public async update(finalizedHeader: BlockHeaderResponse): Promise { + // TODO: do we have to check integrity of data here? when `this.info` says one thing and `this.storage` another const slot = Number(finalizedHeader.header.message.slot); if (this.isNotTimeToRun(slot)) { this.logger.log('No need to run keys indexer'); return; } - this.logger.log(`๐Ÿ”‘ Keys indexer is running`); const stateRoot = finalizedHeader.header.message.state_root; - if (this.info.data.storageStateSlot == 0) { - await this.baseRun(stateRoot, slot); - return; - } // We shouldn't wait for task to finish // to avoid block processing if indexing fails or stuck this.startedAt = Date.now(); - this.baseRun(stateRoot, slot) + this.baseRun(stateRoot, slot, this.updateStorage) .catch((e) => this.logger.error(e)) .finally(() => (this.startedAt = 0)); } - private async baseRun(stateRoot: RootHex, finalizedSlot: Slot): Promise { + private async baseRun( + stateRoot: RootHex, + finalizedSlot: Slot, + stateDataProcessingCallback: (validators: Validators, finalizedSlot: Slot) => Promise, + ): Promise { + this.logger.log(`๐Ÿ”‘ Keys indexer is running`); this.logger.log(`Get validators. State root [${stateRoot}]`); const stateView = await this.consensus.getStateView(stateRoot); this.logger.log(`Total validators count: ${stateView.validators.length}`); // TODO: do we need to store already full withdrawn keys ? - this.info.data.lastValidatorsCount == 0 - ? await this.initStorage(stateView.validators, finalizedSlot) - : await this.updateStorage(stateView.validators, finalizedSlot); + await stateDataProcessingCallback(stateView.validators, finalizedSlot); this.logger.log(`CSM validators count: ${Object.keys(this.storage.data).length}`); this.info.data.storageStateSlot = finalizedSlot; this.info.data.lastValidatorsCount = stateView.validators.length; await this.info.write(); - } - - private async initStorage(validators: Validators, finalizedSlot: Slot): Promise { - this.logger.log(`Init keys data`); - const csmKeys = await this.keysapi.getModuleKeys(this.info.data.moduleId); - this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); - const keysMap = new Map(); - csmKeys.data.keys.forEach((k: any) => keysMap.set(k.key, { ...k })); - for (const [i, v] of validators.getAllReadonlyValues().entries()) { - const keyInfo = keysMap.get('0x'.concat(Buffer.from(v.pubkey).toString('hex'))); - if (!keyInfo) continue; - this.storage.data[i] = { - operatorId: keyInfo.operatorIndex, - keyIndex: keyInfo.index, - pubKey: v.pubkey.toString(), - // TODO: bigint? - withdrawableEpoch: v.withdrawableEpoch, - }; - } - await this.storage.write(); - } - - private async updateStorage(validators: Validators, finalizedSlot: Slot): Promise { - // TODO: should we think about re-using validator indexes? - // TODO: should we think about changing WC for existing old vaidators ? - if (validators.length - this.info.data.lastValidatorsCount == 0) { - this.logger.log(`No new validators in the state`); - return; - } - // TODO: can be better - const vals = validators.getAllReadonlyValues().slice(this.info.data.lastValidatorsCount); - const valKeys = vals.map((v) => '0x'.concat(Buffer.from(v.pubkey).toString('hex'))); - this.logger.log(`New appeared validators count: ${vals.length}`); - const csmKeys = await this.keysapi.findModuleKeys(this.info.data.moduleId, valKeys); - this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); - this.logger.log(`New appeared CSM validators count: ${csmKeys.data.keys.length}`); - for (const csmKey of csmKeys.data.keys) { - for (const [i, v] of vals.entries()) { - if (valKeys[i] != csmKey.key) continue; - const index = i + this.info.data.lastValidatorsCount; - this.storage.data[index] = { - operatorId: csmKey.operatorIndex, - keyIndex: csmKey.index, - pubKey: csmKey.key, - // TODO: bigint? - withdrawableEpoch: v.withdrawableEpoch, - }; - } - } await this.storage.write(); } @@ -227,5 +178,82 @@ export class KeysIndexer implements OnApplicationBootstrap { this.info.data.moduleId = module.id; await this.info.write(); } + + if (this.info.data.storageStateSlot == 0 || this.info.data.lastValidatorsCount == 0) { + this.logger.log(`Init keys data`); + const finalized = await this.consensus.getBeaconHeader('finalized'); + const finalizedSlot = Number(finalized.header.message.slot); + const stateRoot = finalized.header.message.state_root; + await this.baseRun(stateRoot, finalizedSlot, this.initStorage); + } } + + initStorage = async (validators: Validators, finalizedSlot: Slot): Promise => { + const csmKeys = await this.keysapi.getModuleKeys(this.info.data.moduleId); + this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); + const keysMap = new Map(); + csmKeys.data.keys.forEach((k: any) => keysMap.set(k.key, { ...k })); + const iterator = iterateNodesAtDepth( + validators.type.tree_getChunksNode(validators.node), + validators.type.chunkDepth, + 0, + validators.length, + ); + for (let i = 0; i < validators.length; i++) { + const node = iterator.next().value; + const v = validators.type.elementType.tree_toValue(node); + const pubKey = '0x'.concat(Buffer.from(v.pubkey).toString('hex')); + const keyInfo = keysMap.get(pubKey); + if (!keyInfo) continue; + this.storage.data[i] = { + operatorId: keyInfo.operatorIndex, + keyIndex: keyInfo.index, + pubKey: pubKey, + // TODO: bigint? + withdrawableEpoch: v.withdrawableEpoch, + }; + } + }; + + updateStorage = async (validators: Validators, finalizedSlot: Slot): Promise => { + // TODO: should we think about re-using validator indexes? + // TODO: should we think about changing WC for existing old vaidators ? + const appearedValsCount = validators.length - this.info.data.lastValidatorsCount; + if (appearedValsCount == 0) { + this.logger.log(`No new validators in the state`); + return; + } + this.logger.log(`New appeared validators count: ${appearedValsCount}`); + const iterator = iterateNodesAtDepth( + validators.type.tree_getChunksNode(validators.node), + validators.type.chunkDepth, + this.info.data.lastValidatorsCount - 1, + validators.length, + ); + const valKeys = []; + const valWithdrawableEpochs = []; + for (let i = this.info.data.lastValidatorsCount - 1; i < validators.length; i++) { + const node = iterator.next().value; + const v = validators.type.elementType.tree_toValue(node); + valKeys.push('0x'.concat(Buffer.from(v.pubkey).toString('hex'))); + valWithdrawableEpochs.push(v.withdrawableEpoch); + } + // TODO: can be better + const csmKeys = await this.keysapi.findModuleKeys(this.info.data.moduleId, valKeys); + this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); + this.logger.log(`New appeared CSM validators count: ${csmKeys.data.keys.length}`); + for (const csmKey of csmKeys.data.keys) { + for (const [i, v] of valKeys.entries()) { + if (valKeys[i] != csmKey.key) continue; + const index = i + this.info.data.lastValidatorsCount; + this.storage.data[index] = { + operatorId: csmKey.operatorIndex, + keyIndex: csmKey.index, + pubKey: csmKey.key, + // TODO: bigint? + withdrawableEpoch: valWithdrawableEpochs[i], + }; + } + } + }; } From f133b6f0d8979afca18167c2a0a4eb53dae8335d Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Tue, 27 Feb 2024 16:01:49 +0400 Subject: [PATCH 11/15] fix: roots stack data structure --- src/daemon/services/roots-processor.ts | 2 +- src/daemon/services/roots-provider.ts | 8 +++---- src/daemon/services/roots-stack.ts | 30 ++++++++++++++------------ 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/daemon/services/roots-processor.ts b/src/daemon/services/roots-processor.ts index cf48df3..a1a11ee 100644 --- a/src/daemon/services/roots-processor.ts +++ b/src/daemon/services/roots-processor.ts @@ -27,7 +27,7 @@ export class RootsProcessor { const indexerIsOK = this.keysIndexer.eligibleForEveryDuty(rootSlot.slotNumber); if (!indexerIsOK) await this.rootsStack.push(rootSlot); // only new will be pushed await this.handlers.proveIfNeeded(blockRoot, blockInfo, this.keysIndexer.getKey); - if (indexerIsOK) await this.rootsStack.purge(blockRoot); + if (indexerIsOK) await this.rootsStack.purge(rootSlot); await this.rootsStack.setLastProcessed(rootSlot); } } diff --git a/src/daemon/services/roots-provider.ts b/src/daemon/services/roots-provider.ts index 965b182..f0fa9b8 100644 --- a/src/daemon/services/roots-provider.ts +++ b/src/daemon/services/roots-provider.ts @@ -41,13 +41,13 @@ export class RootsProvider { } private async getChild(lastProcessed: RootSlot, finalizedHeader: BlockHeaderResponse): Promise { - this.logger.log(`โฎ๏ธ Last processed root [${lastProcessed.blockRoot}]`); + this.logger.log(`โฎ๏ธ Last processed slot [${lastProcessed.slotNumber}]. Root [${lastProcessed.blockRoot}]`); if (lastProcessed.blockRoot == finalizedHeader.root) return; const diff = Number(finalizedHeader.header.message.slot) - lastProcessed.slotNumber; this.logger.warn(`Diff between last processed and finalized is ${diff} slots`); - const childHeader = await this.consensus.getBeaconHeadersByParentRoot(lastProcessed.blockRoot); - if (!childHeader || !childHeader.finalized) return; - const child = childHeader.data[0].root; + const childHeaders = await this.consensus.getBeaconHeadersByParentRoot(lastProcessed.blockRoot); + if (!childHeaders || !childHeaders.finalized) return; + const child = childHeaders.data[0].root; this.logger.log(`โญ๏ธ Next root to process [${child}]. Child of last processed`); return child; } diff --git a/src/daemon/services/roots-stack.ts b/src/daemon/services/roots-stack.ts index d430ba6..90b5b92 100644 --- a/src/daemon/services/roots-stack.ts +++ b/src/daemon/services/roots-stack.ts @@ -1,44 +1,46 @@ import { Low } from '@huanshiwushuang/lowdb'; import { JSONFile } from '@huanshiwushuang/lowdb/node'; -import { Injectable, OnApplicationBootstrap } from '@nestjs/common'; +import { Injectable, OnModuleInit } from '@nestjs/common'; import { KeysIndexer } from './keys-indexer'; import { RootHex } from '../../common/providers/consensus/response.interface'; -export type RootSlot = { blockRoot: string; slotNumber: number }; +export type RootSlot = { blockRoot: RootHex; slotNumber: number }; type Info = { lastProcessedRootSlot: RootSlot | undefined; }; -type Storage = RootSlot[]; +type Storage = { [slot: number]: RootHex }; @Injectable() -export class RootsStack implements OnApplicationBootstrap { +export class RootsStack implements OnModuleInit { private info: Low; private storage: Low; constructor(protected readonly keysIndexer: KeysIndexer) {} - async onApplicationBootstrap(): Promise { + async onModuleInit(): Promise { await this.initOrReadServiceData(); } public getNextEligible(): RootSlot | undefined { - return this.storage.data.find((s) => this.keysIndexer.eligibleForAnyDuty(s.slotNumber)); + for (const slot in this.storage.data) { + if (this.keysIndexer.eligibleForAnyDuty(Number(slot))) { + return { blockRoot: this.storage.data[slot], slotNumber: Number(slot) }; + } + } } public async push(rs: RootSlot): Promise { - const idx = this.storage.data.findIndex((i) => rs.blockRoot == i.blockRoot); - if (idx !== -1) return; - this.storage.data.push(rs); + if (this.storage.data[rs.slotNumber] !== undefined) return; + this.storage.data[rs.slotNumber] = rs.blockRoot; await this.storage.write(); } - public async purge(blockRoot: RootHex): Promise { - const idx = this.storage.data.findIndex((i) => blockRoot == i.blockRoot); - if (idx == -1) return; - this.storage.data.splice(idx, 1); + public async purge(rs: RootSlot): Promise { + if (this.storage.data[rs.slotNumber] == undefined) return; + delete this.storage.data[rs.slotNumber]; await this.storage.write(); } @@ -55,7 +57,7 @@ export class RootsStack implements OnApplicationBootstrap { this.info = new Low(new JSONFile('.roots-stack-info.json'), { lastProcessedRootSlot: undefined, }); - this.storage = new Low(new JSONFile('.roots-stack-storage.json'), []); + this.storage = new Low(new JSONFile('.roots-stack-storage.json'), {}); await this.info.read(); await this.storage.read(); } From df83e5ffcc2d6815c91590cd5b1cc6edbeca6bfe Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Tue, 27 Feb 2024 16:04:55 +0400 Subject: [PATCH 12/15] fix: linter --- src/daemon/services/keys-indexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon/services/keys-indexer.ts b/src/daemon/services/keys-indexer.ts index 2fa48da..4b93427 100644 --- a/src/daemon/services/keys-indexer.ts +++ b/src/daemon/services/keys-indexer.ts @@ -243,7 +243,7 @@ export class KeysIndexer implements OnModuleInit { this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); this.logger.log(`New appeared CSM validators count: ${csmKeys.data.keys.length}`); for (const csmKey of csmKeys.data.keys) { - for (const [i, v] of valKeys.entries()) { + for (let i = 0; i < valKeys.length; i++) { if (valKeys[i] != csmKey.key) continue; const index = i + this.info.data.lastValidatorsCount; this.storage.data[index] = { From fff984762f25d696a33fc7758d3de2b8ea3fca98 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Wed, 28 Feb 2024 12:00:19 +0400 Subject: [PATCH 13/15] fix: check amount of full withdrawal --- src/common/handlers/handlers.service.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/common/handlers/handlers.service.ts b/src/common/handlers/handlers.service.ts index 2d4baef..8d2de70 100644 --- a/src/common/handlers/handlers.service.ts +++ b/src/common/handlers/handlers.service.ts @@ -16,6 +16,9 @@ type KeyInfoFn = (valIndex: number) => KeyInfo | undefined; @Injectable() export class HandlersService { + // according to the research https://hackmd.io/1wM8vqeNTjqt4pC3XoCUKQ?view#Proposed-solution + private readonly FULL_WITHDRAWAL_MIN_AMOUNT = 8 * 10 ** 18; // 8 ETH + constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, protected readonly consensus: Consensus, @@ -162,15 +165,21 @@ export class HandlersService { keyInfoFn: (valIndex: number) => KeyInfo | undefined, ): Withdrawal[] { const fullWithdrawals = []; - const blockEpoch = Number((Number(blockInfo.message.slot) / 32).toFixed()); + const epoch = Number((Number(blockInfo.message.slot) / 32).toFixed()); const withdrawals = blockInfo.message.body.execution_payload?.withdrawals ?? []; for (const withdrawal of withdrawals) { const keyInfo = keyInfoFn(Number(withdrawal.validator_index)); - if (keyInfo?.withdrawableEpoch != null && keyInfo && blockEpoch >= keyInfo.withdrawableEpoch) { - // TODO: think about sync committee case (balance > 0 after full withdrawal) - fullWithdrawals.push(withdrawal); - } + if (!keyInfo) continue; + if (this.isFullWithdrawal(epoch, keyInfo, withdrawal)) fullWithdrawals.push(withdrawal); } return fullWithdrawals; } + + private isFullWithdrawal(epoch: number, keyInfo: KeyInfo, withdrawal: Withdrawal): boolean { + return ( + keyInfo.withdrawableEpoch != null && + epoch >= keyInfo.withdrawableEpoch && + Number(withdrawal.amount) > this.FULL_WITHDRAWAL_MIN_AMOUNT + ); + } } From 7a571210c98eef99bfe4d28730f41e6f7ec5a9f4 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Thu, 29 Feb 2024 11:13:47 +0400 Subject: [PATCH 14/15] fix: `any` -> particular type --- src/common/providers/base/rest-provider.ts | 40 +++++++++- src/common/providers/consensus/consensus.ts | 75 ++++-------------- src/common/providers/keysapi/keysapi.ts | 37 ++++----- .../providers/keysapi/response.interface.ts | 77 +++++++++++++++++++ src/daemon/services/keys-indexer.ts | 9 ++- src/daemon/services/roots-processor.ts | 2 +- 6 files changed, 156 insertions(+), 84 deletions(-) create mode 100644 src/common/providers/keysapi/response.interface.ts diff --git a/src/common/providers/base/rest-provider.ts b/src/common/providers/base/rest-provider.ts index 412d399..1f46fb5 100644 --- a/src/common/providers/base/rest-provider.ts +++ b/src/common/providers/base/rest-provider.ts @@ -42,7 +42,43 @@ export abstract class BaseRestProvider { // 2. retries // 3. fallbacks - protected async baseGet( + protected async baseJsonGet(base: string, endpoint: string, options?: RequestOptions): Promise { + return (await this.baseGet(base, endpoint, { ...options, streamed: false })) as T; + } + + protected async baseStreamedGet( + base: string, + endpoint: string, + options?: RequestOptions, + ): Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }> { + return (await this.baseGet(base, endpoint, { ...options, streamed: true })) as { + body: BodyReadable; + headers: IncomingHttpHeaders; + }; + } + + protected async baseJsonPost( + base: string, + endpoint: string, + requestBody: any, + options?: RequestOptions, + ): Promise { + return (await this.basePost(base, endpoint, requestBody, { ...options, streamed: false })) as T; + } + + protected async baseStreamedPost( + base: string, + endpoint: string, + requestBody: any, + options?: RequestOptions, + ): Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }> { + return (await this.basePost(base, endpoint, requestBody, { ...options, streamed: true })) as { + body: BodyReadable; + headers: IncomingHttpHeaders; + }; + } + + private async baseGet( base: string, endpoint: string, options?: RequestOptions, @@ -65,7 +101,7 @@ export abstract class BaseRestProvider { return options.streamed ? { body: body, headers: headers } : ((await body.json()) as T); } - protected async basePost( + private async basePost( base: string, endpoint: string, requestBody: any, diff --git a/src/common/providers/consensus/consensus.ts b/src/common/providers/consensus/consensus.ts index a17d10d..1418792 100644 --- a/src/common/providers/consensus/consensus.ts +++ b/src/common/providers/consensus/consensus.ts @@ -1,10 +1,5 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { Inject, Injectable, LoggerService, OnModuleInit, Optional } from '@nestjs/common'; -import { chain } from 'stream-chain'; -import { parser } from 'stream-json'; -import { connectTo } from 'stream-json/Assembler'; -import { IncomingHttpHeaders } from 'undici/types/header'; -import BodyReadable from 'undici/types/readable'; import { BlockHeaderResponse, @@ -13,7 +8,6 @@ import { GenesisResponse, RootHex, StateId, - StateValidatorResponse, } from './response.interface'; import { ConfigService } from '../../config/config.service'; import { PrometheusService } from '../../prometheus/prometheus.service'; @@ -55,7 +49,7 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { ); } - public async onModuleInit(): Promise { + public async onModuleInit(): Promise { // ugly hack to import ESModule to CommonJS project ssz = await eval(`import('@lodestar/types').then((m) => m.ssz)`); this.logger.log(`Getting genesis timestamp`); @@ -68,74 +62,37 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { } public async getGenesis(): Promise { - return (await this.baseGet(this.mainUrl, this.endpoints.genesis)).data as GenesisResponse; + const resp = await this.baseJsonGet<{ data: GenesisResponse }>(this.mainUrl, this.endpoints.genesis); + return resp.data; } public async getBlockInfo(blockId: BlockId): Promise { - return (await this.baseGet(this.mainUrl, this.endpoints.blockInfo(blockId))).data as BlockInfoResponse; + const resp = await this.baseJsonGet<{ data: BlockInfoResponse }>(this.mainUrl, this.endpoints.blockInfo(blockId)); + return resp.data; } public async getBeaconHeader(blockId: BlockId): Promise { - return (await this.baseGet(this.mainUrl, this.endpoints.beaconHeader(blockId))).data as BlockHeaderResponse; + const resp = await this.baseJsonGet<{ data: BlockHeaderResponse }>( + this.mainUrl, + this.endpoints.beaconHeader(blockId), + ); + return resp.data; } public async getBeaconHeadersByParentRoot( parentRoot: RootHex, ): Promise<{ finalized: boolean; data: BlockHeaderResponse[] }> { - return (await this.baseGet(this.mainUrl, this.endpoints.beaconHeadersByParentRoot(parentRoot))) as { - finalized: boolean; - data: BlockHeaderResponse[]; - }; - } - - public async getValidators(stateId: StateId, signal?: AbortSignal): Promise { - const resp: { body: BodyReadable; headers: IncomingHttpHeaders } = await this.baseGet( - this.mainUrl, - this.endpoints.validators(stateId), - { - streamed: true, - signal, - }, - ); - // Progress bar - // TODO: Enable for CLI only - //this.progress.show('Validators from state', resp); - // Data processing - const pipeline = chain([resp.body, parser()]); - return await new Promise((resolve) => { - connectTo(pipeline).on('done', (asm) => resolve(asm.current.data)); - }); - } - - public async getState(stateId: StateId, signal?: AbortSignal): Promise { - const { body } = await this.baseGet<{ body: BodyReadable; headers: IncomingHttpHeaders }>( + return await this.baseJsonGet<{ finalized: boolean; data: BlockHeaderResponse[] }>( this.mainUrl, - this.endpoints.state(stateId), - { - streamed: true, - signal, - }, + this.endpoints.beaconHeadersByParentRoot(parentRoot), ); - // Progress bar - // TODO: Enable for CLI only - //this.progress.show(`State [${stateId}]`, resp); - // Data processing - const pipeline = chain([body, parser()]); - return await new Promise((resolve) => { - connectTo(pipeline).on('done', (asm) => resolve(asm.current)); - }); } public async getStateView(stateId: StateId, signal?: AbortSignal) { - const { body, headers } = await this.baseGet<{ body: BodyReadable; headers: IncomingHttpHeaders }>( - this.mainUrl, - this.endpoints.state(stateId), - { - streamed: true, - signal, - headers: { accept: 'application/octet-stream' }, - }, - ); + const { body, headers } = await this.baseStreamedGet(this.mainUrl, this.endpoints.state(stateId), { + signal, + headers: { accept: 'application/octet-stream' }, + }); const version = headers['eth-consensus-version'] as keyof typeof ForkName; // Progress bar // TODO: Enable for CLI only diff --git a/src/common/providers/keysapi/keysapi.ts b/src/common/providers/keysapi/keysapi.ts index c17bb77..f7cb76b 100644 --- a/src/common/providers/keysapi/keysapi.ts +++ b/src/common/providers/keysapi/keysapi.ts @@ -3,9 +3,8 @@ import { Inject, Injectable, LoggerService, Optional } from '@nestjs/common'; import { chain } from 'stream-chain'; import { parser } from 'stream-json'; import { connectTo } from 'stream-json/Assembler'; -import { IncomingHttpHeaders } from 'undici/types/header'; -import BodyReadable from 'undici/types/readable'; +import { ELBlockSnapshot, ModuleKeys, ModuleKeysFind, Modules, Status } from './response.interface'; import { ConfigService } from '../../config/config.service'; import { PrometheusService } from '../../prometheus/prometheus.service'; import { BaseRestProvider } from '../base/rest-provider'; @@ -35,7 +34,7 @@ export class Keysapi extends BaseRestProvider { ); } - public healthCheck(finalizedTimestamp: number, keysApiMetadata: any): void { + public healthCheck(finalizedTimestamp: number, keysApiMetadata: { elBlockSnapshot: ELBlockSnapshot }): void { if ( finalizedTimestamp - keysApiMetadata.elBlockSnapshot.timestamp > this.config.get('KEYS_INDEXER_KEYAPI_FRESHNESS_PERIOD') @@ -44,23 +43,18 @@ export class Keysapi extends BaseRestProvider { } } - public async getStatus(): Promise { - return await this.baseGet(this.mainUrl, this.endpoints.status); + public async getStatus(): Promise { + return await this.baseJsonGet(this.mainUrl, this.endpoints.status); } - public async getModules(): Promise { - return await this.baseGet(this.mainUrl, this.endpoints.modules); + public async getModules(): Promise { + return await this.baseJsonGet(this.mainUrl, this.endpoints.modules); } - public async getModuleKeys(module_id: string | number, signal?: AbortSignal): Promise { - const resp: { body: BodyReadable; headers: IncomingHttpHeaders } = await this.baseGet( - this.mainUrl, - this.endpoints.moduleKeys(module_id), - { - streamed: true, - signal, - }, - ); + public async getModuleKeys(module_id: string | number, signal?: AbortSignal): Promise { + const resp = await this.baseStreamedGet(this.mainUrl, this.endpoints.moduleKeys(module_id), { + signal, + }); // TODO: ignore depositSignature ? const pipeline = chain([resp.body, parser()]); return await new Promise((resolve) => { @@ -68,7 +62,14 @@ export class Keysapi extends BaseRestProvider { }); } - public async findModuleKeys(module_id: string | number, keysToFind: string[], signal?: AbortSignal): Promise { - return await this.basePost(this.mainUrl, this.endpoints.findModuleKeys(module_id), { pubkeys: keysToFind, signal }); + public async findModuleKeys( + module_id: string | number, + keysToFind: string[], + signal?: AbortSignal, + ): Promise { + return await this.baseJsonPost(this.mainUrl, this.endpoints.findModuleKeys(module_id), { + pubkeys: keysToFind, + signal, + }); } } diff --git a/src/common/providers/keysapi/response.interface.ts b/src/common/providers/keysapi/response.interface.ts new file mode 100644 index 0000000..a99873a --- /dev/null +++ b/src/common/providers/keysapi/response.interface.ts @@ -0,0 +1,77 @@ +export interface Status { + appVersion: string; + chainId: number; + elBlockSnapshot: ELBlockSnapshot; + clBlockSnapshot: CLBlockSnapshot; +} + +export interface Modules { + data: Module[]; + elBlockSnapshot: ELBlockSnapshot; +} + +export interface ModuleKeys { + data: { + keys: Key[]; + module: Module; + }; + meta: { + elBlockSnapshot: ELBlockSnapshot; + }; +} + +export interface ModuleKeysFind { + data: { + keys: Key[]; + }; + meta: { + elBlockSnapshot: ELBlockSnapshot; + }; +} + +export interface ELBlockSnapshot { + blockNumber: number; + blockHash: string; + timestamp: number; +} + +export interface CLBlockSnapshot { + epoch: number; + root: number; + slot: number; + blockNumber: number; + timestamp: number; + blockHash: string; +} + +export interface Module { + nonce: number; + type: string; + // unique id of the module + id: number; + // address of module + stakingModuleAddress: string; + // rewarf fee of the module + moduleFee: number; + // treasury fee + treasuryFee: number; + // target percent of total keys in protocol, in BP + targetShare: number; + // module status if module can not accept the deposits or can participate in further reward distribution + status: number; + // name of module + name: string; + // block.timestamp of the last deposit of the module + lastDepositAt: number; + // block.number of the last deposit of the module + lastDepositBlock: number; +} + +export interface Key { + index: number; + key: string; + depositSignature: string; + used: boolean; + operatorIndex: number; + moduleAddress: string; +} diff --git a/src/daemon/services/keys-indexer.ts b/src/daemon/services/keys-indexer.ts index 4b93427..b960ff4 100644 --- a/src/daemon/services/keys-indexer.ts +++ b/src/daemon/services/keys-indexer.ts @@ -11,6 +11,7 @@ import { KeyInfo } from '../../common/handlers/handlers.service'; import { Consensus } from '../../common/providers/consensus/consensus'; import { BlockHeaderResponse, RootHex, Slot } from '../../common/providers/consensus/response.interface'; import { Keysapi } from '../../common/providers/keysapi/keysapi'; +import { Key, Module } from '../../common/providers/keysapi/response.interface'; type Info = { moduleAddress: string; @@ -168,9 +169,9 @@ export class KeysIndexer implements OnModuleInit { await this.storage.read(); if (this.info.data.moduleId == 0) { - const modules = (await this.keysapi.getModules()).data; - const module = modules.find( - (m: any) => m.stakingModuleAddress.toLowerCase() === this.info.data.moduleAddress.toLowerCase(), + const modulesResp = await this.keysapi.getModules(); + const module = modulesResp.data.find( + (m: Module) => m.stakingModuleAddress.toLowerCase() === this.info.data.moduleAddress.toLowerCase(), ); if (!module) { throw new Error(`Module with address ${this.info.data.moduleAddress} not found`); @@ -192,7 +193,7 @@ export class KeysIndexer implements OnModuleInit { const csmKeys = await this.keysapi.getModuleKeys(this.info.data.moduleId); this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); const keysMap = new Map(); - csmKeys.data.keys.forEach((k: any) => keysMap.set(k.key, { ...k })); + csmKeys.data.keys.forEach((k: Key) => keysMap.set(k.key, { ...k })); const iterator = iterateNodesAtDepth( validators.type.tree_getChunksNode(validators.node), validators.type.chunkDepth, diff --git a/src/daemon/services/roots-processor.ts b/src/daemon/services/roots-processor.ts index a1a11ee..d32fae0 100644 --- a/src/daemon/services/roots-processor.ts +++ b/src/daemon/services/roots-processor.ts @@ -17,7 +17,7 @@ export class RootsProcessor { protected readonly handlers: HandlersService, ) {} - public async process(blockRoot: RootHex): Promise { + public async process(blockRoot: RootHex): Promise { this.logger.log(`๐Ÿ›ƒ Root in processing [${blockRoot}]`); const blockInfo = await this.consensus.getBlockInfo(blockRoot); const rootSlot = { From 35a294f78f4170e636723ebb41b4cbe4bcf8caa6 Mon Sep 17 00:00:00 2001 From: vgorkavenko Date: Mon, 4 Mar 2024 11:47:46 +0400 Subject: [PATCH 15/15] fix: review --- .env.example | 19 ++++---- README.md | 20 ++++----- src/common/config/env.validation.ts | 20 ++++++--- src/common/providers/consensus/consensus.ts | 4 +- src/common/providers/keysapi/keysapi.ts | 6 +-- src/daemon/daemon.service.ts | 8 ++-- src/daemon/services/keys-indexer.ts | 50 ++++++++++++--------- src/daemon/services/roots-processor.ts | 10 ++--- src/daemon/services/roots-stack.ts | 14 +++--- 9 files changed, 81 insertions(+), 70 deletions(-) diff --git a/.env.example b/.env.example index 0184643..4d242b2 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,12 @@ # CLI working mode -#EL_RPC_URLS=https://mainnet.infura.io/v3/... -#CL_API_URLS=https://quiknode.pro/... -#TX_SENDER_PRIVATE_KEY=... +ETH_NETWORK=1 +EL_RPC_URLS=https://mainnet.infura.io/v3/... +CL_API_URLS=https://quiknode.pro/... +LIDO_STAKING_MODULE_ADDRESS=0x9D4AF1Ee19Dad8857db3a45B0374c81c8A1C6320 # Daemon working mode -#ETH_NETWORK=1 -#EL_RPC_URLS=https://mainnet.infura.io/v3/... -#CL_API_URLS=https://quiknode.pro/... -#KEYSAPI_API_URLS=https://keys-api.lido.fi/ -#LIDO_STAKING_MODULE_ADDRESS="0x9D4AF1Ee19Dad8857db3a45B0374c81c8A1C6320" -#TX_SENDER_PRIVATE_KEY=... - +ETH_NETWORK=1 +EL_RPC_URLS=https://mainnet.infura.io/v3/... +CL_API_URLS=https://quiknode.pro/... +KEYSAPI_API_URLS=https://keys-api.lido.fi/ +LIDO_STAKING_MODULE_ADDRESS=0x9D4AF1Ee19Dad8857db3a45B0374c81c8A1C6320 diff --git a/README.md b/README.md index b486231..c5e05d7 100644 --- a/README.md +++ b/README.md @@ -13,23 +13,23 @@ The tool is a daemon that listens to the CL and reports any slashings and withdr The algorithm is as follows: 0. Get the current CL finalized head. -1. Run `KeysIndexer` service to get the current validator set of the CS Module. - > It is necessary at the first startup. All subsequent runs of the indexer will be performed when necessary and independently of the main processing* -2. Choose the next block service to process from `RootsProvider`. +1. Get the current validator set of the CS Module. + > It is necessary at the first startup. All subsequent runs of the indexer will be performed when necessary and independently of the main processing +2. Choose the next block service to process. > The provider chooses the next root with the following priority: - > - Return the root from `RootsStack` service if exists and `KeyIndexer` is helthy enougth to be trusted completely to process this root - > - *When no any processed roots yet* Return a configured root (from `.env` file) or the last finalized root + > - Return the root from roots stack if exists and keys indexer is healthy enough to be trusted completely to process this root + > - *When no any processed roots yet* Return `START_ROOT` or the last finalized root if `START_ROOT` is not set > - Return a finalized child root of the last processed root > - Sleep 12s if nothing to process and **return to the step 0** -3. Run `RootsProcessor` service to process the root. +3. Process the root. > The processor does the following: > - Get the block info from CL by the root - > - If the current state of `KeysIndexer` is outdated (~15-27h behind from the block) to be trusted completely, add the block root to `RootsStack` + > - If the current state of keys indexer is outdated (~15-27h behind from the block) to be trusted completely, add the block root to roots stack > - If the block has a slashing or withdrawal, report it to the CS Module - > - If the current state of `KeysIndexer` is helthy enougth to be trusted completely, remove the root from `RootsStack` + > - If the current state of keys indexer is healthy enough to be trusted completely, remove the root from roots stack -So, according to the algorithm, there are these statements: -1. We always go sequentially by the finalized routs of blocks, taking the next one by the root of the previous one. In this way we avoid missing some blocks. +So, according to the algorithm, there are the following statements: +1. We always go sequentially by the finalized roots of blocks, taking the next one by the root of the previous one. In this way, we avoid missing any blocks. 2. If for some reason the daemon crashes, it will start from the last root running before the crash when it is launched 3. If for some reason KeysAPI crashed or CL node stopped giving validators, we can use the previously successfully received data to guarantee that our slashings will report for another ~15h and withdrawals for ~27h (because of the new validators appearing time and `MIN_VALIDATOR_WITHDRAWABILITY_DELAY`) If any of these time thresholds are breached, we can't be sure that if there was a slashing or a full withdrawal there was definitely not our validator there. That's why we put the root block in the stack just in case, to process it again later when KeysAPI and CL node are well. diff --git a/src/common/config/env.validation.ts b/src/common/config/env.validation.ts index b134f9e..4d242ae 100644 --- a/src/common/config/env.validation.ts +++ b/src/common/config/env.validation.ts @@ -7,6 +7,8 @@ import { IsInt, IsNotEmpty, IsNumber, + IsOptional, + IsString, Max, Min, validateSync, @@ -25,6 +27,9 @@ export enum WorkingMode { CLI = 'cli', } +const MINUTE = 60 * 1000; +const HOUR = 60 * MINUTE; + export class EnvironmentVariables { @IsEnum(Environment) NODE_ENV: Environment = Environment.Development; @@ -32,20 +37,23 @@ export class EnvironmentVariables { @IsEnum(WorkingMode) public WORKING_MODE = WorkingMode.Daemon; + @IsOptional() + @IsString() public START_ROOT?: string; @IsNotEmpty() + @IsString() public LIDO_STAKING_MODULE_ADDRESS: string; @IsNumber() - @Min(30 * 60 * 1000) + @Min(30 * MINUTE) @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) - public KEYS_INDEXER_RUNNING_PERIOD: number = 3 * 60 * 60 * 1000; + public KEYS_INDEXER_RUNNING_PERIOD_MS: number = 3 * HOUR; @IsNumber() @Min(384000) // epoch time in ms @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) - public KEYS_INDEXER_KEYAPI_FRESHNESS_PERIOD: number = 8 * 60 * 60 * 1000; + public KEYS_INDEXER_KEYAPI_FRESHNESS_PERIOD_MS: number = 8 * HOUR; @IsNumber() @Min(1025) @@ -81,7 +89,7 @@ export class EnvironmentVariables { @IsNumber() @Min(1000) @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) - public EL_RPC_RESPONSE_TIMEOUT = 60000; + public EL_RPC_RESPONSE_TIMEOUT_MS = MINUTE; @IsNumber() @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) @@ -99,7 +107,7 @@ export class EnvironmentVariables { @IsNumber() @Min(1000) @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) - public CL_API_RESPONSE_TIMEOUT = 60000; + public CL_API_RESPONSE_TIMEOUT_MS = MINUTE; @IsNumber() @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) @@ -117,7 +125,7 @@ export class EnvironmentVariables { @IsNumber() @Min(1000) @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) - public KEYSAPI_API_RESPONSE_TIMEOUT = 60000; + public KEYSAPI_API_RESPONSE_TIMEOUT_MS = MINUTE; @IsNumber() @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true }) diff --git a/src/common/providers/consensus/consensus.ts b/src/common/providers/consensus/consensus.ts index 1418792..a2ab138 100644 --- a/src/common/providers/consensus/consensus.ts +++ b/src/common/providers/consensus/consensus.ts @@ -42,7 +42,7 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { ) { super( config.get('CL_API_URLS') as Array, - config.get('CL_API_RESPONSE_TIMEOUT'), + config.get('CL_API_RESPONSE_TIMEOUT_MS'), config.get('CL_API_MAX_RETRIES'), logger, prometheus, @@ -100,6 +100,6 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { // Data processing const bodyBites = new Uint8Array(await body.arrayBuffer()); // TODO: high memory usage - return ssz.allForks[version].BeaconState.deserializeToView(bodyBites); + return ssz[version].BeaconState.deserializeToView(bodyBites); } } diff --git a/src/common/providers/keysapi/keysapi.ts b/src/common/providers/keysapi/keysapi.ts index f7cb76b..4c0bd66 100644 --- a/src/common/providers/keysapi/keysapi.ts +++ b/src/common/providers/keysapi/keysapi.ts @@ -18,8 +18,6 @@ export class Keysapi extends BaseRestProvider { findModuleKeys: (module_id: string | number): string => `v1/modules/${module_id}/keys/find`, }; - // TODO: types - constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, @Optional() protected readonly prometheus: PrometheusService, @@ -27,7 +25,7 @@ export class Keysapi extends BaseRestProvider { ) { super( config.get('KEYSAPI_API_URLS') as Array, - config.get('KEYSAPI_API_RESPONSE_TIMEOUT'), + config.get('KEYSAPI_API_RESPONSE_TIMEOUT_MS'), config.get('KEYSAPI_API_MAX_RETRIES'), logger, prometheus, @@ -37,7 +35,7 @@ export class Keysapi extends BaseRestProvider { public healthCheck(finalizedTimestamp: number, keysApiMetadata: { elBlockSnapshot: ELBlockSnapshot }): void { if ( finalizedTimestamp - keysApiMetadata.elBlockSnapshot.timestamp > - this.config.get('KEYS_INDEXER_KEYAPI_FRESHNESS_PERIOD') + this.config.get('KEYS_INDEXER_KEYAPI_FRESHNESS_PERIOD_MS') ) { throw new Error('KeysApi is outdated'); } diff --git a/src/daemon/daemon.service.ts b/src/daemon/daemon.service.ts index 360e6f3..eb58ff9 100644 --- a/src/daemon/daemon.service.ts +++ b/src/daemon/daemon.service.ts @@ -38,13 +38,13 @@ export class DaemonService implements OnApplicationBootstrap { this.logger.log('๐Ÿ—ฟ Get finalized header'); const header = await this.consensus.getBeaconHeader('finalized'); this.logger.log(`๐Ÿ’Ž Finalized slot [${header.header.message.slot}]. Root [${header.root}]`); - await this.keysIndexer.update(header); + this.keysIndexer.update(header); const nextRoot = await this.rootsProvider.getNext(header); if (nextRoot) { await this.rootsProcessor.process(nextRoot); - } else { - this.logger.log(`๐Ÿ’ค Wait for the next finalized root`); - await sleep(12000); + return; } + this.logger.log(`๐Ÿ’ค Wait for the next finalized root`); + await sleep(12000); } } diff --git a/src/daemon/services/keys-indexer.ts b/src/daemon/services/keys-indexer.ts index b960ff4..52ae4ec 100644 --- a/src/daemon/services/keys-indexer.ts +++ b/src/daemon/services/keys-indexer.ts @@ -13,14 +13,14 @@ import { BlockHeaderResponse, RootHex, Slot } from '../../common/providers/conse import { Keysapi } from '../../common/providers/keysapi/keysapi'; import { Key, Module } from '../../common/providers/keysapi/response.interface'; -type Info = { +type KeysIndexerServiceInfo = { moduleAddress: string; moduleId: number; storageStateSlot: number; lastValidatorsCount: number; }; -type Storage = { +type KeysIndexerServiceStorage = { [valIndex: number]: KeyInfo; }; @@ -54,8 +54,8 @@ function Single(target: any, propertyKey: string, descriptor: PropertyDescriptor export class KeysIndexer implements OnModuleInit { private startedAt: number = 0; - private info: Low; - private storage: Low; + private info: Low; + private storage: Low; constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, @@ -73,7 +73,7 @@ export class KeysIndexer implements OnModuleInit { }; @Single - public async update(finalizedHeader: BlockHeaderResponse): Promise { + public update(finalizedHeader: BlockHeaderResponse): void { // TODO: do we have to check integrity of data here? when `this.info` says one thing and `this.storage` another const slot = Number(finalizedHeader.header.message.slot); if (this.isNotTimeToRun(slot)) { @@ -111,32 +111,32 @@ export class KeysIndexer implements OnModuleInit { const storageTimestamp = this.consensus.slotToTimestamp(this.info.data.storageStateSlot) * 1000; return ( this.info.data.storageStateSlot == finalizedSlot || - this.config.get('KEYS_INDEXER_RUNNING_PERIOD') >= Date.now() - storageTimestamp + this.config.get('KEYS_INDEXER_RUNNING_PERIOD_MS') >= Date.now() - storageTimestamp ); } - public eligibleForAnyDuty(slotNumber: Slot): boolean { - return this.eligibleForSlashings(slotNumber) || this.eligibleForFullWithdrawals(slotNumber); + public isTrustedForAnyDuty(slotNumber: Slot): boolean { + return this.isTrustedForSlashings(slotNumber) || this.isTrustedForFullWithdrawals(slotNumber); } - public eligibleForEveryDuty(slotNumber: Slot): boolean { - const eligibleForSlashings = this.eligibleForSlashings(slotNumber); - const eligibleForFullWithdrawals = this.eligibleForFullWithdrawals(slotNumber); - if (!eligibleForSlashings) + public isTrustedForEveryDuty(slotNumber: Slot): boolean { + const trustedForSlashings = this.isTrustedForSlashings(slotNumber); + const trustedForFullWithdrawals = this.isTrustedForFullWithdrawals(slotNumber); + if (!trustedForSlashings) this.logger.warn( '๐Ÿšจ Current keys indexer data might not be ready to detect slashing. ' + 'The root will be processed later again', ); - if (!eligibleForFullWithdrawals) + if (!trustedForFullWithdrawals) this.logger.warn( 'โš ๏ธ Current keys indexer data might not be ready to detect full withdrawal. ' + 'The root will be processed later again', ); - return eligibleForSlashings && eligibleForFullWithdrawals; + return trustedForSlashings && trustedForFullWithdrawals; } - private eligibleForSlashings(slotNumber: Slot): boolean { - // We are ok with oudated indexer for detection slasing + private isTrustedForSlashings(slotNumber: Slot): boolean { + // We are ok with outdated indexer for detection slashing // because of a bunch of delays between deposit and validator appearing // TODO: get constants from node const ETH1_FOLLOW_DISTANCE = 2048; // ~8 hours @@ -146,8 +146,8 @@ export class KeysIndexer implements OnModuleInit { return slotNumber - this.info.data.storageStateSlot <= safeDelay; // ~14.8 hours } - private eligibleForFullWithdrawals(slotNumber: Slot): boolean { - // We are ok with oudated indexer for detection withdrawal + private isTrustedForFullWithdrawals(slotNumber: Slot): boolean { + // We are ok with outdated indexer for detection withdrawal // because of MIN_VALIDATOR_WITHDRAWABILITY_DELAY // TODO: get constants from node const MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 256; @@ -157,14 +157,20 @@ export class KeysIndexer implements OnModuleInit { } private async initOrReadServiceData() { - const defaultInfo: Info = { + const defaultInfo: KeysIndexerServiceInfo = { moduleAddress: this.config.get('LIDO_STAKING_MODULE_ADDRESS'), moduleId: 0, storageStateSlot: 0, lastValidatorsCount: 0, }; - this.info = new Low(new JSONFile('.keys-indexer-info.json'), defaultInfo); - this.storage = new Low(new JSONFile('.keys-indexer-storage.json'), {}); + this.info = new Low( + new JSONFile('.keys-indexer-info.json'), + defaultInfo, + ); + this.storage = new Low( + new JSONFile('.keys-indexer-storage.json'), + {}, + ); await this.info.read(); await this.storage.read(); @@ -202,7 +208,7 @@ export class KeysIndexer implements OnModuleInit { ); for (let i = 0; i < validators.length; i++) { const node = iterator.next().value; - const v = validators.type.elementType.tree_toValue(node); + const v = node.value; const pubKey = '0x'.concat(Buffer.from(v.pubkey).toString('hex')); const keyInfo = keysMap.get(pubKey); if (!keyInfo) continue; diff --git a/src/daemon/services/roots-processor.ts b/src/daemon/services/roots-processor.ts index d32fae0..32991d0 100644 --- a/src/daemon/services/roots-processor.ts +++ b/src/daemon/services/roots-processor.ts @@ -2,7 +2,7 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { Inject, Injectable, LoggerService } from '@nestjs/common'; import { KeysIndexer } from './keys-indexer'; -import { RootsStack } from './roots-stack'; +import { RootSlot, RootsStack } from './roots-stack'; import { HandlersService } from '../../common/handlers/handlers.service'; import { Consensus } from '../../common/providers/consensus/consensus'; import { RootHex } from '../../common/providers/consensus/response.interface'; @@ -20,14 +20,14 @@ export class RootsProcessor { public async process(blockRoot: RootHex): Promise { this.logger.log(`๐Ÿ›ƒ Root in processing [${blockRoot}]`); const blockInfo = await this.consensus.getBlockInfo(blockRoot); - const rootSlot = { + const rootSlot: RootSlot = { blockRoot, slotNumber: Number(blockInfo.message.slot), }; - const indexerIsOK = this.keysIndexer.eligibleForEveryDuty(rootSlot.slotNumber); - if (!indexerIsOK) await this.rootsStack.push(rootSlot); // only new will be pushed + const indexerIsTrusted = this.keysIndexer.isTrustedForEveryDuty(rootSlot.slotNumber); + if (!indexerIsTrusted) await this.rootsStack.push(rootSlot); // only new will be pushed await this.handlers.proveIfNeeded(blockRoot, blockInfo, this.keysIndexer.getKey); - if (indexerIsOK) await this.rootsStack.purge(rootSlot); + if (indexerIsTrusted) await this.rootsStack.purge(rootSlot); await this.rootsStack.setLastProcessed(rootSlot); } } diff --git a/src/daemon/services/roots-stack.ts b/src/daemon/services/roots-stack.ts index 90b5b92..cc24611 100644 --- a/src/daemon/services/roots-stack.ts +++ b/src/daemon/services/roots-stack.ts @@ -7,16 +7,16 @@ import { RootHex } from '../../common/providers/consensus/response.interface'; export type RootSlot = { blockRoot: RootHex; slotNumber: number }; -type Info = { +type RootsStackServiceInfo = { lastProcessedRootSlot: RootSlot | undefined; }; -type Storage = { [slot: number]: RootHex }; +type RootsStackServiceStorage = { [slot: number]: RootHex }; @Injectable() export class RootsStack implements OnModuleInit { - private info: Low; - private storage: Low; + private info: Low; + private storage: Low; constructor(protected readonly keysIndexer: KeysIndexer) {} @@ -26,7 +26,7 @@ export class RootsStack implements OnModuleInit { public getNextEligible(): RootSlot | undefined { for (const slot in this.storage.data) { - if (this.keysIndexer.eligibleForAnyDuty(Number(slot))) { + if (this.keysIndexer.isTrustedForAnyDuty(Number(slot))) { return { blockRoot: this.storage.data[slot], slotNumber: Number(slot) }; } } @@ -54,10 +54,10 @@ export class RootsStack implements OnModuleInit { } private async initOrReadServiceData() { - this.info = new Low(new JSONFile('.roots-stack-info.json'), { + this.info = new Low(new JSONFile('.roots-stack-info.json'), { lastProcessedRootSlot: undefined, }); - this.storage = new Low(new JSONFile('.roots-stack-storage.json'), {}); + this.storage = new Low(new JSONFile('.roots-stack-storage.json'), {}); await this.info.read(); await this.storage.read(); }