feat: common

.env.example

@@ -0,0 +1,13 @@
+# CLI working mode
+#EL_RPC_URLS=https://mainnet.infura.io/v3/...
+#CL_API_URLS=https://quiknode.pro/...
+#TX_SENDER_PRIVATE_KEY=...
+
+# Daemon working mode
+#ETH_NETWORK=1
+#EL_RPC_URLS=https://mainnet.infura.io/v3/...
+#CL_API_URLS=https://quiknode.pro/...
+#KEYSAPI_API_URLS=https://keys-api.lido.fi/
+#LIDO_STAKING_MODULE_ADDRESS="0x9D4AF1Ee19Dad8857db3a45B0374c81c8A1C6320"
+#TX_SENDER_PRIVATE_KEY=...
+

.eslintrc.js

@@ -18,6 +18,13 @@ module.exports = {
     jest: true,
   },
   ignorePatterns: ['.eslintrc.js'],
+  settings: {
+    'import/resolver': {
+      typescript: {
+        project: './tsconfig.json',
+      },
+    },
+  },
   rules: {
     '@typescript-eslint/interface-name-prefix': 'off',
     '@typescript-eslint/explicit-function-return-type': 'off',

.gitignore

@@ -5,6 +5,10 @@
 # ENV
 /.env
 
+# Storage
+/.keys-indexer-*
+/.roots-stack-*
+
 # Logs
 logs
 *.log

.nvmrc

@@ -1 +1 @@
-v18.17.1
+v20.11.0

.prettierrc

@@ -1,4 +1,5 @@
 {
   "singleQuote": true,
-  "trailingComma": "all"
-}
+  "trailingComma": "all",
+  "printWidth": 120
+}

package.json

@@ -22,15 +22,21 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
+    "@huanshiwushuang/lowdb": "^6.0.2",
     "@lido-nestjs/logger": "^1.3.2",
     "@nestjs/common": "^10.0.0",
     "@nestjs/config": "^3.1.1",
     "@nestjs/core": "^10.0.0",
     "@nestjs/platform-express": "^10.0.0",
+    "@types/cli-progress": "^3.11.5",
+    "cli-progress": "^3.12.0",
     "nest-commander": "^3.12.5",
     "nest-winston": "^1.9.4",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
+    "stream-chain": "^2.2.5",
+    "stream-json": "^1.8.0",
+    "undici": "^6.4.0",
     "winston": "^3.11.0"
   },
   "devDependencies": {
@@ -42,12 +48,16 @@
     "@swc/jest": "^0.2.30",
     "@types/express": "^4.17.17",
     "@types/jest": "^29.5.2",
+    "@types/lowdb": "1.0.0",
     "@types/node": "^20.3.1",
+    "@types/stream-chain": "^2.0.4",
+    "@types/stream-json": "^1.7.7",
     "@types/supertest": "^6.0.0",
     "@typescript-eslint/eslint-plugin": "^6.0.0",
     "@typescript-eslint/parser": "^6.0.0",
     "eslint": "^8.42.0",
     "eslint-config-prettier": "^9.0.0",
+    "eslint-import-resolver-typescript": "^3.6.1",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-prettier": "^5.0.0",
     "jest": "^29.5.0",

src/cli/cli.service.ts

@@ -1,16 +1,9 @@
 import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
-import {
-  Inject,
-  Injectable,
-  LoggerService,
-  OnApplicationBootstrap,
-} from '@nestjs/common';
+import { Inject, Injectable, LoggerService, OnApplicationBootstrap } from '@nestjs/common';
 
 @Injectable()
 export class CliService implements OnApplicationBootstrap {
-  constructor(
-    @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
-  ) {}
+  constructor(@Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService) {}
   async onApplicationBootstrap() {
     this.logger.log('Working mode: CLI');
   }

src/common/config/config.service.ts

@@ -7,16 +7,10 @@ export class ConfigService extends ConfigServiceSource<EnvironmentVariables> {
    * List of env variables that should be hidden
    */
   public get secrets(): string[] {
-    return [
-      ...this.get('EL_RPC_URLS'),
-      ...this.get('CL_API_URLS'),
-      ...this.get('KEYSAPI_API_URLS'),
-    ];
+    return [...this.get('EL_RPC_URLS'), ...this.get('CL_API_URLS'), ...this.get('KEYSAPI_API_URLS')];
   }
 
-  public get<T extends keyof EnvironmentVariables>(
-    key: T,
-  ): EnvironmentVariables[T] {
+  public get<T extends keyof EnvironmentVariables>(key: T): EnvironmentVariables[T] {
     return super.get(key, { infer: true }) as EnvironmentVariables[T];
   }
 }

src/common/config/env.validation.ts

@@ -32,6 +32,21 @@ export class EnvironmentVariables {
   @IsEnum(WorkingMode)
   public WORKING_MODE = WorkingMode.Daemon;
 
+  public START_ROOT?: string;
+
+  @IsNotEmpty()
+  public LIDO_STAKING_MODULE_ADDRESS: string;
+
+  @IsNumber()
+  @Min(30 * 60 * 1000)
+  @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
+  public KEYS_INDEXER_RUNNING_PERIOD: number = 3 * 60 * 60 * 1000;
+
+  @IsNumber()
+  @Min(384000) // epoch time in ms
+  @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
+  public KEYS_INDEXER_KEYAPI_FRESHNESS_PERIOD: number = 8 * 60 * 60 * 1000;
+
   @IsNumber()
   @Min(1025)
   @Max(65535)

src/common/handlers/handlers.module.ts

@@ -4,5 +4,6 @@ import { HandlersService } from './handlers.service';
 
 @Module({
   providers: [HandlersService],
+  exports: [HandlersService],
 })
 export class HandlersModule {}

src/common/handlers/handlers.service.ts

@@ -1,4 +1,126 @@
-import { Injectable } from '@nestjs/common';
+import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
+import { Inject, Injectable, LoggerService } from '@nestjs/common';
+
+import { BlockInfoResponse, RootHex } from '../providers/consensus/response.interface';
+
+export interface KeyInfo {
+  operatorId: number;
+  keyIndex: number;
+  pubKey: string;
+  withdrawableEpoch: number;
+}
+
+type KeyInfoFn = (valIndex: number) => KeyInfo | undefined;
 
 @Injectable()
-export class HandlersService {}
+export class HandlersService {
+  constructor(@Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService) {}
+
+  public async prove(blockRoot: RootHex, blockInfo: BlockInfoResponse, keyInfoFn: KeyInfoFn): Promise<void> {
+    const slashings = await this.getUnprovenSlashings(blockRoot, blockInfo, keyInfoFn);
+    const withdrawals = await this.getUnprovenWithdrawals(blockRoot, blockInfo, keyInfoFn);
+    if (!slashings.length && !withdrawals.length) return;
+    const payload = await this.buildProvePayload(slashings, withdrawals);
+    // TODO: ask before sending if CLI or daemon in watch mode
+    await this.sendProves(payload);
+  }
+
+  private async buildProvePayload(slashings: string[], withdrawals: string[]): Promise<any> {
+    // TODO: implement
+    //  this.consensus.getState(...)
+    return {};
+  }
+  private async sendProves(payload: any): Promise<void> {
+    // TODO: implement
+  }
+
+  private async getUnprovenSlashings(
+    blockRoot: RootHex,
+    blockInfo: BlockInfoResponse,
+    keyInfoFn: KeyInfoFn,
+  ): Promise<string[]> {
+    const slashings = [
+      ...this.getSlashedProposers(blockInfo, keyInfoFn),
+      ...this.getSlashedAttesters(blockInfo, keyInfoFn),
+    ];
+    if (!slashings.length) return [];
+    const unproven = [];
+    for (const slashing of slashings) {
+      // TODO: implement
+      //  const proved = await this.execution.isSlashingProved(slashing);
+      const proved = false;
+      if (!proved) unproven.push(slashing);
+    }
+    if (!unproven.length) {
+      this.logger.log(`No slashings to prove. Root [${blockRoot}]`);
+      return [];
+    }
+    this.logger.warn(`🔍 Unproven slashings: ${unproven}`);
+    return unproven;
+  }
+
+  private async getUnprovenWithdrawals(
+    blockRoot: RootHex,
+    blockInfo: BlockInfoResponse,
+    keyInfoFn: KeyInfoFn,
+  ): Promise<string[]> {
+    const withdrawals = this.getFullWithdrawals(blockInfo, keyInfoFn);
+    if (!withdrawals.length) return [];
+    const unproven = [];
+    for (const withdrawal of withdrawals) {
+      // TODO: implement
+      //  const proved = await this.execution.isSlashingProved(slashing);
+      const proved = false;
+      if (!proved) unproven.push(withdrawal);
+    }
+    if (!unproven.length) {
+      this.logger.log(`No full withdrawals to prove. Root [${blockRoot}]`);
+      return [];
+    }
+    this.logger.warn(`🔍 Unproven full withdrawals: ${unproven}`);
+    return unproven;
+  }
+
+  private getSlashedAttesters(
+    blockInfo: BlockInfoResponse,
+    keyInfoFn: (valIndex: number) => KeyInfo | undefined,
+  ): string[] {
+    const slashed = [];
+    for (const att of blockInfo.message.body.attester_slashings) {
+      const accused = att.attestation_1.attesting_indices.filter((x) =>
+        att.attestation_2.attesting_indices.includes(x),
+      );
+      slashed.push(...accused.filter((item) => keyInfoFn(Number(item))));
+    }
+    return slashed;
+  }
+
+  private getSlashedProposers(
+    blockInfo: BlockInfoResponse,
+    keyInfoFn: (valIndex: number) => KeyInfo | undefined,
+  ): string[] {
+    const slashed = [];
+    for (const prop of blockInfo.message.body.proposer_slashings) {
+      if (keyInfoFn(Number(prop.signed_header_1.proposer_index))) {
+        slashed.push(prop.signed_header_1.proposer_index);
+      }
+    }
+    return slashed;
+  }
+
+  private getFullWithdrawals(
+    blockInfo: BlockInfoResponse,
+    keyInfoFn: (valIndex: number) => KeyInfo | undefined,
+  ): string[] {
+    const fullWithdrawals = [];
+    const blockEpoch = Number(blockInfo.message.slot) / 32;
+    const withdrawals = blockInfo.message.body.execution_payload.withdrawals;
+    for (const withdrawal of withdrawals) {
+      const keyInfo = keyInfoFn(Number(withdrawal.validator_index));
+      if (keyInfo && blockEpoch >= keyInfo.withdrawableEpoch) {
+        fullWithdrawals.push(withdrawal.validator_index);
+      }
+    }
+    return fullWithdrawals;
+  }
+}

src/common/logger/logger.module.ts

@@ -1,8 +1,4 @@
-import {
-  LoggerModule as Logger,
-  jsonTransport,
-  simpleTransport,
-} from '@lido-nestjs/logger';
+import { LoggerModule as Logger, jsonTransport, simpleTransport } from '@lido-nestjs/logger';
 import { Module } from '@nestjs/common';
 
 import { ConfigModule } from '../config/config.module';
@@ -20,9 +16,7 @@ import { LogFormat } from '../config/interfaces';
         const format = configService.get('LOG_FORMAT');
         const isJSON = format === LogFormat.JSON;
 
-        const transports = isJSON
-          ? jsonTransport({ secrets })
-          : simpleTransport({ secrets });
+        const transports = isJSON ? jsonTransport({ secrets }) : simpleTransport({ secrets });
 
         return { level, transports };
       },

src/common/providers/base/rest-provider.ts

@@ -0,0 +1,92 @@
+import { LoggerService } from '@nestjs/common';
+import { request } from 'undici';
+import { IncomingHttpHeaders } from 'undici/types/header';
+import BodyReadable from 'undici/types/readable';
+
+import { PrometheusService } from '../../prometheus/prometheus.service';
+
+export interface RequestPolicy {
+  timeout: number;
+  maxRetries: number;
+  fallbacks: Array<string>;
+}
+
+export interface RequestOptions {
+  streamed?: boolean;
+  requestPolicy?: RequestPolicy;
+  signal?: AbortSignal;
+}
+
+export abstract class BaseRestProvider {
+  protected readonly mainUrl: string;
+  protected readonly requestPolicy: RequestPolicy;
+
+  protected constructor(
+    urls: Array<string>,
+    responseTimeout: number,
+    maxRetries: number,
+    protected readonly logger: LoggerService,
+    protected readonly prometheus?: PrometheusService,
+  ) {
+    this.mainUrl = urls[0];
+    this.requestPolicy = {
+      timeout: responseTimeout,
+      maxRetries: maxRetries,
+      fallbacks: urls.slice(1),
+    };
+  }
+
+  // TODO: Request should have:
+  //  1. metrics (if it is daemon mode)
+  //  2. retries
+  //  3. fallbacks
+
+  protected async baseGet<T>(
+    base: string,
+    endpoint: string,
+    options?: RequestOptions,
+  ): Promise<T | { body: BodyReadable; headers: IncomingHttpHeaders }> {
+    options = {
+      streamed: false,
+      requestPolicy: this.requestPolicy,
+      ...options,
+    } as RequestOptions;
+    const { body, headers, statusCode } = await request(new URL(endpoint, base), {
+      method: 'GET',
+      headersTimeout: (options.requestPolicy as RequestPolicy).timeout,
+      signal: options.signal,
+    });
+    if (statusCode !== 200) {
+      const hostname = new URL(base).hostname;
+      throw new Error(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`);
+    }
+    return options.streamed ? { body: body, headers: headers } : ((await body.json()) as T);
+  }
+
+  protected async basePost<T>(
+    base: string,
+    endpoint: string,
+    requestBody: any,
+    options?: RequestOptions,
+  ): Promise<T | { body: BodyReadable; headers: IncomingHttpHeaders }> {
+    options = {
+      streamed: false,
+      requestPolicy: this.requestPolicy,
+      ...options,
+    } as RequestOptions;
+    const { body, headers, statusCode } = await request(new URL(endpoint, base), {
+      method: 'POST',
+      headersTimeout: (options.requestPolicy as RequestPolicy).timeout,
+      signal: options.signal,
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(requestBody),
+    });
+    if (statusCode !== 200) {
+      const hostname = new URL(base).hostname;
+      throw new Error(`Request failed with status code [${statusCode}] on host [${hostname}]: ${endpoint}`);
+    }
+    return options.streamed ? { body: body, headers: headers } : ((await body.json()) as T);
+  }
+}