feat: blocks processing (#1)

* feat: common

* feat: daemon

* chore: add missing libs

* chore: add desc to README

* chore: add statements

* feat: use SSZ state representation from node

* fix: review and handlers

* fix: add fork selector for state

* fix: handlers. a bit

* fix: keys-indexer init\update data

* fix: roots stack data structure

* fix: linter

* fix: check amount of full withdrawal

* fix: `any` -> particular type

* fix: review

.env.example

@@ -0,0 +1,12 @@
+# CLI working mode
+ETH_NETWORK=1
+EL_RPC_URLS=https://mainnet.infura.io/v3/...
+CL_API_URLS=https://quiknode.pro/...
+LIDO_STAKING_MODULE_ADDRESS=0x9D4AF1Ee19Dad8857db3a45B0374c81c8A1C6320
+
+# Daemon working mode
+ETH_NETWORK=1
+EL_RPC_URLS=https://mainnet.infura.io/v3/...
+CL_API_URLS=https://quiknode.pro/...
+KEYSAPI_API_URLS=https://keys-api.lido.fi/
+LIDO_STAKING_MODULE_ADDRESS=0x9D4AF1Ee19Dad8857db3a45B0374c81c8A1C6320

.eslintrc.js

@@ -18,6 +18,13 @@ module.exports = {
     jest: true,
   },
   ignorePatterns: ['.eslintrc.js'],
+  settings: {
+    'import/resolver': {
+      typescript: {
+        project: './tsconfig.json',
+      },
+    },
+  },
   rules: {
     '@typescript-eslint/interface-name-prefix': 'off',
     '@typescript-eslint/explicit-function-return-type': 'off',

.gitignore

@@ -5,6 +5,10 @@
 # ENV
 /.env
 
+# Storage
+/.keys-indexer-*
+/.roots-stack-*
+
 # Logs
 logs
 *.log

.nvmrc

@@ -1 +1 @@
-v18.17.1
+v20.11.0

.prettierrc

@@ -1,4 +1,5 @@
 {
   "singleQuote": true,
-  "trailingComma": "all"
-}
+  "trailingComma": "all",
+  "printWidth": 120
+}

README.md

@@ -7,6 +7,33 @@
 
 Tool for reporting Slashings and Withdrawals for Lido Community Staking Module
 
+### Daemon working mode
+
+The tool is a daemon that listens to the CL and reports any slashings and withdrawals to the Lido Community Staking Module.
+
+The algorithm is as follows:
+0. Get the current CL finalized head.
+1. Get the current validator set of the CS Module.
+   > It is necessary at the first startup. All subsequent runs of the indexer will be performed when necessary and independently of the main processing
+2. Choose the next block service to process.
+   > The provider chooses the next root with the following priority:
+   > - Return the root from roots stack if exists and keys indexer is healthy enough to be trusted completely to process this root
+   > - *When no any processed roots yet* Return `START_ROOT` or the last finalized root if `START_ROOT` is not set
+   > - Return a finalized child root of the last processed root
+   > - Sleep 12s if nothing to process and **return to the step 0**
+3. Process the root.
+   > The processor does the following:
+   > - Get the block info from CL by the root
+   > - If the current state of keys indexer is outdated (~15-27h behind from the block) to be trusted completely, add the block root to roots stack
+   > - If the block has a slashing or withdrawal, report it to the CS Module
+   > - If the current state of keys indexer is healthy enough to be trusted completely, remove the root from roots stack
+
+So, according to the algorithm, there are the following statements:
+1. We always go sequentially by the finalized roots of blocks, taking the next one by the root of the previous one. In this way, we avoid missing any blocks.
+2. If for some reason the daemon crashes, it will start from the last root running before the crash when it is launched
+3. If for some reason KeysAPI crashed or CL node stopped giving validators, we can use the previously successfully received data to guarantee that our slashings will report for another ~15h and withdrawals for ~27h (because of the new validators appearing time and `MIN_VALIDATOR_WITHDRAWABILITY_DELAY`)
+If any of these time thresholds are breached, we can't be sure that if there was a slashing or a full withdrawal there was definitely not our validator there. That's why we put the root block in the stack just in case, to process it again later when KeysAPI and CL node are well.
+
 ## Installation
 
 ```bash

package.json

@@ -22,32 +22,46 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
+    "@huanshiwushuang/lowdb": "^6.0.2",
     "@lido-nestjs/logger": "^1.3.2",
+    "@lodestar/params": "^1.16.0",
+    "@lodestar/types": "^1.15.0",
     "@nestjs/common": "^10.0.0",
     "@nestjs/config": "^3.1.1",
     "@nestjs/core": "^10.0.0",
     "@nestjs/platform-express": "^10.0.0",
+    "@types/cli-progress": "^3.11.5",
+    "class-transformer": "^0.5.1",
+    "class-validator": "^0.14.1",
+    "cli-progress": "^3.12.0",
     "nest-commander": "^3.12.5",
     "nest-winston": "^1.9.4",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
+    "stream-chain": "^2.2.5",
+    "stream-json": "^1.8.0",
+    "undici": "^6.4.0",
     "winston": "^3.11.0"
   },
   "devDependencies": {
     "@nestjs/cli": "^10.0.0",
     "@nestjs/schematics": "^10.0.0",
     "@nestjs/testing": "^10.0.0",
-    "@swc/cli": "^0.1.63",
-    "@swc/core": "^1.3.104",
+    "@swc/cli": "^0.3.9",
+    "@swc/core": "^1.4.0",
     "@swc/jest": "^0.2.30",
     "@types/express": "^4.17.17",
     "@types/jest": "^29.5.2",
+    "@types/lowdb": "1.0.0",
     "@types/node": "^20.3.1",
+    "@types/stream-chain": "^2.0.4",
+    "@types/stream-json": "^1.7.7",
     "@types/supertest": "^6.0.0",
     "@typescript-eslint/eslint-plugin": "^6.0.0",
     "@typescript-eslint/parser": "^6.0.0",
     "eslint": "^8.42.0",
     "eslint-config-prettier": "^9.0.0",
+    "eslint-import-resolver-typescript": "^3.6.1",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-prettier": "^5.0.0",
     "jest": "^29.5.0",
@@ -58,7 +72,7 @@
     "ts-loader": "^9.4.3",
     "ts-node": "^10.9.1",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.1.3"
+    "typescript": "^5.3.3"
   },
   "jest": {
     "moduleFileExtensions": [

src/cli/cli.service.ts

@@ -1,16 +1,9 @@
 import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
-import {
-  Inject,
-  Injectable,
-  LoggerService,
-  OnApplicationBootstrap,
-} from '@nestjs/common';
+import { Inject, Injectable, LoggerService, OnApplicationBootstrap } from '@nestjs/common';
 
 @Injectable()
 export class CliService implements OnApplicationBootstrap {
-  constructor(
-    @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
-  ) {}
+  constructor(@Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService) {}
   async onApplicationBootstrap() {
     this.logger.log('Working mode: CLI');
   }

src/common/config/config.service.ts

@@ -7,16 +7,10 @@ export class ConfigService extends ConfigServiceSource<EnvironmentVariables> {
    * List of env variables that should be hidden
    */
   public get secrets(): string[] {
-    return [
-      ...this.get('EL_RPC_URLS'),
-      ...this.get('CL_API_URLS'),
-      ...this.get('KEYSAPI_API_URLS'),
-    ];
+    return [...this.get('EL_RPC_URLS'), ...this.get('CL_API_URLS'), ...this.get('KEYSAPI_API_URLS')];
   }
 
-  public get<T extends keyof EnvironmentVariables>(
-    key: T,
-  ): EnvironmentVariables[T] {
+  public get<T extends keyof EnvironmentVariables>(key: T): EnvironmentVariables[T] {
     return super.get(key, { infer: true }) as EnvironmentVariables[T];
   }
 }

src/common/config/env.validation.ts

@@ -7,6 +7,8 @@ import {
   IsInt,
   IsNotEmpty,
   IsNumber,
+  IsOptional,
+  IsString,
   Max,
   Min,
   validateSync,
@@ -25,13 +27,34 @@ export enum WorkingMode {
   CLI = 'cli',
 }
 
+const MINUTE = 60 * 1000;
+const HOUR = 60 * MINUTE;
+
 export class EnvironmentVariables {
   @IsEnum(Environment)
   NODE_ENV: Environment = Environment.Development;
 
   @IsEnum(WorkingMode)
   public WORKING_MODE = WorkingMode.Daemon;
 
+  @IsOptional()
+  @IsString()
+  public START_ROOT?: string;
+
+  @IsNotEmpty()
+  @IsString()
+  public LIDO_STAKING_MODULE_ADDRESS: string;
+
+  @IsNumber()
+  @Min(30 * MINUTE)
+  @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
+  public KEYS_INDEXER_RUNNING_PERIOD_MS: number = 3 * HOUR;
+
+  @IsNumber()
+  @Min(384000) // epoch time in ms
+  @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
+  public KEYS_INDEXER_KEYAPI_FRESHNESS_PERIOD_MS: number = 8 * HOUR;
+
   @IsNumber()
   @Min(1025)
   @Max(65535)
@@ -66,7 +89,7 @@ export class EnvironmentVariables {
   @IsNumber()
   @Min(1000)
   @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
-  public EL_RPC_RESPONSE_TIMEOUT = 60000;
+  public EL_RPC_RESPONSE_TIMEOUT_MS = MINUTE;
 
   @IsNumber()
   @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
@@ -84,7 +107,7 @@ export class EnvironmentVariables {
   @IsNumber()
   @Min(1000)
   @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
-  public CL_API_RESPONSE_TIMEOUT = 60000;
+  public CL_API_RESPONSE_TIMEOUT_MS = MINUTE;
 
   @IsNumber()
   @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
@@ -102,7 +125,7 @@ export class EnvironmentVariables {
   @IsNumber()
   @Min(1000)
   @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
-  public KEYSAPI_API_RESPONSE_TIMEOUT = 60000;
+  public KEYSAPI_API_RESPONSE_TIMEOUT_MS = MINUTE;
 
   @IsNumber()
   @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })

src/common/handlers/handlers.module.ts

@@ -1,8 +1,11 @@
 import { Module } from '@nestjs/common';
 
 import { HandlersService } from './handlers.service';
+import { ProvidersModule } from '../providers/providers.module';
 
 @Module({
+  imports: [ProvidersModule],
   providers: [HandlersService],
+  exports: [HandlersService],
 })
 export class HandlersModule {}