Merge branch 'main' into dependabot/npm_and_yarn/braces-3.0.3

README.md

@@ -105,7 +105,7 @@ So, according to the algorithm, there are the following statements:
 
 `--validator-index` - Validator index in the Consensus Layer
 
-`--block` - Block number (slot or root of block on the Consensus Layer)
+`--block` - Block number (slot or root of block on the Consensus Layer which contains the validator withdrawal or evidence of slashing)
 
 `--help` - Show help
 

docker-compose.yml

@@ -17,6 +17,7 @@ services:
     container_name: prover-daemon
     restart: unless-stopped
     environment:
+      - HTTP_PORT=${HTTP_PORT:-8080}
       - DRY_RUN=${DRY_RUN:-false}
       - CHAIN_ID=${CHAIN_ID}
       - KEYSAPI_API_URLS=${KEYSAPI_API_URLS}

src/common/prometheus/prometheus.constants.ts

@@ -15,6 +15,8 @@ export const METRIC_OUTGOING_KEYSAPI_REQUESTS_COUNT = `outgoing_keysapi_requests
 export const METRIC_TASK_DURATION_SECONDS = `task_duration_seconds`;
 export const METRIC_TASK_RESULT_COUNT = `task_result_count`;
 
+export const METRIC_HIGH_GAS_FEE_INTERRUPTIONS_COUNT = `high_gas_fee_interruptions_count`;
+
 export const METRIC_DATA_ACTUALITY = `data_actuality`;
 export const METRIC_LAST_PROCESSED_SLOT_NUMBER = `last_processed_slot_number`;
 export const METRIC_ROOTS_STACK_SIZE = `roots_stack_size`;

src/common/prometheus/prometheus.service.ts

@@ -7,6 +7,7 @@ import { Metric, Options } from './interfaces';
 import {
   METRICS_PREFIX,
   METRIC_BUILD_INFO,
+  METRIC_HIGH_GAS_FEE_INTERRUPTIONS_COUNT,
   METRIC_OUTGOING_CL_REQUESTS_COUNT,
   METRIC_OUTGOING_CL_REQUESTS_DURATION_SECONDS,
   METRIC_OUTGOING_EL_REQUESTS_COUNT,
@@ -119,6 +120,16 @@ export class PrometheusService {
     help: 'Count of passed or failed tasks',
     labelNames: ['name', 'status'],
   });
+
+  public highGasFeeInterruptionsCount = this.getOrCreateMetric('Counter', {
+    name: METRIC_HIGH_GAS_FEE_INTERRUPTIONS_COUNT,
+    help: 'Count of high gas fee interruptions',
+  });
+
+  public txSendingErrors = this.getOrCreateMetric('Counter', {
+    name: 'tx_sending_errors',
+    help: 'Count of transaction sending errors',
+  });
 }
 
 export function TrackCLRequest(target: any, propertyKey: string, descriptor: PropertyDescriptor) {

src/common/prover/duties/slashings.ts

@@ -52,7 +52,7 @@ export class SlashingsService {
     this.logger.log(`Building slashing proof payloads`);
     const payloads = this.buildSlashingsProofPayloads(finalizedHeader, nextHeaderTs, stateView, slashings);
     for (const payload of payloads) {
-      this.logger.warn(`📡 Sending slashing proof payload for validator index: ${payload.witness.validatorIndex}`);
+      this.logger.log(`📡 Sending slashing proof payload for validator index: ${payload.witness.validatorIndex}`);
       await this.verifier.sendSlashingProof(payload);
     }
   }
@@ -96,8 +96,9 @@ export class SlashingsService {
   ): Generator<SlashingProofPayload> {
     for (const [valIndex, keyInfo] of Object.entries(slashings)) {
       const validator = stateView.validators.get(Number(valIndex));
+      this.logger.log(`Generating validator [${valIndex}] proof`);
       const validatorProof = generateValidatorProof(stateView, Number(valIndex));
-      // verify validator proof
+      this.logger.log('Verifying validator proof locally');
       verifyProof(stateView.hashTreeRoot(), validatorProof.gindex, validatorProof.witnesses, validator.hashTreeRoot());
       yield {
         keyIndex: keyInfo.keyIndex,

src/common/prover/duties/withdrawals.ts

@@ -97,7 +97,7 @@ export class WithdrawalsService {
       withdrawals,
     );
     for (const payload of payloads) {
-      this.logger.warn(`📡 Sending withdrawal proof payload for validator index: ${payload.witness.validatorIndex}`);
+      this.logger.log(`📡 Sending withdrawal proof payload for validator index: ${payload.witness.validatorIndex}`);
       await this.verifier.sendWithdrawalProof(payload);
     }
   }
@@ -130,7 +130,7 @@ export class WithdrawalsService {
       withdrawals,
     );
     for (const payload of payloads) {
-      this.logger.warn(
+      this.logger.log(
         `📡 Sending historical withdrawal proof payload for validator index: ${payload.witness.validatorIndex}`,
       );
       await this.verifier.sendHistoricalWithdrawalProof(payload);
@@ -166,15 +166,17 @@ export class WithdrawalsService {
         this.logger.warn(`Validator ${valIndex} is not full withdrawn. Just huge amount of ETH. Skipped`);
         continue;
       }
+      this.logger.log(`Generating validator [${valIndex}] proof`);
       const validatorProof = generateValidatorProof(stateView, Number(valIndex));
+      this.logger.log('Generating withdrawal proof');
       const withdrawalProof = generateWithdrawalProof(
         stateView,
         currentBlockView,
         keyWithWithdrawalInfo.withdrawal.offset,
       );
-      // verify validator proof
+      this.logger.log('Verifying validator proof locally');
       verifyProof(stateView.hashTreeRoot(), validatorProof.gindex, validatorProof.witnesses, validator.hashTreeRoot());
-      // verify withdrawal proof
+      this.logger.log('Verifying withdrawal proof locally');
       verifyProof(
         stateView.hashTreeRoot(),
         withdrawalProof.gindex,
@@ -236,26 +238,29 @@ export class WithdrawalsService {
         this.logger.warn(`Validator ${valIndex} is not full withdrawn. Just huge amount of ETH. Skipped`);
         continue;
       }
+      this.logger.log(`Generating validator [${valIndex}] proof`);
       const validatorProof = generateValidatorProof(stateWithWdsView, Number(valIndex));
+      this.logger.log('Generating withdrawal proof');
       const withdrawalProof = generateWithdrawalProof(
         stateWithWdsView,
         blockWithWdsView,
         keyWithWithdrawalInfo.withdrawal.offset,
       );
+      this.logger.log('Generating historical state proof');
       const historicalStateProof = generateHistoricalStateProof(
         finalizedStateView,
         summaryStateView,
         summaryIndex,
         rootIndexInSummary,
       );
-      // verify validator proof
+      this.logger.log('Verifying validator proof locally');
       verifyProof(
         stateWithWdsView.hashTreeRoot(),
         validatorProof.gindex,
         validatorProof.witnesses,
         validator.hashTreeRoot(),
       );
-      // verify withdrawal proof
+      this.logger.log('Verifying withdrawal proof locally');
       verifyProof(
         stateWithWdsView.hashTreeRoot(),
         withdrawalProof.gindex,
@@ -266,7 +271,7 @@ export class WithdrawalsService {
           .get(keyWithWithdrawalInfo.withdrawal.offset)
           .hashTreeRoot(),
       );
-      // verify historical state proof
+      this.logger.log('Verifying historical state proof locally');
       verifyProof(
         finalizedStateView.hashTreeRoot(),
         historicalStateProof.gindex,

src/common/providers/execution/execution.ts

@@ -1,12 +1,15 @@
+import { TransactionResponse } from '@ethersproject/abstract-provider';
 import { MAX_BLOCKCOUNT, SimpleFallbackJsonRpcBatchProvider } from '@lido-nestjs/execution';
 import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
 import { Inject, Injectable, LoggerService, Optional } from '@nestjs/common';
 import { PopulatedTransaction, Wallet, utils } from 'ethers';
 import { InquirerService } from 'nest-commander';
+import { promise as spinnerFor } from 'ora-classic';
 
 import { bigIntMax, bigIntMin, percentile } from './utils/common';
 import { ConfigService } from '../../config/config.service';
 import { WorkingMode } from '../../config/env.validation';
+import { PrometheusService } from '../../prometheus/prometheus.service';
 
 class ErrorWithContext extends Error {
   public readonly context: any;
@@ -35,6 +38,7 @@ export class Execution {
   constructor(
     @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
     protected readonly config: ConfigService,
+    @Optional() protected readonly prometheus: PrometheusService,
     @Optional() protected readonly inquirerService: InquirerService,
     public readonly provider: SimpleFallbackJsonRpcBatchProvider,
   ) {
@@ -47,15 +51,27 @@ export class Execution {
     populateTxCallback: (...payload: any[]) => Promise<PopulatedTransaction>,
     payload: any[],
   ): Promise<void> {
-    try {
-      await this._execute(emulateTxCallback, populateTxCallback, payload);
-    } catch (e) {
-      if (e instanceof NoSignerError || e instanceof DryRunError) {
-        this.logger.warn(e);
+    // endless loop to retry transaction execution in case of high gas fee
+    while (true) {
+      try {
+        await this._execute(emulateTxCallback, populateTxCallback, payload);
         return;
+      } catch (e) {
+        if (e instanceof NoSignerError || e instanceof DryRunError) {
+          this.logger.warn(e);
+          return;
+        }
+        if (!this.isCLI() && e instanceof HighGasFeeError) {
+          this.prometheus.highGasFeeInterruptionsCount.inc();
+          this.logger.warn(e);
+          this.logger.warn('Retrying in 1 minute...');
+          await new Promise((resolve) => setTimeout(resolve, 60 * 1000));
+          continue;
+        }
+        this.prometheus.txSendingErrors.inc();
+        this.logger.error(e);
+        throw e;
       }
-      this.logger.error(e);
-      throw e;
     }
   }
 
@@ -67,12 +83,13 @@ export class Execution {
     this.logger.debug!(payload);
     const tx = await populateTxCallback(...payload);
     let context: { payload: any[]; tx?: any } = { payload, tx };
+    this.logger.log('Emulating call');
     try {
       await emulateTxCallback(...payload);
-      this.logger.log('✅ Emulated call succeeded');
     } catch (e) {
       throw new EmulatedCallError(e, context);
     }
+    this.logger.log('✅ Emulated call succeeded');
     if (!this.signer) {
       throw new NoSignerError('No specified signer. Only emulated calls are available', context);
     }
@@ -88,7 +105,7 @@ export class Execution {
       throw new DryRunError('Dry run mode is enabled. Transaction is prepared, but not sent', context);
     }
     const isFeePerGasAcceptable = await this.isFeePerGasAcceptable();
-    if (this.config.get('WORKING_MODE') == WorkingMode.CLI) {
+    if (this.isCLI()) {
       const opts = await this.inquirerService.ask('tx-execution', {} as { sendingConfirmed: boolean });
       if (!opts.sendingConfirmed) {
         throw new UserCancellationError('Transaction is not sent due to user cancellation', context);
@@ -99,13 +116,29 @@ export class Execution {
       }
     }
     const signed = await this.signer.signTransaction(populated);
+    let submitted: TransactionResponse;
     try {
-      const submitted = await this.provider.sendTransaction(signed);
-      await submitted.wait();
+      const submittedPromise = this.provider.sendTransaction(signed);
+      let msg = `Sending transaction with nonce ${populated.nonce} and gasLimit: ${populated.gasLimit}, maxFeePerGas: ${populated.maxFeePerGas}, maxPriorityFeePerGas: ${populated.maxPriorityFeePerGas}`;
+      if (this.isCLI()) {
+        spinnerFor(submittedPromise, { text: msg });
+      } else {
+        this.logger.log(msg);
+      }
+      submitted = await submittedPromise;
+      this.logger.log(`Transaction sent to mempool. Hash: ${submitted.hash}`);
+      const waitingPromise = submitted.wait();
+      msg = `Waiting until the transaction has been mined`;
+      if (this.isCLI()) {
+        spinnerFor(waitingPromise, { text: msg });
+      } else {
+        this.logger.log(msg);
+      }
+      await waitingPromise;
     } catch (e) {
       throw new SendTransactionError(e, context);
     }
-    this.logger.log('✅ Transaction succeeded');
+    this.logger.log(`✅ Transaction succeeded! Hash: ${submitted?.hash}`);
   }
 
   //
@@ -183,4 +216,8 @@ export class Execution {
     ];
     this.lastFeeHistoryBlockNumber = latestBlockNumber;
   }
+
+  private isCLI(): boolean {
+    return this.config.get('WORKING_MODE') == WorkingMode.CLI;
+  }
 }

src/daemon/daemon.module.ts

@@ -6,13 +6,14 @@ import { RootsProcessor } from './services/roots-processor';
 import { RootsProvider } from './services/roots-provider';
 import { RootsStack } from './services/roots-stack';
 import { ConfigModule } from '../common/config/config.module';
+import { HealthModule } from '../common/health/health.module';
 import { LoggerModule } from '../common/logger/logger.module';
 import { PrometheusModule } from '../common/prometheus/prometheus.module';
 import { ProverModule } from '../common/prover/prover.module';
 import { ProvidersModule } from '../common/providers/providers.module';
 
 @Module({
-  imports: [LoggerModule, ConfigModule, PrometheusModule, ProvidersModule, ProverModule],
+  imports: [LoggerModule, ConfigModule, HealthModule, PrometheusModule, ProvidersModule, ProverModule],
   providers: [DaemonService, KeysIndexer, RootsProvider, RootsProcessor, RootsStack],
   exports: [DaemonService],
 })

src/daemon/services/roots-stack.ts

@@ -41,9 +41,9 @@ export class RootsStack implements OnModuleInit, OnApplicationBootstrap {
   }
 
   public getNextEligible(): RootSlot | undefined {
-    for (const slot in this.storage.data) {
-      if (this.keysIndexer.isTrustedForAnyDuty(Number(slot))) {
-        return { blockRoot: this.storage.data[slot], slotNumber: Number(slot) };
+    for (const slot of Object.keys(this.storage.data).map(Number).sort()) {
+      if (this.keysIndexer.isTrustedForAnyDuty(slot)) {
+        return { blockRoot: this.storage.data[slot], slotNumber: slot };
       }
     }
   }