feat: reworked algorithm calculation of withdrawal frame validators

how-estimation-works.md

@@ -85,14 +85,21 @@ where `unfinalized` is the amount of the withdrawal request considered summed wi
 
 If there is not enough ether to fulfill the withdrawal request (`unfinalized > totalBuffer`), the previous case might be appended with the known validators are to be withdrawn (when the `withdrawable_epoch` is assigned).
 
-It's needed to select the Lido-participating validators which are already in process of withdrawal and group them by `withdrawable_epoch` to `frameBalances`, allowing to find the oracle report frame containing enough funds from:
+It's needed to select the Lido-participating validators which are already in process of withdrawal and group them by calculated frame of expected withdrawal to `frameBalances`, allowing to find the oracle report frame containing enough funds from:
 
 - buffer (`totalBuffer`)
 - projectedRewards (`rewardsPerEpoch * epochsTillTheFrame`)
-- frameBalances (`object { [frame]: [sum of balances of validators with withdrawable_epoch for certain frame] }`)
+- frameBalances (`object { [frame]: [sum of balances of validators with calculated withdrawal frame] }`)
 
-So the final formula for that case looks like this:
-`frame (which has engough validator balances) + sweepingMean`. More about `sweepingMean` [here](#sweeping mean).
+#### Algorithm of calculation withdrawal frame of validators:
+
+1. Withdrawals sweep cursor goes from 0 to the last validator index in infinite loop.
+2. When the cursor reaches a withdrawable validator, it withdraws ETH from that validator.
+3. The cursor can withdraw from a maximum of 16 validators per slot.
+4. We assume that all validators in network have to something to withdraw (partially or fully)
+5. `percentOfActiveValidators` is used to exclude inactive validators from the queue, ensuring more accurate calculations.
+6. Formula to get number of slots to wait is `(number of validators to withdraw before cursor get index of validator) / 16`
+7. By knowing number slots we can calculate frame of withdrawal 
 
 ---
 

src/common/execution-provider/execution-provider.service.ts

@@ -1,10 +1,17 @@
 import { SimpleFallbackJsonRpcBatchProvider } from '@lido-nestjs/execution';
 import { CHAINS } from '@lido-nestjs/constants';
 import { Injectable } from '@nestjs/common';
+import { ethers } from 'ethers';
+import { ConfigService } from '@nestjs/config';
+import { PrometheusService } from '../prometheus';
 
 @Injectable()
 export class ExecutionProviderService {
-  constructor(protected readonly provider: SimpleFallbackJsonRpcBatchProvider) {}
+  constructor(
+    protected readonly provider: SimpleFallbackJsonRpcBatchProvider,
+    protected readonly prometheusService: PrometheusService,
+    protected readonly configService: ConfigService,
+  ) {}
 
   /**
    * Returns network name
@@ -22,4 +29,19 @@ export class ExecutionProviderService {
     const { chainId } = await this.provider.getNetwork();
     return chainId;
   }
+
+  // using ethers.JsonRpcProvider direct request to "eth_getBlockByNumber"
+  // default @ethersproject provider getBlock does not contain "withdrawals" property
+  public async getLatestWithdrawals(): Promise<Array<{ validatorIndex: string }>> {
+    const endTimer = this.prometheusService.elRpcRequestDuration.startTimer();
+    try {
+      const provider = new ethers.JsonRpcProvider(this.configService.get('EL_RPC_URLS')[0]);
+      const block = await provider.send('eth_getBlockByNumber', ['latest', false]);
+      endTimer({ result: 'success' });
+      return block.withdrawals;
+    } catch (error) {
+      endTimer({ result: 'error' });
+      throw error;
+    }
+  }
 }

src/http/validators/validators.service.ts

@@ -15,7 +15,7 @@ export class ValidatorsService {
     const lastUpdatedAt = this.validatorsServiceStorage.getLastUpdate();
     const maxExitEpoch = Number(this.validatorsServiceStorage.getMaxExitEpoch());
     const frameBalancesBigNumber = this.validatorsServiceStorage.getFrameBalances();
-    const totalValidators = this.validatorsServiceStorage.getTotal();
+    const totalValidators = this.validatorsServiceStorage.getActiveValidatorsCount();
     const currentFrame = this.genesisTimeService.getFrameOfEpoch(this.genesisTimeService.getCurrentEpoch());
 
     const frameBalances = Object.keys(frameBalancesBigNumber).reduce((acc, item) => {

src/jobs/validators/utils/get-validator-withdrawal-timestamp.ts

@@ -0,0 +1,44 @@
+import { BigNumber } from '@ethersproject/bignumber';
+import { WITHDRAWALS_VALIDATORS_PER_SLOT } from '../validators.constants';
+import { SECONDS_PER_SLOT } from 'common/genesis-time';
+
+/*
+#### Algorithm of calculation withdrawal frame of validators:
+
+1. Withdrawals sweep cursor goes from 0 to the last validator index in infinite loop.
+2. When the cursor reaches a withdrawable validator, it withdraws ETH from that validator.
+3. The cursor can withdraw from a maximum of 16 validators per slot.
+4. We assume that all validators in network have to something to withdraw (partially or fully)
+5. `percentOfActiveValidators` is used to exclude inactive validators from the queue, ensuring more accurate calculations.
+6. Formula to get number of slots to wait is `(number of validators to withdraw before cursor get index of validator) / 16`
+7. By knowing number slots we can calculate frame of withdrawal
+
+Examples:
+
+1. If the current cursor is 50 and the total number of validators is 100,
+   then if we want to know when the validator with index 75 will be withdrawn:
+   (75 - 50) / 16 = 2 slots.
+
+2. If the current cursor is 50 and the total number of validators is 100,
+   and we want to know when the validator with index 25 will be withdrawn
+   (since the cursor will go to the end and start from 0):
+   (100 - 50 + 25) / 16 = 5 slots.
+
+*/
+export function getValidatorWithdrawalTimestamp(
+  index: BigNumber,
+  lastWithdrawalValidatorIndex: BigNumber,
+  activeValidatorCount: number,
+  totalValidatorsCount: number,
+) {
+  const diff = index.sub(lastWithdrawalValidatorIndex);
+  const percentOfActiveValidators = activeValidatorCount / totalValidatorsCount;
+  const lengthQueueValidators = diff.lt(0)
+    ? BigNumber.from(activeValidatorCount).sub(lastWithdrawalValidatorIndex).add(index)
+    : diff;
+
+  const slots = lengthQueueValidators.div(BigNumber.from(WITHDRAWALS_VALIDATORS_PER_SLOT));
+  const seconds = slots.toNumber() * SECONDS_PER_SLOT * percentOfActiveValidators;
+
+  return Date.now() + seconds * 1000;
+}

src/jobs/validators/validators.constants.ts

@@ -9,3 +9,5 @@ export const ORACLE_REPORTS_CRON_BY_CHAIN_ID = {
   [CHAINS.Mainnet]: '30 4/8 * * *', // 4 utc, 12 utc, 20 utc
   [CHAINS.Holesky]: CronExpression.EVERY_3_HOURS, // happens very often, not necessary sync in testnet
 };
+
+export const WITHDRAWALS_VALIDATORS_PER_SLOT = 16;

src/jobs/validators/validators.service.ts

@@ -4,20 +4,22 @@ import { LOGGER_PROVIDER, LoggerService } from 'common/logger';
 import { JobService } from 'common/job';
 import { ConfigService } from 'common/config';
 import { ConsensusProviderService } from 'common/consensus-provider';
+import { ExecutionProviderService } from 'common/execution-provider';
 import { GenesisTimeService } from 'common/genesis-time';
 import { OneAtTime } from '@lido-nestjs/decorators';
 import { ValidatorsStorageService } from 'storage';
 import { FAR_FUTURE_EPOCH, ORACLE_REPORTS_CRON_BY_CHAIN_ID, MAX_SEED_LOOKAHEAD } from './validators.constants';
 import { BigNumber } from '@ethersproject/bignumber';
 import { processValidatorsStream } from 'jobs/validators/utils/validators-stream';
-import { unblock } from '../../common/utils/unblock';
+import { unblock } from 'common/utils/unblock';
 import { LidoKeysService } from './lido-keys';
 import { ResponseValidatorsData, Validator } from './validators.types';
-import { parseGweiToWei } from '../../common/utils/parse-gwei-to-big-number';
+import { parseGweiToWei } from 'common/utils/parse-gwei-to-big-number';
 import { ValidatorsCacheService } from 'storage/validators/validators-cache.service';
 import { CronExpression } from '@nestjs/schedule';
-import { PrometheusService } from '../../common/prometheus';
-import { stringifyFrameBalances } from '../../common/validators/strigify-frame-balances';
+import { PrometheusService } from 'common/prometheus';
+import { stringifyFrameBalances } from 'common/validators/strigify-frame-balances';
+import { getValidatorWithdrawalTimestamp } from './utils/get-validator-withdrawal-timestamp';
 
 export class ValidatorsService {
   static SERVICE_LOG_NAME = 'validators';
@@ -27,6 +29,7 @@ export class ValidatorsService {
 
     protected readonly prometheusService: PrometheusService,
     protected readonly consensusProviderService: ConsensusProviderService,
+    protected readonly executionProviderService: ExecutionProviderService,
     protected readonly configService: ConfigService,
     protected readonly jobService: JobService,
     protected readonly validatorsStorageService: ValidatorsStorageService,
@@ -65,12 +68,12 @@ export class ValidatorsService {
         const data: ResponseValidatorsData = await processValidatorsStream(stream);
         const currentEpoch = this.genesisTimeService.getCurrentEpoch();
 
-        let totalValidators = 0;
+        let activeValidatorCount = 0;
         let latestEpoch = `${currentEpoch + MAX_SEED_LOOKAHEAD + 1}`;
 
         for (const item of data) {
           if (['active_ongoing', 'active_exiting', 'active_slashed'].includes(item.status)) {
-            totalValidators++;
+            activeValidatorCount++;
           }
 
           if (item.validator.exit_epoch !== FAR_FUTURE_EPOCH.toString()) {
@@ -81,7 +84,9 @@ export class ValidatorsService {
 
           await unblock();
         }
-        this.validatorsStorageService.setTotal(totalValidators);
+
+        this.validatorsStorageService.setActiveValidatorsCount(activeValidatorCount);
+        this.validatorsStorageService.setTotalValidatorsCount(data.length);
         this.validatorsStorageService.setMaxExitEpoch(latestEpoch);
         this.validatorsStorageService.setLastUpdate(Math.floor(Date.now() / 1000));
 
@@ -92,7 +97,7 @@ export class ValidatorsService {
         const currentFrame = this.genesisTimeService.getFrameOfEpoch(this.genesisTimeService.getCurrentEpoch());
         this.logger.log('End update validators', {
           service: ValidatorsService.SERVICE_LOG_NAME,
-          totalValidators,
+          activeValidatorCount,
           latestEpoch,
           frameBalances: stringifyFrameBalances(frameBalances),
           currentFrame,
@@ -113,12 +118,18 @@ export class ValidatorsService {
   protected async getLidoValidatorsWithdrawableBalances(validators: Validator[]) {
     const keysData = await this.lidoKeys.fetchLidoKeysData();
     const lidoValidators = await this.lidoKeys.getLidoValidatorsByKeys(keysData.data, validators);
-
+    const lastWithdrawalValidatorIndex = await this.getLastWithdrawalValidatorIndex();
     const frameBalances = {};
 
     for (const item of lidoValidators) {
       if (item.validator.withdrawable_epoch !== FAR_FUTURE_EPOCH.toString() && BigNumber.from(item.balance).gt(0)) {
-        const frame = this.genesisTimeService.getFrameOfEpoch(Number(item.validator.withdrawable_epoch));
+        const withdrawalTimestamp = getValidatorWithdrawalTimestamp(
+          BigNumber.from(item.index),
+          lastWithdrawalValidatorIndex,
+          this.validatorsStorageService.getActiveValidatorsCount(),
+          this.validatorsStorageService.getTotalValidatorsCount(),
+        );
+        const frame = this.genesisTimeService.getFrameByTimestamp(withdrawalTimestamp) + 1;
         const prevBalance = frameBalances[frame];
         const balance = parseGweiToWei(item.balance);
         frameBalances[frame] = prevBalance ? prevBalance.add(balance) : BigNumber.from(balance);
@@ -129,4 +140,9 @@ export class ValidatorsService {
 
     return frameBalances;
   }
+
+  protected async getLastWithdrawalValidatorIndex() {
+    const withdrawals = await this.executionProviderService.getLatestWithdrawals();
+    return BigNumber.from(withdrawals[withdrawals.length - 1].validatorIndex);
+  }
 }

src/storage/validators/validators-cache.service.ts

@@ -50,7 +50,7 @@ export class ValidatorsCacheService {
         return;
       }
 
-      this.validatorsStorage.setTotal(Number(data[0]));
+      this.validatorsStorage.setActiveValidatorsCount(Number(data[0]));
       this.validatorsStorage.setMaxExitEpoch(data[1]);
       this.validatorsStorage.setLastUpdate(Number(data[2]));
       this.validatorsStorage.setFrameBalances(this.parseFrameBalances(data[3]));
@@ -73,7 +73,7 @@ export class ValidatorsCacheService {
 
     await mkdir(ValidatorsCacheService.CACHE_DIR, { recursive: true });
     const data = [
-      this.validatorsStorage.getTotal(),
+      this.validatorsStorage.getActiveValidatorsCount(),
       this.validatorsStorage.getMaxExitEpoch(),
       this.validatorsStorage.getLastUpdate(),
       stringifyFrameBalances(this.validatorsStorage.getFrameBalances()),

src/storage/validators/validators.service.ts

@@ -4,7 +4,8 @@ import { BigNumber } from '@ethersproject/bignumber';
 @Injectable()
 export class ValidatorsStorageService {
   protected maxExitEpoch: string;
-  protected total: number;
+  protected activeValidatorsCount: number;
+  protected totalValidatorsCount: number;
   protected lastUpdate: number;
   protected frameBalances: Record<string, BigNumber>;
 
@@ -20,8 +21,8 @@ export class ValidatorsStorageService {
    * Get total validators
    * @returns total validators number
    */
-  public getTotal(): number {
-    return this.total;
+  public getActiveValidatorsCount(): number {
+    return this.activeValidatorsCount;
   }
 
   /**
@@ -42,10 +43,10 @@ export class ValidatorsStorageService {
 
   /**
    * Updates total validators
-   * @param total - total validators number
+   * @param activeValidatorsCount - total validators number
    */
-  public setTotal(total: number): void {
-    this.total = total;
+  public setActiveValidatorsCount(activeValidatorsCount: number): void {
+    this.activeValidatorsCount = activeValidatorsCount;
   }
 
   /**
@@ -71,4 +72,12 @@ export class ValidatorsStorageService {
   public setFrameBalances(frameBalances: Record<string, BigNumber>): void {
     this.frameBalances = frameBalances;
   }
+
+  public setTotalValidatorsCount(totalValidatorsCount: number) {
+    this.totalValidatorsCount = totalValidatorsCount;
+  }
+
+  public getTotalValidatorsCount() {
+    return this.totalValidatorsCount;
+  }
 }

src/waiting-time/utils/calculate-frame-by-validator-balances.ts

@@ -1,18 +1,14 @@
 import { BigNumber } from '@ethersproject/bignumber';
 
-import { calculateSweepingMean } from './calculate-sweeping-mean';
-
 type calculateFrameByValidatorBalancesArgs = {
   unfinilized: BigNumber;
   rewardsPerFrame: BigNumber;
   currentFrame: number;
-  totalValidators: number;
   frameBalances: Record<string, BigNumber>;
-  epochPerFrame: number;
 };
 
 export const calculateFrameByValidatorBalances = (args: calculateFrameByValidatorBalancesArgs): number | null => {
-  const { frameBalances, unfinilized, totalValidators, epochPerFrame, rewardsPerFrame, currentFrame } = args;
+  const { frameBalances, unfinilized, rewardsPerFrame, currentFrame } = args;
   let unfinalizedAmount = unfinilized;
   let lastFrame = BigNumber.from(currentFrame);
 
@@ -38,13 +34,5 @@ export const calculateFrameByValidatorBalances = (args: calculateFrameByValidato
     }
   }
 
-  if (result === null) return null;
-
-  const sweepingMean = calculateSweepingMean(totalValidators).toNumber();
-  const framesOfSweepingMean = Math.ceil(sweepingMean / epochPerFrame);
-
-  const resultFrame = result.add(framesOfSweepingMean).toNumber();
-
-  // If withdrawable_epoch is less than current frame, should return next frame
-  return resultFrame < currentFrame ? currentFrame + 1 : resultFrame;
+  return result === null ? null : result.toNumber();
 };

src/waiting-time/waiting-time.service.spec.ts

@@ -104,7 +104,7 @@ describe('WaitingTimeService', () => {
         {
           provide: ValidatorsStorageService,
           useValue: {
-            getTotal: jest.fn(),
+            getActiveValidatorsCount: jest.fn(),
             getFrameBalances: jest.fn(),
           },
         },
@@ -142,7 +142,7 @@ describe('WaitingTimeService', () => {
     jest.spyOn(genesisTimeService, 'getFrameByTimestamp').mockImplementation(getFrameByTimestampMock);
     jest.spyOn(genesisTimeService, 'timeToWithdrawalFrame').mockImplementation(timeToWithdrawalFrameMock);
     jest.spyOn(rewardsStorage, 'getRewardsPerFrame').mockReturnValue(rewardsPerFrame);
-    jest.spyOn(validatorsStorage, 'getTotal').mockReturnValue(10000);
+    jest.spyOn(validatorsStorage, 'getActiveValidatorsCount').mockReturnValue(10000);
     jest.spyOn(validatorsStorage, 'getFrameBalances').mockReturnValue({});
     jest.spyOn(service, 'getFrameIsBunker').mockReturnValue(null);
   });

src/waiting-time/waiting-time.service.ts

@@ -221,14 +221,10 @@ export class WaitingTimeService {
 
     // loop over all known frames with balances of withdrawing validators
     const frameBalances = this.validators.getFrameBalances();
-    const epochPerFrame = this.contractConfig.getEpochsPerFrame();
-    const totalValidators = this.validators.getTotal();
     const rewardsPerFrame = this.rewardsStorage.getRewardsPerFrame();
     const valueFrameValidatorsBalance = calculateFrameByValidatorBalances({
       unfinilized: unfinalized.sub(fullBuffer),
       frameBalances,
-      epochPerFrame,
-      totalValidators,
       currentFrame,
       rewardsPerFrame,
     });
@@ -263,7 +259,7 @@ export class WaitingTimeService {
     latestEpoch: string,
   ): Promise<number> {
     // latest epoch of most late to exit validators
-    const totalValidators = this.validators.getTotal();
+    const totalValidators = this.validators.getActiveValidatorsCount();
 
     const churnLimit = Math.max(MIN_PER_EPOCH_CHURN_LIMIT, totalValidators / CHURN_LIMIT_QUOTIENT);
     const epochPerFrame = this.contractConfig.getEpochsPerFrame();
@@ -466,7 +462,7 @@ export class WaitingTimeService {
   public calculateRequestTimeSimple(unfinalizedETH: BigNumber): number {
     const currentEpoch = this.genesisTimeService.getCurrentEpoch();
     const latestEpoch = this.validators.getMaxExitEpoch();
-    const totalValidators = this.validators.getTotal();
+    const totalValidators = this.validators.getActiveValidatorsCount();
 
     const churnLimit = Math.max(MIN_PER_EPOCH_CHURN_LIMIT, totalValidators / CHURN_LIMIT_QUOTIENT);