Merge pull request #36 from lidofinance/develop

develop to main

.github/workflows/ci-dev.yml

@@ -0,0 +1,29 @@
+name: CI Dev
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - develop
+    paths-ignore:
+      - ".github/**"
+
+permissions: {}
+
+jobs:
+  # test:
+  #   ...
+
+  deploy:
+    runs-on: ubuntu-latest
+    # needs: test
+    name: Build and deploy
+    steps:
+      - name: Testnet deploy
+        uses: lidofinance/dispatch-workflow@v1
+        env:
+          APP_ID: ${{ secrets.APP_ID }}
+          APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }}
+          TARGET_REPO: "lidofinance/infra-mainnet"
+          TARGET_WORKFLOW: "deploy_testnet_csm_prover_tool.yaml"
+          TARGET: "develop"

.github/workflows/ci-staging.yml

@@ -0,0 +1,30 @@
+name: CI Staging
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - ".github/**"
+
+permissions: {}
+
+jobs:
+  # test:
+  #   ...
+
+  deploy:
+    runs-on: ubuntu-latest
+    # needs: test
+    name: Build and deploy
+    steps:
+      - name: Staging deploy
+        uses: lidofinance/dispatch-workflow@v1
+        env:
+          APP_ID: ${{ secrets.APP_ID }}
+          APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }}
+          TARGET_REPO: "lidofinance/infra-mainnet"
+          TARGET_WORKFLOW: "deploy_staging_mainnet_csm_prover_tool.yaml"
+          TARGET: "main"
+

Dockerfile.daemon → Dockerfile

@@ -1,4 +1,9 @@
-FROM node:20.12.1-alpine as building
+FROM node:20.12.1-bookworm-slim AS building
+
+RUN apt-get update && apt-get install -y --no-install-recommends -qq \
+    curl=7.88.1-10+deb12u7 \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
@@ -9,7 +14,7 @@ COPY ./src ./src
 RUN yarn install --frozen-lockfile --non-interactive && yarn cache clean && yarn typechain
 RUN yarn build
 
-FROM node:20.12.1-alpine
+FROM building AS production
 
 WORKDIR /app
 
@@ -22,6 +27,6 @@ RUN mkdir -p ./storage/ && chown -R node:node ./storage/
 USER node
 
 HEALTHCHECK --interval=120s --timeout=60s --retries=3 \
-  CMD sh -c "wget -nv -t1 --spider http://localhost:$HTTP_PORT/health" || exit 1
+    CMD curl -f http://localhost:$HTTP_PORT/health || exit 1
 
 CMD ["yarn", "start:prod"]

Dockerfile.cli

@@ -1,9 +1,9 @@
-FROM node:20.12.1-alpine as building
+FROM node:20.12.1-alpine AS building
 
 WORKDIR /app
 
 COPY package.json yarn.lock ./
-COPY ./tsconfig*.json ./
+COPY ./tsconfig*.json ./nest-cli.json ./.swcrc ./
 COPY ./src ./src
 
 RUN yarn install --frozen-lockfile --non-interactive && yarn cache clean && yarn typechain

docker-compose.yml

@@ -45,7 +45,7 @@ services:
   daemon:
     build:
       context: .
-      dockerfile: ./Dockerfile.daemon
+      dockerfile: ./Dockerfile
     container_name: prover-daemon
     restart: unless-stopped
     environment:

src/common/providers/base/rest-provider.ts

@@ -3,7 +3,7 @@ import { request } from 'undici';
 import { IncomingHttpHeaders } from 'undici/types/header';
 import BodyReadable from 'undici/types/readable';
 
-import { RequestOptions, RequestPolicy, rejectDelay, retrier } from './utils/func';
+import { RequestOptions, RequestPolicy, rejectDelay, retrier, urljoin } from './utils/func';
 import { PrometheusService } from '../../prometheus';
 
 export type RetryOptions = RequestOptions &
@@ -97,7 +97,7 @@ export abstract class BaseRestProvider {
       requestPolicy: this.requestPolicy,
       ...options,
     } as RequestOptions;
-    const { body, headers, statusCode } = await request(new URL(endpoint, base), {
+    const { body, headers, statusCode } = await request(urljoin(base, endpoint), {
       method: 'GET',
       headersTimeout: (options.requestPolicy as RequestPolicy).timeout,
       signal: options.signal,
@@ -123,7 +123,7 @@ export abstract class BaseRestProvider {
       requestPolicy: this.requestPolicy,
       ...options,
     } as RequestOptions;
-    const { body, headers, statusCode } = await request(new URL(endpoint, base), {
+    const { body, headers, statusCode } = await request(urljoin(base, endpoint), {
       method: 'POST',
       headersTimeout: (options.requestPolicy as RequestPolicy).timeout,
       signal: options.signal,

src/common/providers/base/utils/func.ts

@@ -56,3 +56,69 @@ export const retrier = (
     }
   };
 };
+
+function normalize(strArray: string[]) {
+  const resultArray = [];
+  if (strArray.length === 0) {
+    return '';
+  }
+
+  if (typeof strArray[0] !== 'string') {
+    throw new TypeError('Url must be a string. Received ' + strArray[0]);
+  }
+
+  // If the first part is a plain protocol, we combine it with the next part.
+  if (strArray[0].match(/^[^/:]+:\/*$/) && strArray.length > 1) {
+    const first = strArray.shift();
+    strArray[0] = first + strArray[0];
+  }
+
+  // There must be two or three slashes in the file protocol, two slashes in anything else.
+  if (strArray[0].match(/^file:\/\/\//)) {
+    strArray[0] = strArray[0].replace(/^([^/:]+):\/*/, '$1:///');
+  } else {
+    strArray[0] = strArray[0].replace(/^([^/:]+):\/*/, '$1://');
+  }
+
+  for (let i = 0; i < strArray.length; i++) {
+    let component = strArray[i];
+
+    if (typeof component !== 'string') {
+      throw new TypeError('Url must be a string. Received ' + component);
+    }
+
+    if (component === '') {
+      continue;
+    }
+
+    if (i > 0) {
+      // Removing the starting slashes for each component but the first.
+      component = component.replace(/^[\/]+/, '');
+    }
+    if (i < strArray.length - 1) {
+      // Removing the ending slashes for each component but the last.
+      component = component.replace(/[\/]+$/, '');
+    } else {
+      // For the last component we will combine multiple slashes to a single one.
+      component = component.replace(/[\/]+$/, '/');
+    }
+
+    resultArray.push(component);
+  }
+
+  let str = resultArray.join('/');
+  // Each input component is now separated by a single slash except the possible first plain protocol part.
+
+  // remove trailing slash before parameters or hash
+  str = str.replace(/\/(\?|&|#[^!])/g, '$1');
+
+  // replace ? in parameters with &
+  const parts = str.split('?');
+  str = parts.shift() + (parts.length > 0 ? '?' : '') + parts.join('&');
+
+  return str;
+}
+
+export function urljoin(...args: string[]): string {
+  return normalize(args);
+}

src/daemon/services/keys-indexer.ts

@@ -114,7 +114,11 @@ export class KeysIndexer implements OnModuleInit, OnApplicationBootstrap {
     // We shouldn't wait for task to finish
     // to avoid block processing if indexing fails or stuck
     this.startedAt = Date.now();
-    this.baseRun(stateRoot, slot, this.updateStorage)
+    this.baseRun(
+      stateRoot,
+      slot,
+      async (validators, finalizedSlot) => await this.updateStorage(validators, finalizedSlot),
+    )
       .catch((e) => this.logger.error(e))
       .finally(() => (this.startedAt = 0));
   }
@@ -131,10 +135,11 @@ export class KeysIndexer implements OnModuleInit, OnApplicationBootstrap {
     const stateView = this.consensus.stateToView(state.bodyBytes, state.forkName);
     this.logger.log(`Total validators count: ${stateView.validators.length}`);
     // TODO: do we need to store already full withdrawn keys ?
+    const currValidatorsCount = stateView.validators.length;
     await stateDataProcessingCallback(stateView.validators, finalizedSlot);
     this.logger.log(`CSM validators count: ${Object.keys(this.storage.data).length}`);
     this.info.data.storageStateSlot = finalizedSlot;
-    this.info.data.lastValidatorsCount = stateView.validators.length;
+    this.info.data.lastValidatorsCount = currValidatorsCount;
     await this.info.write();
     await this.storage.write();
   }
@@ -224,22 +229,27 @@ export class KeysIndexer implements OnModuleInit, OnApplicationBootstrap {
       const finalized = await this.consensus.getBeaconHeader('finalized');
       const finalizedSlot = Number(finalized.header.message.slot);
       const stateRoot = finalized.header.message.state_root;
-      await this.baseRun(stateRoot, finalizedSlot, this.initStorage);
+      await this.baseRun(
+        stateRoot,
+        finalizedSlot,
+        async (validators, finalizedSlot) => await this.initStorage(validators, finalizedSlot),
+      );
     }
   }
 
-  initStorage = async (validators: Validators, finalizedSlot: Slot): Promise<void> => {
+  private async initStorage(validators: Validators, finalizedSlot: Slot): Promise<void> {
     const csmKeys = await this.keysapi.getModuleKeys(this.info.data.moduleId);
     this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta);
     const keysMap = new Map<string, { operatorIndex: number; index: number }>();
     csmKeys.data.keys.forEach((k: Key) => keysMap.set(k.key, { ...k }));
+    const valLength = validators.length;
     const iterator = iterateNodesAtDepth(
       validators.type.tree_getChunksNode(validators.node),
       validators.type.chunkDepth,
       0,
-      validators.length,
+      valLength,
     );
-    for (let i = 0; i < validators.length; i++) {
+    for (let i = 0; i < valLength; i++) {
       const node = iterator.next().value;
       const v = node.value;
       const pubKey = toHex(v.pubkey);
@@ -251,12 +261,14 @@ export class KeysIndexer implements OnModuleInit, OnApplicationBootstrap {
         pubKey: pubKey,
       };
     }
-  };
+    iterator.return && iterator.return();
+  }
 
-  updateStorage = async (validators: Validators, finalizedSlot: Slot): Promise<void> => {
+  private async updateStorage(validators: Validators, finalizedSlot: Slot): Promise<void> {
     // TODO: should we think about re-using validator indexes?
     // TODO: should we think about changing WC for existing old vaidators ?
-    const appearedValsCount = validators.length - this.info.data.lastValidatorsCount;
+    const valLength = validators.length;
+    const appearedValsCount = valLength - this.info.data.lastValidatorsCount;
     if (appearedValsCount == 0) {
       this.logger.log(`No new validators in the state`);
       return;
@@ -269,7 +281,7 @@ export class KeysIndexer implements OnModuleInit, OnApplicationBootstrap {
       appearedValsCount,
     );
     const valKeys = [];
-    for (let i = this.info.data.lastValidatorsCount; i < validators.length; i++) {
+    for (let i = this.info.data.lastValidatorsCount; i < valLength; i++) {
       const node = iterator.next().value;
       const v = validators.type.elementType.tree_toValue(node);
       valKeys.push(toHex(v.pubkey));
@@ -278,8 +290,9 @@ export class KeysIndexer implements OnModuleInit, OnApplicationBootstrap {
     const csmKeys = await this.keysapi.findModuleKeys(this.info.data.moduleId, valKeys);
     this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta);
     this.logger.log(`New appeared CSM validators count: ${csmKeys.data.keys.length}`);
+    const valKeysLength = valKeys.length;
     for (const csmKey of csmKeys.data.keys) {
-      for (let i = 0; i < valKeys.length; i++) {
+      for (let i = 0; i < valKeysLength; i++) {
         if (valKeys[i] != csmKey.key) continue;
         const index = i + this.info.data.lastValidatorsCount;
         this.storage.data[index] = {
@@ -289,7 +302,8 @@ export class KeysIndexer implements OnModuleInit, OnApplicationBootstrap {
         };
       }
     }
-  };
+    iterator.return && iterator.return();
+  }
 
   private setMetrics() {
     const info = () => this.info.data;