chore: fix failing sim tests (#6389)

* Disable /ipfs/id/push/1.0.0

* Fix the peer disconnect errors

* Align genesis time for logging

* Fix genesis state issue

* Increase timeotu for keystore unit test

* Remove unintended protocol

* Fix the test timeout

* Update stopChildProcess

* Update the @chainsafe/libp2p-identify

* Compute effective genesis delay

* Update workflow file to make it consistent

* Increase the genesis delay calculation

.github/workflows/test-sim.yml

@@ -58,27 +58,41 @@ jobs:
         if: steps.cache-deps.outputs.cache-hit == 'true'
       # </common-build>
 
+      - name: Download required docker images before running tests
+        run: |
+          docker pull ${{env.GETH_DOCKER_IMAGE}}
+          docker pull ${{env.LIGHTHOUSE_DOCKER_IMAGE}}
+          docker pull ${{env.NETHERMIND_DOCKER_IMAGE}}
+
       - name: Sim tests multifork
         run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:multifork
         working-directory: packages/cli
         env: 
           GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}
 
       - name: Sim tests endpoints
-        run: yarn test:sim:endpoints
+        run: DEBUG='${{github.event.inputs.debug}}'  yarn test:sim:endpoints
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}        
 
       - name: Sim tests deneb
-        run: yarn test:sim:deneb
+        run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:deneb
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}        
 
       - name: Sim tests backup eth provider
-        run: yarn test:sim:backup_eth_provider
+        run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:backup_eth_provider
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}        
 
       - name: Sim tests mixed client
         run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:mixedclient
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}        
 
       - name: Upload debug log test files for "packages/cli"
         if: ${{ always() }}

packages/beacon-node/package.json

@@ -102,6 +102,7 @@
     "@chainsafe/enr": "^3.0.0",
     "@chainsafe/libp2p-gossipsub": "^11.2.0",
     "@chainsafe/libp2p-noise": "^14.1.0",
+    "@chainsafe/libp2p-identify": "^1.0.0",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/ssz": "^0.14.0",

packages/beacon-node/src/network/libp2p/index.ts

@@ -1,7 +1,8 @@
 import {PeerId} from "@libp2p/interface";
 import {Registry} from "prom-client";
 import {ENR} from "@chainsafe/enr";
-import {identify} from "@libp2p/identify";
+// TODO: We should use this fork until https://github.com/libp2p/js-libp2p/pull/2387
+import {identify} from "@chainsafe/libp2p-identify";
 import {bootstrap} from "@libp2p/bootstrap";
 import {mdns} from "@libp2p/mdns";
 import {createLibp2p} from "libp2p";

packages/cli/test/sim/backup_eth_provider.test.ts

@@ -16,6 +16,7 @@ const {estimatedTimeoutMs, forkConfig} = defineSimTestConfig({
   ALTAIR_FORK_EPOCH: altairForkEpoch,
   BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
   runTillEpoch: runTillEpoch + syncWaitEpoch,
+  initialNodes: 3,
 });
 
 const env = await SimulationEnvironment.initWithDefaults(

packages/cli/test/sim/deneb.test.ts

@@ -18,12 +18,13 @@ const {estimatedTimeoutMs, forkConfig} = defineSimTestConfig({
   ALTAIR_FORK_EPOCH: altairForkEpoch,
   BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
   runTillEpoch: runTillEpoch + syncWaitEpoch,
+  initialNodes: 2,
 });
 
 const env = await SimulationEnvironment.initWithDefaults(
   {
-    id: "multi-fork",
-    logsDir: path.join(logFilesDir, "multi-fork"),
+    id: "deneb",
+    logsDir: path.join(logFilesDir, "deneb"),
     forkConfig,
   },
   [

packages/cli/test/sim/endpoints.test.ts

@@ -16,6 +16,7 @@ const {estimatedTimeoutMs, forkConfig} = defineSimTestConfig({
   ALTAIR_FORK_EPOCH: altairForkEpoch,
   BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
   runTillEpoch: 2,
+  initialNodes: 1,
 });
 
 const env = await SimulationEnvironment.initWithDefaults(

packages/cli/test/sim/mixed_client.test.ts

@@ -17,6 +17,7 @@ const {estimatedTimeoutMs, forkConfig} = defineSimTestConfig({
   BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
   CAPELLA_FORK_EPOCH: capellaForkEpoch,
   runTillEpoch: runTillEpoch + syncWaitEpoch,
+  initialNodes: 2,
 });
 
 const env = await SimulationEnvironment.initWithDefaults(

packages/cli/test/sim/multi_fork.test.ts

@@ -27,6 +27,7 @@ const {estimatedTimeoutMs, forkConfig} = defineSimTestConfig({
   BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
   CAPELLA_FORK_EPOCH: capellaForkEpoch,
   runTillEpoch: runTillEpoch + syncWaitEpoch,
+  initialNodes: 5,
 });
 
 const env = await SimulationEnvironment.initWithDefaults(

packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts

@@ -1,6 +1,6 @@
 import fs from "node:fs";
 import {randomBytes} from "node:crypto";
-import {describe, it, expect, beforeEach} from "vitest";
+import {describe, it, expect, beforeEach, vi} from "vitest";
 import tmp from "tmp";
 import {Keystore} from "@chainsafe/bls-keystore";
 import bls from "@chainsafe/bls";
@@ -12,6 +12,7 @@ import {LocalKeystoreDefinition} from "../../../../../src/cmds/validator/keymana
 const numberOfSigners = 10;
 
 describe("keystoreCache", () => {
+  vi.setConfig({testTimeout: 10000, hookTimeout: 50000});
   let definitions: LocalKeystoreDefinition[];
   let signers: SignerLocal[];
   let secretKeys: Uint8Array[];
@@ -50,7 +51,7 @@ describe("keystoreCache", () => {
       passwords.push(password);
       secretKeys.push(secretKey.toBytes());
     }
-  }, 50000);
+  });
 
   describe("writeKeystoreCache", () => {
     it("should write a valid keystore cache file", async () => {

packages/cli/test/utils/simulation/utils/index.ts

@@ -17,18 +17,44 @@ export const avg = (arr: number[]): number => {
   return arr.length === 0 ? 0 : arr.reduce((p, c) => p + c, 0) / arr.length;
 };
 
+function getGenesisDelaySlots(initialNodes?: number): number {
+  if (process.env.GENESIS_DELAY_SLOTS) {
+    const genesisDelaySlots = parseInt(process.env.GENESIS_DELAY_SLOTS);
+    // If custom job is invoked and want to use default genesis delay then provider -1 as value
+    if (genesisDelaySlots >= 0) return genesisDelaySlots;
+  }
+
+  if (initialNodes == null) return 40;
+  // Considering each node consists of EN, BN, VC and KM
+  // EN - Execution Node - 15s
+  const execution = 15;
+  // BN - Beacon Node - 15s
+  const beacon = 15;
+  // VC - Validator Client - 10s
+  const validator = 10;
+  // KM - Key Manager - 3s
+  const keyManager = 3;
+  // Initial script launch time - 10s
+  const initialLaunchScript = 10;
+
+  return Math.ceil(
+    ((execution + beacon + validator + keyManager) * initialNodes + initialLaunchScript) / SIM_TESTS_SECONDS_PER_SLOT
+  );
+}
+
 export function defineSimTestConfig(
   opts: Partial<ChainConfig> & {
     cliqueSealingPeriod?: number;
     additionalSlotsForTTD?: number;
     runTillEpoch: number;
+    // Used to calculate genesis delay
+    initialNodes?: number;
   }
 ): {
   estimatedTimeoutMs: number;
   forkConfig: ChainForkConfig;
 } {
-  const genesisDelaySeconds =
-    (process.env.GENESIS_DELAY_SLOTS ? parseInt(process.env.GENESIS_DELAY_SLOTS) : 40) * SIM_TESTS_SECONDS_PER_SLOT;
+  const genesisDelaySeconds = getGenesisDelaySlots(opts.initialNodes) * SIM_TESTS_SECONDS_PER_SLOT;
 
   const estimatedTimeoutMs =
     getEstimatedTimeInSecForRun({

packages/cli/test/utils/simulation/validator_clients/index.ts

@@ -14,14 +14,14 @@ export async function createValidatorNode<V extends ValidatorClient>(
     "id" | "paths" | "forkConfig" | "nodeIndex" | "genesisTime" | "runner" | "beaconUrls"
   >
 ): Promise<ValidatorNode> {
-  const {runner, forkConfig} = options;
+  const {runner} = options;
   const clId = `${options.id}-${client}`;
 
   const opts: ValidatorGeneratorOptions = {
     ...options,
     id: clId,
     keys: options.keys ?? {type: "no-keys"},
-    genesisTime: options.genesisTime + forkConfig.GENESIS_DELAY,
+    genesisTime: options.genesisTime,
     clientOptions: options.clientOptions ?? {},
     address: "127.0.0.1",
   };

packages/cli/test/utils/simulation/validator_clients/lighthouse.ts

@@ -39,7 +39,6 @@ export const generateLighthouseValidatorNode: ValidatorNodeGenerator<ValidatorCl
     "beacon-nodes": beaconUrls[0],
     "debug-level": "debug",
     "init-slashing-protection": null,
-    "allow-unsynced": null,
     http: null,
     "unencrypted-http-transport": null,
     "http-address": "0.0.0.0",

packages/cli/test/utils/simulation/validator_clients/lodestar.ts

@@ -34,7 +34,7 @@ export const generateLodestarValidatorNode: ValidatorNodeGenerator<ValidatorClie
     "keymanager.address": "127.0.0.1",
     "keymanager.port": ports.validator.keymanagerPort,
     logPrefix: id,
-    logFormatGenesisTime: genesisTime,
+    logFormatGenesisTime: `${genesisTime}`,
     logLevel: LogLevel.debug,
     logFile: "none",
     importKeystores: keystoresDir,

packages/test-utils/src/childProcess.ts

@@ -79,6 +79,19 @@ export async function execChildProcess(cmd: string | string[], options?: ExecChi
   });
 }
 
+/**
+ * Check if process with given pid is running
+ */
+export function isPidRunning(pid: number): boolean {
+  try {
+    // Signal 0 is a special signal that checks if the process exists
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 export const stopChildProcess = async (
   childProcess: childProcess.ChildProcess,
   signal: NodeJS.Signals | number = "SIGTERM"
@@ -87,11 +100,20 @@ export const stopChildProcess = async (
     return;
   }
 
-  return new Promise((resolve, reject) => {
+  const pid = childProcess.pid;
+
+  await new Promise((resolve, reject) => {
     childProcess.once("error", reject);
-    childProcess.once("close", resolve);
+    // We use `exit` instead of `close` as multiple processes can share same `stdio`
+    childProcess.once("exit", resolve);
     childProcess.kill(signal);
   });
+
+  if (pid != null && isPidRunning(pid)) {
+    // Wait for sometime and try to kill this time
+    await sleep(500);
+    await stopChildProcess(childProcess, "SIGKILL");
+  }
 };
 
 /**

yarn.lock

@@ -385,6 +385,23 @@
     uint8arraylist "^2.4.8"
     uint8arrays "^5.0.1"
 
+"@chainsafe/libp2p-identify@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@chainsafe/libp2p-identify/-/libp2p-identify-1.0.0.tgz#28191e619715a87c140d8b516ee85cb7d39e41e0"
+  integrity sha512-X+VWUC0xeCFIulE4BU5M8FmTxZ/OKzku+9/1UaX2EG1LcqQkCDrPi6CCODbE0SraqImG4aVHRbiCFWxKEfE8wQ==
+  dependencies:
+    "@libp2p/interface" "^1.1.2"
+    "@libp2p/interface-internal" "^1.0.7"
+    "@libp2p/peer-id" "^4.0.5"
+    "@libp2p/peer-record" "^7.0.7"
+    "@multiformats/multiaddr" "^12.1.10"
+    "@multiformats/multiaddr-matcher" "^1.1.0"
+    it-protobuf-stream "^1.1.1"
+    protons-runtime "^5.0.0"
+    uint8arraylist "^2.4.7"
+    uint8arrays "^5.0.0"
+    wherearewe "^2.0.1"
+
 "@chainsafe/libp2p-noise@^14.1.0":
   version "14.1.0"
   resolved "https://registry.yarnpkg.com/@chainsafe/libp2p-noise/-/libp2p-noise-14.1.0.tgz#4084a448cec73a941fddfc94751f6ce2e23c07cd"
@@ -1629,6 +1646,21 @@
     uint8arraylist "^2.4.3"
     uint8arrays "^5.0.0"
 
+"@libp2p/peer-record@^7.0.7":
+  version "7.0.7"
+  resolved "https://registry.yarnpkg.com/@libp2p/peer-record/-/peer-record-7.0.7.tgz#e55145b2509592696f42ff73c38f813efbbbc688"
+  integrity sha512-RsggFJVAWQBA2z+ZJsK5nKHDKLmSd89IhFiE5GyImedQFiMkJz/gDFROzfNF2NdOyEBNdRy5SmC9scNFRQQD9A==
+  dependencies:
+    "@libp2p/crypto" "^4.0.1"
+    "@libp2p/interface" "^1.1.2"
+    "@libp2p/peer-id" "^4.0.5"
+    "@libp2p/utils" "^5.2.3"
+    "@multiformats/multiaddr" "^12.1.10"
+    protons-runtime "^5.0.0"
+    uint8-varint "^2.0.2"
+    uint8arraylist "^2.4.7"
+    uint8arrays "^5.0.0"
+
 "@libp2p/peer-store@^10.0.5":
   version "10.0.5"
   resolved "https://registry.yarnpkg.com/@libp2p/peer-store/-/peer-store-10.0.5.tgz#b969d1707f5dcbf4d110e099270de285b075aa02"
@@ -8054,6 +8086,16 @@ it-protobuf-stream@^1.0.2:
     protons-runtime "^5.0.0"
     uint8arraylist "^2.4.1"
 
+it-protobuf-stream@^1.1.1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/it-protobuf-stream/-/it-protobuf-stream-1.1.2.tgz#4444d78fcae0fce949b4cbea622bf1d92667e64f"
+  integrity sha512-epZBuG+7cPaTxCR/Lf3ApshBdA9qfflGPQLfLLrp9VQ0w67Z2xo4H+SLLetav57/29oPtAXwVaoyemg99JOWzA==
+  dependencies:
+    it-length-prefixed-stream "^1.0.0"
+    it-stream-types "^2.0.1"
+    protons-runtime "^5.0.0"
+    uint8arraylist "^2.4.1"
+
 it-pushable@^3.0.0, it-pushable@^3.1.2, it-pushable@^3.2.0, it-pushable@^3.2.1, it-pushable@^3.2.3:
   version "3.2.3"
   resolved "https://registry.yarnpkg.com/it-pushable/-/it-pushable-3.2.3.tgz#e2b80aed90cfbcd54b620c0a0785e546d4e5f334"