test(prover): improve test coverage

packages/cli/package.json

@@ -99,6 +99,7 @@
     "@types/got": "^9.6.12",
     "@types/inquirer": "^9.0.3",
     "@types/lodash": "^4.14.192",
-    "@types/yargs": "^17.0.24"
+    "@types/yargs": "^17.0.24",
+    "@lodestar/test-utils": "^1.9.1"
   }
 }

packages/cli/test/e2e/blsToExecutionchange.test.ts

@@ -4,36 +4,41 @@ import {ApiError, getClient} from "@lodestar/api";
 import {config} from "@lodestar/config/default";
 import {interopSecretKey} from "@lodestar/state-transition";
 import {toHexString} from "@chainsafe/ssz";
+import {execCliCommand, spawnCliCommand, stopChildProcess} from "@lodestar/test-utils";
 import {testFilesDir} from "../utils.js";
-import {describeCliTest, execCli} from "../utils/childprocRunner.js";
-import {itDone} from "../utils/runUtils.js";
 
-describeCliTest("bLSToExecutionChange cmd", function ({spawnCli}) {
+describe("bLSToExecutionChange cmd", function () {
   this.timeout("60s");
 
-  itDone("Perform bLSToExecutionChange", async function (done) {
+  it("Perform bLSToExecutionChange", async () => {
     const restPort = 9596;
 
-    const devBnProc = spawnCli({pipeStdToParent: false, logPrefix: "dev"}, [
-      // ⏎
-      "dev",
-      `--dataDir=${path.join(testFilesDir, "dev-bls-to-execution-change")}`,
-      "--genesisValidators=8",
-      "--startValidators=0..7",
-      "--rest",
-      `--rest.port=${restPort}`,
-      // Speed up test to make genesis happen faster
-      "--params.SECONDS_PER_SLOT=2",
-    ]);
+    const devBnProc = await spawnCliCommand(
+      "packages/cli/bin/lodestar.js",
+      [
+        "dev",
+        `--dataDir=${path.join(testFilesDir, "dev-bls-to-execution-change")}`,
+        "--genesisValidators=8",
+        "--startValidators=0..7",
+        "--rest",
+        `--rest.port=${restPort}`,
+        // Speed up test to make genesis happen faster
+        "--params.SECONDS_PER_SLOT=2",
+      ],
+      {pipeStdioToParent: false, logPrefix: "dev"}
+    );
+
     // Exit early if process exits
     devBnProc.on("exit", (code) => {
       if (code !== null && code > 0) {
-        done(Error(`devBnProc process exited with code ${code}`));
+        throw new Error(`devBnProc process exited with code ${code}`);
       }
     });
 
     const baseUrl = `http://127.0.0.1:${restPort}`;
-    const client = getClient({baseUrl}, {config});
+    // To cleanup the event stream connection
+    const httpClientController = new AbortController();
+    const client = getClient({baseUrl, getAbortSignal: () => httpClientController.signal}, {config});
 
     // Wait for beacon node API to be available + genesis
     await retry(
@@ -57,8 +62,7 @@ describeCliTest("bLSToExecutionChange cmd", function ({spawnCli}) {
     // 2 0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4
     // 3 0x88c141df77cd9d8d7a71a75c826c41a9c9f03c6ee1b180f3e7852f6a280099ded351b58d66e653
 
-    await execCli([
-      // ⏎
+    await execCliCommand("packages/cli/bin/lodestar.js", [
       "validator",
       "bls-to-execution-change",
       "--network=dev",
@@ -80,8 +84,9 @@ describeCliTest("bLSToExecutionChange cmd", function ({spawnCli}) {
       throw Error("Invalid message generated");
     }
 
+    httpClientController.abort();
     devBnProc.kill("SIGINT");
     await sleep(1000);
-    devBnProc.kill("SIGKILL");
+    await stopChildProcess(devBnProc, "SIGKILL");
   });
 });

packages/cli/test/e2e/importFromFsDirect.test.ts

@@ -1,14 +1,16 @@
 import fs from "node:fs";
 import path from "node:path";
 import {rimraf} from "rimraf";
+import {getMochaContext} from "@lodestar/test-utils/mocha";
 import {testFilesDir} from "../utils.js";
-import {describeCliTest} from "../utils/childprocRunner.js";
-import {getAfterEachCallbacks} from "../utils/runUtils.js";
 import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js";
-import {expectKeys, getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js";
+import {expectKeys, startValidatorWithKeyManager} from "../utils/validator.js";
 import {getKeystoresStr} from "../utils/keystores.js";
 
-describeCliTest("import from fs same cmd as validate", function ({spawnCli}) {
+describe("import from fs same cmd as validate", function () {
+  const testContext = getMochaContext(this);
+  this.timeout("30s");
+
   const dataDir = path.join(testFilesDir, "import-and-validate-test");
   const importFromDir = path.join(dataDir, "eth2.0_deposit_out");
   const passphraseFilepath = path.join(importFromDir, "password.text");
@@ -18,9 +20,6 @@ describeCliTest("import from fs same cmd as validate", function ({spawnCli}) {
     rimraf.sync(importFromDir);
   });
 
-  const afterEachCallbacks = getAfterEachCallbacks();
-  const itKeymanagerStep = getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir});
-
   const passphrase = "AAAAAAAA0000000000";
   const keyCount = 2;
   const pubkeys = cachedPubkeysHex.slice(0, keyCount);
@@ -38,18 +37,23 @@ describeCliTest("import from fs same cmd as validate", function ({spawnCli}) {
   });
 
   // Check that there are not keys loaded without adding extra args `--importKeystores`
-  itKeymanagerStep("run 'validator' check keys are loaded", async function (keymanagerClient) {
+  it("run 'validator' there are no keys loaded", async () => {
+    const {keymanagerClient} = await startValidatorWithKeyManager([], {
+      dataDir,
+      logPrefix: "case-1",
+      testContext,
+    });
+
     await expectKeys(keymanagerClient, [], "Wrong listKeys response data");
   });
 
   // Run validator with extra arguments to load keystores in same step
-  itKeymanagerStep(
-    "run 'validator' check keys are loaded",
-    async function (keymanagerClient) {
-      await expectKeys(keymanagerClient, pubkeys, "Wrong listKeys response data");
-    },
-    {
-      validatorCmdExtraArgs: [`--importKeystores=${importFromDir}`, `--importKeystoresPassword=${passphraseFilepath}`],
-    }
-  );
+  it("run 'validator' check keys are loaded", async () => {
+    const {keymanagerClient} = await startValidatorWithKeyManager(
+      [`--importKeystores=${importFromDir}`, `--importKeystoresPassword=${passphraseFilepath}`],
+      {dataDir, logPrefix: "case-2", testContext}
+    );
+
+    await expectKeys(keymanagerClient, pubkeys, "Wrong listKeys response data");
+  });
 });

packages/cli/test/e2e/importFromFsPreStep.test.ts

@@ -2,14 +2,17 @@ import fs from "node:fs";
 import path from "node:path";
 import {rimraf} from "rimraf";
 import {expect} from "chai";
+import {getMochaContext} from "@lodestar/test-utils/mocha";
+import {execCliCommand} from "@lodestar/test-utils";
 import {testFilesDir} from "../utils.js";
-import {describeCliTest, execCli} from "../utils/childprocRunner.js";
-import {getAfterEachCallbacks} from "../utils/runUtils.js";
 import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js";
-import {expectKeys, getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js";
+import {expectKeys, startValidatorWithKeyManager} from "../utils/validator.js";
 import {getKeystoresStr} from "../utils/keystores.js";
 
-describeCliTest("import from fs then validate", function ({spawnCli}) {
+describe("import from fs then validate", function () {
+  const testContext = getMochaContext(this);
+  this.timeout("30s");
+
   const dataDir = path.join(testFilesDir, "import-then-validate-test");
   const importFromDir = path.join(dataDir, "eth2.0_deposit_out");
   const passphraseFilepath = path.join(importFromDir, "password.text");
@@ -19,9 +22,6 @@ describeCliTest("import from fs then validate", function ({spawnCli}) {
     rimraf.sync(importFromDir);
   });
 
-  const afterEachCallbacks = getAfterEachCallbacks();
-  const itKeymanagerStep = getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir});
-
   const passphrase = "AAAAAAAA0000000000";
   const keyCount = 2;
   const pubkeys = cachedPubkeysHex.slice(0, keyCount);
@@ -37,8 +37,7 @@ describeCliTest("import from fs then validate", function ({spawnCli}) {
       fs.writeFileSync(path.join(importFromDir, `keystore_${i}.json`), keystoresStr[i]);
     }
 
-    const stdout = await execCli([
-      // ⏎
+    const stdout = await execCliCommand("packages/cli/bin/lodestar.js", [
       "validator import",
       `--dataDir ${dataDir}`,
       `--importKeystores ${importFromDir}`,
@@ -54,18 +53,16 @@ describeCliTest("import from fs then validate", function ({spawnCli}) {
     fs.mkdirSync(path.join(dataDir, "keystores"), {recursive: true});
     fs.mkdirSync(path.join(dataDir, "secrets"), {recursive: true});
 
-    const stdout = await execCli([
-      // ⏎
-      "validator list",
-      `--dataDir ${dataDir}`,
-    ]);
+    const stdout = await execCliCommand("packages/cli/bin/lodestar.js", ["validator list", `--dataDir ${dataDir}`]);
 
     for (let i = 0; i < keyCount; i++) {
       expect(stdout).includes(pubkeys[i], `stdout should include imported pubkey[${i}]`);
     }
   });
 
-  itKeymanagerStep("run 'validator' check keys are loaded", async function (keymanagerClient) {
+  it("run 'validator' check keys are loaded", async function () {
+    const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext});
+
     await expectKeys(keymanagerClient, pubkeys, "Wrong listKeys response data");
   });
 });

packages/cli/test/e2e/importKeystoresFromApi.test.ts

@@ -5,23 +5,24 @@ import {DeletionStatus, getClient, ImportStatus} from "@lodestar/api/keymanager"
 import {config} from "@lodestar/config/default";
 import {Interchange} from "@lodestar/validator";
 import {ApiError, HttpStatusCode} from "@lodestar/api";
+import {bufferStderr, spawnCliCommand} from "@lodestar/test-utils";
+import {getMochaContext} from "@lodestar/test-utils/mocha";
 import {testFilesDir} from "../utils.js";
-import {bufferStderr, describeCliTest} from "../utils/childprocRunner.js";
 import {cachedPubkeysHex, cachedSeckeysHex} from "../utils/cachedKeys.js";
-import {expectDeepEquals, getAfterEachCallbacks} from "../utils/runUtils.js";
-import {expectKeys, getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js";
+import {expectDeepEquals} from "../utils/runUtils.js";
+import {expectKeys, startValidatorWithKeyManager} from "../utils/validator.js";
 import {getKeystoresStr} from "../utils/keystores.js";
 
-describeCliTest("import keystores from api", function ({spawnCli}) {
+describe("import keystores from api", function () {
+  const testContext = getMochaContext(this);
+  this.timeout("30s");
+
   const dataDir = path.join(testFilesDir, "import-keystores-test");
 
   before("Clean dataDir", () => {
     rimraf.sync(dataDir);
   });
 
-  const afterEachCallbacks = getAfterEachCallbacks();
-  const itKeymanagerStep = getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir});
-
   /** Generated from  const sk = bls.SecretKey.fromKeygen(Buffer.alloc(32, 0xaa)); */
   const passphrase = "AAAAAAAA0000000000";
   const keyCount = 2;
@@ -55,7 +56,8 @@ describeCliTest("import keystores from api", function ({spawnCli}) {
 
   const slashingProtectionStr = JSON.stringify(slashingProtection);
 
-  itKeymanagerStep("run 'validator' and import remote keys from API", async function (keymanagerClient) {
+  it("run 'validator' and import remote keys from API", async () => {
+    const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext});
     // Produce and encrypt keystores
     const keystoresStr = await getKeystoresStr(passphrase, secretKeys);
 
@@ -84,16 +86,14 @@ describeCliTest("import keystores from api", function ({spawnCli}) {
     );
 
     // Attempt to run a second process and expect the keystore lock to throw
-    const vcProc2 = spawnCli({pipeStdToParent: true, logPrefix: "vc-2"}, [
-      // ⏎
-      "validator",
-      `--dataDir=${dataDir}`,
-    ]);
+    const validator = await spawnCliCommand("packages/cli/bin/lodestar.js", ["validator", "--dataDir", dataDir], {
+      logPrefix: "vc-2",
+    });
 
     await new Promise<void>((resolve, reject) => {
       // logger.error is printed to stdout, Yargs errors are printed in stderr
-      const vcProc2Stderr = bufferStderr(vcProc2);
-      vcProc2.on("exit", (code) => {
+      const vcProc2Stderr = bufferStderr(validator);
+      validator.on("exit", (code) => {
         if (code !== null && code > 0) {
           // process should exit with code > 0, and an error related to locks. Sample error:
           // vc 351591:  ✖ Error: EEXIST: file already exists, open '/tmp/tmp-351554-dMctEAj7sJIz/import-keystores-test/keystores/0x8be678633e927aa0435addad5dcd5283fef6110d91362519cd6d43e61f6c017d724fa579cc4b2972134e050b6ba120c0/voting-keystore.json.lock'
@@ -111,7 +111,9 @@ describeCliTest("import keystores from api", function ({spawnCli}) {
     });
   });
 
-  itKeymanagerStep("run 'validator' check keys are loaded + delete", async function (keymanagerClient) {
+  it("run 'validator' check keys are loaded + delete", async function () {
+    const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext});
+
     // Check that keys imported in previous it() are still there
     await expectKeys(keymanagerClient, pubkeys, "Wrong listKeys before deleting");
 
@@ -128,13 +130,16 @@ describeCliTest("import keystores from api", function ({spawnCli}) {
     await expectKeys(keymanagerClient, [], "Wrong listKeys after deleting");
   });
 
-  itKeymanagerStep("different process check no keys are loaded", async function (keymanagerClient) {
+  it("different process check no keys are loaded", async function () {
+    const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext});
     // After deleting there should be no keys
     await expectKeys(keymanagerClient, [], "Wrong listKeys");
   });
 
-  itKeymanagerStep("reject calls without bearerToken", async function (_, {keymanagerUrl}) {
-    const keymanagerClientNoAuth = getClient({baseUrl: keymanagerUrl, bearerToken: undefined}, {config});
+  it("reject calls without bearerToken", async function () {
+    await startValidatorWithKeyManager([], {dataDir, testContext});
+
+    const keymanagerClientNoAuth = getClient({baseUrl: "http://localhost:38011", bearerToken: undefined}, {config});
     const res = await keymanagerClientNoAuth.listRemoteKeys();
     expect(res.ok).to.be.false;
     expect(res.error?.code).to.be.eql(HttpStatusCode.UNAUTHORIZED);

packages/cli/test/e2e/importRemoteKeysFromApi.test.ts

@@ -4,27 +4,40 @@ import {expect} from "chai";
 import {Api, DeleteRemoteKeyStatus, getClient, ImportRemoteKeyStatus} from "@lodestar/api/keymanager";
 import {config} from "@lodestar/config/default";
 import {ApiError, HttpStatusCode} from "@lodestar/api";
+import {getMochaContext} from "@lodestar/test-utils/mocha";
 import {testFilesDir} from "../utils.js";
-import {describeCliTest} from "../utils/childprocRunner.js";
 import {cachedPubkeysHex} from "../utils/cachedKeys.js";
-import {expectDeepEquals, getAfterEachCallbacks} from "../utils/runUtils.js";
-import {getKeymanagerTestRunner} from "../utils/keymanagerTestRunners.js";
+import {expectDeepEquals} from "../utils/runUtils.js";
+import {startValidatorWithKeyManager} from "../utils/validator.js";
+
+const url = "https://remote.signer";
+
+async function expectKeys(keymanagerClient: Api, expectedPubkeys: string[], message: string): Promise<void> {
+  const remoteKeys = await keymanagerClient.listRemoteKeys();
+  ApiError.assert(remoteKeys);
+  expectDeepEquals(
+    remoteKeys.response.data,
+    expectedPubkeys.map((pubkey) => ({pubkey, url, readonly: false})),
+    message
+  );
+}
+
+describe("import remoteKeys from api", function () {
+  const testContext = getMochaContext(this);
+  this.timeout("30s");
 
-describeCliTest("import remoteKeys from api", function ({spawnCli}) {
   const dataDir = path.join(testFilesDir, "import-remoteKeys-test");
 
   before("Clean dataDir", () => {
     rimraf.sync(dataDir);
   });
 
-  const afterEachCallbacks = getAfterEachCallbacks();
-  const itKeymanagerStep = getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, dataDir});
-
   /** Generated from  const sk = bls.SecretKey.fromKeygen(Buffer.alloc(32, 0xaa)); */
-  const url = "https://remote.signer";
   const pubkeysToAdd = [cachedPubkeysHex[0], cachedPubkeysHex[1]];
 
-  itKeymanagerStep("run 'validator' and import remote keys from API", async function (keymanagerClient) {
+  it("run 'validator' and import remote keys from API", async () => {
+    const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext});
+
     // Wrap in retry since the API may not be listening yet
     await expectKeys(keymanagerClient, [], "Wrong listRemoteKeys before importing");
 
@@ -50,7 +63,8 @@ describeCliTest("import remoteKeys from api", function ({spawnCli}) {
     );
   });
 
-  itKeymanagerStep("run 'validator' check keys are loaded + delete", async function (keymanagerClient) {
+  it("run 'validator' check keys are loaded + delete", async function () {
+    const {keymanagerClient} = await startValidatorWithKeyManager([], {dataDir, testContext});
     // Check that keys imported in previous it() are still there
     await expectKeys(keymanagerClient, pubkeysToAdd, "Wrong listRemoteKeys before deleting");
 
@@ -67,20 +81,12 @@ describeCliTest("import remoteKeys from api", function ({spawnCli}) {
     await expectKeys(keymanagerClient, [], "Wrong listRemoteKeys after deleting");
   });
 
-  itKeymanagerStep("reject calls without bearerToken", async function (_, {keymanagerUrl}) {
+  it("reject calls without bearerToken", async function () {
+    await startValidatorWithKeyManager([], {dataDir, testContext});
+    const keymanagerUrl = "http://localhost:38011";
     const keymanagerClientNoAuth = getClient({baseUrl: keymanagerUrl, bearerToken: undefined}, {config});
     const res = await keymanagerClientNoAuth.listRemoteKeys();
     expect(res.ok).to.be.false;
     expect(res.error?.code).to.be.eql(HttpStatusCode.UNAUTHORIZED);
   });
-
-  async function expectKeys(keymanagerClient: Api, expectedPubkeys: string[], message: string): Promise<void> {
-    const remoteKeys = await keymanagerClient.listRemoteKeys();
-    ApiError.assert(remoteKeys);
-    expectDeepEquals(
-      remoteKeys.response.data,
-      expectedPubkeys.map((pubkey) => ({pubkey, url, readonly: false})),
-      message
-    );
-  }
 });