Merge branch 'unstable' into nh/sim-merge-test

.github/FUNDING.yml

@@ -1 +1 @@
-custom: https://gitcoin.co/grants/6034/lodestar-typescript-eth-consensus-client-by-chains
+custom: https://etherscan.io/address/0xb4da52336092db22fe8e036866d59c6488604f89

.github/workflows/test-sim.yml

@@ -23,9 +23,9 @@ on:
         default: 40
 
 env:
-  GETH_DOCKER_IMAGE: ethereum/client-go:v1.11.6
+  GETH_DOCKER_IMAGE: ethereum/client-go:v1.13.11
   LIGHTHOUSE_DOCKER_IMAGE: sigp/lighthouse:latest-amd64-modern-dev
-  NETHERMIND_DOCKER_IMAGE: nethermind/nethermind:1.18.0
+  NETHERMIND_DOCKER_IMAGE: nethermind/nethermind:1.25.3
 
 jobs:
   tests-sim:
@@ -58,27 +58,41 @@ jobs:
         if: steps.cache-deps.outputs.cache-hit == 'true'
       # </common-build>
 
+      - name: Download required docker images before running tests
+        run: |
+          docker pull ${{env.GETH_DOCKER_IMAGE}}
+          docker pull ${{env.LIGHTHOUSE_DOCKER_IMAGE}}
+          docker pull ${{env.NETHERMIND_DOCKER_IMAGE}}
+
       - name: Sim tests multifork
         run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:multifork
         working-directory: packages/cli
         env: 
           GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}
 
       - name: Sim tests endpoints
-        run: yarn test:sim:endpoints
+        run: DEBUG='${{github.event.inputs.debug}}'  yarn test:sim:endpoints
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}        
 
       - name: Sim tests deneb
-        run: yarn test:sim:deneb
+        run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:deneb
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}        
 
       - name: Sim tests backup eth provider
-        run: yarn test:sim:backup_eth_provider
+        run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:backup_eth_provider
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}        
 
       - name: Sim tests mixed client
         run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:mixedclient
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}        
 
       - name: Upload debug log test files for "packages/cli"
         if: ${{ always() }}

CONTRIBUTING.md

@@ -32,6 +32,14 @@ To run tests:
 - :test_tube: Run `yarn check-types` to check TypeScript types.
 - :test_tube: Run `yarn lint` to run the linter (ESLint).
 
+Note that to run `test:e2e`, first ensure that the environment is correctly setup by running the `run_e2e_env.sh` script.
+
+```sh
+GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6  NETHERMIND_DOCKER_IMAGE=nethermind/nethermind:1.18.0 ./scripts/run_e2e_env.sh start
+```
+
+Similarly, run `yarn download-spec-tests` before running `yarn test:spec`.
+
 Contributing to tests:
 
 - Test must not depend on external live resources, such that running tests for a commit must be deterministic:
@@ -53,11 +61,12 @@ If you observe following error running any of the test files that means you are
 
 - To fix errors always focus on passing all minimal tests first without running mainnet tests.
 - Spec tests often compare full expected vs actual states in JSON format.
-- A single logical error can cause many spec tests to fail. To focus on a single test at a time you can use vitest's option `--bail` to stop at the first failed test
-- To then run only that failed test you can run against a specific file as use vitest's filters to run only one case
+- A single logical error can cause many spec tests to fail. To focus on a single test at a time you can use vitest's option `--bail 1` to stop at the first failed test
+- To then run only that failed test you can run against a specific file as use vitest's filters option `-t <pattern>` to run only one case
 
 ```sh
-LODESTAR_PRESET=minimal yarn vitest --run --config vitest.config.spec.ts test/spec/phase0/sanity.test.ts
+cd packages/beacon-node
+LODESTAR_PRESET=minimal yarn vitest --run --bail 1 --config vitest.spec.config.ts test/spec/presets/sanity.test.ts -t attester_slashing
 ```
 
 ## Docker

Dockerfile

@@ -39,9 +39,9 @@ FROM node:20-alpine
 WORKDIR /usr/app
 COPY --from=build_deps /usr/app .
 
-# NodeJS applications have a default memory limit of 2.5GB.
-# This limit is bit tight for a Prater node, it is recommended to raise the limit
+# NodeJS applications have a default memory limit of 4GB on most machines.
+# This limit is bit tight for a Mainnet node, it is recommended to raise the limit
 # since memory may spike during certain network conditions.
-ENV NODE_OPTIONS=--max-old-space-size=4096
+ENV NODE_OPTIONS=--max-old-space-size=8192
 
 ENTRYPOINT ["node", "./packages/cli/bin/lodestar"]

docker-compose.yml

@@ -11,11 +11,11 @@ services:
       - "9000:9000" # P2P port
     #  - "9596:9596" # REST API port
     command: beacon --dataDir /data --rest --rest.address 0.0.0.0 --metrics --logFile /logs/beacon.log --logFileLevel debug --logFileDailyRotate 5
-    # NodeJS applications have a default memory limit of 2.5GB.
-    # This limit is bit tight for a Prater node, it is recommended to raise the limit
+    # NodeJS applications have a default memory limit of 4GB on most machines.
+    # This limit is bit tight for a Mainnet node, it is recommended to raise the limit
     # since memory may spike during certain network conditions.
     environment:
-      NODE_OPTIONS: --max-old-space-size=4096
+      NODE_OPTIONS: --max-old-space-size=8192
 
   prometheus:
     build: docker/prometheus

docs/pages/getting-started/starting-a-node.md

@@ -47,7 +47,7 @@ Use the `--JsonRpc.JwtSecretFile /data/jwtsecret` flag to configure the secret.
 Use the `--engine-jwt-secret=<FILE>` flag to configure the secret. Use their documentation [here](https://besu.hyperledger.org/en/stable/Reference/CLI/CLI-Syntax/#engine-jwt-secret).
 
 **For Erigon:**
-Use the `--authrpc.jwtsecret` flag to configure the secret. Use their documentation [here](https://github.com/ledgerwatch/erigon#authentication-api).
+Use the `--authrpc.jwtsecret` flag to configure the secret. Use their documentation [here](https://github.com/ledgerwatch/erigon?tab=readme-ov-file#beacon-chain-consensus-layer).
 
 ## Run a beacon node
 

docs/pages/tools/flamegraphs.md

@@ -18,7 +18,7 @@ Next we need to update the Lodestar service by modifying the start script. We ne
 ```sh
 node \
   --perf-basic-prof \
-  --max-old-space-size=4096 \
+  --max-old-space-size=8192 \
   /usr/src/lodestar/packages/cli/bin/lodestar \
   beacon \
   --rcConfig /home/devops/beacon/rcconfig.yml

docs/pages/validator-management/vc-configuration.md

@@ -97,13 +97,14 @@ With produceBlockV3 (enabled automatically after the Deneb hard fork), the `--bu
 With Lodestar's `--builder.selection` validator options, you can select:
 
 - `maxprofit`: Default setting for Lodestar set at `--builder.boostFactor=100`. This default setting will always choose the more profitable block. Using this option, you may customize your `--builder.boostFactor` to your preference. Examples of its usage are below.
+- `executionalways`: An alias of `--builder.boostFactor=0`, which will select the local execution block, unless it fails to produce due to an error or a delay in the response from the execution client.
 - `executiononly`: Beacon node will be requested to produce local execution block even if builder relays are configured. This option will always select the local execution block and will error if it couldn't produce one.
 - `builderalways`: An alias of `--builder.boostFactor=18446744073709551615` (2**64 - 1), which will select the builder block, unless the builder block fails to produce. The builder block may fail to produce if it's not available, not timely or there is an indication of censorship via `shouldOverrideBuilder` from the execution payload response.
 - `builderonly`: Generally used for distributed validators (DVs). No execution block production will be triggered. Therefore, if a builder block is not produced, the API will fail and _no block will be produced_.
 
 #### Calculating builder boost factor with examples
 
-To calculate the builder boost factor setting, you need to know what percentage you will accept a builder block for against a local execution block using the following formula: `100*100/(100+percentage)`.
+To calculate the builder boost factor setting, you need to know what percentage you will accept a builder block for against a local execution block using the following formula: `100*100/(100+percentage)`. The value passed to `--builder.boostFactor` must be a valid number without decimals.
 
 Example 1: I will only accept a builder block with 25% more value than the local execution block.
 ```

lerna.json

@@ -4,7 +4,7 @@
   ],
   "npmClient": "yarn",
   "useNx": true,
-  "version": "1.14.0",
+  "version": "1.15.1",
   "stream": true,
   "command": {
     "version": {

lodestar

@@ -1,7 +1,7 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 # Convenience script to run the lodestar binary from built source
 #
-# ./lodestar.sh beacon --network prater
+# ./lodestar.sh beacon --network mainnet
 
-node --trace-deprecation --max-old-space-size=4096 ./packages/cli/bin/lodestar.js "$@"
+node --trace-deprecation --max-old-space-size=8192 ./packages/cli/bin/lodestar.js "$@"

package.json

@@ -31,6 +31,7 @@
     "test:browsers": "lerna run test:browsers",
     "test:e2e": "lerna run test:e2e --concurrency 1",
     "test:e2e:sim": "lerna run test:e2e:sim",
+    "download-spec-tests": "lerna run download-spec-tests",
     "test:spec": "lerna run test:spec",
     "test-coverage:unit": "c8 --config .c8rc.json --report-dir coverage/unit/ --all npm run test:unit",
     "test-coverage:browsers": "c8 --config .c8rc.json --report-dir coverage/browsers/ --all npm run test:browsers",

packages/api/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.14.0",
+  "version": "1.15.1",
   "type": "module",
   "exports": {
     ".": {
@@ -63,18 +63,17 @@
     "coverage": "codecov -F lodestar-api",
     "lint": "eslint --color --ext .ts src/ test/",
     "lint:fix": "yarn run lint --fix",
-    "pretest": "yarn run check-types",
-    "test": "yarn test:unit && yarn test:e2e",
+    "test": "yarn test:unit",
     "test:unit": "vitest --run --dir test/unit/",
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
     "@chainsafe/ssz": "^0.14.0",
-    "@lodestar/config": "^1.14.0",
-    "@lodestar/params": "^1.14.0",
-    "@lodestar/types": "^1.14.0",
-    "@lodestar/utils": "^1.14.0",
+    "@lodestar/config": "^1.15.1",
+    "@lodestar/params": "^1.15.1",
+    "@lodestar/types": "^1.15.1",
+    "@lodestar/utils": "^1.15.1",
     "eventsource": "^2.0.2",
     "qs": "^6.11.1"
   },

packages/api/src/beacon/routes/beacon/block.ts

@@ -229,8 +229,12 @@ export type Api = {
    * Retrieves BlobSidecar included in requested block.
    * @param blockId Block identifier.
    * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \<slot\>, \<hex encoded blockRoot with 0x prefix\>.
+   * @param indices Array of indices for blob sidecars to request for in the specified block. Returns all blob sidecars in the block if not specified.
    */
-  getBlobSidecars(blockId: BlockId): Promise<
+  getBlobSidecars(
+    blockId: BlockId,
+    indices?: number[]
+  ): Promise<
     ApiClientResponse<{
       [HttpStatusCode.OK]: {executionOptimistic: ExecutionOptimistic; data: deneb.BlobSidecars};
     }>
@@ -270,7 +274,7 @@ export type ReqTypes = {
   publishBlockV2: {body: unknown; query: {broadcast_validation?: string}};
   publishBlindedBlock: {body: unknown};
   publishBlindedBlockV2: {body: unknown; query: {broadcast_validation?: string}};
-  getBlobSidecars: BlockIdOnlyReq;
+  getBlobSidecars: {params: {block_id: string}; query: {indices?: number[]}};
 };
 
 export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api, ReqTypes> {
@@ -356,7 +360,17 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
         query: {broadcast_validation: Schema.String},
       },
     },
-    getBlobSidecars: blockIdOnlyReq,
+    getBlobSidecars: {
+      writeReq: (block_id, indices) => ({
+        params: {block_id: String(block_id)},
+        query: {indices},
+      }),
+      parseReq: ({params, query}) => [params.block_id, query.indices],
+      schema: {
+        params: {block_id: Schema.StringRequired},
+        query: {indices: Schema.UintArray},
+      },
+    },
   };
 }
 

packages/api/src/beacon/routes/validator.ts

@@ -43,6 +43,7 @@ import {ExecutionOptimistic} from "./beacon/block.js";
 
 export enum BuilderSelection {
   BuilderAlways = "builderalways",
+  ExecutionAlways = "executionalways",
   MaxProfit = "maxprofit",
   /** Only activate builder flow for DVT block proposal protocols */
   BuilderOnly = "builderonly",

packages/api/test/unit/beacon/genericServerTest/config.test.ts

@@ -9,9 +9,7 @@ import {testData} from "../testData/config.js";
 /* eslint-disable @typescript-eslint/naming-convention */
 
 describe("beacon / config", () => {
-  describe("Run generic server test", () => {
-    runGenericServerTest<Api, ReqTypes>(config, getClient, getRoutes, testData);
-  });
+  runGenericServerTest<Api, ReqTypes>(config, getClient, getRoutes, testData);
 
   it("Serialize Partial Spec object", () => {
     const returnTypes = getReturnTypes();

packages/api/test/unit/beacon/genericServerTest/debug.test.ts

@@ -1,5 +1,6 @@
-import {describe, it, expect, MockInstance} from "vitest";
+import {describe, it, expect, MockInstance, beforeAll, afterAll} from "vitest";
 import {toHexString} from "@chainsafe/ssz";
+import {FastifyInstance} from "fastify";
 import {ssz} from "@lodestar/types";
 import {config} from "@lodestar/config/default";
 import {Api, ReqTypes, routesData} from "../../../../src/beacon/routes/debug.js";
@@ -13,19 +14,28 @@ import {testData} from "../testData/debug.js";
 
 describe(
   "beacon / debug",
-  function () {
-    describe("Run generic server test", () => {
-      runGenericServerTest<Api, ReqTypes>(config, getClient, getRoutes, testData);
-    });
+  () => {
+    runGenericServerTest<Api, ReqTypes>(config, getClient, getRoutes, testData);
 
     // Get state by SSZ
 
     describe("getState() in SSZ format", () => {
-      const {baseUrl, server} = getTestServer();
       const mockApi = getMockApi<Api>(routesData);
-      for (const route of Object.values(getRoutes(config, mockApi))) {
-        registerRoute(server, route);
-      }
+      let baseUrl: string;
+      let server: FastifyInstance;
+
+      beforeAll(async () => {
+        const res = getTestServer();
+        server = res.server;
+        for (const route of Object.values(getRoutes(config, mockApi))) {
+          registerRoute(server, route);
+        }
+        baseUrl = await res.start();
+      });
+
+      afterAll(async () => {
+        if (server !== undefined) await server.close();
+      });
 
       for (const method of ["getState" as const, "getStateV2" as const]) {
         it(method, async () => {

packages/api/test/unit/beacon/genericServerTest/events.test.ts

@@ -1,4 +1,5 @@
-import {describe, it, expect, beforeEach, afterEach} from "vitest";
+import {describe, it, expect, beforeEach, afterEach, beforeAll, afterAll} from "vitest";
+import {FastifyInstance} from "fastify";
 import {sleep} from "@lodestar/utils";
 import {Api, routesData, EventType, BeaconEvent} from "../../../../src/beacon/routes/events.js";
 import {getClient} from "../../../../src/beacon/client/events.js";
@@ -8,11 +9,23 @@ import {getMockApi, getTestServer} from "../../../utils/utils.js";
 import {eventTestData} from "../testData/events.js";
 
 describe("beacon / events", () => {
-  const {baseUrl, server} = getTestServer();
   const mockApi = getMockApi<Api>(routesData);
-  for (const route of Object.values(getRoutes(mockApi))) {
-    registerRoute(server, route);
-  }
+  let server: FastifyInstance;
+  let baseUrl: string;
+
+  beforeAll(async () => {
+    const res = getTestServer();
+    server = res.server;
+    for (const route of Object.values(getRoutes(mockApi))) {
+      registerRoute(server, route);
+    }
+
+    baseUrl = await res.start();
+  });
+
+  afterAll(async () => {
+    if (server !== undefined) await server.close();
+  });
 
   let controller: AbortController;
   beforeEach(() => {

packages/api/test/unit/beacon/oapiSpec.test.ts

@@ -130,12 +130,6 @@ const ignoredProperties: Record<string, IgnoredProperty> = {
    */
   getHealth: {request: ["query.syncing_status"]},
 
-  /**
-   * https://github.com/ChainSafe/lodestar/issues/6185
-   *  - must have required property 'query'
-   */
-  getBlobSidecars: {request: ["query"]},
-
   /* 
    https://github.com/ChainSafe/lodestar/issues/4638 
    /query - must have required property 'skip_randao_verification'

packages/api/test/unit/beacon/testData/beacon.ts

@@ -71,7 +71,7 @@ export const testData: GenericServerTestCases<Api> = {
     res: undefined,
   },
   getBlobSidecars: {
-    args: ["head"],
+    args: ["head", [0]],
     res: {executionOptimistic: true, data: ssz.deneb.BlobSidecars.defaultValue()},
   },