deps: update libp2p to 0.45.x (#5506)

* deps: update libp2p

* Fix libp2p components

* Fix e2e tests

* fix merge error

* chore: bump libp2p to 0.45.6

* chore: fix yarn lock

* Update to gossipsub 9.0.0

* Fix linter error

* Rely entirely on lodestar's peer manager to prune connections

* Update libp2p and gossipsub

* Remove unused Libp2pEvent members

package.json

@@ -67,7 +67,7 @@
     "karma-spec-reporter": "^0.0.36",
     "karma-webpack": "^5.0.0",
     "lerna": "^6.6.1",
-    "libp2p": "0.42.2",
+    "libp2p": "0.45.9",
     "mocha": "^10.2.0",
     "node-gyp": "^9.3.1",
     "npm-run-all": "^4.1.5",

packages/beacon-node/package.json

@@ -98,27 +98,27 @@
     "@chainsafe/as-sha256": "^0.3.1",
     "@chainsafe/bls": "7.1.1",
     "@chainsafe/blst": "^0.2.9",
-    "@chainsafe/discv5": "^3.0.0",
-    "@chainsafe/libp2p-gossipsub": "^6.2.0",
-    "@chainsafe/libp2p-noise": "^11.0.4",
+    "@chainsafe/discv5": "^4.0.0",
+    "@chainsafe/libp2p-gossipsub": "^9.1.0",
+    "@chainsafe/libp2p-noise": "^12.0.1",
     "@chainsafe/persistent-merkle-tree": "^0.5.0",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/ssz": "^0.10.2",
     "@chainsafe/threads": "^1.11.0",
     "@ethersproject/abi": "^5.7.0",
     "@fastify/bearer-auth": "^9.0.0",
     "@fastify/cors": "^8.2.1",
-    "@libp2p/bootstrap": "^6.0.3",
-    "@libp2p/interface-connection": "^3.0.2",
-    "@libp2p/interface-connection-manager": "^1.3.0",
-    "@libp2p/interface-peer-id": "^2.0.1",
-    "@libp2p/interface-pubsub": "^3.0.0",
-    "@libp2p/interface-registrar": "^2.0.8",
-    "@libp2p/mdns": "^6.0.0",
-    "@libp2p/mplex": "^7.1.3",
-    "@libp2p/peer-id-factory": "^2.0.1",
-    "@libp2p/prometheus-metrics": "^1.1.3",
-    "@libp2p/tcp": "6.1.0",
+    "@libp2p/bootstrap": "^8.0.0",
+    "@libp2p/interface-connection": "^5.1.0",
+    "@libp2p/interface-connection-manager": "^3.0.1",
+    "@libp2p/interface-peer-id": "^2.0.2",
+    "@libp2p/interface-pubsub": "^4.0.1",
+    "@libp2p/interface-registrar": "^2.0.12",
+    "@libp2p/mdns": "^8.0.0",
+    "@libp2p/mplex": "^8.0.3",
+    "@libp2p/peer-id-factory": "^2.0.3",
+    "@libp2p/prometheus-metrics": "^1.1.4",
+    "@libp2p/tcp": "7.0.1",
     "@lodestar/api": "^1.9.1",
     "@lodestar/config": "^1.9.1",
     "@lodestar/db": "^1.9.1",
@@ -131,21 +131,20 @@
     "@lodestar/types": "^1.9.1",
     "@lodestar/utils": "^1.9.1",
     "@lodestar/validator": "^1.9.1",
-    "@multiformats/multiaddr": "^11.0.0",
+    "@multiformats/multiaddr": "^12.1.3",
     "@types/datastore-level": "^3.0.0",
     "buffer-xor": "^2.0.2",
     "c-kzg": "^2.1.0",
     "cross-fetch": "^3.1.4",
-    "datastore-core": "^8.0.1",
-    "datastore-level": "^9.0.1",
+    "datastore-core": "^9.1.1",
+    "datastore-level": "^10.1.1",
     "deepmerge": "^4.3.1",
     "fastify": "^4.19.0",
-    "gc-stats": "^1.4.0",
-    "interface-datastore": "^7.0.0",
-    "it-all": "^3.0.1",
-    "it-pipe": "^2.0.5",
+    "interface-datastore": "^8.2.0",
+    "it-all": "^3.0.2",
+    "it-pipe": "^3.0.1",
     "jwt-simple": "0.5.6",
-    "libp2p": "0.42.2",
+    "libp2p": "0.45.9",
     "prom-client": "^14.2.0",
     "qs": "^6.11.1",
     "snappyjs": "^0.7.0",

packages/beacon-node/src/constants/network.ts

@@ -43,8 +43,8 @@ export const GOODBYE_KNOWN_CODES: Record<string, string> = {
   251: "Peer banned this node",
 };
 
-/** Until js-libp2p types its events */
+/** Until js-libp2p exports an enum for its events */
 export enum Libp2pEvent {
-  peerConnect = "peer:connect",
-  peerDisconnect = "peer:disconnect",
+  connectionOpen = "connection:open",
+  connectionClose = "connection:close",
 }

packages/beacon-node/src/network/core/networkCore.ts

@@ -353,7 +353,7 @@ export class NetworkCore implements INetworkCore {
   }
 
   getConnectionsByPeer(): Map<string, Connection[]> {
-    return getConnectionsMap(this.libp2p.connectionManager);
+    return getConnectionsMap(this.libp2p);
   }
 
   async getConnectedPeers(): Promise<PeerIdStr[]> {
@@ -368,7 +368,7 @@ export class NetworkCore implements INetworkCore {
 
   async connectToPeer(peerIdStr: PeerIdStr, multiaddrStrArr: MultiaddrStr[]): Promise<void> {
     const peer = peerIdFromString(peerIdStr);
-    await this.libp2p.peerStore.addressBook.add(peer, multiaddrStrArr.map(multiaddr));
+    await this.libp2p.peerStore.merge(peer, {multiaddrs: multiaddrStrArr.map(multiaddr)});
     await this.libp2p.dial(peer);
   }
 

packages/beacon-node/src/network/gossip/gossipsub.ts

@@ -7,7 +7,6 @@ import {ATTESTATION_SUBNET_COUNT, ForkName, SYNC_COMMITTEE_SUBNET_COUNT} from "@
 import {Logger, Map2d, Map2dArr} from "@lodestar/utils";
 
 import {RegistryMetricCreator} from "../../metrics/index.js";
-import {peerIdFromString} from "../../util/peerId.js";
 import {PeersData} from "../peers/peersData.js";
 import {ClientKind} from "../peers/client.js";
 import {GOSSIP_MAX_SIZE, GOSSIP_MAX_SIZE_BELLATRIX} from "../../constants/network.js";
@@ -89,7 +88,7 @@ export class Eth2Gossipsub extends GossipSub {
 
     // Gossipsub parameters defined here:
     // https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub
-    super(modules.libp2p, {
+    super(modules.libp2p.services.components, {
       globalSignaturePolicy: SignaturePolicy.StrictNoSign,
       allowPublishToZeroPeers: allowPublishToZeroPeers,
       D: gossipsubD ?? GOSSIP_D,
@@ -300,8 +299,7 @@ export class Eth2Gossipsub extends GossipSub {
     // Without this we'll have huge event loop lag
     // See https://github.com/ChainSafe/lodestar/issues/5604
     setTimeout(() => {
-      // TODO: reportMessageValidationResult should take PeerIdStr since it only uses string version
-      this.reportMessageValidationResult(data.msgId, peerIdFromString(data.propagationSource), data.acceptance);
+      this.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance);
     }, 0);
   }
 }

packages/beacon-node/src/network/interface.ts

@@ -1,7 +1,6 @@
 import {Libp2p as ILibp2p} from "libp2p";
 import {Connection} from "@libp2p/interface-connection";
-import {Registrar} from "@libp2p/interface-registrar";
-import {ConnectionManager} from "@libp2p/interface-connection-manager";
+import {Components} from "libp2p/components";
 import {Slot, SlotRootHex, allForks, altair, capella, deneb, phase0} from "@lodestar/types";
 import {PeerIdStr} from "../util/peerId.js";
 import {INetworkEventBus} from "./events.js";
@@ -66,4 +65,21 @@ export interface INetwork extends INetworkCorePublic {
 export type PeerDirection = Connection["stat"]["direction"];
 export type PeerStatus = Connection["stat"]["status"];
 
-export type Libp2p = ILibp2p & {connectionManager: ConnectionManager; registrar: Registrar};
+export type LodestarComponents = Pick<
+  Components,
+  | "peerId"
+  | "events"
+  | "addressManager"
+  | "peerStore"
+  | "upgrader"
+  | "registrar"
+  | "connectionManager"
+  | "transportManager"
+  | "connectionGater"
+  | "contentRouting"
+  | "peerRouting"
+  | "datastore"
+  | "connectionProtector"
+  | "metrics"
+>;
+export type Libp2p = ILibp2p<{components: LodestarComponents}>;

packages/beacon-node/src/network/nodejs/bundle.ts

@@ -1,4 +1,5 @@
 import {createLibp2p} from "libp2p";
+import {identifyService} from "libp2p/identify";
 import {tcp} from "@libp2p/tcp";
 import {mplex} from "@libp2p/mplex";
 import {bootstrap} from "@libp2p/bootstrap";
@@ -42,7 +43,7 @@ export async function createNodejsLibp2p(options: Libp2pOptions): Promise<Libp2p
       peerDiscovery.push(mdns());
     }
   }
-  return (await createLibp2p({
+  return createLibp2p({
     peerId: options.peerId,
     addresses: {
       listen: options.addresses.listen,
@@ -71,52 +72,42 @@ export async function createNodejsLibp2p(options: Libp2pOptions): Promise<Libp2p
     connectionManager: {
       // dialer config
       maxParallelDials: 100,
-      maxAddrsToDial: 4,
-      maxDialsPerPeer: 2,
+      maxPeerAddrsToDial: 4,
+      maxParallelDialsPerPeer: 2,
       dialTimeout: 30_000,
 
-      autoDial: false,
-      // DOCS: the maximum number of connections libp2p is willing to have before it starts disconnecting.
-      // If ConnectionManager.size > maxConnections calls _maybeDisconnectOne() which will sort peers disconnect
-      // the one with the least `_peerValues`. That's a custom peer generalized score that's not used, so it always
-      // has the same value in current Lodestar usage.
-      maxConnections: options.maxConnections,
-      // DOCS: the minimum number of connections below which libp2p not activate preemptive disconnections.
-      // If ConnectionManager.size < minConnections, it won't prune peers in _maybeDisconnectOne(). If autoDial is
-      // off it doesn't have any effect in behaviour.
-      minConnections: options.minConnections,
+      // Rely entirely on lodestar's peer manager to prune connections
+      //maxConnections: options.maxConnections,
+      // DOCS: There is no way to turn off autodial other than setting minConnections to 0
+      minConnections: 0,
     },
     datastore: options.datastore,
-    nat: {
-      // libp2p usage of nat-api is broken as shown in this issue. https://github.com/ChainSafe/lodestar/issues/2996
-      // Also, unnsolicited usage of UPnP is not great, and should be customizable with flags
-      enabled: false,
-    },
-    relay: {
-      enabled: false,
-      hop: {
-        enabled: false,
-        active: false,
-      },
-      advertise: {
-        enabled: false,
-        ttl: 0,
-        bootDelay: 0,
-      },
-      autoRelay: {
-        enabled: false,
-        maxListeners: 0,
-      },
-    },
-
-    identify: {
-      host: {
+    services: {
+      identify: identifyService({
         agentVersion: options.hideAgentVersion
           ? ""
           : options.lodestarVersion
           ? `lodestar/${options.lodestarVersion}`
           : "lodestar",
-      },
+      }),
+      // individual components are specified because the components object is a Proxy
+      // and passing it here directly causes problems downstream, not to mention is slowwww
+      components: (components: Components) => ({
+        peerId: components.peerId,
+        events: components.events,
+        addressManager: components.addressManager,
+        peerStore: components.peerStore,
+        upgrader: components.upgrader,
+        registrar: components.registrar,
+        connectionManager: components.connectionManager,
+        transportManager: components.transportManager,
+        connectionGater: components.connectionGater,
+        contentRouting: components.contentRouting,
+        peerRouting: components.peerRouting,
+        datastore: components.datastore,
+        connectionProtector: components.connectionProtector,
+        metrics: components.metrics,
+      }),
     },
-  })) as Libp2p;
+  });
 }

packages/beacon-node/src/network/nodejs/noise.ts

@@ -1,6 +1,6 @@
 import type {ConnectionEncrypter} from "@libp2p/interface-connection-encrypter";
 import {newInstance, ChaCha20Poly1305} from "@chainsafe/as-chacha20poly1305";
-import {ICryptoInterface, noise, stablelib} from "@chainsafe/libp2p-noise";
+import {ICryptoInterface, noise, pureJsCrypto} from "@chainsafe/libp2p-noise";
 import {digest} from "@chainsafe/as-sha256";
 
 type Bytes = Uint8Array;
@@ -11,7 +11,7 @@ const asImpl = new ChaCha20Poly1305(ctx);
 
 // same to stablelib but we use as-chacha20poly1305 and as-sha256
 const lodestarCrypto: ICryptoInterface = {
-  ...stablelib,
+  ...pureJsCrypto,
   hashSHA256(data: Uint8Array): Uint8Array {
     return digest(data);
   },

packages/beacon-node/src/network/peers/datastore.ts

@@ -57,15 +57,15 @@ export class Eth2PeerDataStore extends BaseDatastore {
     return this._dbDatastore.close();
   }
 
-  async put(key: Key, val: Uint8Array): Promise<void> {
+  async put(key: Key, val: Uint8Array): Promise<Key> {
     return this._put(key, val, false);
   }
 
   /**
    * Same interface to put with "fromDb" option, if this item is updated back from db
    * Move oldest items from memory data store to db if it's over this._maxMemoryItems
    */
-  async _put(key: Key, val: Uint8Array, fromDb = false): Promise<void> {
+  async _put(key: Key, val: Uint8Array, fromDb = false): Promise<Key> {
     while (this._memoryDatastore.size >= this._maxMemoryItems) {
       // it's likely this is called only 1 time
       await this.pruneMemoryDatastore();
@@ -83,6 +83,7 @@ export class Eth2PeerDataStore extends BaseDatastore {
     }
 
     if (!fromDb) await this._addDirtyItem(keyStr);
+    return key;
   }
 
   /**

packages/beacon-node/src/network/peers/discover.ts

@@ -9,7 +9,7 @@ import {LoggerNode} from "@lodestar/logger/node";
 import {NetworkCoreMetrics} from "../core/metrics.js";
 import {Libp2p} from "../interface.js";
 import {ENRKey, SubnetType} from "../metadata.js";
-import {getConnectionsMap, getDefaultDialer, prettyPrintPeerId} from "../util.js";
+import {getConnectionsMap, prettyPrintPeerId} from "../util.js";
 import {Discv5Worker} from "../discv5/index.js";
 import {LodestarDiscv5Opts} from "../discv5/types.js";
 import {deserializeEnrSubnets, zeroAttnets, zeroSyncnets} from "./utils/enrSubnetsDeserialize.js";
@@ -163,12 +163,17 @@ export class PeerDiscovery {
     const cachedENRsToDial = new Map<PeerIdStr, CachedENR>();
     // Iterate in reverse to consider first the most recent ENRs
     const cachedENRsReverse: CachedENR[] = [];
+    const pendingDials = new Set(
+      this.libp2p.services.components.connectionManager
+        .getDialQueue()
+        .map((pendingDial) => pendingDial.peerId?.toString())
+    );
     for (const [id, cachedENR] of this.cachedENRs.entries()) {
       if (
         // time expired or
         Date.now() - cachedENR.addedUnixMs > MAX_CACHED_ENR_AGE_MS ||
         // already dialing
-        getDefaultDialer(this.libp2p).pendingDials.has(id)
+        pendingDials.has(id)
       ) {
         this.cachedENRs.delete(id);
       } else {
@@ -324,7 +329,11 @@ export class PeerDiscovery {
       }
 
       // Ignore dialing peers
-      if (getDefaultDialer(this.libp2p).pendingDials.has(peerId.toString())) {
+      if (
+        this.libp2p.services.components.connectionManager
+          .getDialQueue()
+          .find((pendingDial) => pendingDial.peerId && pendingDial.peerId.equals(peerId))
+      ) {
         return DiscoveredPeerStatus.already_dialing;
       }
 
@@ -391,7 +400,7 @@ export class PeerDiscovery {
 
     // Must add the multiaddrs array to the address book before dialing
     // https://github.com/libp2p/js-libp2p/blob/aec8e3d3bb1b245051b60c2a890550d262d5b062/src/index.js#L638
-    await this.libp2p.peerStore.addressBook.add(peerId, [multiaddrTCP]);
+    await this.libp2p.peerStore.merge(peerId, {multiaddrs: [multiaddrTCP]});
 
     // Note: PeerDiscovery adds the multiaddrTCP beforehand
     const peerIdShort = prettyPrintPeerId(peerId);
@@ -415,7 +424,7 @@ export class PeerDiscovery {
 
   /** Check if there is 1+ open connection with this peer */
   private isPeerConnected(peerIdStr: PeerIdStr): boolean {
-    const connections = getConnectionsMap(this.libp2p.connectionManager).get(peerIdStr);
+    const connections = getConnectionsMap(this.libp2p).get(peerIdStr);
     return Boolean(connections && connections.some((connection) => connection.stat.status === "OPEN"));
   }
 }