feat: update libp2p (#6285)

* wip * chore: update discv5 and enr * chore: bump dependencies * chore: fix import statement * chore: update libp2p dependencies * chore: yarn.lock cleanup * chore: update discv5 * feat: revamp libp2p dashboard * Clean up metrics registry type casts * chore: fix lint / build errors * chore: more import updates * chore: more yarn.lock cleanup * chore: update gossipsub --------- Co-authored-by: Nico Flaig <[email protected]>
ChainSafe · Jan 16, 2024 · 9eb9cce · 9eb9cce · github-actions · Jan 16, 2024
1 parent 4ec413c
commit 9eb9cce
Show file tree

Hide file tree

Showing 65 changed files with 1,083 additions and 932 deletions.
diff --git a/dashboards/lodestar_libp2p.json b/dashboards/lodestar_libp2p.json
diff --git a/package.json b/package.json
@@ -81,7 +81,7 @@
     "karma-spec-reporter": "^0.0.36",
     "karma-webpack": "^5.0.0",
     "lerna": "^7.3.0",
-    "libp2p": "0.46.12",
+    "libp2p": "1.1.1",
     "mocha": "^10.2.0",
     "node-gyp": "^9.4.0",
     "npm-run-all": "^4.1.5",

diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json
@@ -99,9 +99,10 @@
     "@chainsafe/as-sha256": "^0.3.1",
     "@chainsafe/bls": "7.1.1",
     "@chainsafe/blst": "^0.2.9",
-    "@chainsafe/discv5": "^5.1.0",
-    "@chainsafe/libp2p-gossipsub": "^10.1.1",
-    "@chainsafe/libp2p-noise": "^13.0.1",
+    "@chainsafe/discv5": "^7.1.0",
+    "@chainsafe/enr": "^2.0.2",
+    "@chainsafe/libp2p-gossipsub": "^11.1.0",
+    "@chainsafe/libp2p-noise": "^14.1.0",
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/ssz": "^0.14.0",
@@ -111,14 +112,15 @@
     "@fastify/cors": "^8.2.1",
     "@fastify/swagger": "^8.10.0",
     "@fastify/swagger-ui": "^1.9.3",
-    "@libp2p/bootstrap": "^9.0.7",
-    "@libp2p/interface": "^0.1.2",
-    "@libp2p/mdns": "^9.0.9",
-    "@libp2p/mplex": "^9.0.7",
-    "@libp2p/peer-id": "^3.0.2",
-    "@libp2p/peer-id-factory": "^3.0.4",
-    "@libp2p/prometheus-metrics": "^2.0.7",
-    "@libp2p/tcp": "8.0.8",
+    "@libp2p/bootstrap": "^10.0.10",
+    "@libp2p/identify": "^1.0.9",
+    "@libp2p/interface": "^1.1.1",
+    "@libp2p/mdns": "^10.0.10",
+    "@libp2p/mplex": "^10.0.10",
+    "@libp2p/peer-id": "^4.0.4",
+    "@libp2p/peer-id-factory": "^4.0.3",
+    "@libp2p/prometheus-metrics": "^3.0.10",
+    "@libp2p/tcp": "9.0.10",
     "@lodestar/api": "^1.14.0",
     "@lodestar/config": "^1.14.0",
     "@lodestar/db": "^1.14.0",
@@ -139,20 +141,20 @@
     "datastore-level": "^10.1.1",
     "deepmerge": "^4.3.1",
     "fastify": "^4.19.0",
-    "interface-datastore": "^8.2.0",
-    "it-all": "^3.0.2",
+    "interface-datastore": "^8.2.7",
+    "it-all": "^3.0.4",
     "it-pipe": "^3.0.1",
     "jwt-simple": "0.5.6",
-    "libp2p": "0.46.12",
+    "libp2p": "1.1.1",
     "multiformats": "^11.0.1",
     "prom-client": "^15.1.0",
     "qs": "^6.11.1",
     "snappyjs": "^0.7.0",
     "strict-event-emitter-types": "^2.0.0",
     "systeminformation": "^5.17.12",
-    "uint8-varint": "^2.0.1",
-    "uint8arraylist": "^2.4.3",
-    "uint8arrays": "^4.0.3",
+    "uint8-varint": "^2.0.2",
+    "uint8arraylist": "^2.4.7",
+    "uint8arrays": "^4.0.9",
     "xxhash-wasm": "1.0.2"
   },
   "devDependencies": {

diff --git a/packages/beacon-node/src/api/impl/node/utils.ts b/packages/beacon-node/src/api/impl/node/utils.ts
@@ -1,4 +1,4 @@
-import {Connection, StreamStatus} from "@libp2p/interface/connection";
+import {Connection, StreamStatus} from "@libp2p/interface";
 import {routes} from "@lodestar/api";
 
 /**

diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts
@@ -1,10 +1,9 @@
-import {PeerId} from "@libp2p/interface/peer-id";
+import {Connection, PeerId} from "@libp2p/interface";
 import {multiaddr} from "@multiformats/multiaddr";
-import {Connection} from "@libp2p/interface/connection";
 import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types";
 import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/dist/src/score/peer-score.js";
 import {fromHexString} from "@chainsafe/ssz";
-import {ENR} from "@chainsafe/discv5";
+import {ENR} from "@chainsafe/enr";
 import {routes} from "@lodestar/api";
 import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";

diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts
@@ -3,7 +3,7 @@ import worker_threads from "node:worker_threads";
 import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/dist/src/score/peer-score.js";
 import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types";
 import {ModuleThread, Thread, Worker, spawn} from "@chainsafe/threads";
-import {PeerId} from "@libp2p/interface/peer-id";
+import {PeerId} from "@libp2p/interface";
 import {exportToProtobuf} from "@libp2p/peer-id-factory";
 import {routes} from "@lodestar/api";
 import {BeaconConfig, chainConfigToJson} from "@lodestar/config";

diff --git a/packages/beacon-node/src/network/discv5/index.ts b/packages/beacon-node/src/network/discv5/index.ts
@@ -1,8 +1,8 @@
 import EventEmitter from "events";
-import {PeerId} from "@libp2p/interface/peer-id";
+import {PeerId} from "@libp2p/interface";
 import StrictEventEmitter from "strict-event-emitter-types";
 import {exportToProtobuf} from "@libp2p/peer-id-factory";
-import {createKeypairFromPeerId, ENR, ENRData, IKeypair, SignableENR} from "@chainsafe/discv5";
+import {createPrivateKeyFromPeerId, ENR, ENRData, SignableENR} from "@chainsafe/enr";
 import {spawn, Thread, Worker} from "@chainsafe/threads";
 import {chainConfigFromJson, chainConfigToJson, BeaconConfig} from "@lodestar/config";
 import {LoggerNode} from "@lodestar/logger/node";
@@ -25,7 +25,7 @@ export type Discv5Events = {
  * Wrapper class abstracting the details of discv5 worker instantiation and message-passing
  */
 export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<EventEmitter, Discv5Events>}) {
-  private readonly keypair: IKeypair;
+  private readonly keypair;
   private readonly subscription: {unsubscribe: () => void};
   private closed = false;
 
@@ -35,7 +35,7 @@ export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<E
   ) {
     super();
 
-    this.keypair = createKeypairFromPeerId(this.opts.peerId);
+    this.keypair = createPrivateKeyFromPeerId(this.opts.peerId);
     this.subscription = workerApi.discovered().subscribe((enrObj) => this.onDiscovered(enrObj));
   }
 
@@ -80,7 +80,7 @@ export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<E
 
   async enr(): Promise<SignableENR> {
     const obj = await this.workerApi.enr();
-    return new SignableENR(obj.kvs, obj.seq, this.keypair);
+    return new SignableENR(obj.kvs, obj.seq, this.keypair.privateKey);
   }
 
   setEnrValue(key: string, value: Uint8Array): Promise<void> {

diff --git a/packages/beacon-node/src/network/discv5/types.ts b/packages/beacon-node/src/network/discv5/types.ts
@@ -1,4 +1,5 @@
-import {Discv5, ENRData, SignableENRData} from "@chainsafe/discv5";
+import {Discv5} from "@chainsafe/discv5";
+import {ENRData, SignableENRData} from "@chainsafe/enr";
 import {Observable} from "@chainsafe/threads/observable";
 import {ChainConfig} from "@lodestar/config";
 import {LoggerNodeOpts} from "@lodestar/logger/node";

diff --git a/packages/beacon-node/src/network/discv5/utils.ts b/packages/beacon-node/src/network/discv5/utils.ts
@@ -1,4 +1,4 @@
-import {ENR} from "@chainsafe/discv5";
+import {ENR} from "@chainsafe/enr";
 import {BeaconConfig} from "@lodestar/config";
 import {ENRKey} from "../metadata.js";
 

diff --git a/packages/beacon-node/src/network/discv5/worker.ts b/packages/beacon-node/src/network/discv5/worker.ts
@@ -5,15 +5,8 @@ import {createFromProtobuf} from "@libp2p/peer-id-factory";
 import {Multiaddr, multiaddr} from "@multiformats/multiaddr";
 import {expose} from "@chainsafe/threads/worker";
 import {Observable, Subject} from "@chainsafe/threads/observable";
-import {
-  createKeypairFromPeerId,
-  Discv5,
-  ENR,
-  ENRData,
-  IDiscv5CreateOptions,
-  SignableENR,
-  SignableENRData,
-} from "@chainsafe/discv5";
+import {Discv5} from "@chainsafe/discv5";
+import {createPrivateKeyFromPeerId, ENR, ENRData, SignableENR, SignableENRData} from "@chainsafe/enr";
 import {createBeaconConfig} from "@lodestar/config";
 import {getNodeLogger} from "@lodestar/logger/node";
 import {Gauge} from "@lodestar/utils";
@@ -51,20 +44,20 @@ if (workerData.metrics) {
 }
 
 const peerId = await createFromProtobuf(workerData.peerIdProto);
-const keypair = createKeypairFromPeerId(peerId);
+const keypair = createPrivateKeyFromPeerId(peerId);
 
 const config = createBeaconConfig(workerData.chainConfig, workerData.genesisValidatorsRoot);
 
 // Initialize discv5
 const discv5 = Discv5.create({
-  enr: SignableENR.decodeTxt(workerData.enr, keypair),
+  enr: SignableENR.decodeTxt(workerData.enr, keypair.privateKey),
   peerId,
   bindAddrs: {
     ip4: (workerData.bindAddrs.ip4 ? multiaddr(workerData.bindAddrs.ip4) : undefined) as Multiaddr,
     ip6: workerData.bindAddrs.ip6 ? multiaddr(workerData.bindAddrs.ip6) : undefined,
   },
   config: workerData.config,
-  metricsRegistry: metricsRegistry as IDiscv5CreateOptions["metricsRegistry"],
+  metricsRegistry,
 });
 
 // Load boot enrs

diff --git a/packages/beacon-node/src/network/events.ts b/packages/beacon-node/src/network/events.ts
@@ -1,6 +1,5 @@
 import {EventEmitter} from "events";
-import {PeerId} from "@libp2p/interface/peer-id";
-import {TopicValidatorResult} from "@libp2p/interface/pubsub";
+import {PeerId, TopicValidatorResult} from "@libp2p/interface";
 import {phase0, RootHex} from "@lodestar/types";
 import {BlockInput} from "../chain/blocks/types.js";
 import {StrictEventEmitterSingleArg} from "../util/strictEvents.js";

diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts
@@ -1,6 +1,6 @@
 import {compress, uncompress} from "snappyjs";
 import xxhashFactory from "xxhash-wasm";
-import {Message} from "@libp2p/interface/pubsub";
+import {Message} from "@libp2p/interface";
 import {digest} from "@chainsafe/as-sha256";
 import {RPC} from "@chainsafe/libp2p-gossipsub/message";
 import {intToBytes, toHex} from "@lodestar/utils";

diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts
@@ -1,5 +1,5 @@
 import {Libp2p} from "libp2p";
-import {Message, TopicValidatorResult} from "@libp2p/interface/pubsub";
+import {Message, TopicValidatorResult} from "@libp2p/interface";
 import {PeerIdStr} from "@chainsafe/libp2p-gossipsub/types";
 import {ForkName} from "@lodestar/params";
 import {allForks, altair, capella, deneb, phase0, Slot} from "@lodestar/types";

diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts
@@ -1,5 +1,20 @@
 import {Libp2p as ILibp2p} from "libp2p";
-import {Components} from "libp2p/components";
+import {
+  Libp2pEvents,
+  ComponentLogger,
+  NodeInfo,
+  ConnectionProtector,
+  ConnectionGater,
+  ContentRouting,
+  TypedEventTarget,
+  Metrics,
+  PeerId,
+  PeerRouting,
+  PeerStore,
+  Upgrader,
+} from "@libp2p/interface";
+import type {AddressManager, ConnectionManager, Registrar, TransportManager} from "@libp2p/interface-internal";
+import type {Datastore} from "interface-datastore";
 import {Slot, SlotRootHex, allForks, altair, capella, deneb, phase0} from "@lodestar/types";
 import {PeerIdStr} from "../util/peerId.js";
 import {INetworkEventBus} from "./events.js";
@@ -64,21 +79,23 @@ export interface INetwork extends INetworkCorePublic {
   writeDiscv5HeapSnapshot(prefix: string, dirpath: string): Promise<string>;
 }
 
-export type LodestarComponents = Pick<
-  Components,
-  | "peerId"
-  | "events"
-  | "addressManager"
-  | "peerStore"
-  | "upgrader"
-  | "registrar"
-  | "connectionManager"
-  | "transportManager"
-  | "connectionGater"
-  | "contentRouting"
-  | "peerRouting"
-  | "datastore"
-  | "connectionProtector"
-  | "metrics"
->;
+export type LodestarComponents = {
+  peerId: PeerId;
+  nodeInfo: NodeInfo;
+  logger: ComponentLogger;
+  events: TypedEventTarget<Libp2pEvents>;
+  addressManager: AddressManager;
+  peerStore: PeerStore;
+  upgrader: Upgrader;
+  registrar: Registrar;
+  connectionManager: ConnectionManager;
+  transportManager: TransportManager;
+  connectionGater: ConnectionGater;
+  contentRouting: ContentRouting;
+  peerRouting: PeerRouting;
+  datastore: Datastore;
+  connectionProtector?: ConnectionProtector;
+  metrics?: Metrics;
+};
+
 export type Libp2p = ILibp2p<{components: LodestarComponents}>;
diff --git a/packages/beacon-node/src/network/libp2p/index.ts b/packages/beacon-node/src/network/libp2p/index.ts
@@ -1,18 +1,17 @@
-import {PeerId} from "@libp2p/interface/peer-id";
+import {PeerId} from "@libp2p/interface";
 import {Registry} from "prom-client";
-import {ENR} from "@chainsafe/discv5";
-import type {Components} from "libp2p/components";
-import {identifyService} from "libp2p/identify";
+import {ENR} from "@chainsafe/enr";
+import {identify} from "@libp2p/identify";
 import {bootstrap} from "@libp2p/bootstrap";
 import {mdns} from "@libp2p/mdns";
 import {createLibp2p} from "libp2p";
 import {mplex} from "@libp2p/mplex";
 import {prometheusMetrics} from "@libp2p/prometheus-metrics";
 import {tcp} from "@libp2p/tcp";
+import {noise} from "@chainsafe/libp2p-noise";
 import {defaultNetworkOptions, NetworkOptions} from "../options.js";
 import {Eth2PeerDataStore} from "../peers/datastore.js";
-import {Libp2p} from "../interface.js";
-import {createNoise} from "./noise.js";
+import {Libp2p, LodestarComponents} from "../interface.js";
 
 export type NodeJsLibp2pOpts = {
   peerStoreDir?: string;
@@ -70,7 +69,7 @@ export async function createNodeJsLibp2p(
       listen: localMultiaddrs,
       announce: [],
     },
-    connectionEncryption: [createNoise()],
+    connectionEncryption: [noise()],
     // Reject connections when the server's connection count gets high
     transports: [
       tcp({
@@ -98,7 +97,6 @@ export async function createNodeJsLibp2p(
       // dialer config
       maxParallelDials: 100,
       maxPeerAddrsToDial: 4,
-      maxParallelDialsPerPeer: 2,
       dialTimeout: 30_000,
 
       // Rely entirely on lodestar's peer manager to prune connections
@@ -111,13 +109,15 @@ export async function createNodeJsLibp2p(
     },
     datastore,
     services: {
-      identify: identifyService({
+      identify: identify({
         agentVersion: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar",
       }),
       // individual components are specified because the components object is a Proxy
       // and passing it here directly causes problems downstream, not to mention is slowwww
-      components: (components: Components) => ({
+      components: (components: LodestarComponents) => ({
         peerId: components.peerId,
+        nodeInfo: components.nodeInfo,
+        logger: components.logger,
         events: components.events,
         addressManager: components.addressManager,
         peerStore: components.peerStore,