Enable using an engine endpoint for eth1 deposit/merge tracker (#3949)

* Enable using an engine endpoint for eth1 deposit/merge tracker

* update CI merge to run eth tracker on engine

* update the nethermind build target

* unit test

* rearrange jwt secret write files

* fix jwt file availability

.github/workflows/test-sim-merge.yml

@@ -3,8 +3,8 @@ name: Sim merge tests
 on: [pull_request, push]
 
 env:
-  GETH_COMMIT: e0e8bf31c5d44f7de33ce774b221debf2c42256c
-  NETHERMIND_COMMIT: 29ffe2630afb69120004ff79bde70d37a3b80b7b
+  GETH_COMMIT: bb5633c5ee3975ce016636066ec790054ec469e4
+  NETHERMIND_COMMIT: 00b50532543824dbac65e8b7ab09484e44992c27
 
 jobs:
   sim-merge-tests:
@@ -51,7 +51,7 @@ jobs:
           EL_BINARY_DIR: ../../go-ethereum/build/bin
           EL_SCRIPT_DIR: kiln/geth
           ENGINE_PORT: 8551
-          EL_PORT: 8545
+          ETH_PORT: 8545
           TX_SCENARIOS: simple
 
       # Install Nethermind merge interop
@@ -69,8 +69,8 @@ jobs:
         env:
           EL_BINARY_DIR: ../../nethermind/src/Nethermind/Nethermind.Runner
           EL_SCRIPT_DIR: kiln/nethermind
-          EL_PORT: 8550
           ENGINE_PORT: 8551
+          ETH_PORT: 8545
 
       - name: Upload debug log test files
         if: ${{ always() }}

kiln/devnets/kiln.vars

@@ -12,7 +12,7 @@ BESU_IMAGE=hyperledger/besu:develop
 
 LODESTAR_IMAGE=chainsafe/lodestar:next
 
-LODESTAR_EXTRA_ARGS="--eth1.providerUrls http://127.0.0.1:8545 --execution.urls http://127.0.0.1:8551 --api.rest.enabled --api.rest.host 0.0.0.0 --api.rest.api '*'"
+LODESTAR_EXTRA_ARGS="--execution.urls http://127.0.0.1:8551 --api.rest.enabled --api.rest.host 0.0.0.0 --api.rest.api '*'"
 
 LODESTAR_VALIDATOR_ARGS='--network kiln  --fromMnemonic "lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow" --mnemonicIndexes 0..5'
 

kiln/geth/post-merge.sh

@@ -5,4 +5,4 @@ currentDir=$(pwd)
 
 . $scriptDir/common-setup.sh
 
-$EL_BINARY_DIR/geth --http --ws -http.api "engine,net,eth" --datadir $DATA_DIR --allow-insecure-unlock --unlock $pubKey --password $DATA_DIR/password.txt --authrpc.jwtsecret $currentDir/$DATA_DIR/jwtsecret
+$EL_BINARY_DIR/geth --http -http.api "engine,net,eth,miner" --http.port $ETH_PORT --authrpc.port $ENGINE_PORT --authrpc.jwtsecret $currentDir/$DATA_DIR/jwtsecret --datadir $DATA_DIR --allow-insecure-unlock --unlock $pubKey --password $DATA_DIR/password.txt

kiln/geth/pre-merge.sh

@@ -5,4 +5,4 @@ currentDir=$(pwd)
 
 . $scriptDir/common-setup.sh
 
-$EL_BINARY_DIR/geth --http --ws -http.api "engine,net,eth,miner" --datadir $DATA_DIR --allow-insecure-unlock --unlock $pubKey --password $DATA_DIR/password.txt --nodiscover --mine --authrpc.jwtsecret $currentDir/$DATA_DIR/jwtsecret
+$EL_BINARY_DIR/geth --http -http.api "engine,net,eth,miner" --http.port $ETH_PORT --authrpc.port $ENGINE_PORT --authrpc.jwtsecret $currentDir/$DATA_DIR/jwtsecret --datadir $DATA_DIR --allow-insecure-unlock --unlock $pubKey --password $DATA_DIR/password.txt --nodiscover --mine

kiln/gethdocker/Dockerfile

@@ -9,5 +9,5 @@ RUN cd /go-ethereum && go run build/ci.go install ./cmd/geth
 FROM alpine:latest
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
-EXPOSE 8545 8546 30303 30303/udp
+EXPOSE 8545 8551 30303 30303/udp
 ENTRYPOINT ["geth"]

kiln/gethdocker/README.md

@@ -12,5 +12,5 @@ docker build  . --tag geth:kiln
 
 ```bash
 cd packages/lodestar
-EL_BINARY_DIR=geth:kiln EL_SCRIPT_DIR=kiln/gethdocker EL_PORT=8545 ENGINE_PORT=8551 TX_SCENARIOS=simple yarn mocha test/sim/merge-interop.test.ts
+EL_BINARY_DIR=geth:kiln EL_SCRIPT_DIR=kiln/gethdocker ETH_PORT=8545 ENGINE_PORT=8551 TX_SCENARIOS=simple yarn mocha test/sim/merge-interop.test.ts
 ```

kiln/gethdocker/post-merge.sh

@@ -5,4 +5,4 @@ currentDir=$(pwd)
 
 . $scriptDir/common-setup.sh
 
-docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --network host -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --http --ws -http.api "engine,net,eth" --allow-insecure-unlock --unlock $pubKey --password /data/password.txt --datadir /data --authrpc.jwtsecret /data/jwtsecret
+docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --network host -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --http -http.api "engine,net,eth,miner" --http.port $ETH_PORT --authrpc.port $ENGINE_PORT --authrpc.jwtsecret /data/jwtsecret --allow-insecure-unlock --unlock $pubKey --password /data/password.txt --datadir /data

kiln/gethdocker/pre-merge.sh

@@ -6,4 +6,4 @@ currentDir=$(pwd)
 . $scriptDir/common-setup.sh
 
 # EL_BINARY_DIR refers to the local docker image build from kiln/gethdocker folder
-docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --network host -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --http --ws -http.api "engine,net,eth,miner" --allow-insecure-unlock --unlock $pubKey --password /data/password.txt --datadir /data --nodiscover --mine --authrpc.jwtsecret /data/jwtsecret
+docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --network host -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --http -http.api "engine,net,eth,miner" --http.port $ETH_PORT --authrpc.port $ENGINE_PORT --authrpc.jwtsecret /data/jwtsecret --allow-insecure-unlock --unlock $pubKey --password /data/password.txt --datadir /data --nodiscover --mine

kiln/nethermind/post-merge.sh

@@ -6,4 +6,4 @@ currentDir=$(pwd)
 . $scriptDir/common-setup.sh
 
 cd $EL_BINARY_DIR
-dotnet run -c Release -- --config themerge_kiln_testvectors --Merge.TerminalTotalDifficulty $TTD --JsonRpc.JwtSecretFile $currentDir/$DATA_DIR/jwtsecret
+dotnet run -c Release -- --config themerge_kiln_testvectors --Merge.TerminalTotalDifficulty $TTD --JsonRpc.JwtSecretFile $currentDir/$DATA_DIR/jwtsecret --JsonRpc.Enabled true --JsonRpc.Host 0.0.0.0 --JsonRpc.AdditionalRpcUrls "http://localhost:$ETH_PORT|http|net;eth;subscribe;engine;web3;client|no-auth,http://localhost:$ENGINE_PORT|http|eth;engine"

kiln/nethermind/pre-merge.sh

@@ -6,4 +6,4 @@ currentDir=$(pwd)
 . $scriptDir/common-setup.sh
 
 cd $EL_BINARY_DIR
-dotnet run -c Release -- --config themerge_kiln_m2 --Merge.TerminalTotalDifficulty $TTD --JsonRpc.JwtSecretFile $currentDir/$DATA_DIR/jwtsecret 
+dotnet run -c Release -- --config themerge_kiln_m2 --Merge.TerminalTotalDifficulty $TTD --JsonRpc.JwtSecretFile $currentDir/$DATA_DIR/jwtsecret --Merge.Enabled true  --Init.DiagnosticMode=None --JsonRpc.Enabled true --JsonRpc.Host 0.0.0.0 --JsonRpc.AdditionalRpcUrls "http://localhost:$ETH_PORT|http|net;eth;subscribe;engine;web3;client|no-auth,http://localhost:$ENGINE_PORT|http|eth;engine"

packages/cli/src/options/beaconNodeOptions/eth1.ts

@@ -1,5 +1,7 @@
+import fs from "node:fs";
 import {defaultOptions, IBeaconNodeOptions} from "@chainsafe/lodestar";
-import {ICliCommandOptions} from "../../util";
+import {ICliCommandOptions, extractJwtHexSecret} from "../../util";
+import {ExecutionEngineArgs} from "./execution";
 
 export interface IEth1Args {
   "eth1.enabled": boolean;
@@ -10,17 +12,30 @@ export interface IEth1Args {
   "eth1.unsafeAllowDepositDataOverwrite": boolean;
 }
 
-export function parseArgs(args: IEth1Args): IBeaconNodeOptions["eth1"] {
+export function parseArgs(args: IEth1Args & Partial<ExecutionEngineArgs>): IBeaconNodeOptions["eth1"] {
   // Support deprecated flag 'eth1.providerUrl' only if 'eth1.providerUrls' is not defined
   // Safe default to '--eth1.providerUrl' only if it's defined. Prevent returning providerUrls: [undefined]
+  let jwtSecretHex: string | undefined;
   let providerUrls = args["eth1.providerUrls"];
-  if (providerUrls !== undefined && args["eth1.providerUrl"]) {
+  if (providerUrls === undefined && args["eth1.providerUrl"]) {
     providerUrls = [args["eth1.providerUrl"]];
   }
 
+  // If no providerUrls are explicitly provided, we should pick the execution endpoint
+  // because as per Kiln spec v2.1, execution *must* host the `eth_` methods necessary
+  // for deposit and merge trackers on engine endpoints as well protected by a
+  // jwt auth mechanism.
+  if (providerUrls === undefined && args["execution.urls"]) {
+    providerUrls = args["execution.urls"];
+    jwtSecretHex = args["jwt-secret"]
+      ? extractJwtHexSecret(fs.readFileSync(args["jwt-secret"], "utf-8").trim())
+      : undefined;
+  }
+
   return {
     enabled: args["eth1.enabled"],
-    providerUrls: providerUrls,
+    providerUrls,
+    jwtSecretHex,
     depositContractDeployBlock: args["eth1.depositContractDeployBlock"],
     disableEth1DepositDataTracker: args["eth1.disableEth1DepositDataTracker"],
     unsafeAllowDepositDataOverwrite: args["eth1.unsafeAllowDepositDataOverwrite"],
@@ -43,7 +58,8 @@ export const options: ICliCommandOptions<IEth1Args> = {
   },
 
   "eth1.providerUrls": {
-    description: "Urls to Eth1 node with enabled rpc",
+    description:
+      "Urls to Eth1 node with enabled rpc. If not explicity provided and execution endpoint provided via execution.urls, it will use execution.urls. Otherwise will try connecting on the specified default(s)",
     type: "array",
     defaultDescription: defaultOptions.eth1.providerUrls.join(" "),
     group: "eth1",

packages/cli/test/unit/options/beaconNodeOptions.test.ts

@@ -1,7 +1,9 @@
 import {expect} from "chai";
+import fs from "node:fs";
 import {IBeaconNodeOptions} from "@chainsafe/lodestar";
 import {LogLevel, RecursivePartial} from "@chainsafe/lodestar-utils";
 import {parseBeaconNodeArgs, IBeaconNodeArgs} from "../../../src/options/beaconNodeOptions";
+import {getTestdirPath} from "../../utils";
 
 describe("options / beaconNodeOptions", () => {
   it("Should parse BeaconNodeArgs", () => {
@@ -129,4 +131,28 @@ describe("options / beaconNodeOptions", () => {
     const options = parseBeaconNodeArgs(beaconNodeArgsPartial);
     expect(options).to.deep.equal(expectedOptions);
   });
+
+  it("Should use execution endpoint & jwt for eth1", () => {
+    const jwtSecretFile = getTestdirPath("./jwtsecret");
+    const jwtSecretHex = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d";
+    fs.writeFileSync(jwtSecretFile, jwtSecretHex, {encoding: "utf8"});
+
+    // Cast to match the expected fully defined type
+    const beaconNodeArgsPartial = {
+      "eth1.enabled": true,
+      "execution.urls": ["http://my.node:8551"],
+      "jwt-secret": jwtSecretFile,
+    } as IBeaconNodeArgs;
+
+    const expectedOptions: RecursivePartial<IBeaconNodeOptions> = {
+      eth1: {
+        enabled: true,
+        providerUrls: ["http://my.node:8551"],
+        jwtSecretHex,
+      },
+    };
+
+    const options = parseBeaconNodeArgs(beaconNodeArgsPartial);
+    expect(options.eth1).to.deep.equal(expectedOptions.eth1);
+  });
 });

packages/lodestar/src/eth1/options.ts

@@ -2,6 +2,11 @@ export type Eth1Options = {
   enabled: boolean;
   disableEth1DepositDataTracker?: boolean;
   providerUrls: string[];
+  /**
+   * jwtSecretHex is the jwt secret if the eth1 modules should ping the jwt auth
+   * protected engine endpoints.
+   */
+  jwtSecretHex?: string;
   depositContractDeployBlock?: number;
   unsafeAllowDepositDataOverwrite: boolean;
 };

packages/lodestar/src/eth1/provider/eth1Provider.ts

@@ -2,6 +2,8 @@ import {toHexString} from "@chainsafe/ssz";
 import {phase0} from "@chainsafe/lodestar-types";
 import {AbortSignal} from "@chainsafe/abort-controller";
 import {IChainConfig} from "@chainsafe/lodestar-config";
+import {fromHex} from "@chainsafe/lodestar-utils";
+
 import {chunkifyInclusiveRange} from "../../util/chunkify";
 import {linspace} from "../../util/numpy";
 import {retry} from "../../util/retry";
@@ -43,7 +45,7 @@ export class Eth1Provider implements IEth1Provider {
 
   constructor(
     config: Pick<IChainConfig, "DEPOSIT_CONTRACT_ADDRESS">,
-    opts: Pick<Eth1Options, "depositContractDeployBlock" | "providerUrls">,
+    opts: Pick<Eth1Options, "depositContractDeployBlock" | "providerUrls" | "jwtSecretHex">,
     signal?: AbortSignal
   ) {
     this.deployBlock = opts.depositContractDeployBlock ?? 0;
@@ -52,6 +54,7 @@ export class Eth1Provider implements IEth1Provider {
       signal,
       // Don't fallback with is truncated error. Throw early and let the retry on this class handle it
       shouldNotFallback: isJsonRpcTruncatedError,
+      jwtSecret: opts.jwtSecretHex ? fromHex(opts.jwtSecretHex) : undefined,
     });
   }
 

packages/lodestar/test/sim/merge-interop.test.ts

@@ -28,15 +28,15 @@ import {bytesToData, dataToBytes, quantityToNum} from "../../src/eth1/provider/u
 // EL_BINARY_DIR: File path to locate the EL executable
 // EL_SCRIPT_DIR: Directory in packages/lodestar for the EL client, from where to
 // execute post-merge/pre-merge EL scenario scripts
-// EL_PORT: EL port on localhost for hosting both engine & json rpc endpoints
+// ETH_PORT: EL port on localhost hosting non auth protected eth_ methods
 // ENGINE_PORT: Specify the port on which an jwt auth protected engine api is being hosted,
 //   typically by default at 8551 for geth. Some ELs could host it as same port as eth_ apis,
 //   but just with the engine_ methods protected. In that case this param can be skipped
 // TX_SCENARIOS: comma seprated transaction scenarios this EL client build supports
 // Example:
 // ```
 // $ EL_BINARY_DIR=/home/lion/Code/eth2.0/merge-interop/go-ethereum/build/bin \
-//   EL_SCRIPT_DIR=kiln/geth EL_PORT=8545 ENGINE_PORT=8551 TX_SCENARIOS=simple \
+//   EL_SCRIPT_DIR=kiln/geth ETH_PORT=8545 ENGINE_PORT=8551 TX_SCENARIOS=simple \
 //   ../../node_modules/.bin/mocha test/sim/merge.test.ts
 // ```
 
@@ -52,8 +52,10 @@ describe("executionEngine / ExecutionEngineHttp", function () {
   this.timeout("10min");
 
   const dataPath = fs.mkdtempSync("lodestar-test-merge-interop");
-  const jsonRpcPort = process.env.EL_PORT;
-  const enginePort = process.env.ENGINE_PORT ?? jsonRpcPort;
+  const jsonRpcPort = process.env.ETH_PORT;
+  const enginePort = process.env.ENGINE_PORT;
+
+  /** jsonRpcUrl is used only for eth transactions or to check if EL online/offline */
   const jsonRpcUrl = `http://localhost:${jsonRpcPort}`;
   const engineApiUrl = `http://localhost:${enginePort}`;
 
@@ -115,9 +117,9 @@ describe("executionEngine / ExecutionEngineHttp", function () {
   // $ ./go-ethereum/build/bin/geth --catalyst --datadir "~/ethereum/taunus" init genesis.json
   // $ ./build/bin/geth --catalyst --http --ws -http.api "engine" --datadir "~/ethereum/taunus" console
   async function runEL(elScript: string, ttd: number): Promise<{genesisBlockHash: string}> {
-    if (!process.env.EL_BINARY_DIR || !process.env.EL_SCRIPT_DIR || !process.env.EL_PORT) {
+    if (!process.env.EL_BINARY_DIR || !process.env.EL_SCRIPT_DIR || !process.env.ENGINE_PORT || !process.env.ETH_PORT) {
       throw Error(
-        `EL ENV must be provided, EL_BINARY_DIR: ${process.env.EL_BINARY_DIR}, EL_SCRIPT_DIR: ${process.env.EL_SCRIPT_DIR}, EL_PORT: ${process.env.EL_PORT}`
+        `EL ENV must be provided, EL_BINARY_DIR: ${process.env.EL_BINARY_DIR}, EL_SCRIPT_DIR: ${process.env.EL_SCRIPT_DIR}, ENGINE_PORT: ${process.env.ENGINE_PORT}, ETH_PORT: ${process.env.ETH_PORT}`
       );
     }
 
@@ -136,7 +138,7 @@ describe("executionEngine / ExecutionEngineHttp", function () {
     await waitForELOnline(jsonRpcUrl, controller.signal);
 
     // Fetch genesis block hash
-    const genesisBlockHash = await getGenesisBlockHash(jsonRpcUrl, controller.signal);
+    const genesisBlockHash = await getGenesisBlockHash({providerUrl: engineApiUrl, jwtSecretHex}, controller.signal);
     return {genesisBlockHash};
   }
 
@@ -319,7 +321,8 @@ describe("executionEngine / ExecutionEngineHttp", function () {
         api: {rest: {enabled: true} as RestApiOptions},
         sync: {isSingleNode: true},
         network: {allowPublishToZeroPeers: true, discv5: null},
-        eth1: {enabled: true, providerUrls: [jsonRpcUrl]},
+        // Now eth deposit/merge tracker methods directly available on engine endpoints
+        eth1: {enabled: true, providerUrls: [engineApiUrl], jwtSecretHex},
         executionEngine: {urls: [engineApiUrl], jwtSecretHex},
       },
       validatorCount: validatorClientCount * validatorsPerClient,
@@ -418,7 +421,7 @@ describe("executionEngine / ExecutionEngineHttp", function () {
 
     // Assertions to make sure the end state is good
     // 1. The proper head is set
-    const rpc = new Eth1Provider({DEPOSIT_CONTRACT_ADDRESS: ZERO_HASH}, {providerUrls: [jsonRpcUrl]});
+    const rpc = new Eth1Provider({DEPOSIT_CONTRACT_ADDRESS: ZERO_HASH}, {providerUrls: [engineApiUrl], jwtSecretHex});
     const consensusHead = bn.chain.forkChoice.getHead();
     const executionHeadBlockNumber = await rpc.getBlockNumber();
     const executionHeadBlock = await rpc.getBlockByNumber(executionHeadBlockNumber);
@@ -502,10 +505,13 @@ async function isPortInUse(port: number): Promise<boolean> {
   });
 }
 
-async function getGenesisBlockHash(url: string, signal: AbortSignal): Promise<string> {
+async function getGenesisBlockHash(
+  {providerUrl, jwtSecretHex}: {providerUrl: string; jwtSecretHex?: string},
+  signal: AbortSignal
+): Promise<string> {
   const eth1Provider = new Eth1Provider(
     ({DEPOSIT_CONTRACT_ADDRESS: ZERO_HASH} as Partial<IChainConfig>) as IChainConfig,
-    {providerUrls: [url]},
+    {providerUrls: [providerUrl], jwtSecretHex},
     signal
   );