test: stabilize sim tests (#6312)

* Fix sim test genesis delay * Update the defaults for the all sim tests * Increase the limit for slotImportTolerance for unknown block sync * Update the additional slots for TTD * Adjust the ttd time * Adjust the ttd * Adjust the ttd * Update the ttd calculation * Add timestamp to report * Update the slot to check the merge transition * Update ttd calculations * Increase the genesis delay * Update the shangai ttd calculation * Increase the slot tolerance * Disable broadcast validation for unknown block tests * Fix lint errors
ChainSafe · Jan 17, 2024 · 912ec09 · 912ec09
1 parent d7ca290
commit 912ec09
Show file tree

Hide file tree

Showing 13 changed files with 154 additions and 224 deletions.
diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml
@@ -16,6 +16,11 @@ on:
         description: Runtime DEBUG value
         required: false
         default: ""
+      genesisDelaySlots:
+        description: 'Number of slots delay before genesis'
+        required: true
+        type: number
+        default: 40
 
 env:
   GETH_DOCKER_IMAGE: ethereum/client-go:v1.11.6
@@ -56,6 +61,8 @@ jobs:
       - name: Sim tests multifork
         run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:multifork
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}
 
       - name: Sim tests endpoints
         run: yarn test:sim:endpoints

diff --git a/packages/cli/test/scripts/e2e_test_env.ts b/packages/cli/test/scripts/e2e_test_env.ts
@@ -2,37 +2,25 @@
 import path from "node:path";
 import {BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
 import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
-import {getEstimatedTTD, logFilesDir} from "../utils/simulation/utils/index.js";
+import {defineSimTestConfig, logFilesDir} from "../utils/simulation/utils/index.js";
 import {connectAllNodes} from "../utils/simulation/utils/network.js";
 
-const secondsPerSlot = 4;
-const cliqueSealingPeriod = 5;
-const genesisDelaySeconds = 30 * secondsPerSlot;
 const altairForkEpoch = 1;
 const bellatrixForkEpoch = 2;
 const capellaForkEpoch = 3;
-// Make sure bellatrix started before TTD reach
-const additionalSlotsForTTD = 2;
 
-const ttd = getEstimatedTTD({
-  genesisDelaySeconds,
-  bellatrixForkEpoch,
-  secondsPerSlot,
-  cliqueSealingPeriod,
-  additionalSlots: additionalSlotsForTTD,
+const {forkConfig} = defineSimTestConfig({
+  ALTAIR_FORK_EPOCH: altairForkEpoch,
+  BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
+  CAPELLA_FORK_EPOCH: capellaForkEpoch,
+  runTillEpoch: Infinity,
 });
 
 const env = await SimulationEnvironment.initWithDefaults(
   {
     id: "e2e-test-env",
     logsDir: path.join(logFilesDir, "e2e-test-env"),
-    chainConfig: {
-      ALTAIR_FORK_EPOCH: altairForkEpoch,
-      BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
-      CAPELLA_FORK_EPOCH: capellaForkEpoch,
-      GENESIS_DELAY: genesisDelaySeconds,
-      TERMINAL_TOTAL_DIFFICULTY: ttd,
-    },
+    forkConfig,
   },
   [
     {id: "node-1", beacon: BeaconClient.Lodestar, execution: ExecutionClient.Geth, keysCount: 32, mining: true},

diff --git a/packages/cli/test/sim/backup_eth_provider.test.ts b/packages/cli/test/sim/backup_eth_provider.test.ts
@@ -1,53 +1,28 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
 import {activePreset} from "@lodestar/params";
+import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
 import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
-import {CLIQUE_SEALING_PERIOD, SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js";
 import {AssertionMatch, BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
-import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
-import {
-  getEstimatedTimeInSecForRun,
-  getEstimatedTTD,
-  logFilesDir,
-  replaceIpFromUrl,
-} from "../utils/simulation/utils/index.js";
+import {defineSimTestConfig, logFilesDir, replaceIpFromUrl} from "../utils/simulation/utils/index.js";
 import {connectAllNodes, waitForSlot} from "../utils/simulation/utils/network.js";
 
-const genesisDelaySeconds = 20 * SIM_TESTS_SECONDS_PER_SLOT;
 const altairForkEpoch = 2;
 const bellatrixForkEpoch = 4;
-// Make sure bellatrix started before TTD reach
-const additionalSlotsForTTD = activePreset.SLOTS_PER_EPOCH - 2;
 const runTillEpoch = 6;
 const syncWaitEpoch = 2;
 
-const runTimeoutMs =
-  getEstimatedTimeInSecForRun({
-    genesisDelaySeconds,
-    secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
-    runTill: runTillEpoch + syncWaitEpoch,
-    // After adding Nethermind its took longer to complete
-    graceExtraTimeFraction: 0.3,
-  }) * 1000;
-
-const ttd = getEstimatedTTD({
-  genesisDelaySeconds,
-  bellatrixForkEpoch: bellatrixForkEpoch,
-  secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
-  cliqueSealingPeriod: CLIQUE_SEALING_PERIOD,
-  additionalSlots: additionalSlotsForTTD,
+const {estimatedTimeoutMs, forkConfig} = defineSimTestConfig({
+  ALTAIR_FORK_EPOCH: altairForkEpoch,
+  BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
+  runTillEpoch: runTillEpoch + syncWaitEpoch,
 });
 
 const env = await SimulationEnvironment.initWithDefaults(
   {
     id: "backup-eth-provider",
     logsDir: path.join(logFilesDir, "backup-eth-provider"),
-    chainConfig: {
-      ALTAIR_FORK_EPOCH: altairForkEpoch,
-      BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
-      GENESIS_DELAY: genesisDelaySeconds,
-      TERMINAL_TOTAL_DIFFICULTY: ttd,
-    },
+    forkConfig,
   },
   [{id: "node-1", beacon: BeaconClient.Lodestar, execution: ExecutionClient.Geth, keysCount: 32, mining: true}]
 );
@@ -88,7 +63,7 @@ const node3 = await env.createNodePair({
 env.nodes.push(node2);
 env.nodes.push(node3);
 
-await env.start({runTimeoutMs});
+await env.start({runTimeoutMs: estimatedTimeoutMs});
 await connectAllNodes(env.nodes);
 
 await waitForSlot(env.clock.getLastSlotOfEpoch(1), env.nodes, {silent: true, env});

diff --git a/packages/cli/test/sim/deneb.test.ts b/packages/cli/test/sim/deneb.test.ts
@@ -1,50 +1,30 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
+import {ApiError} from "@lodestar/api";
 import {activePreset} from "@lodestar/params";
 import {toHex, toHexString} from "@lodestar/utils";
-import {ApiError} from "@lodestar/api";
+import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
 import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
-import {CLIQUE_SEALING_PERIOD, SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js";
 import {AssertionMatch, BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
-import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
-import {getEstimatedTimeInSecForRun, getEstimatedTTD, logFilesDir} from "../utils/simulation/utils/index.js";
+import {defineSimTestConfig, logFilesDir} from "../utils/simulation/utils/index.js";
 import {connectAllNodes, connectNewNode, waitForNodeSync, waitForSlot} from "../utils/simulation/utils/network.js";
 
-const genesisDelaySeconds = 20 * SIM_TESTS_SECONDS_PER_SLOT;
 const altairForkEpoch = 2;
 const bellatrixForkEpoch = 4;
-// Make sure bellatrix started before TTD reach
-const additionalSlotsForTTD = activePreset.SLOTS_PER_EPOCH - 2;
 const runTillEpoch = 6;
 const syncWaitEpoch = 2;
 
-const runTimeoutMs =
-  getEstimatedTimeInSecForRun({
-    genesisDelaySeconds,
-    secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
-    runTill: runTillEpoch + syncWaitEpoch,
-    // After adding Nethermind its took longer to complete
-    graceExtraTimeFraction: 0.3,
-  }) * 1000;
-
-const ttd = getEstimatedTTD({
-  genesisDelaySeconds,
-  bellatrixForkEpoch: bellatrixForkEpoch,
-  secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
-  cliqueSealingPeriod: CLIQUE_SEALING_PERIOD,
-  additionalSlots: additionalSlotsForTTD,
+const {estimatedTimeoutMs, forkConfig} = defineSimTestConfig({
+  ALTAIR_FORK_EPOCH: altairForkEpoch,
+  BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
+  runTillEpoch: runTillEpoch + syncWaitEpoch,
 });
 
 const env = await SimulationEnvironment.initWithDefaults(
   {
     id: "multi-fork",
     logsDir: path.join(logFilesDir, "multi-fork"),
-    chainConfig: {
-      ALTAIR_FORK_EPOCH: altairForkEpoch,
-      BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
-      GENESIS_DELAY: genesisDelaySeconds,
-      TERMINAL_TOTAL_DIFFICULTY: ttd,
-    },
+    forkConfig,
   },
   [
     {id: "node-1", beacon: BeaconClient.Lodestar, execution: ExecutionClient.Mock, keysCount: 32},
@@ -59,7 +39,7 @@ env.tracker.register({
   },
 });
 
-await env.start({runTimeoutMs});
+await env.start({runTimeoutMs: estimatedTimeoutMs});
 await connectAllNodes(env.nodes);
 
 // The `TTD` will be reach around `start of bellatrixForkEpoch + additionalSlotsForMerge` slot

diff --git a/packages/cli/test/sim/endpoints.test.ts b/packages/cli/test/sim/endpoints.test.ts
@@ -1,37 +1,28 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
-import {expect} from "chai";
 import {toHexString} from "@chainsafe/ssz";
-import {routes} from "@lodestar/api";
-import {ApiError} from "@lodestar/api";
-import {BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
+import {expect} from "chai";
+import {ApiError, routes} from "@lodestar/api";
 import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
-import {getEstimatedTimeInSecForRun, logFilesDir} from "../utils/simulation/utils/index.js";
+import {BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
+import {defineSimTestConfig, logFilesDir} from "../utils/simulation/utils/index.js";
 import {waitForSlot} from "../utils/simulation/utils/network.js";
-import {SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js";
 
-const genesisDelaySeconds = 10 * SIM_TESTS_SECONDS_PER_SLOT;
 const altairForkEpoch = 2;
 const bellatrixForkEpoch = 4;
 const validatorCount = 2;
-const runTimeoutMs =
-  getEstimatedTimeInSecForRun({
-    genesisDelaySeconds,
-    secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
-    runTill: 2,
-    // After adding Nethermind its took longer to complete
-    graceExtraTimeFraction: 0.1,
-  }) * 1000;
+
+const {estimatedTimeoutMs, forkConfig} = defineSimTestConfig({
+  ALTAIR_FORK_EPOCH: altairForkEpoch,
+  BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
+  runTillEpoch: 2,
+});
 
 const env = await SimulationEnvironment.initWithDefaults(
   {
     id: "beacon-endpoints",
     logsDir: path.join(logFilesDir, "beacon-endpoints"),
-    chainConfig: {
-      ALTAIR_FORK_EPOCH: altairForkEpoch,
-      BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
-      GENESIS_DELAY: genesisDelaySeconds,
-    },
+    forkConfig,
   },
   [
     {
@@ -43,7 +34,7 @@ const env = await SimulationEnvironment.initWithDefaults(
     },
   ]
 );
-await env.start({runTimeoutMs});
+await env.start({runTimeoutMs: estimatedTimeoutMs});
 
 const node = env.nodes[0].beacon;
 await waitForSlot(2, env.nodes, {env, silent: true});

diff --git a/packages/cli/test/sim/mixed_client.test.ts b/packages/cli/test/sim/mixed_client.test.ts
@@ -2,48 +2,28 @@
 import path from "node:path";
 import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
 import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
-import {CLIQUE_SEALING_PERIOD, SIM_TESTS_SECONDS_PER_SLOT} from "../utils/simulation/constants.js";
 import {AssertionMatch, BeaconClient, ExecutionClient, ValidatorClient} from "../utils/simulation/interfaces.js";
-import {getEstimatedTTD, getEstimatedTimeInSecForRun, logFilesDir} from "../utils/simulation/utils/index.js";
+import {defineSimTestConfig, logFilesDir} from "../utils/simulation/utils/index.js";
 import {connectAllNodes, waitForSlot} from "../utils/simulation/utils/network.js";
 
-const genesisDelaySeconds = 20 * SIM_TESTS_SECONDS_PER_SLOT;
 const altairForkEpoch = 2;
 const bellatrixForkEpoch = 4;
 const capellaForkEpoch = 6;
-// Make sure bellatrix started before TTD reach
-const additionalSlotsForTTD = 2;
 const runTillEpoch = 8;
 const syncWaitEpoch = 2;
 
-const runTimeoutMs =
-  getEstimatedTimeInSecForRun({
-    genesisDelaySeconds,
-    secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
-    runTill: runTillEpoch + syncWaitEpoch,
-    // After adding Nethermind its took longer to complete
-    graceExtraTimeFraction: 0.3,
-  }) * 1000;
-
-const ttd = getEstimatedTTD({
-  genesisDelaySeconds,
-  bellatrixForkEpoch,
-  secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT,
-  cliqueSealingPeriod: CLIQUE_SEALING_PERIOD,
-  additionalSlots: additionalSlotsForTTD,
+const {estimatedTimeoutMs, forkConfig} = defineSimTestConfig({
+  ALTAIR_FORK_EPOCH: altairForkEpoch,
+  BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
+  CAPELLA_FORK_EPOCH: capellaForkEpoch,
+  runTillEpoch: runTillEpoch + syncWaitEpoch,
 });
 
 const env = await SimulationEnvironment.initWithDefaults(
   {
     id: "mixed-clients",
     logsDir: path.join(logFilesDir, "mixed-clients"),
-    chainConfig: {
-      ALTAIR_FORK_EPOCH: altairForkEpoch,
-      BELLATRIX_FORK_EPOCH: bellatrixForkEpoch,
-      CAPELLA_FORK_EPOCH: capellaForkEpoch,
-      GENESIS_DELAY: genesisDelaySeconds,
-      TERMINAL_TOTAL_DIFFICULTY: ttd,
-    },
+    forkConfig,
   },
   [
     {
@@ -86,7 +66,7 @@ env.tracker.register({
   },
 });
 
-await env.start({runTimeoutMs});
+await env.start({runTimeoutMs: estimatedTimeoutMs});
 await connectAllNodes(env.nodes);
 
 await waitForSlot(env.clock.getLastSlotOfEpoch(capellaForkEpoch + 1), env.nodes, {env, silent: true});