diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index db7e00d2820..e6b050408c9 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -43,9 +43,6 @@ jobs: - name: Download spec tests run: yarn download-spec-tests working-directory: packages/lodestar - - name: Check spec tests - run: yarn check-spec-tests - working-directory: packages/lodestar # Run them in different steps to quickly identifying which command failed # Otherwise just doing `yarn test:spec` you can't tell which specific suite failed @@ -53,21 +50,9 @@ jobs: - name: Spec tests general run: yarn test:spec-general working-directory: packages/lodestar - - name: Spec tests phase0-minimal - run: yarn test:spec-phase0-minimal - working-directory: packages/lodestar - - name: Spec tests phase0-mainnet - run: yarn test:spec-phase0-mainnet - working-directory: packages/lodestar - - name: Spec tests altair-minimal - run: yarn test:spec-altair-minimal - working-directory: packages/lodestar - - name: Spec tests altair-mainnet - run: yarn test:spec-altair-mainnet - working-directory: packages/lodestar - - name: Spec tests bellatrix-minimal - run: yarn test:spec-bellatrix-minimal + - name: Spec tests minimal + run: yarn test:spec-minimal working-directory: packages/lodestar - - name: Spec tests bellatrix-mainnet - run: yarn test:spec-bellatrix-mainnet + - name: Spec tests mainnet + run: yarn test:spec-mainnet working-directory: packages/lodestar diff --git a/packages/lodestar/package.json b/packages/lodestar/package.json index ccb74e2efa7..3453eeab845 100644 --- a/packages/lodestar/package.json +++ b/packages/lodestar/package.json @@ -44,19 +44,10 @@ "test:sim:merge-interop": "mocha 'test/sim/merge-interop.test.ts'", "download-spec-tests": "node -r ts-node/register test/spec/downloadTests.ts", "check-spec-tests": "mocha test/spec/checkCoverage.ts", - "test:bls": "mocha 'test/spec/bls/**/*.test.ts'", - "test:ssz-generic": "mocha 'test/spec/ssz/generic/index.test.ts'", - "test:spec-general": "yarn test:bls && yarn test:ssz-generic", - "test:spec": "yarn test:spec-general && yarn test:bls && yarn test:spec-phase0 && yarn test:spec-altair && yarn test:spec-bellatrix", - "test:spec-phase0-minimal": "LODESTAR_PRESET=minimal mocha --config .mocharc.spec.js 'test/spec/phase0/**/*.test.ts'", - "test:spec-phase0-mainnet": "LODESTAR_PRESET=mainnet mocha --config .mocharc.spec.js 'test/spec/phase0/**/*.test.ts'", - "test:spec-altair-minimal": "LODESTAR_PRESET=minimal mocha --config .mocharc.spec.js 'test/spec/altair/**/*.test.ts'", - "test:spec-altair-mainnet": "LODESTAR_PRESET=mainnet mocha --config .mocharc.spec.js 'test/spec/altair/**/*.test.ts'", - "test:spec-bellatrix-minimal": "LODESTAR_PRESET=minimal mocha --config .mocharc.spec.js 'test/spec/bellatrix/**/*.test.ts'", - "test:spec-bellatrix-mainnet": "LODESTAR_PRESET=mainnet mocha --config .mocharc.spec.js 'test/spec/bellatrix/**/*.test.ts'", - "test:spec-phase0": "yarn test:spec-phase0-minimal && yarn test:spec-phase0-mainnet", - "test:spec-altair": "yarn test:spec-altair-minimal && yarn test:spec-altair-mainnet", - "test:spec-bellatrix": "yarn test:spec-bellatrix-minimal && yarn test:spec-bellatrix-mainnet", + "test:spec-general": "mocha --config .mocharc.spec.js 'test/spec/general/**/*.test.ts'", + "test:spec-minimal": "LODESTAR_PRESET=minimal mocha --config .mocharc.spec.js 'test/spec/presets/**/*.test.ts'", + "test:spec-mainnet": "LODESTAR_PRESET=mainnet mocha --config .mocharc.spec.js 'test/spec/presets/**/*.test.ts'", + "test:spec": "yarn test:spec-general && yarn test:spec-minimal && test:spec-mainnet", "check-readme": "typescript-docs-verifier" }, "dependencies": { diff --git a/packages/lodestar/test/spec/allForks/epochProcessing.ts b/packages/lodestar/test/spec/allForks/epochProcessing.ts deleted file mode 100644 index b4caefe7ee9..00000000000 --- a/packages/lodestar/test/spec/allForks/epochProcessing.ts +++ /dev/null @@ -1,67 +0,0 @@ -import {join} from "node:path"; -import fs from "node:fs"; - -import { - CachedBeaconStateAllForks, - EpochProcess, - BeaconStateAllForks, - beforeProcessEpoch, -} from "@chainsafe/lodestar-beacon-state-transition"; -import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; -import {ssz} from "@chainsafe/lodestar-types"; -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; -import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; -import {getConfig} from "./util"; -import {IBaseSpecTest} from "../type"; - -export type EpochProcessFn = (state: CachedBeaconStateAllForks, epochProcess: EpochProcess) => void; - -/** - * https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/epoch_processing/README.md - */ -type EpochProcessingStateTestCase = IBaseSpecTest & { - pre: BeaconStateAllForks; - post: BeaconStateAllForks; -}; - -/** - * @param fork - * @param epochProcessFns Describe with which function to run each directory of tests - */ -export function epochProcessing(fork: ForkName, epochProcessFns: Record): void { - const rootDir = join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/epoch_processing`); - for (const testDir of fs.readdirSync(rootDir)) { - const epochProcessFn = epochProcessFns[testDir]; - if (epochProcessFn === undefined) { - throw Error(`No epochProcessFn for ${testDir}`); - } - - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/epoch_processing/${testDir}`, - join(rootDir, `${testDir}/pyspec_tests`), - (testcase) => { - const stateTB = testcase.pre.clone(); - const state = createCachedBeaconStateTest(stateTB, getConfig(fork)); - - const epochProcess = beforeProcessEpoch(state); - epochProcessFn(state, epochProcess); - state.commit(); - - return state; - }, - { - inputTypes: inputTypeSszTreeViewDU, - sszTypes: { - pre: ssz[fork].BeaconState, - post: ssz[fork].BeaconState, - }, - getExpected: (testCase) => testCase.post, - expectFunc: (testCase, expected, actual) => { - expectEqualBeaconState(fork, expected, actual); - }, - } - ); - } -} diff --git a/packages/lodestar/test/spec/allForks/fork.ts b/packages/lodestar/test/spec/allForks/fork.ts deleted file mode 100644 index d3641c940e7..00000000000 --- a/packages/lodestar/test/spec/allForks/fork.ts +++ /dev/null @@ -1,43 +0,0 @@ -import {join} from "node:path"; -import {allForks, phase0, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; -import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; -import {createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; -import {ssz} from "@chainsafe/lodestar-types"; -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest} from "../type"; -import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; -import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; - -export function fork(forkConfig: Partial, pre: ForkName, fork: Exclude): void { - const testConfig = createIChainForkConfig(forkConfig); - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/fork/fork`, - join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/fork/fork/pyspec_tests`), - (testcase) => { - const preState = createCachedBeaconStateTest(testcase.pre, testConfig); - return allForks.upgradeStateByFork[fork](preState); - }, - { - inputTypes: inputTypeSszTreeViewDU, - sszTypes: { - pre: ssz[pre].BeaconState, - post: ssz[fork].BeaconState, - }, - - timeout: 10000, - shouldError: (testCase) => testCase.post === undefined, - getExpected: (testCase) => testCase.post, - expectFunc: (testCase, expected, actual) => { - expectEqualBeaconState(fork, expected, actual); - }, - } - ); -} - -type PostBeaconState = Exclude; - -interface IUpgradeStateCase extends IBaseSpecTest { - pre: BeaconStateAllForks; - post: PostBeaconState; -} diff --git a/packages/lodestar/test/spec/allForks/forkChoice.ts b/packages/lodestar/test/spec/allForks/forkChoice.ts deleted file mode 100644 index 21c195ecbb8..00000000000 --- a/packages/lodestar/test/spec/allForks/forkChoice.ts +++ /dev/null @@ -1,426 +0,0 @@ -import {join} from "node:path"; -import {expect} from "chai"; -import { - phase0, - allForks, - computeEpochAtSlot, - CachedBeaconStateAllForks, - ZERO_HASH, - getEffectiveBalanceIncrementsZeroInactive, - computeStartSlotAtEpoch, - EffectiveBalanceIncrements, - BeaconStateAllForks, - bellatrix, -} from "@chainsafe/lodestar-beacon-state-transition"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import {initializeForkChoice} from "@chainsafe/lodestar/src/chain/forkChoice"; -import { - CheckpointStateCache, - toCheckpointHex, - toCheckpointKey, -} from "@chainsafe/lodestar/src/chain/stateCache/stateContextCheckpointsCache"; -import {ChainEventEmitter} from "@chainsafe/lodestar/src/chain/emitter"; -import {toHexString} from "@chainsafe/ssz"; -import { - CheckpointWithHex, - ForkChoiceError, - ForkChoiceErrorCode, - IForkChoice, - assertValidTerminalPowBlock, - ExecutionStatus, - PowBlockHex, -} from "@chainsafe/lodestar-fork-choice"; -import {ssz, RootHex} from "@chainsafe/lodestar-types"; -import {bnToNum} from "@chainsafe/lodestar-utils"; -import {ACTIVE_PRESET, SLOTS_PER_EPOCH, ForkName} from "@chainsafe/lodestar-params"; -import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; -import {testLogger} from "../../utils/logger"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {getConfig} from "./util"; - -/* eslint-disable @typescript-eslint/naming-convention */ - -/* eslint-disable @typescript-eslint/naming-convention */ - -const ANCHOR_STATE_FILE_NAME = "anchor_state"; -const ANCHOR_BLOCK_FILE_NAME = "anchor_block"; -const BLOCK_FILE_NAME = "^(block)_([0-9a-zA-Z]+)$"; -const POW_BLOCK_FILE_NAME = "^(pow_block)_([0-9a-zA-Z]+)$"; -const ATTESTATION_FILE_NAME = "^(attestation)_([0-9a-zA-Z])+$"; - -const logger = testLogger("spec-test"); -export function forkChoiceTest(fork: ForkName, testFolders: string[] = ["get_head", "on_block", "ex_ante"]): void { - for (const testFolder of testFolders) { - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/fork_choice/${testFolder}`, - join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/fork_choice/${testFolder}/pyspec_tests`), - (testcase) => { - const {steps, anchorState} = testcase; - const currentSlot = anchorState.slot; - const config = getConfig(fork); - let state = createCachedBeaconStateTest(anchorState, config); - - const emitter = new ChainEventEmitter(); - const forkchoice = initializeForkChoice(config, emitter, currentSlot, state, true); - - const checkpointStateCache = new CheckpointStateCache({}); - const stateCache = new Map(); - cacheState(state, stateCache); - - /** This is to track test's tickTime to be used in proposer boost */ - let tickTime = 0; - - for (const [i, step] of steps.entries()) { - if (isTick(step)) { - tickTime = bnToNum(step.tick); - const currentSlot = Math.floor(tickTime / config.SECONDS_PER_SLOT); - logger.debug("Step tick", {currentSlot, valid: Boolean(step.valid), time: tickTime}); - forkchoice.updateTime(currentSlot); - } - - // attestation step - else if (isAttestation(step)) { - logger.debug("Step attestation", {root: step.attestation, valid: Boolean(step.valid)}); - const attestation = testcase.attestations.get(step.attestation); - if (!attestation) throw Error(`No attestation ${step.attestation}`); - forkchoice.onAttestation(state.epochCtx.getIndexedAttestation(attestation)); - } - - // block step - else if (isBlock(step)) { - logger.debug("Step block", {root: step.block, valid: Boolean(step.valid)}); - const validBlock = Boolean(step.valid ?? true); - - const signedBlock = testcase.blocks.get(step.block); - if (!signedBlock) throw Error(`No block ${step.block}`); - - // Log the BeaconBlock root instead of the SignedBeaconBlock root, forkchoice references BeaconBlock roots - const blockRoot = config - .getForkTypes(signedBlock.message.slot) - .BeaconBlock.hashTreeRoot(signedBlock.message); - logger.debug("Step block", {slot: signedBlock.message.slot, root: toHexString(blockRoot)}); - - const preState = stateCache.get(toHexString(signedBlock.message.parentRoot)); - if (!preState) { - continue; - // should not throw error, on_block_bad_parent_root test wants this - } - const blockDelaySec = (tickTime - preState.genesisTime) % config.SECONDS_PER_SLOT; - const isMergeTransitionBlock = - bellatrix.isBellatrixStateType(preState) && - bellatrix.isBellatrixBlockBodyType(signedBlock.message.body) && - bellatrix.isMergeTransitionBlock(preState, signedBlock.message.body); - - try { - if (isMergeTransitionBlock) { - const mergeBlock = signedBlock.message as bellatrix.BeaconBlock; - - const powBlockRootHex = toHexString(mergeBlock.body.executionPayload.parentHash); - const powBlock = serializePowBlock(testcase.powBlocks.get(`pow_block_${powBlockRootHex}`)); - const powBlockParent = serializePowBlock( - powBlock && testcase.powBlocks.get(`pow_block_${powBlock.parentHash}`) - ); - assertValidTerminalPowBlock(config, mergeBlock, { - executionStatus: powBlock !== undefined ? ExecutionStatus.Valid : ExecutionStatus.Syncing, - powBlock, - powBlockParent, - }); - } - - state = runStateTranstion(preState, signedBlock, forkchoice, checkpointStateCache, blockDelaySec); - // TODO: May be part of runStateTranstion, necessary to commit again? - state.commit(); - cacheState(state, stateCache); - } catch (e) { - if (validBlock) throw e; - } - } - - // checks step - else if (isCheck(step)) { - // Forkchoice head is computed lazily only on request - const head = forkchoice.updateHead(); - const proposerBootRoot = forkchoice.getProposerBoostRoot(); - - if (step.checks.head !== undefined) { - expect(head.slot).to.be.equal(bnToNum(step.checks.head.slot), `Invalid head slot at step ${i}`); - expect(head.blockRoot).to.be.equal(step.checks.head.root, `Invalid head root at step ${i}`); - } - if (step.checks.proposer_boost_root !== undefined) { - expect(proposerBootRoot).to.be.equal( - step.checks.proposer_boost_root, - `Invalid proposer boost root at step ${i}` - ); - } - // time in spec mapped to Slot in our forkchoice implementation. - // Compare in slots because proposer boost steps doesn't always come on - // slot boundary. - if (step.checks.time !== undefined && step.checks.time > 0) - expect(forkchoice.getTime()).to.be.equal( - Math.floor(bnToNum(step.checks.time) / config.SECONDS_PER_SLOT), - `Invalid forkchoice time at step ${i}` - ); - if (step.checks.justified_checkpoint) { - expect(toSpecTestCheckpoint(forkchoice.getJustifiedCheckpoint())).to.be.deep.equal( - step.checks.justified_checkpoint, - `Invalid justified checkpoint at step ${i}` - ); - } - if (step.checks.finalized_checkpoint) { - expect(toSpecTestCheckpoint(forkchoice.getFinalizedCheckpoint())).to.be.deep.equal( - step.checks.finalized_checkpoint, - `Invalid finalized checkpoint at step ${i}` - ); - } - if (step.checks.best_justified_checkpoint) { - expect(toSpecTestCheckpoint(forkchoice.getBestJustifiedCheckpoint())).to.be.deep.equal( - step.checks.best_justified_checkpoint, - `Invalid best justified checkpoint at step ${i}` - ); - } - } - } - }, - { - inputTypes: { - meta: InputType.YAML, - steps: InputType.YAML, - }, - sszTypes: { - [ANCHOR_STATE_FILE_NAME]: ssz[fork].BeaconState, - [ANCHOR_BLOCK_FILE_NAME]: ssz[fork].BeaconBlock, - [BLOCK_FILE_NAME]: ssz[fork].SignedBeaconBlock, - [POW_BLOCK_FILE_NAME]: ssz.bellatrix.PowBlock, - [ATTESTATION_FILE_NAME]: ssz.phase0.Attestation, - }, - mapToTestCase: (t: Record) => { - // t has input file name as key - const blocks = new Map(); - const powBlocks = new Map(); - const attestations = new Map(); - for (const key in t) { - const blockMatch = key.match(BLOCK_FILE_NAME); - if (blockMatch) { - blocks.set(key, t[key]); - } - const powBlockMatch = key.match(POW_BLOCK_FILE_NAME); - if (powBlockMatch) { - powBlocks.set(key, t[key]); - } - const attMatch = key.match(ATTESTATION_FILE_NAME); - if (attMatch) { - attestations.set(key, t[key]); - } - } - return { - meta: t["meta"] as IForkChoiceTestCase["meta"], - anchorState: t[ANCHOR_STATE_FILE_NAME] as IForkChoiceTestCase["anchorState"], - anchorBlock: t[ANCHOR_BLOCK_FILE_NAME] as IForkChoiceTestCase["anchorBlock"], - steps: t["steps"] as IForkChoiceTestCase["steps"], - blocks, - powBlocks, - attestations, - }; - }, - timeout: 10000, - // eslint-disable-next-line @typescript-eslint/no-empty-function - expectFunc: () => {}, - } - ); - } -} - -function runStateTranstion( - preState: CachedBeaconStateAllForks, - signedBlock: allForks.SignedBeaconBlock, - forkchoice: IForkChoice, - checkpointCache: CheckpointStateCache, - blockDelaySec: number -): CachedBeaconStateAllForks { - const preSlot = preState.slot; - const postSlot = signedBlock.message.slot - 1; - let preEpoch = computeEpochAtSlot(preSlot); - let postState = preState.clone(); - for ( - let nextEpochSlot = computeStartSlotAtEpoch(preEpoch + 1); - nextEpochSlot <= postSlot; - nextEpochSlot += SLOTS_PER_EPOCH - ) { - postState = allForks.processSlots(postState, nextEpochSlot, null); - cacheCheckpointState(postState, checkpointCache); - } - preEpoch = postState.epochCtx.epoch; - postState = allForks.stateTransition(postState, signedBlock, { - verifyStateRoot: true, - verifyProposer: false, - verifySignatures: false, - }); - const postEpoch = postState.epochCtx.epoch; - if (postEpoch > preEpoch) { - cacheCheckpointState(postState, checkpointCache); - } - // same logic like in state transition https://github.com/ChainSafe/lodestar/blob/f6778740075fe2b75edf94d1db0b5691039cb500/packages/lodestar/src/chain/blocks/stateTransition.ts#L101 - let justifiedBalances: EffectiveBalanceIncrements | undefined; - const checkpointHex = toCheckpointHex(postState.currentJustifiedCheckpoint); - const justifiedState = checkpointCache.get(checkpointHex); - if ( - postState.currentJustifiedCheckpoint.epoch > forkchoice.getJustifiedCheckpoint().epoch || - postState.finalizedCheckpoint.epoch > forkchoice.getFinalizedCheckpoint().epoch - ) { - if (!justifiedState) { - const checkpointHexKey = toCheckpointKey(checkpointHex); - const cachedCps = checkpointCache.dumpCheckpointKeys().join(", "); - throw Error(`No justifiedState for checkpoint ${checkpointHexKey}. Available: ${cachedCps}`); - } - justifiedBalances = getEffectiveBalanceIncrementsZeroInactive(justifiedState); - } - - try { - forkchoice.onBlock(signedBlock.message, postState, { - blockDelaySec, - justifiedBalances, - }); - for (const attestation of signedBlock.message.body.attestations) { - try { - const indexedAttestation = postState.epochCtx.getIndexedAttestation(attestation); - forkchoice.onAttestation(indexedAttestation); - } catch (e) { - if (e instanceof ForkChoiceError && e.type.code === ForkChoiceErrorCode.INVALID_ATTESTATION) { - logger.debug("INVALID_ATTESTATION onAttestation", e.type.err); - } else { - logger.error("Error onAttestation", {}, e as Error); - } - } - } - } catch (e) { - if (e instanceof ForkChoiceError && e.type.code === ForkChoiceErrorCode.INVALID_BLOCK) { - logger.debug("INVALID_BLOCK onBlock", e.type.err); - } else { - logger.error("Error onBlock", {}, e as Error); - } - } - return postState; -} - -function cacheCheckpointState(checkpointState: CachedBeaconStateAllForks, checkpointCache: CheckpointStateCache): void { - const slot = checkpointState.slot; - if (slot % SLOTS_PER_EPOCH !== 0) { - throw new Error(`Invalid checkpoint state slot ${checkpointState.slot}`); - } - const blockHeader = ssz.phase0.BeaconBlockHeader.clone(checkpointState.latestBlockHeader); - if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { - blockHeader.stateRoot = checkpointState.hashTreeRoot(); - } - const cp: phase0.Checkpoint = { - root: ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader), - epoch: computeEpochAtSlot(slot), - }; - checkpointCache.add(cp, checkpointState); -} - -function cacheState(wrappedState: CachedBeaconStateAllForks, stateCache: Map): void { - const blockHeader = ssz.phase0.BeaconBlockHeader.clone(wrappedState.latestBlockHeader); - if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { - blockHeader.stateRoot = wrappedState.hashTreeRoot(); - } - const blockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader); - stateCache.set(toHexString(blockRoot), wrappedState); -} - -function toSpecTestCheckpoint(checkpoint: CheckpointWithHex): SpecTestCheckpoint { - return { - epoch: BigInt(checkpoint.epoch), - root: checkpoint.rootHex, - }; -} - -type Step = OnTick | OnAttestation | OnBlock | OnPowBlock | Checks; - -type SpecTestCheckpoint = {epoch: BigInt; root: string}; - -// This test executes steps in sequence. There may be multiple items of the following types: -// on_tick execution step - -type OnTick = { - /** to execute `on_tick(store, time)` */ - tick: bigint; - /** optional, default to `true`. */ - valid?: number; -}; - -type OnAttestation = { - /** the name of the `attestation_<32-byte-root>.ssz_snappy` file. To execute `on_attestation(store, attestation)` */ - attestation: string; - /** optional, default to `true`. */ - valid?: number; -}; - -type OnBlock = { - /** the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` */ - block: string; - /** optional, default to `true`. */ - valid?: number; -}; - -type OnPowBlock = { - /** - * the name of the `pow_block_<32-byte-root>.ssz_snappy` file. To - * execute `on_pow_block(store, block)` - */ - pow_block: string; -}; - -type Checks = { - /** Value in the ForkChoice store to verify it's correct after being mutated by another step */ - checks: { - head?: { - slot: bigint; - root: string; - }; - time?: bigint; - justified_checkpoint?: SpecTestCheckpoint; - finalized_checkpoint?: SpecTestCheckpoint; - best_justified_checkpoint?: SpecTestCheckpoint; - proposer_boost_root?: RootHex; - }; -}; - -interface IForkChoiceTestCase { - meta?: { - description?: string; - bls_setting: BigInt; - }; - anchorState: BeaconStateAllForks; - anchorBlock: allForks.BeaconBlock; - steps: Step[]; - blocks: Map; - powBlocks: Map; - attestations: Map; -} - -function isTick(step: Step): step is OnTick { - return (step as OnTick).tick > 0; -} - -function isAttestation(step: Step): step is OnAttestation { - return typeof (step as OnAttestation).attestation === "string"; -} - -function isBlock(step: Step): step is OnBlock { - return typeof (step as OnBlock).block === "string"; -} - -function isCheck(step: Step): step is Checks { - return typeof (step as Checks).checks === "object"; -} - -function serializePowBlock(powBlock: bellatrix.PowBlock | undefined): PowBlockHex | undefined { - if (powBlock) { - return { - blockHash: toHexString(powBlock.blockHash), - parentHash: toHexString(powBlock.parentHash), - totalDifficulty: BigInt(powBlock.totalDifficulty), - }; - } - return; -} diff --git a/packages/lodestar/test/spec/allForks/operations.ts b/packages/lodestar/test/spec/allForks/operations.ts deleted file mode 100644 index 766f24ed1a0..00000000000 --- a/packages/lodestar/test/spec/allForks/operations.ts +++ /dev/null @@ -1,77 +0,0 @@ -import fs from "node:fs"; -import {join} from "node:path"; -import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import {ssz} from "@chainsafe/lodestar-types"; -import {Type} from "@chainsafe/ssz"; -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; -import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; -import {IBaseSpecTest} from "../type"; -import {getConfig} from "./util"; - -/* eslint-disable @typescript-eslint/naming-convention */ - -export type BlockProcessFn = (state: T, testCase: any) => void; - -export type OperationsTestCase = IBaseSpecTest & { - pre: BeaconStateAllForks; - post: BeaconStateAllForks; - execution: {execution_valid: boolean}; -}; - -export function operations( - fork: ForkName, - operationFns: Record>, - sszTypes?: Record> -): void { - const rootDir = join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/operations`); - const testDirs = fs - .readdirSync(rootDir, {withFileTypes: true}) - // Ignore the .DS_Store and ._.DS_Store artificat files by filtering directories - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - for (const testDir of testDirs) { - const operationFn = operationFns[testDir]; - if (operationFn === undefined) { - throw Error(`No operationFn for ${testDir}`); - } - - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/operations/${testDir}`, - join(rootDir, `${testDir}/pyspec_tests`), - (testcase) => { - const state = testcase.pre.clone(); - const cachedState = createCachedBeaconStateTest(state, getConfig(fork)); - operationFn(cachedState as T, testcase); - state.commit(); - return state; - }, - { - inputTypes: {...inputTypeSszTreeViewDU, execution: InputType.YAML}, - sszTypes: { - pre: ssz[fork].BeaconState, - post: ssz[fork].BeaconState, - attestation: ssz.phase0.Attestation, - attester_slashing: ssz.phase0.AttesterSlashing, - block: ssz[fork].BeaconBlock, - deposit: ssz.phase0.Deposit, - proposer_slashing: ssz.phase0.ProposerSlashing, - voluntary_exit: ssz.phase0.SignedVoluntaryExit, - // Altair - sync_aggregate: ssz.altair.SyncAggregate, - // Bellatrix - execution_payload: ssz.bellatrix.ExecutionPayload, - // Provide types for new objects - ...sszTypes, - }, - shouldError: (testCase) => testCase.post === undefined, - getExpected: (testCase) => testCase.post, - expectFunc: (testCase, expected, actual) => { - expectEqualBeaconState(fork, expected, actual); - }, - } - ); - } -} diff --git a/packages/lodestar/test/spec/allForks/rewards.ts b/packages/lodestar/test/spec/allForks/rewards.ts deleted file mode 100644 index 1cc4a3617d3..00000000000 --- a/packages/lodestar/test/spec/allForks/rewards.ts +++ /dev/null @@ -1,151 +0,0 @@ -import fs from "node:fs"; -import {join} from "node:path"; -import {expect} from "chai"; -import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; -import { - altair, - phase0, - CachedBeaconStatePhase0, - BeaconStateAllForks, - BeaconStateAltair, - beforeProcessEpoch, -} from "@chainsafe/lodestar-beacon-state-transition"; -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; -import {VectorCompositeType} from "@chainsafe/ssz"; -import {ssz} from "@chainsafe/lodestar-types"; -import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest} from "../type"; -import {inputTypeSszTreeViewDU} from "../util"; -import {getConfig} from "./util"; - -/* eslint-disable @typescript-eslint/naming-convention */ - -export function rewards(fork: ForkName): void { - switch (fork) { - case ForkName.phase0: - return rewardsPhase0(fork); - default: - return rewardsAltair(fork); - } -} - -const deltasType = new VectorCompositeType(ssz.phase0.Balances, 2); - -export function rewardsPhase0(fork: ForkName): void { - const rootDir = join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/rewards`); - for (const testDir of fs.readdirSync(rootDir)) { - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/rewards/${testDir}`, - join(rootDir, `${testDir}/pyspec_tests`), - (testcase) => { - const config = getConfig(fork); - const wrappedState = createCachedBeaconStateTest(testcase.pre, config); - const epochProcess = beforeProcessEpoch(wrappedState); - return phase0.getAttestationDeltas(wrappedState as CachedBeaconStatePhase0, epochProcess); - }, - { - inputTypes: inputTypeSszTreeViewDU, - sszTypes: { - pre: ssz[fork].BeaconState, - source_deltas: deltasType, - target_deltas: deltasType, - head_deltas: deltasType, - inclusion_delay_deltas: deltasType, - inactivity_penalty_deltas: deltasType, - }, - timeout: 100000000, - getExpected: (testCase) => - sumDeltas([ - testCase.source_deltas, - testCase.target_deltas, - testCase.head_deltas, - testCase.inclusion_delay_deltas, - testCase.inactivity_penalty_deltas, - ]), - expectFunc: (testCase, expected, actual) => { - expect(actual).to.deep.equal(expected); - }, - } - ); - } -} - -export function rewardsAltair(fork: ForkName): void { - const rootDir = join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/rewards`); - for (const testDir of fs.readdirSync(rootDir)) { - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/rewards/${testDir}`, - join(rootDir, `${testDir}/pyspec_tests`), - (testcase) => { - const config = getConfig(fork); - const state = createCachedBeaconStateTest(testcase.pre as BeaconStateAltair, config); - const epochProcess = beforeProcessEpoch(state); - // To debug this test and get granular results you can tweak inputs to get more granular results - // - // TIMELY_HEAD_FLAG_INDEX -> FLAG_PREV_HEAD_ATTESTER_OR_UNSLASHED - // TIMELY_SOURCE_FLAG_INDEX -> FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED - // TIMELY_TARGET_FLAG_INDEX -> FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED - // - // - To get head_deltas set TIMELY_SOURCE_FLAG_INDEX | TIMELY_TARGET_FLAG_INDEX to false - // - To get source_deltas set TIMELY_HEAD_FLAG_INDEX | TIMELY_TARGET_FLAG_INDEX to false - // - To get target_deltas set TIMELY_HEAD_FLAG_INDEX | TIMELY_SOURCE_FLAG_INDEX to false - // + set all inactivityScores to zero - // - To get inactivity_penalty_deltas set TIMELY_HEAD_FLAG_INDEX | TIMELY_SOURCE_FLAG_INDEX to false - // + set PARTICIPATION_FLAG_WEIGHTS[TIMELY_TARGET_FLAG_INDEX] to zero - return altair.getRewardsAndPenalties(state, epochProcess); - }, - { - inputTypes: inputTypeSszTreeViewDU, - sszTypes: { - pre: ssz[fork].BeaconState, - head_deltas: deltasType, - source_deltas: deltasType, - target_deltas: deltasType, - inactivity_penalty_deltas: deltasType, - }, - getExpected: (testCase) => - sumDeltas([ - testCase.head_deltas, - testCase.source_deltas, - testCase.target_deltas, - testCase.inactivity_penalty_deltas, - ]), - expectFunc: (testCase, expected, actual) => { - expect(actual).to.deep.equal(expected); - }, - } - ); - } -} - -type Deltas = [number[], number[]]; - -interface RewardTestCasePhase0 extends IBaseSpecTest { - pre: BeaconStateAllForks; - source_deltas: Deltas; - target_deltas: Deltas; - head_deltas: Deltas; - inclusion_delay_deltas: Deltas; - inactivity_penalty_deltas: Deltas; -} - -interface RewardTestCaseAltair extends IBaseSpecTest { - pre: BeaconStateAllForks; - head_deltas: Deltas; - source_deltas: Deltas; - target_deltas: Deltas; - inactivity_penalty_deltas: Deltas; -} - -function sumDeltas(deltasArr: Deltas[]): Deltas { - const totalDeltas: Deltas = [[], []]; - for (const deltas of deltasArr) { - for (const n of [0, 1]) { - for (let i = 0; i < deltas[n].length; i++) { - totalDeltas[n][i] = (totalDeltas[n][i] ?? 0) + deltas[n][i]; - } - } - } - return totalDeltas; -} diff --git a/packages/lodestar/test/spec/allForks/ssz_static.ts b/packages/lodestar/test/spec/allForks/ssz_static.ts deleted file mode 100644 index a8df955874b..00000000000 --- a/packages/lodestar/test/spec/allForks/ssz_static.ts +++ /dev/null @@ -1,65 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import {ssz} from "@chainsafe/lodestar-types"; -import {Type} from "@chainsafe/ssz"; -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {replaceUintTypeWithUintBigintType} from "../utils/replaceUintTypeWithUintBigintType"; -import {parseSszStaticTestcase} from "../utils/sszTestCaseParser"; -import {runValidSszTest} from "../utils/runValidSszTest"; - -// ssz_static -// | Attestation -// | case_0 -// | roots.yaml -// | serialized.ssz_snappy -// | value.yaml -// -// Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md - -/* eslint-disable - @typescript-eslint/naming-convention, - @typescript-eslint/no-unsafe-assignment, - @typescript-eslint/no-unsafe-call, - @typescript-eslint/no-unsafe-member-access, - no-console -*/ - -// eslint-disable-next-line -type Types = Record>; - -export function sszStatic(fork: ForkName): void { - const rootDir = path.join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/ssz_static`); - for (const typeName of fs.readdirSync(rootDir)) { - /* eslint-disable @typescript-eslint/strict-boolean-expressions */ - const type = (ssz[fork] as Types)[typeName] || (ssz.altair as Types)[typeName] || (ssz.phase0 as Types)[typeName]; - if (!type) { - throw Error(`No type for ${typeName}`); - } - - testStatic(typeName, type, fork, ACTIVE_PRESET); - /* eslint-enable @typescript-eslint/strict-boolean-expressions */ - } -} - -function testStatic(typeName: string, sszType: Type, forkName: ForkName, preset: string): void { - const typeDir = path.join(SPEC_TEST_LOCATION, `tests/${preset}/${forkName}/ssz_static/${typeName}`); - - for (const caseName of fs.readdirSync(typeDir)) { - describe(`${preset}/${forkName}/ssz_static/${typeName}/${caseName}`, () => { - const sszTypeNoUint = replaceUintTypeWithUintBigintType(sszType); - const caseDir = path.join(typeDir, caseName); - for (const testId of fs.readdirSync(caseDir)) { - it(testId, function () { - // Mainnet must deal with big full states and hash each one multiple times - if (preset === "mainnet") { - this.timeout(30 * 1000); - } - - const testData = parseSszStaticTestcase(path.join(caseDir, testId)); - runValidSszTest(sszTypeNoUint, testData); - }); - } - }); - } -} diff --git a/packages/lodestar/test/spec/allForks/util.ts b/packages/lodestar/test/spec/allForks/util.ts deleted file mode 100644 index 17bd4ee5023..00000000000 --- a/packages/lodestar/test/spec/allForks/util.ts +++ /dev/null @@ -1,16 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {config as phase0Config} from "../phase0/util"; -import {config as altairConfig} from "../altair/util"; -import {config as bellatrixConfig} from "../bellatrix/util"; - -export function getConfig(fork: ForkName): IChainForkConfig { - switch (fork) { - case ForkName.phase0: - return phase0Config; - case ForkName.altair: - return altairConfig; - case ForkName.bellatrix: - return bellatrixConfig; - } -} diff --git a/packages/lodestar/test/spec/altair/epoch_processing.test.ts b/packages/lodestar/test/spec/altair/epoch_processing.test.ts deleted file mode 100644 index 0c1d765b9e2..00000000000 --- a/packages/lodestar/test/spec/altair/epoch_processing.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import {allForks, altair} from "@chainsafe/lodestar-beacon-state-transition"; -import {phase0} from "@chainsafe/lodestar-beacon-state-transition"; -import {ForkName} from "@chainsafe/lodestar-params"; -import {EpochProcessFn, epochProcessing} from "../allForks/epochProcessing"; - -/* eslint-disable @typescript-eslint/naming-convention */ - -epochProcessing(ForkName.altair, { - effective_balance_updates: allForks.processEffectiveBalanceUpdates, - eth1_data_reset: allForks.processEth1DataReset, - historical_roots_update: allForks.processHistoricalRootsUpdate, - inactivity_updates: altair.processInactivityUpdates as EpochProcessFn, - justification_and_finalization: allForks.processJustificationAndFinalization, - participation_flag_updates: altair.processParticipationFlagUpdates as EpochProcessFn, - participation_record_updates: (phase0.processParticipationRecordUpdates as unknown) as EpochProcessFn, - randao_mixes_reset: allForks.processRandaoMixesReset, - registry_updates: allForks.processRegistryUpdates, - rewards_and_penalties: altair.processRewardsAndPenalties as EpochProcessFn, - slashings: altair.processSlashings as EpochProcessFn, - slashings_reset: allForks.processSlashingsReset, - sync_committee_updates: altair.processSyncCommitteeUpdates as EpochProcessFn, -}); diff --git a/packages/lodestar/test/spec/altair/finality.test.ts b/packages/lodestar/test/spec/altair/finality.test.ts deleted file mode 100644 index 5d4beb06711..00000000000 --- a/packages/lodestar/test/spec/altair/finality.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {finality} from "../allForks/finality"; - -finality(ForkName.altair); diff --git a/packages/lodestar/test/spec/altair/fork.test.ts b/packages/lodestar/test/spec/altair/fork.test.ts deleted file mode 100644 index 78e190cab38..00000000000 --- a/packages/lodestar/test/spec/altair/fork.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {fork} from "../allForks/fork"; - -fork({}, ForkName.phase0, ForkName.altair); diff --git a/packages/lodestar/test/spec/altair/fork_choice.test.ts b/packages/lodestar/test/spec/altair/fork_choice.test.ts deleted file mode 100644 index c59a9a404e0..00000000000 --- a/packages/lodestar/test/spec/altair/fork_choice.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {forkChoiceTest} from "../allForks/forkChoice"; - -forkChoiceTest(ForkName.altair); diff --git a/packages/lodestar/test/spec/altair/genesis.test.ts b/packages/lodestar/test/spec/altair/genesis.test.ts deleted file mode 100644 index 06025ec3bf9..00000000000 --- a/packages/lodestar/test/spec/altair/genesis.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {genesis} from "../allForks/genesis"; - -genesis(ForkName.altair); diff --git a/packages/lodestar/test/spec/altair/merkle.test.ts b/packages/lodestar/test/spec/altair/merkle.test.ts deleted file mode 100644 index 144efb9d082..00000000000 --- a/packages/lodestar/test/spec/altair/merkle.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {merkle} from "../allForks/merkle"; - -merkle(ForkName.altair); diff --git a/packages/lodestar/test/spec/altair/operations.test.ts b/packages/lodestar/test/spec/altair/operations.test.ts deleted file mode 100644 index a8fa46774b0..00000000000 --- a/packages/lodestar/test/spec/altair/operations.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import {CachedBeaconStateAltair, allForks, altair} from "@chainsafe/lodestar-beacon-state-transition"; -import {phase0, ssz} from "@chainsafe/lodestar-types"; -import {ForkName} from "@chainsafe/lodestar-params"; -import {IBaseSpecTest, shouldVerify} from "../type"; -import {operations, BlockProcessFn} from "../allForks/operations"; - -/* eslint-disable @typescript-eslint/naming-convention */ - -// Define above to re-use in sync_aggregate and sync_aggregate_random -const sync_aggregate: BlockProcessFn = ( - state, - testCase: IBaseSpecTest & {sync_aggregate: altair.SyncAggregate} -) => { - const block = ssz.altair.BeaconBlock.defaultValue(); - - // processSyncAggregate() needs the full block to get the slot - block.slot = state.slot; - block.body.syncAggregate = ssz.altair.SyncAggregate.toViewDU(testCase["sync_aggregate"]); - - altair.processSyncAggregate(state, block); -}; - -operations(ForkName.altair, { - attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { - altair.processAttestations(state, [testCase.attestation]); - }, - - attester_slashing: (state, testCase: IBaseSpecTest & {attester_slashing: phase0.AttesterSlashing}) => { - altair.processAttesterSlashing(state, testCase.attester_slashing, shouldVerify(testCase)); - }, - - block_header: (state, testCase: IBaseSpecTest & {block: altair.BeaconBlock}) => { - allForks.processBlockHeader(state, testCase.block); - }, - - deposit: (state, testCase: IBaseSpecTest & {deposit: phase0.Deposit}) => { - altair.processDeposit(state, testCase.deposit); - }, - - proposer_slashing: (state, testCase: IBaseSpecTest & {proposer_slashing: phase0.ProposerSlashing}) => { - altair.processProposerSlashing(state, testCase.proposer_slashing); - }, - - sync_aggregate, - sync_aggregate_random: sync_aggregate, - - voluntary_exit: (state, testCase: IBaseSpecTest & {voluntary_exit: phase0.SignedVoluntaryExit}) => { - altair.processVoluntaryExit(state, testCase.voluntary_exit); - }, -}); diff --git a/packages/lodestar/test/spec/altair/random.test.ts b/packages/lodestar/test/spec/altair/random.test.ts deleted file mode 100644 index 3f2072a0aca..00000000000 --- a/packages/lodestar/test/spec/altair/random.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; -import {sanityBlock} from "../allForks/sanity"; - -sanityBlock(ForkName.altair, `/tests/${ACTIVE_PRESET}/${ForkName.altair}/random/random/pyspec_tests`); diff --git a/packages/lodestar/test/spec/altair/rewards.test.ts b/packages/lodestar/test/spec/altair/rewards.test.ts deleted file mode 100644 index 001735eac36..00000000000 --- a/packages/lodestar/test/spec/altair/rewards.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {rewards} from "../allForks/rewards"; - -rewards(ForkName.altair); diff --git a/packages/lodestar/test/spec/altair/sanity.test.ts b/packages/lodestar/test/spec/altair/sanity.test.ts deleted file mode 100644 index c0635779b27..00000000000 --- a/packages/lodestar/test/spec/altair/sanity.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {sanity} from "../allForks/sanity"; - -sanity(ForkName.altair); diff --git a/packages/lodestar/test/spec/altair/ssz_static.test.ts b/packages/lodestar/test/spec/altair/ssz_static.test.ts deleted file mode 100644 index 75018f9d090..00000000000 --- a/packages/lodestar/test/spec/altair/ssz_static.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {sszStatic} from "../allForks/ssz_static"; - -sszStatic(ForkName.altair); diff --git a/packages/lodestar/test/spec/altair/transition.test.ts b/packages/lodestar/test/spec/altair/transition.test.ts deleted file mode 100644 index 86793b20707..00000000000 --- a/packages/lodestar/test/spec/altair/transition.test.ts +++ /dev/null @@ -1,5 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {transition} from "../allForks/transition"; - -// eslint-disable-next-line @typescript-eslint/naming-convention -transition((forkEpoch) => ({ALTAIR_FORK_EPOCH: forkEpoch}), ForkName.phase0, ForkName.altair); diff --git a/packages/lodestar/test/spec/altair/util.ts b/packages/lodestar/test/spec/altair/util.ts deleted file mode 100644 index c286b6620c1..00000000000 --- a/packages/lodestar/test/spec/altair/util.ts +++ /dev/null @@ -1,12 +0,0 @@ -import {altair} from "@chainsafe/lodestar-types"; -import {createIChainForkConfig} from "@chainsafe/lodestar-config"; -import {IBaseSpecTest} from "../type"; - -export interface IAltairStateTestCase extends IBaseSpecTest { - pre: altair.BeaconState; - post: altair.BeaconState; -} - -/** Config with `ALTAIR_FORK_EPOCH: 0` */ -// eslint-disable-next-line @typescript-eslint/naming-convention -export const config = createIChainForkConfig({ALTAIR_FORK_EPOCH: 0}); diff --git a/packages/lodestar/test/spec/bellatrix/epoch_processing.test.ts b/packages/lodestar/test/spec/bellatrix/epoch_processing.test.ts deleted file mode 100644 index c3a76f5b0b0..00000000000 --- a/packages/lodestar/test/spec/bellatrix/epoch_processing.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -import {allForks, altair, bellatrix} from "@chainsafe/lodestar-beacon-state-transition"; -import {processParticipationRecordUpdates} from "@chainsafe/lodestar-beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates"; -import {ForkName} from "@chainsafe/lodestar-params"; -import {EpochProcessFn, epochProcessing} from "../allForks/epochProcessing"; - -/* eslint-disable @typescript-eslint/naming-convention */ - -// NOTE: Exact same code as for altair - -epochProcessing(ForkName.bellatrix, { - effective_balance_updates: allForks.processEffectiveBalanceUpdates, - eth1_data_reset: allForks.processEth1DataReset, - historical_roots_update: allForks.processHistoricalRootsUpdate, - inactivity_updates: altair.processInactivityUpdates as EpochProcessFn, - justification_and_finalization: allForks.processJustificationAndFinalization, - participation_flag_updates: altair.processParticipationFlagUpdates as EpochProcessFn, - participation_record_updates: (processParticipationRecordUpdates as unknown) as EpochProcessFn, - randao_mixes_reset: allForks.processRandaoMixesReset, - registry_updates: allForks.processRegistryUpdates, - rewards_and_penalties: altair.processRewardsAndPenalties as EpochProcessFn, - slashings: bellatrix.processSlashings as EpochProcessFn, - slashings_reset: allForks.processSlashingsReset, - sync_committee_updates: altair.processSyncCommitteeUpdates as EpochProcessFn, -}); diff --git a/packages/lodestar/test/spec/bellatrix/finality.test.ts b/packages/lodestar/test/spec/bellatrix/finality.test.ts deleted file mode 100644 index 0770bd7b818..00000000000 --- a/packages/lodestar/test/spec/bellatrix/finality.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {finality} from "../allForks/finality"; - -finality(ForkName.bellatrix); diff --git a/packages/lodestar/test/spec/bellatrix/fork.test.ts b/packages/lodestar/test/spec/bellatrix/fork.test.ts deleted file mode 100644 index 8cb000c79a9..00000000000 --- a/packages/lodestar/test/spec/bellatrix/fork.test.ts +++ /dev/null @@ -1,5 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {fork} from "../allForks/fork"; - -// eslint-disable-next-line @typescript-eslint/naming-convention -fork({ALTAIR_FORK_EPOCH: 0}, ForkName.altair, ForkName.bellatrix); diff --git a/packages/lodestar/test/spec/bellatrix/fork_choice.test.ts b/packages/lodestar/test/spec/bellatrix/fork_choice.test.ts deleted file mode 100644 index fede5636e20..00000000000 --- a/packages/lodestar/test/spec/bellatrix/fork_choice.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {forkChoiceTest} from "../allForks/forkChoice"; - -forkChoiceTest(ForkName.bellatrix, ["get_head", "on_block", "ex_ante", "on_merge_block"]); diff --git a/packages/lodestar/test/spec/bellatrix/genesis.test.ts b/packages/lodestar/test/spec/bellatrix/genesis.test.ts deleted file mode 100644 index da0b4d1e6da..00000000000 --- a/packages/lodestar/test/spec/bellatrix/genesis.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {genesis} from "../allForks/genesis"; - -genesis(ForkName.bellatrix); diff --git a/packages/lodestar/test/spec/bellatrix/merkle.test.ts b/packages/lodestar/test/spec/bellatrix/merkle.test.ts deleted file mode 100644 index 190b11adbf0..00000000000 --- a/packages/lodestar/test/spec/bellatrix/merkle.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {merkle} from "../allForks/merkle"; - -merkle(ForkName.bellatrix); diff --git a/packages/lodestar/test/spec/bellatrix/operations.test.ts b/packages/lodestar/test/spec/bellatrix/operations.test.ts deleted file mode 100644 index 012fb92a301..00000000000 --- a/packages/lodestar/test/spec/bellatrix/operations.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { - CachedBeaconStateAllForks, - CachedBeaconStateAltair, - CachedBeaconStateBellatrix, - allForks, - altair, - bellatrix, -} from "@chainsafe/lodestar-beacon-state-transition"; -import {phase0, ssz} from "@chainsafe/lodestar-types"; -import {ForkName} from "@chainsafe/lodestar-params"; -import {IBaseSpecTest, shouldVerify} from "../type"; -import {operations, BlockProcessFn} from "../allForks/operations"; -// eslint-disable-next-line no-restricted-imports -import {processExecutionPayload} from "@chainsafe/lodestar-beacon-state-transition/lib/bellatrix/block/processExecutionPayload"; - -/* eslint-disable @typescript-eslint/naming-convention */ - -// Define above to re-use in sync_aggregate and sync_aggregate_random -const sync_aggregate: BlockProcessFn = ( - state, - testCase: IBaseSpecTest & {sync_aggregate: altair.SyncAggregate} -) => { - const block = ssz.altair.BeaconBlock.defaultValue(); - - // processSyncAggregate() needs the full block to get the slot - block.slot = state.slot; - block.body.syncAggregate = ssz.altair.SyncAggregate.toViewDU(testCase["sync_aggregate"]); - - altair.processSyncAggregate((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, block); -}; - -operations(ForkName.bellatrix, { - attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { - altair.processAttestations((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, [testCase.attestation]); - }, - - attester_slashing: (state, testCase: IBaseSpecTest & {attester_slashing: phase0.AttesterSlashing}) => { - bellatrix.processAttesterSlashing(state, testCase.attester_slashing, shouldVerify(testCase)); - }, - - block_header: (state, testCase: IBaseSpecTest & {block: altair.BeaconBlock}) => { - allForks.processBlockHeader(state, testCase.block); - }, - - deposit: (state, testCase: IBaseSpecTest & {deposit: phase0.Deposit}) => { - altair.processDeposit((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, testCase.deposit); - }, - - proposer_slashing: (state, testCase: IBaseSpecTest & {proposer_slashing: phase0.ProposerSlashing}) => { - bellatrix.processProposerSlashing(state, testCase.proposer_slashing); - }, - - sync_aggregate, - sync_aggregate_random: sync_aggregate, - - voluntary_exit: (state, testCase: IBaseSpecTest & {voluntary_exit: phase0.SignedVoluntaryExit}) => { - altair.processVoluntaryExit( - (state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, - testCase.voluntary_exit - ); - }, - - execution_payload: ( - state, - testCase: IBaseSpecTest & {execution_payload: bellatrix.ExecutionPayload; execution: {execution_valid: boolean}} - ) => { - processExecutionPayload( - (state as CachedBeaconStateAllForks) as CachedBeaconStateBellatrix, - testCase.execution_payload, - {notifyNewPayload: () => testCase.execution.execution_valid} - ); - }, -}); diff --git a/packages/lodestar/test/spec/bellatrix/random.test.ts b/packages/lodestar/test/spec/bellatrix/random.test.ts deleted file mode 100644 index c7762ef0235..00000000000 --- a/packages/lodestar/test/spec/bellatrix/random.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; -import {sanityBlock} from "../allForks/sanity"; - -sanityBlock(ForkName.bellatrix, `/tests/${ACTIVE_PRESET}/${ForkName.bellatrix}/random/random/pyspec_tests`); diff --git a/packages/lodestar/test/spec/bellatrix/rewards.test.ts b/packages/lodestar/test/spec/bellatrix/rewards.test.ts deleted file mode 100644 index 20f95268e7d..00000000000 --- a/packages/lodestar/test/spec/bellatrix/rewards.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {rewards} from "../allForks/rewards"; - -rewards(ForkName.bellatrix); diff --git a/packages/lodestar/test/spec/bellatrix/sanity.test.ts b/packages/lodestar/test/spec/bellatrix/sanity.test.ts deleted file mode 100644 index f8bdd42b0a4..00000000000 --- a/packages/lodestar/test/spec/bellatrix/sanity.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {sanity} from "../allForks/sanity"; - -sanity(ForkName.bellatrix); diff --git a/packages/lodestar/test/spec/bellatrix/ssz_static.test.ts b/packages/lodestar/test/spec/bellatrix/ssz_static.test.ts deleted file mode 100644 index 46797129475..00000000000 --- a/packages/lodestar/test/spec/bellatrix/ssz_static.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {sszStatic} from "../allForks/ssz_static"; - -sszStatic(ForkName.bellatrix); diff --git a/packages/lodestar/test/spec/bellatrix/transition.test.ts b/packages/lodestar/test/spec/bellatrix/transition.test.ts deleted file mode 100644 index 45ffa7671a0..00000000000 --- a/packages/lodestar/test/spec/bellatrix/transition.test.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {transition} from "../allForks/transition"; - -transition( - // eslint-disable-next-line @typescript-eslint/naming-convention - (forkEpoch) => ({ALTAIR_FORK_EPOCH: 0, BELLATRIX_FORK_EPOCH: forkEpoch}), - ForkName.altair, - ForkName.bellatrix -); diff --git a/packages/lodestar/test/spec/bellatrix/util.ts b/packages/lodestar/test/spec/bellatrix/util.ts deleted file mode 100644 index 71217493c78..00000000000 --- a/packages/lodestar/test/spec/bellatrix/util.ts +++ /dev/null @@ -1,16 +0,0 @@ -import {bellatrix} from "@chainsafe/lodestar-types"; -import {createIChainForkConfig} from "@chainsafe/lodestar-config"; -import {IBaseSpecTest} from "../type"; - -export interface IBellatrixStateTestCase extends IBaseSpecTest { - pre: bellatrix.BeaconState; - post: bellatrix.BeaconState; -} - -/** Config with `ALTAIR_FORK_EPOCH: 0, BELLATRIX_FORK_EPOCH: 0` */ -export const config = createIChainForkConfig({ - /* eslint-disable @typescript-eslint/naming-convention */ - ALTAIR_FORK_EPOCH: 0, - BELLATRIX_FORK_EPOCH: 0, - TERMINAL_TOTAL_DIFFICULTY: BigInt("115792089237316195423570985008687907853269984665640564039457584007913129638912"), -}); diff --git a/packages/lodestar/test/spec/bls/aggregate.test.ts b/packages/lodestar/test/spec/bls/aggregate.test.ts deleted file mode 100644 index b848e79e315..00000000000 --- a/packages/lodestar/test/spec/bls/aggregate.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -import path from "node:path"; -import bls from "@chainsafe/bls"; -// eslint-disable-next-line no-restricted-imports -import {EmptyAggregateError} from "@chainsafe/bls/lib/errors"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; - -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest} from "../type"; - -interface IAggregateSigsTestCase extends IBaseSpecTest { - data: { - input: string[]; - output: string; - }; -} - -describeDirectorySpecTest( - "bls/aggregate/small", - path.join(SPEC_TEST_LOCATION, "tests/general/phase0/bls/aggregate/small"), - (testCase) => { - try { - const signatures = testCase.data.input; - const agg = bls.aggregateSignatures(signatures.map(fromHexString)); - return toHexString(agg); - } catch (e) { - if (e instanceof EmptyAggregateError) return null; - throw e; - } - }, - { - inputTypes: {data: InputType.YAML}, - getExpected: (testCase) => testCase.data.output, - } -); diff --git a/packages/lodestar/test/spec/bls/aggregate_verify.test.ts b/packages/lodestar/test/spec/bls/aggregate_verify.test.ts deleted file mode 100644 index ff93f8fe09e..00000000000 --- a/packages/lodestar/test/spec/bls/aggregate_verify.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import path from "node:path"; -import bls from "@chainsafe/bls"; -import {fromHexString} from "@chainsafe/ssz"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib"; - -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest} from "../type"; - -interface IAggregateSigsVerifyTestCase extends IBaseSpecTest { - data: { - input: { - pubkeys: string[]; - messages: string[]; - signature: string; - }; - output: boolean; - }; -} - -describeDirectorySpecTest( - "bls/aggregate_verify/small", - path.join(SPEC_TEST_LOCATION, "tests/general/phase0/bls/aggregate_verify/small"), - (testCase) => { - const {pubkeys, messages, signature} = testCase.data.input; - return bls.verifyMultiple(pubkeys.map(fromHexString), messages.map(fromHexString), fromHexString(signature)); - }, - { - inputTypes: {data: InputType.YAML}, - getExpected: (testCase) => testCase.data.output, - } -); diff --git a/packages/lodestar/test/spec/bls/fast_aggregate_verify.test.ts b/packages/lodestar/test/spec/bls/fast_aggregate_verify.test.ts deleted file mode 100644 index ab1f0f16c5a..00000000000 --- a/packages/lodestar/test/spec/bls/fast_aggregate_verify.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import path from "node:path"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib"; -import bls, {CoordType} from "@chainsafe/bls"; -import {fromHexString} from "@chainsafe/ssz"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest} from "../type"; - -interface IAggregateSigsVerifyTestCase extends IBaseSpecTest { - data: { - input: { - pubkeys: string[]; - message: string; - signature: string; - }; - output: boolean; - }; -} - -describeDirectorySpecTest( - "bls/fast_aggregate_verify/small", - path.join(SPEC_TEST_LOCATION, "tests/general/phase0/bls/fast_aggregate_verify/small"), - (testCase) => { - const {pubkeys, message, signature} = testCase.data.input; - try { - return bls.Signature.fromBytes(fromHexString(signature), undefined, true).verifyAggregate( - pubkeys.map((hex) => bls.PublicKey.fromBytes(fromHexString(hex), CoordType.jacobian, true)), - fromHexString(message) - ); - } catch (e) { - return false; - } - }, - { - inputTypes: {data: InputType.YAML}, - getExpected: (testCase) => testCase.data.output, - } -); diff --git a/packages/lodestar/test/spec/bls/sign.test.ts b/packages/lodestar/test/spec/bls/sign.test.ts deleted file mode 100644 index 93564fc12cf..00000000000 --- a/packages/lodestar/test/spec/bls/sign.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import path from "node:path"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import bls from "@chainsafe/bls"; -// eslint-disable-next-line no-restricted-imports -import {ZeroSecretKeyError} from "@chainsafe/bls/lib/errors"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest} from "../type"; - -interface ISignMessageTestCase extends IBaseSpecTest { - data: { - input: { - privkey: string; - message: string; - }; - output: string; - }; -} - -describeDirectorySpecTest( - "bls/sign/small", - path.join(SPEC_TEST_LOCATION, "tests/general/phase0/bls/sign/small"), - (testCase) => { - try { - const {privkey, message} = testCase.data.input; - const signature = bls.sign(fromHexString(privkey), fromHexString(message)); - return toHexString(signature); - } catch (e) { - if (e instanceof ZeroSecretKeyError) return null; - else throw e; - } - }, - { - inputTypes: {data: InputType.YAML}, - getExpected: (testCase) => testCase.data.output, - } -); diff --git a/packages/lodestar/test/spec/bls/verify.test.ts b/packages/lodestar/test/spec/bls/verify.test.ts deleted file mode 100644 index bf01dbb9f2d..00000000000 --- a/packages/lodestar/test/spec/bls/verify.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import path from "node:path"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util/lib"; -import bls from "@chainsafe/bls"; -import {fromHexString} from "@chainsafe/ssz"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest} from "../type"; - -interface IVerifyTestCase extends IBaseSpecTest { - data: { - input: { - pubkey: string; - message: string; - signature: string; - }; - output: boolean; - }; -} - -describeDirectorySpecTest( - "bls/verify/small", - path.join(SPEC_TEST_LOCATION, "tests/general/phase0/bls/verify/small"), - (testCase) => { - const {pubkey, message, signature} = testCase.data.input; - return bls.verify(fromHexString(pubkey), fromHexString(message), fromHexString(signature)); - }, - { - inputTypes: {data: InputType.YAML}, - getExpected: (testCase) => testCase.data.output, - } -); diff --git a/packages/lodestar/test/spec/checkCoverage.ts b/packages/lodestar/test/spec/checkCoverage.ts deleted file mode 100644 index 67878cb30fe..00000000000 --- a/packages/lodestar/test/spec/checkCoverage.ts +++ /dev/null @@ -1,127 +0,0 @@ -import {expect} from "chai"; -import fs from "node:fs"; -import path from "node:path"; -import {SPEC_TEST_LOCATION} from "./specTestVersioning"; - -// TEMP TEMP -const forksToIgnore = new Set([]); - -// This test ensures that we are covering all available spec tests. -// The directory structure is organized first by preset, then by fork. -// The presets mainnet and minimal have the same directory structure. -// -// spec-tests/ -// ├── tests -// │ ├── general -// │ │ └── phase0 -// │ ├── mainnet -// │ │ ├── altair -// │ │ ├── bellatrix -// │ │ └── phase0 -// │ └── minimal -// │ ├── altair -// │ ├── bellatrix -// │ └── phase0 -// -// Each fork has the same structure but adding extra tests for added functionality -// -// | phase0 | altair | bellatrix | -// | ---------------- | ---------------- | ---------------- | -// | epoch_processing | epoch_processing | epoch_processing | -// | finality | finality | finality -// | - | fork | - -// | fork_choice | fork_choice | fork_choice -// | genesis | genesis | - -// | operations | operations | operations -// | rewards | rewards | rewards -// | sanity | sanity | sanity -// | shuffling | - | - -// | ssz_static | ssz_static | ssz_static -// | - | transition | - -// -// ------------ -// -// Lodestar spec test organization mixes mainnet and minimal preset in the same file. -// Tests are then organized by fork and follow the same naming structure as the spec tests. - -const knownPresets = ["mainnet", "minimal"]; -const knownForks = ["altair", "bellatrix", "phase0"]; -const lodestarTests = path.join(__dirname, "../spec"); - -const missingTests = new Set(); - -const specTestsTestPath = path.join(SPEC_TEST_LOCATION, "tests"); -const specTestsTestLs = fs.readdirSync(specTestsTestPath); -expect(specTestsTestLs).to.deep.equal(["general", ...knownPresets], "New dir in spec-tests/tests"); - -for (const preset of knownPresets) { - const presetDirPath = path.join(specTestsTestPath, preset); - const presetDirLs = fs - .readdirSync(presetDirPath, {withFileTypes: true}) - // Ignore the .DS_Store and ._.DS_Store artificat files by filtering directories - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - expect(presetDirLs).to.deep.equal(knownForks, `New fork in spec-tests/tests/${preset}`); - - for (const fork of knownForks) { - if (forksToIgnore.has(fork)) continue; - ensureDirTestCoverage(presetDirPath, fork); - } -} - -if (missingTests.size > 0) { - throw Error(`Some spec tests are not covered: \n${Array.from(missingTests.values()).join("\n")}`); -} - -/** - * Ensure there are Lodestar spec tests for all EF spec tests - * @param rootTestDir EF spec test root dir: /spec-tests/tests/minimal/ - * @param testRelDir /altair/ - */ -function ensureDirTestCoverage(rootTestDir: string, testRelDir: string): void { - // spec-tests/tests/mainnet/phase0/ - // ├── epoch_processing - // ├── finality - // ├── fork_choice - // ├── genesis - // ├── operations - // ├── rewards - // ├── sanity - // ├── shuffling - // └── ssz_static - const testGroups = fs - .readdirSync(path.join(rootTestDir, testRelDir), {withFileTypes: true}) - // Ignore the .DS_Store and ._.DS_Store artificat files by filtering directories - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - for (const testGroup of testGroups) { - const testDir = path.join(lodestarTests, testRelDir, testGroup); - const testFile = testDir + ".test.ts"; - if (existsDir(testDir)) { - // Check next dir level - ensureDirTestCoverage(rootTestDir, path.join(testRelDir, testGroup)); - } else if (existsFile(testFile)) { - // Is file, assume it covers all cases - } else { - missingTests.add(path.relative(lodestarTests, testDir)); - } - } -} - -function existsDir(p: string): boolean { - try { - return fs.lstatSync(p).isDirectory(); - } catch (e) { - if ((e as {code: string}).code === "ENOENT") return false; - throw e; - } -} - -function existsFile(p: string): boolean { - try { - return fs.lstatSync(p).isFile(); - } catch (e) { - if ((e as {code: string}).code === "ENOENT") return false; - throw e; - } -} diff --git a/packages/lodestar/test/spec/general/bls.ts b/packages/lodestar/test/spec/general/bls.ts new file mode 100644 index 00000000000..ff0a14f1db2 --- /dev/null +++ b/packages/lodestar/test/spec/general/bls.ts @@ -0,0 +1,170 @@ +import bls, {CoordType, Signature} from "@chainsafe/bls"; +import {InputType} from "@chainsafe/lodestar-spec-test-util"; +import {toHexString} from "@chainsafe/lodestar-utils"; +import {fromHexString} from "@chainsafe/ssz"; +import {TestRunnerFn} from "../utils/types"; + +/* eslint-disable @typescript-eslint/naming-convention */ + +const testFnByType: Record any> = { + aggregate, + aggregate_verify, + eth_aggregate_pubkeys, + eth_fast_aggregate_verify, + fast_aggregate_verify, + sign, + verify, +}; + +const G2_POINT_AT_INFINITY = + "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"; +const G1_POINT_AT_INFINITY = + "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"; + +export const blsTestRunner: TestRunnerFn = (fork, testName) => { + return { + testFunction: ({data}) => { + const testFn = testFnByType[testName]; + if (testFn === undefined) { + throw Error(`Unknown bls test ${testName}`); + } + + try { + return testFn(data.input) as unknown; + } catch (e) { + const {message} = e as Error; + if (message.includes("BLST_ERROR") || message === "EMPTY_AGGREGATE_ARRAY" || message === "ZERO_SECRET_KEY") { + return null; + } else { + throw e; + } + } + }, + options: { + inputTypes: {data: InputType.YAML}, + getExpected: (testCase) => testCase.data.output, + }, + }; +}; + +type BlsTestCase = { + data: { + input: unknown; + output: unknown; + }; +}; + +/** + * ``` + * input: List[BLS Signature] -- list of input BLS signatures + * output: BLS Signature -- expected output, single BLS signature or empty. + * ``` + */ +function aggregate(input: string[]): string { + const pks = input.map((pkHex) => Signature.fromHex(pkHex)); + const agg = bls.Signature.aggregate(pks); + return agg.toHex(); +} + +/** + * ``` + * input: + * pubkeys: List[BLS Pubkey] -- the pubkeys + * messages: List[bytes32] -- the messages + * signature: BLS Signature -- the signature to verify against pubkeys and messages + * output: bool -- true (VALID) or false (INVALID) + * ``` + */ +function aggregate_verify(input: {pubkeys: string[]; messages: string[]; signature: string}): boolean { + const {pubkeys, messages, signature} = input; + return bls.verifyMultiple(pubkeys.map(fromHexString), messages.map(fromHexString), fromHexString(signature)); +} + +/** + * ``` + * input: List[BLS Signature] -- list of input BLS signatures + * output: BLS Signature -- expected output, single BLS signature or empty. + * ``` + */ +function eth_aggregate_pubkeys(input: string[]): string | null { + // Don't add this checks in the source as beacon nodes check the pubkeys for inf when onboarding + for (const pk of input) { + if (pk === G1_POINT_AT_INFINITY) return null; + } + + const agg = bls.aggregatePublicKeys(input.map((hex) => fromHexString(hex))); + return toHexString(agg); +} + +/** + * ``` + * input: + * pubkeys: List[BLS Pubkey] -- list of input BLS pubkeys + * message: bytes32 -- the message + * signature: BLS Signature -- the signature to verify against pubkeys and message + * output: bool -- true (VALID) or false (INVALID) + * ``` + */ +function eth_fast_aggregate_verify(input: {pubkeys: string[]; message: string; signature: string}): boolean { + const {pubkeys, message, signature} = input; + + if (pubkeys.length === 0 && signature === G2_POINT_AT_INFINITY) { + return true; + } + + // Don't add this checks in the source as beacon nodes check the pubkeys for inf when onboarding + for (const pk of pubkeys) { + if (pk === G1_POINT_AT_INFINITY) return false; + } + + return bls.verifyAggregate( + pubkeys.map((hex) => fromHexString(hex)), + fromHexString(message), + fromHexString(signature) + ); +} + +/** + * ``` + * input: + * pubkeys: List[BLS Pubkey] -- list of input BLS pubkeys + * message: bytes32 -- the message + * signature: BLS Signature -- the signature to verify against pubkeys and message + * output: bool -- true (VALID) or false (INVALID) + * ``` + */ +function fast_aggregate_verify(input: {pubkeys: string[]; message: string; signature: string}): boolean | null { + const {pubkeys, message, signature} = input; + try { + return bls.Signature.fromBytes(fromHexString(signature), undefined, true).verifyAggregate( + pubkeys.map((hex) => bls.PublicKey.fromBytes(fromHexString(hex), CoordType.jacobian, true)), + fromHexString(message) + ); + } catch (e) { + return false; + } +} + +/** + * input: + * privkey: bytes32 -- the private key used for signing + * message: bytes32 -- input message to sign (a hash) + * output: BLS Signature -- expected output, single BLS signature or empty. + */ +function sign(input: {privkey: string; message: string}): string | null { + const {privkey, message} = input; + const signature = bls.sign(fromHexString(privkey), fromHexString(message)); + return toHexString(signature); +} + +/** + * input: + * pubkey: bytes48 -- the pubkey + * message: bytes32 -- the message + * signature: bytes96 -- the signature to verify against pubkey and message + * output: bool -- VALID or INVALID + */ +function verify(input: {pubkey: string; message: string; signature: string}): boolean { + const {pubkey, message, signature} = input; + return bls.verify(fromHexString(pubkey), fromHexString(message), fromHexString(signature)); +} diff --git a/packages/lodestar/test/spec/general/index.test.ts b/packages/lodestar/test/spec/general/index.test.ts new file mode 100644 index 00000000000..cb260d82b70 --- /dev/null +++ b/packages/lodestar/test/spec/general/index.test.ts @@ -0,0 +1,11 @@ +import {RunnerType} from "../utils/types"; +import {specTestIterator} from "../utils/specTestIterator"; +import {blsTestRunner} from "./bls"; +import {sszGeneric} from "./ssz_generic"; + +/* eslint-disable @typescript-eslint/naming-convention */ + +specTestIterator("general", { + bls: {type: RunnerType.default, fn: blsTestRunner}, + ssz_generic: {type: RunnerType.custom, fn: sszGeneric}, +}); diff --git a/packages/lodestar/test/spec/ssz/generic/index.test.ts b/packages/lodestar/test/spec/general/ssz_generic.ts similarity index 51% rename from packages/lodestar/test/spec/ssz/generic/index.test.ts rename to packages/lodestar/test/spec/general/ssz_generic.ts index 7fd0755cf01..c13202e8c80 100644 --- a/packages/lodestar/test/spec/ssz/generic/index.test.ts +++ b/packages/lodestar/test/spec/general/ssz_generic.ts @@ -1,28 +1,32 @@ -import {expect} from "chai"; -import path from "node:path"; import fs from "node:fs"; -import {SPEC_TEST_LOCATION} from "../../specTestVersioning"; -import {parseSszGenericValidTestcase, parseSszGenericInvalidTestcase} from "../../utils/sszTestCaseParser"; -import {runValidSszTest} from "../../utils/runValidSszTest"; -import {getTestType} from "./types"; +import path from "node:path"; +import {expect} from "chai"; +import {TestRunnerCustom} from "../utils/types"; +import {parseSszGenericInvalidTestcase, parseSszGenericValidTestcase} from "../utils/sszTestCaseParser"; +import {runValidSszTest} from "../utils/runValidSszTest"; +import {getTestType} from "./ssz_generic_types"; -const rootGenericSszPath = path.join(SPEC_TEST_LOCATION, "tests", "general", "phase0", "ssz_generic"); +/* eslint-disable @typescript-eslint/naming-convention */ -for (const testType of fs.readdirSync(rootGenericSszPath)) { - const testTypePath = path.join(rootGenericSszPath, testType); +// Mapping of sszGeneric() fn arguments to the path in spec tests +// +// / config / fork / test runner / test handler / test suite / test case +// +// tests / general / phase0 / ssz_generic / basic_vector / valid / vec_bool_1_max/meta.yaml +// - describe(`${testType} invalid`, () => { - const invalidCasesPath = path.join(testTypePath, "invalid"); - for (const invalidCase of fs.readdirSync(invalidCasesPath)) { - it(invalidCase, () => { +export const sszGeneric: TestRunnerCustom = (fork, typeName, testSuite, testSuiteDirpath) => { + if (testSuite === "invalid") { + for (const testCase of fs.readdirSync(testSuiteDirpath)) { + it(testCase, () => { // TODO: Strong type errors and assert that the entire it() throws known errors - if (invalidCase.endsWith("_0")) { - expect(() => getTestType(testType, invalidCase), "Must throw constructing type").to.throw(); + if (testCase.endsWith("_0")) { + expect(() => getTestType(typeName, testSuite), "Must throw constructing type").to.throw(); return; } - const type = getTestType(testType, invalidCase); - const testData = parseSszGenericInvalidTestcase(path.join(invalidCasesPath, invalidCase)); + const type = getTestType(typeName, testCase); + const testData = parseSszGenericInvalidTestcase(path.join(testSuiteDirpath, testCase)); /* eslint-disable no-console */ if (process.env.DEBUG) { @@ -36,21 +40,18 @@ for (const testType of fs.readdirSync(rootGenericSszPath)) { expect(() => type.deserialize(testData.serialized), "Must throw on deserialize").to.throw(); }); } - }); - - describe(`${testType} valid`, () => { - const validCasesPath = path.join(testTypePath, "valid"); - for (const validCase of fs.readdirSync(validCasesPath)) { + } else if (testSuite === "valid") { + for (const testCase of fs.readdirSync(testSuiteDirpath)) { // NOTE: ComplexTestStruct tests are not correctly generated. // where deserialized .d value is D: '0x00'. However the tests guide mark that field as D: Bytes[256]. // Those test won't be fixed since most implementations staticly compile types. - if (validCase.startsWith("ComplexTestStruct")) { + if (testCase.startsWith("ComplexTestStruct")) { continue; } - it(validCase, () => { - const type = getTestType(testType, validCase); - const testData = parseSszGenericValidTestcase(path.join(validCasesPath, validCase)); + it(testCase, () => { + const type = getTestType(typeName, testCase); + const testData = parseSszGenericValidTestcase(path.join(testSuiteDirpath, testCase)); runValidSszTest(type, { root: testData.root, serialized: testData.serialized, @@ -58,5 +59,7 @@ for (const testType of fs.readdirSync(rootGenericSszPath)) { }); }); } - }); -} + } else { + throw Error(`Unknown ssz_generic testSuite ${testSuite}`); + } +}; diff --git a/packages/lodestar/test/spec/ssz/generic/types.ts b/packages/lodestar/test/spec/general/ssz_generic_types.ts similarity index 97% rename from packages/lodestar/test/spec/ssz/generic/types.ts rename to packages/lodestar/test/spec/general/ssz_generic_types.ts index 387e198867c..cd95a5dec79 100644 --- a/packages/lodestar/test/spec/ssz/generic/types.ts +++ b/packages/lodestar/test/spec/general/ssz_generic_types.ts @@ -107,6 +107,10 @@ const vecElementTypes = { uint256, }; +/** + * @param testType `"basic_vector" | "bitvector" | "containers"` + * @param testCase `"vec_bool_1_max" | "bitvec_2_zero"` + */ export function getTestType(testType: string, testCase: string): Type { switch (testType) { // `vec_{element type}_{length}` diff --git a/packages/lodestar/test/spec/phase0/epoch_processing.test.ts b/packages/lodestar/test/spec/phase0/epoch_processing.test.ts deleted file mode 100644 index 103be5bec05..00000000000 --- a/packages/lodestar/test/spec/phase0/epoch_processing.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import {allForks, phase0} from "@chainsafe/lodestar-beacon-state-transition"; -import {processParticipationRecordUpdates} from "@chainsafe/lodestar-beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates"; -import {ForkName} from "@chainsafe/lodestar-params"; -import {EpochProcessFn, epochProcessing} from "../allForks/epochProcessing"; - -/* eslint-disable @typescript-eslint/naming-convention */ - -epochProcessing(ForkName.phase0, { - effective_balance_updates: allForks.processEffectiveBalanceUpdates, - eth1_data_reset: allForks.processEth1DataReset, - historical_roots_update: allForks.processHistoricalRootsUpdate, - justification_and_finalization: allForks.processJustificationAndFinalization, - participation_record_updates: (processParticipationRecordUpdates as unknown) as EpochProcessFn, - randao_mixes_reset: allForks.processRandaoMixesReset, - registry_updates: allForks.processRegistryUpdates, - rewards_and_penalties: phase0.processRewardsAndPenalties as EpochProcessFn, - slashings: phase0.processSlashings as EpochProcessFn, - slashings_reset: allForks.processSlashingsReset, -}); diff --git a/packages/lodestar/test/spec/phase0/finality.test.ts b/packages/lodestar/test/spec/phase0/finality.test.ts deleted file mode 100644 index 8848ad8cedd..00000000000 --- a/packages/lodestar/test/spec/phase0/finality.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {finality} from "../allForks/finality"; - -finality(ForkName.phase0); diff --git a/packages/lodestar/test/spec/phase0/fork_choice.test.ts b/packages/lodestar/test/spec/phase0/fork_choice.test.ts deleted file mode 100644 index 9f58c7a1984..00000000000 --- a/packages/lodestar/test/spec/phase0/fork_choice.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {forkChoiceTest} from "../allForks/forkChoice"; - -forkChoiceTest(ForkName.phase0); diff --git a/packages/lodestar/test/spec/phase0/genesis.test.ts b/packages/lodestar/test/spec/phase0/genesis.test.ts deleted file mode 100644 index 4b43bd08df9..00000000000 --- a/packages/lodestar/test/spec/phase0/genesis.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {genesis} from "../allForks/genesis"; - -genesis(ForkName.phase0); diff --git a/packages/lodestar/test/spec/phase0/operations.test.ts b/packages/lodestar/test/spec/phase0/operations.test.ts deleted file mode 100644 index c1cbe1ff29c..00000000000 --- a/packages/lodestar/test/spec/phase0/operations.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import {CachedBeaconStatePhase0, allForks, phase0} from "@chainsafe/lodestar-beacon-state-transition"; -import {ForkName} from "@chainsafe/lodestar-params"; -import {IBaseSpecTest, shouldVerify} from "../type"; -import {operations} from "../allForks/operations"; - -/* eslint-disable @typescript-eslint/naming-convention */ - -/** Describe with which function to run each directory of tests */ -operations(ForkName.phase0, { - attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { - phase0.processAttestation(state, testCase.attestation); - }, - - attester_slashing: (state, testCase: IBaseSpecTest & {attester_slashing: phase0.AttesterSlashing}) => { - phase0.processAttesterSlashing(state, testCase.attester_slashing, shouldVerify(testCase)); - }, - - block_header: (state, testCase: IBaseSpecTest & {block: phase0.BeaconBlock}) => { - allForks.processBlockHeader(state, testCase.block); - }, - - deposit: (state, testCase: IBaseSpecTest & {deposit: phase0.Deposit}) => { - phase0.processDeposit(state, testCase.deposit); - }, - - proposer_slashing: (state, testCase: IBaseSpecTest & {proposer_slashing: phase0.ProposerSlashing}) => { - phase0.processProposerSlashing(state, testCase.proposer_slashing); - }, - - voluntary_exit: (state, testCase: IBaseSpecTest & {voluntary_exit: phase0.SignedVoluntaryExit}) => { - phase0.processVoluntaryExit(state, testCase.voluntary_exit); - }, -}); diff --git a/packages/lodestar/test/spec/phase0/random.test.ts b/packages/lodestar/test/spec/phase0/random.test.ts deleted file mode 100644 index b252eee62fc..00000000000 --- a/packages/lodestar/test/spec/phase0/random.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; -import {sanityBlock} from "../allForks/sanity"; - -sanityBlock(ForkName.phase0, `/tests/${ACTIVE_PRESET}/${ForkName.phase0}/random/random/pyspec_tests`); diff --git a/packages/lodestar/test/spec/phase0/rewards.test.ts b/packages/lodestar/test/spec/phase0/rewards.test.ts deleted file mode 100644 index 917b98dc4e9..00000000000 --- a/packages/lodestar/test/spec/phase0/rewards.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {rewards} from "../allForks/rewards"; - -rewards(ForkName.phase0); diff --git a/packages/lodestar/test/spec/phase0/sanity.test.ts b/packages/lodestar/test/spec/phase0/sanity.test.ts deleted file mode 100644 index db10bfc37b7..00000000000 --- a/packages/lodestar/test/spec/phase0/sanity.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {sanity} from "../allForks/sanity"; - -sanity(ForkName.phase0); diff --git a/packages/lodestar/test/spec/phase0/shuffling.test.ts b/packages/lodestar/test/spec/phase0/shuffling.test.ts deleted file mode 100644 index e54a177ef21..00000000000 --- a/packages/lodestar/test/spec/phase0/shuffling.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import {join} from "node:path"; -import {unshuffleList} from "@chainsafe/lodestar-beacon-state-transition"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import {ACTIVE_PRESET} from "@chainsafe/lodestar-params"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest} from "../type"; -import {bnToNum, fromHex} from "@chainsafe/lodestar-utils"; - -describeDirectorySpecTest( - `${ACTIVE_PRESET}/phase0/shuffling/`, - join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/phase0/shuffling/core/shuffle`), - (testcase) => { - const seed = fromHex(testcase.mapping.seed); - const output = Array.from({length: bnToNum(testcase.mapping.count)}, (_, i) => i); - unshuffleList(output, seed); - return output; - }, - { - inputTypes: {mapping: InputType.YAML}, - timeout: 10000, - getExpected: (testCase) => testCase.mapping.mapping.map((value) => bnToNum(value)), - } -); - -interface IShufflingTestCase extends IBaseSpecTest { - mapping: { - seed: string; - count: bigint; - mapping: bigint[]; - }; -} diff --git a/packages/lodestar/test/spec/phase0/ssz_static.test.ts b/packages/lodestar/test/spec/phase0/ssz_static.test.ts deleted file mode 100644 index 4b46a05c855..00000000000 --- a/packages/lodestar/test/spec/phase0/ssz_static.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {ForkName} from "@chainsafe/lodestar-params"; -import {sszStatic} from "../allForks/ssz_static"; - -sszStatic(ForkName.phase0); diff --git a/packages/lodestar/test/spec/phase0/util.ts b/packages/lodestar/test/spec/phase0/util.ts deleted file mode 100644 index b45cf82a9b1..00000000000 --- a/packages/lodestar/test/spec/phase0/util.ts +++ /dev/null @@ -1,10 +0,0 @@ -import {phase0} from "@chainsafe/lodestar-types"; -import {config} from "@chainsafe/lodestar-config/default"; -import {IBaseSpecTest} from "../type"; - -export interface IPhase0StateTestCase extends IBaseSpecTest { - pre: phase0.BeaconState; - post: phase0.BeaconState; -} - -export {config}; diff --git a/packages/lodestar/test/spec/presets/epoch_processing.ts b/packages/lodestar/test/spec/presets/epoch_processing.ts new file mode 100644 index 00000000000..33464d0bf68 --- /dev/null +++ b/packages/lodestar/test/spec/presets/epoch_processing.ts @@ -0,0 +1,114 @@ +import { + CachedBeaconStateAllForks, + EpochProcess, + BeaconStateAllForks, + beforeProcessEpoch, + allForks, + phase0, + altair, + bellatrix, +} from "@chainsafe/lodestar-beacon-state-transition"; +import {ssz} from "@chainsafe/lodestar-types"; +import {ForkName} from "@chainsafe/lodestar-params"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState"; +import {getConfig} from "../utils/getConfig"; +import {TestRunnerFn} from "../utils/types"; + +export type EpochProcessFn = (state: CachedBeaconStateAllForks, epochProcess: EpochProcess) => void; + +/* eslint-disable @typescript-eslint/naming-convention */ + +const epochProcessFnByFork: Record> = { + [ForkName.phase0]: { + effective_balance_updates: allForks.processEffectiveBalanceUpdates, + eth1_data_reset: allForks.processEth1DataReset, + historical_roots_update: allForks.processHistoricalRootsUpdate, + justification_and_finalization: allForks.processJustificationAndFinalization, + participation_record_updates: phase0.processParticipationRecordUpdates as EpochProcessFn, + randao_mixes_reset: allForks.processRandaoMixesReset, + registry_updates: allForks.processRegistryUpdates, + rewards_and_penalties: phase0.processRewardsAndPenalties as EpochProcessFn, + slashings: phase0.processSlashings as EpochProcessFn, + slashings_reset: allForks.processSlashingsReset, + }, + + [ForkName.altair]: { + effective_balance_updates: allForks.processEffectiveBalanceUpdates, + eth1_data_reset: allForks.processEth1DataReset, + historical_roots_update: allForks.processHistoricalRootsUpdate, + inactivity_updates: altair.processInactivityUpdates as EpochProcessFn, + justification_and_finalization: allForks.processJustificationAndFinalization, + participation_flag_updates: altair.processParticipationFlagUpdates as EpochProcessFn, + participation_record_updates: phase0.processParticipationRecordUpdates as EpochProcessFn, + randao_mixes_reset: allForks.processRandaoMixesReset, + registry_updates: allForks.processRegistryUpdates, + rewards_and_penalties: altair.processRewardsAndPenalties as EpochProcessFn, + slashings: altair.processSlashings as EpochProcessFn, + slashings_reset: allForks.processSlashingsReset, + sync_committee_updates: altair.processSyncCommitteeUpdates as EpochProcessFn, + }, + + [ForkName.bellatrix]: { + effective_balance_updates: allForks.processEffectiveBalanceUpdates, + eth1_data_reset: allForks.processEth1DataReset, + historical_roots_update: allForks.processHistoricalRootsUpdate, + inactivity_updates: altair.processInactivityUpdates as EpochProcessFn, + justification_and_finalization: allForks.processJustificationAndFinalization, + participation_flag_updates: altair.processParticipationFlagUpdates as EpochProcessFn, + participation_record_updates: phase0.processParticipationRecordUpdates as EpochProcessFn, + randao_mixes_reset: allForks.processRandaoMixesReset, + registry_updates: allForks.processRegistryUpdates, + rewards_and_penalties: altair.processRewardsAndPenalties as EpochProcessFn, + slashings: bellatrix.processSlashings as EpochProcessFn, + slashings_reset: allForks.processSlashingsReset, + sync_committee_updates: altair.processSyncCommitteeUpdates as EpochProcessFn, + }, +}; + +/** + * https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/epoch_processing/README.md + */ +type EpochProcessingTestCase = { + meta?: {bls_setting?: bigint}; + pre: BeaconStateAllForks; + post: BeaconStateAllForks; +}; + +/** + * @param fork + * @param epochProcessFns Describe with which function to run each directory of tests + */ +export const epochProcessing: TestRunnerFn = (fork, testName) => { + const config = getConfig(fork); + const epochProcessFns = epochProcessFnByFork[fork]; + + const epochProcessFn = epochProcessFns[testName]; + if (epochProcessFn === undefined) { + throw Error(`No epochProcessFn for ${testName}`); + } + + return { + testFunction: (testcase) => { + const stateTB = testcase.pre.clone(); + const state = createCachedBeaconStateTest(stateTB, config); + + const epochProcess = beforeProcessEpoch(state); + epochProcessFn(state, epochProcess); + state.commit(); + + return state; + }, + options: { + inputTypes: inputTypeSszTreeViewDU, + sszTypes: { + pre: ssz[fork].BeaconState, + post: ssz[fork].BeaconState, + }, + getExpected: (testCase) => testCase.post, + expectFunc: (testCase, expected, actual) => { + expectEqualBeaconState(fork, expected, actual); + }, + }, + }; +}; diff --git a/packages/lodestar/test/spec/allForks/finality.ts b/packages/lodestar/test/spec/presets/finality.ts similarity index 68% rename from packages/lodestar/test/spec/allForks/finality.ts rename to packages/lodestar/test/spec/presets/finality.ts index 460148618c4..b8adf211ebc 100644 --- a/packages/lodestar/test/spec/allForks/finality.ts +++ b/packages/lodestar/test/spec/presets/finality.ts @@ -1,22 +1,16 @@ -import {join} from "node:path"; import {allForks, altair, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; -import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import {bellatrix, ssz} from "@chainsafe/lodestar-types"; -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; +import {ForkName} from "@chainsafe/lodestar-params"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest, shouldVerify} from "../type"; -import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; -import {getConfig} from "./util"; -import {generateBlocksSZZTypeMapping} from "./sanity"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState"; +import {shouldVerify, TestRunnerFn} from "../utils/types"; +import {getConfig} from "../utils/getConfig"; /* eslint-disable @typescript-eslint/naming-convention */ -export function finality(fork: ForkName): void { - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/finality/finality`, - join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/finality/finality/pyspec_tests`), - (testcase) => { +export const finality: TestRunnerFn = (fork) => { + return { + testFunction: (testcase) => { let state = createCachedBeaconStateTest(testcase.pre, getConfig(fork)); const verify = shouldVerify(testcase); for (let i = 0; i < testcase.meta.blocks_count; i++) { @@ -32,7 +26,7 @@ export function finality(fork: ForkName): void { state.commit(); return state; }, - { + options: { inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, @@ -45,8 +39,18 @@ export function finality(fork: ForkName): void { expectFunc: (testCase, expected, actual) => { expectEqualBeaconState(fork, expected, actual); }, - } - ); + }, + }; +}; + +type BlocksSZZTypeMapping = Record; + +export function generateBlocksSZZTypeMapping(fork: ForkName, n: number): BlocksSZZTypeMapping { + const blocksMapping: BlocksSZZTypeMapping = {}; + for (let i = 0; i < n; i++) { + blocksMapping[`blocks_${i}`] = ssz[fork].SignedBeaconBlock; + } + return blocksMapping; } /** @@ -56,7 +60,7 @@ export function finality(fork: ForkName): void { * ``` * https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/finality/README.md */ -interface IFinalityTestCase extends IBaseSpecTest { +type FinalityTestCase = { [k: string]: altair.SignedBeaconBlock | unknown | null | undefined; meta: { blocks_count: number; @@ -64,4 +68,4 @@ interface IFinalityTestCase extends IBaseSpecTest { }; pre: BeaconStateAllForks; post?: BeaconStateAllForks; -} +}; diff --git a/packages/lodestar/test/spec/presets/fork.ts b/packages/lodestar/test/spec/presets/fork.ts new file mode 100644 index 00000000000..491144a723b --- /dev/null +++ b/packages/lodestar/test/spec/presets/fork.ts @@ -0,0 +1,51 @@ +import {allForks, phase0, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {ssz} from "@chainsafe/lodestar-types"; +import {ForkName} from "@chainsafe/lodestar-params"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {TestRunnerFn} from "../utils/types"; +import {createIChainForkConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; + +export const fork: TestRunnerFn = (forkNext) => { + if (forkNext === ForkName.phase0) { + throw Error("fork phase0 not supported"); + } + + const config = createIChainForkConfig({}); + const forkPrev = getPreviousFork(config, forkNext); + + return { + testFunction: (testcase) => { + const preState = createCachedBeaconStateTest(testcase.pre, config); + return allForks.upgradeStateByFork[forkNext](preState); + }, + options: { + inputTypes: inputTypeSszTreeViewDU, + sszTypes: { + pre: ssz[forkPrev].BeaconState, + post: ssz[forkNext].BeaconState, + }, + + timeout: 10000, + shouldError: (testCase) => testCase.post === undefined, + getExpected: (testCase) => testCase.post, + expectFunc: (testCase, expected, actual) => { + expectEqualBeaconState(forkNext, expected, actual); + }, + }, + }; +}; + +type ForkStateCase = { + pre: BeaconStateAllForks; + post: Exclude; +}; + +export function getPreviousFork(config: IChainForkConfig, fork: ForkName): ForkName { + // Find the previous fork + const forkIndex = config.forksAscendingEpochOrder.findIndex((f) => f.name === fork); + if (forkIndex < 1) { + throw Error(`Fork ${fork} not found`); + } + return config.forksAscendingEpochOrder[forkIndex - 1].name; +} diff --git a/packages/lodestar/test/spec/presets/fork_choice.ts b/packages/lodestar/test/spec/presets/fork_choice.ts new file mode 100644 index 00000000000..0a17fdeed5e --- /dev/null +++ b/packages/lodestar/test/spec/presets/fork_choice.ts @@ -0,0 +1,421 @@ +import {expect} from "chai"; +import { + phase0, + allForks, + computeEpochAtSlot, + CachedBeaconStateAllForks, + ZERO_HASH, + getEffectiveBalanceIncrementsZeroInactive, + computeStartSlotAtEpoch, + EffectiveBalanceIncrements, + BeaconStateAllForks, + bellatrix, +} from "@chainsafe/lodestar-beacon-state-transition"; +import {InputType} from "@chainsafe/lodestar-spec-test-util"; +import {initializeForkChoice} from "@chainsafe/lodestar/src/chain/forkChoice"; +import { + CheckpointStateCache, + toCheckpointHex, + toCheckpointKey, +} from "@chainsafe/lodestar/src/chain/stateCache/stateContextCheckpointsCache"; +import {ChainEventEmitter} from "@chainsafe/lodestar/src/chain/emitter"; +import {toHexString} from "@chainsafe/ssz"; +import { + CheckpointWithHex, + ForkChoiceError, + ForkChoiceErrorCode, + IForkChoice, + assertValidTerminalPowBlock, + ExecutionStatus, + PowBlockHex, +} from "@chainsafe/lodestar-fork-choice"; +import {ssz, RootHex} from "@chainsafe/lodestar-types"; +import {bnToNum} from "@chainsafe/lodestar-utils"; +import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {testLogger} from "../../utils/logger"; +import {getConfig} from "../utils/getConfig"; +import {TestRunnerFn} from "../utils/types"; + +/* eslint-disable @typescript-eslint/naming-convention */ + +/* eslint-disable @typescript-eslint/naming-convention */ + +const ANCHOR_STATE_FILE_NAME = "anchor_state"; +const ANCHOR_BLOCK_FILE_NAME = "anchor_block"; +const BLOCK_FILE_NAME = "^(block)_([0-9a-zA-Z]+)$"; +const POW_BLOCK_FILE_NAME = "^(pow_block)_([0-9a-zA-Z]+)$"; +const ATTESTATION_FILE_NAME = "^(attestation)_([0-9a-zA-Z])+$"; + +const logger = testLogger("spec-test"); + +export const forkChoiceTest: TestRunnerFn = (fork) => { + return { + testFunction: (testcase) => { + const {steps, anchorState} = testcase; + const currentSlot = anchorState.slot; + const config = getConfig(fork); + let state = createCachedBeaconStateTest(anchorState, config); + + const emitter = new ChainEventEmitter(); + const forkchoice = initializeForkChoice(config, emitter, currentSlot, state, true); + + const checkpointStateCache = new CheckpointStateCache({}); + const stateCache = new Map(); + cacheState(state, stateCache); + + /** This is to track test's tickTime to be used in proposer boost */ + let tickTime = 0; + + for (const [i, step] of steps.entries()) { + if (isTick(step)) { + tickTime = bnToNum(step.tick); + const currentSlot = Math.floor(tickTime / config.SECONDS_PER_SLOT); + logger.debug("Step tick", {currentSlot, valid: Boolean(step.valid), time: tickTime}); + forkchoice.updateTime(currentSlot); + } + + // attestation step + else if (isAttestation(step)) { + logger.debug("Step attestation", {root: step.attestation, valid: Boolean(step.valid)}); + const attestation = testcase.attestations.get(step.attestation); + if (!attestation) throw Error(`No attestation ${step.attestation}`); + forkchoice.onAttestation(state.epochCtx.getIndexedAttestation(attestation)); + } + + // block step + else if (isBlock(step)) { + logger.debug("Step block", {root: step.block, valid: Boolean(step.valid)}); + const validBlock = Boolean(step.valid ?? true); + + const signedBlock = testcase.blocks.get(step.block); + if (!signedBlock) throw Error(`No block ${step.block}`); + + // Log the BeaconBlock root instead of the SignedBeaconBlock root, forkchoice references BeaconBlock roots + const blockRoot = config.getForkTypes(signedBlock.message.slot).BeaconBlock.hashTreeRoot(signedBlock.message); + logger.debug("Step block", {slot: signedBlock.message.slot, root: toHexString(blockRoot)}); + + const preState = stateCache.get(toHexString(signedBlock.message.parentRoot)); + if (!preState) { + continue; + // should not throw error, on_block_bad_parent_root test wants this + } + const blockDelaySec = (tickTime - preState.genesisTime) % config.SECONDS_PER_SLOT; + const isMergeTransitionBlock = + bellatrix.isBellatrixStateType(preState) && + bellatrix.isBellatrixBlockBodyType(signedBlock.message.body) && + bellatrix.isMergeTransitionBlock(preState, signedBlock.message.body); + + try { + if (isMergeTransitionBlock) { + const mergeBlock = signedBlock.message as bellatrix.BeaconBlock; + + const powBlockRootHex = toHexString(mergeBlock.body.executionPayload.parentHash); + const powBlock = serializePowBlock(testcase.powBlocks.get(`pow_block_${powBlockRootHex}`)); + const powBlockParent = serializePowBlock( + powBlock && testcase.powBlocks.get(`pow_block_${powBlock.parentHash}`) + ); + assertValidTerminalPowBlock(config, mergeBlock, { + executionStatus: powBlock !== undefined ? ExecutionStatus.Valid : ExecutionStatus.Syncing, + powBlock, + powBlockParent, + }); + } + + state = runStateTranstion(preState, signedBlock, forkchoice, checkpointStateCache, blockDelaySec); + // TODO: May be part of runStateTranstion, necessary to commit again? + state.commit(); + cacheState(state, stateCache); + } catch (e) { + if (validBlock) throw e; + } + } + + // checks step + else if (isCheck(step)) { + // Forkchoice head is computed lazily only on request + const head = forkchoice.updateHead(); + const proposerBootRoot = forkchoice.getProposerBoostRoot(); + + if (step.checks.head !== undefined) { + expect(head.slot).to.be.equal(bnToNum(step.checks.head.slot), `Invalid head slot at step ${i}`); + expect(head.blockRoot).to.be.equal(step.checks.head.root, `Invalid head root at step ${i}`); + } + if (step.checks.proposer_boost_root !== undefined) { + expect(proposerBootRoot).to.be.equal( + step.checks.proposer_boost_root, + `Invalid proposer boost root at step ${i}` + ); + } + // time in spec mapped to Slot in our forkchoice implementation. + // Compare in slots because proposer boost steps doesn't always come on + // slot boundary. + if (step.checks.time !== undefined && step.checks.time > 0) + expect(forkchoice.getTime()).to.be.equal( + Math.floor(bnToNum(step.checks.time) / config.SECONDS_PER_SLOT), + `Invalid forkchoice time at step ${i}` + ); + if (step.checks.justified_checkpoint) { + expect(toSpecTestCheckpoint(forkchoice.getJustifiedCheckpoint())).to.be.deep.equal( + step.checks.justified_checkpoint, + `Invalid justified checkpoint at step ${i}` + ); + } + if (step.checks.finalized_checkpoint) { + expect(toSpecTestCheckpoint(forkchoice.getFinalizedCheckpoint())).to.be.deep.equal( + step.checks.finalized_checkpoint, + `Invalid finalized checkpoint at step ${i}` + ); + } + if (step.checks.best_justified_checkpoint) { + expect(toSpecTestCheckpoint(forkchoice.getBestJustifiedCheckpoint())).to.be.deep.equal( + step.checks.best_justified_checkpoint, + `Invalid best justified checkpoint at step ${i}` + ); + } + } + } + }, + + options: { + inputTypes: { + meta: InputType.YAML, + steps: InputType.YAML, + }, + sszTypes: { + [ANCHOR_STATE_FILE_NAME]: ssz[fork].BeaconState, + [ANCHOR_BLOCK_FILE_NAME]: ssz[fork].BeaconBlock, + [BLOCK_FILE_NAME]: ssz[fork].SignedBeaconBlock, + [POW_BLOCK_FILE_NAME]: ssz.bellatrix.PowBlock, + [ATTESTATION_FILE_NAME]: ssz.phase0.Attestation, + }, + mapToTestCase: (t: Record) => { + // t has input file name as key + const blocks = new Map(); + const powBlocks = new Map(); + const attestations = new Map(); + for (const key in t) { + const blockMatch = key.match(BLOCK_FILE_NAME); + if (blockMatch) { + blocks.set(key, t[key]); + } + const powBlockMatch = key.match(POW_BLOCK_FILE_NAME); + if (powBlockMatch) { + powBlocks.set(key, t[key]); + } + const attMatch = key.match(ATTESTATION_FILE_NAME); + if (attMatch) { + attestations.set(key, t[key]); + } + } + return { + meta: t["meta"] as ForkChoiceTestCase["meta"], + anchorState: t[ANCHOR_STATE_FILE_NAME] as ForkChoiceTestCase["anchorState"], + anchorBlock: t[ANCHOR_BLOCK_FILE_NAME] as ForkChoiceTestCase["anchorBlock"], + steps: t["steps"] as ForkChoiceTestCase["steps"], + blocks, + powBlocks, + attestations, + }; + }, + timeout: 10000, + // eslint-disable-next-line @typescript-eslint/no-empty-function + expectFunc: () => {}, + }, + }; +}; + +function runStateTranstion( + preState: CachedBeaconStateAllForks, + signedBlock: allForks.SignedBeaconBlock, + forkchoice: IForkChoice, + checkpointCache: CheckpointStateCache, + blockDelaySec: number +): CachedBeaconStateAllForks { + const preSlot = preState.slot; + const postSlot = signedBlock.message.slot - 1; + let preEpoch = computeEpochAtSlot(preSlot); + let postState = preState.clone(); + for ( + let nextEpochSlot = computeStartSlotAtEpoch(preEpoch + 1); + nextEpochSlot <= postSlot; + nextEpochSlot += SLOTS_PER_EPOCH + ) { + postState = allForks.processSlots(postState, nextEpochSlot, null); + cacheCheckpointState(postState, checkpointCache); + } + preEpoch = postState.epochCtx.epoch; + postState = allForks.stateTransition(postState, signedBlock, { + verifyStateRoot: true, + verifyProposer: false, + verifySignatures: false, + }); + const postEpoch = postState.epochCtx.epoch; + if (postEpoch > preEpoch) { + cacheCheckpointState(postState, checkpointCache); + } + // same logic like in state transition https://github.com/ChainSafe/lodestar/blob/f6778740075fe2b75edf94d1db0b5691039cb500/packages/lodestar/src/chain/blocks/stateTransition.ts#L101 + let justifiedBalances: EffectiveBalanceIncrements | undefined; + const checkpointHex = toCheckpointHex(postState.currentJustifiedCheckpoint); + const justifiedState = checkpointCache.get(checkpointHex); + if ( + postState.currentJustifiedCheckpoint.epoch > forkchoice.getJustifiedCheckpoint().epoch || + postState.finalizedCheckpoint.epoch > forkchoice.getFinalizedCheckpoint().epoch + ) { + if (!justifiedState) { + const checkpointHexKey = toCheckpointKey(checkpointHex); + const cachedCps = checkpointCache.dumpCheckpointKeys().join(", "); + throw Error(`No justifiedState for checkpoint ${checkpointHexKey}. Available: ${cachedCps}`); + } + justifiedBalances = getEffectiveBalanceIncrementsZeroInactive(justifiedState); + } + + try { + forkchoice.onBlock(signedBlock.message, postState, { + blockDelaySec, + justifiedBalances, + }); + for (const attestation of signedBlock.message.body.attestations) { + try { + const indexedAttestation = postState.epochCtx.getIndexedAttestation(attestation); + forkchoice.onAttestation(indexedAttestation); + } catch (e) { + if (e instanceof ForkChoiceError && e.type.code === ForkChoiceErrorCode.INVALID_ATTESTATION) { + logger.debug("INVALID_ATTESTATION onAttestation", e.type.err); + } else { + logger.error("Error onAttestation", {}, e as Error); + } + } + } + } catch (e) { + if (e instanceof ForkChoiceError && e.type.code === ForkChoiceErrorCode.INVALID_BLOCK) { + logger.debug("INVALID_BLOCK onBlock", e.type.err); + } else { + logger.error("Error onBlock", {}, e as Error); + } + } + return postState; +} + +function cacheCheckpointState(checkpointState: CachedBeaconStateAllForks, checkpointCache: CheckpointStateCache): void { + const slot = checkpointState.slot; + if (slot % SLOTS_PER_EPOCH !== 0) { + throw new Error(`Invalid checkpoint state slot ${checkpointState.slot}`); + } + const blockHeader = ssz.phase0.BeaconBlockHeader.clone(checkpointState.latestBlockHeader); + if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { + blockHeader.stateRoot = checkpointState.hashTreeRoot(); + } + const cp: phase0.Checkpoint = { + root: ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader), + epoch: computeEpochAtSlot(slot), + }; + checkpointCache.add(cp, checkpointState); +} + +function cacheState(wrappedState: CachedBeaconStateAllForks, stateCache: Map): void { + const blockHeader = ssz.phase0.BeaconBlockHeader.clone(wrappedState.latestBlockHeader); + if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { + blockHeader.stateRoot = wrappedState.hashTreeRoot(); + } + const blockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader); + stateCache.set(toHexString(blockRoot), wrappedState); +} + +function toSpecTestCheckpoint(checkpoint: CheckpointWithHex): SpecTestCheckpoint { + return { + epoch: BigInt(checkpoint.epoch), + root: checkpoint.rootHex, + }; +} + +type Step = OnTick | OnAttestation | OnBlock | OnPowBlock | Checks; + +type SpecTestCheckpoint = {epoch: BigInt; root: string}; + +// This test executes steps in sequence. There may be multiple items of the following types: +// on_tick execution step + +type OnTick = { + /** to execute `on_tick(store, time)` */ + tick: bigint; + /** optional, default to `true`. */ + valid?: number; +}; + +type OnAttestation = { + /** the name of the `attestation_<32-byte-root>.ssz_snappy` file. To execute `on_attestation(store, attestation)` */ + attestation: string; + /** optional, default to `true`. */ + valid?: number; +}; + +type OnBlock = { + /** the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` */ + block: string; + /** optional, default to `true`. */ + valid?: number; +}; + +type OnPowBlock = { + /** + * the name of the `pow_block_<32-byte-root>.ssz_snappy` file. To + * execute `on_pow_block(store, block)` + */ + pow_block: string; +}; + +type Checks = { + /** Value in the ForkChoice store to verify it's correct after being mutated by another step */ + checks: { + head?: { + slot: bigint; + root: string; + }; + time?: bigint; + justified_checkpoint?: SpecTestCheckpoint; + finalized_checkpoint?: SpecTestCheckpoint; + best_justified_checkpoint?: SpecTestCheckpoint; + proposer_boost_root?: RootHex; + }; +}; + +type ForkChoiceTestCase = { + meta?: { + description?: string; + bls_setting: BigInt; + }; + anchorState: BeaconStateAllForks; + anchorBlock: allForks.BeaconBlock; + steps: Step[]; + blocks: Map; + powBlocks: Map; + attestations: Map; +}; + +function isTick(step: Step): step is OnTick { + return (step as OnTick).tick > 0; +} + +function isAttestation(step: Step): step is OnAttestation { + return typeof (step as OnAttestation).attestation === "string"; +} + +function isBlock(step: Step): step is OnBlock { + return typeof (step as OnBlock).block === "string"; +} + +function isCheck(step: Step): step is Checks { + return typeof (step as Checks).checks === "object"; +} + +function serializePowBlock(powBlock: bellatrix.PowBlock | undefined): PowBlockHex | undefined { + if (powBlock) { + return { + blockHash: toHexString(powBlock.blockHash), + parentHash: toHexString(powBlock.parentHash), + totalDifficulty: BigInt(powBlock.totalDifficulty), + }; + } + return; +} diff --git a/packages/lodestar/test/spec/allForks/genesis.ts b/packages/lodestar/test/spec/presets/genesis.ts similarity index 71% rename from packages/lodestar/test/spec/allForks/genesis.ts rename to packages/lodestar/test/spec/presets/genesis.ts index a80ae6f2b72..c6459984d54 100644 --- a/packages/lodestar/test/spec/allForks/genesis.ts +++ b/packages/lodestar/test/spec/presets/genesis.ts @@ -1,7 +1,6 @@ -import {join} from "node:path"; import {expect} from "chai"; import {phase0, Root, ssz, bellatrix, TimeSeconds} from "@chainsafe/lodestar-types"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; +import {InputType} from "@chainsafe/lodestar-spec-test-util"; import { BeaconStateAllForks, createEmptyEpochContextImmutableData, @@ -9,22 +8,27 @@ import { isValidGenesisState, } from "@chainsafe/lodestar-beacon-state-transition"; import {bnToNum} from "@chainsafe/lodestar-utils"; -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {expectEqualBeaconState} from "../util"; -import {IBaseSpecTest} from "../type"; -import {getConfig} from "./util"; +import {expectEqualBeaconState} from "../utils/expectEqualBeaconState"; +import {TestRunnerFn} from "../utils/types"; +import {getConfig} from "../utils/getConfig"; // The aim of the genesis tests is to provide a baseline to test genesis-state initialization and test if the // proposed genesis-validity conditions are working. /* eslint-disable @typescript-eslint/naming-convention */ -export function genesis(fork: ForkName): void { - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/genesis/initialization`, - join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/genesis/initialization/pyspec_tests`), - (testcase) => { +export const genesis: TestRunnerFn = (fork, testName, testSuite) => { + const testFn = genesisTestFns[testName]; + if (testFn === undefined) { + throw Error(`Unknown genesis test ${testName}`); + } + + return testFn(fork, testName, testSuite); +}; + +const genesisInitialization: TestRunnerFn = (fork) => { + return { + testFunction: (testcase) => { const deposits: phase0.Deposit[] = []; for (let i = 0; i < testcase.meta.deposits_count; i++) { deposits.push(testcase[`deposits_${i}`] as phase0.Deposit); @@ -39,8 +43,8 @@ export function genesis(fork: ForkName): void { return initializeBeaconStateFromEth1( getConfig(fork), immutableData, - ssz.Root.fromJson((testcase.eth1 as IGenesisInitCase).eth1_block_hash), - bnToNum((testcase.eth1 as IGenesisInitCase).eth1_timestamp), + ssz.Root.fromJson((testcase.eth1 as GenesisInitCase).eth1_block_hash), + bnToNum((testcase.eth1 as GenesisInitCase).eth1_timestamp), deposits, undefined, testcase["execution_payload_header"] && @@ -56,7 +60,7 @@ export function genesis(fork: ForkName): void { // ``` // {deposits_count: 64} // ``` - { + options: { inputTypes: {meta: InputType.YAML, eth1: InputType.YAML}, sszTypes: { eth1_block_hash: ssz.Root, @@ -70,21 +74,16 @@ export function genesis(fork: ForkName): void { expectFunc: (testCase, expected, actual) => { expectEqualBeaconState(fork, expected, actual); }, - } - ); - - interface IGenesisValidityTestCase extends IBaseSpecTest { - is_valid: boolean; - genesis: BeaconStateAllForks; - } + }, + }; +}; - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/genesis/validity`, - join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/genesis/validity/pyspec_tests`), - (testcase) => { +const genesisValidity: TestRunnerFn = (fork) => { + return { + testFunction: (testcase) => { return isValidGenesisState(getConfig(fork), testcase.genesis); }, - { + options: { inputTypes: { is_valid: InputType.YAML, genesis: InputType.SSZ_SNAPPY, @@ -96,9 +95,14 @@ export function genesis(fork: ForkName): void { expectFunc: (testCase, expected, actual) => { expect(actual).to.be.equal(expected, "isValidGenesisState is not" + expected); }, - } - ); -} + }, + }; +}; + +const genesisTestFns: Record> = { + initialization: genesisInitialization, + validity: genesisValidity, +}; function generateDepositSSZTypeMapping(n: number): Record { const depositMappings: Record = {}; @@ -108,7 +112,12 @@ function generateDepositSSZTypeMapping(n: number): Record( - `${ACTIVE_PRESET}/${fork}/transition`, - join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/merkle/single_proof/pyspec_tests`), - (testcase) => { +export const merkle: TestRunnerFn = (fork) => { + return { + testFunction: (testcase) => { const {proof: specTestProof, state} = testcase; const stateRoot = state.hashTreeRoot(); const leaf = fromHexString(specTestProof.leaf); @@ -24,7 +19,7 @@ export function merkle(fork: ForkName): void { const leafIndex = Number(specTestProof.leaf_index); const depth = Math.floor(Math.log2(leafIndex)); const verified = verifyMerkleBranch(leaf, branch, depth, leafIndex % 2 ** depth, stateRoot); - expect(verified, "cannot verify merkle branch").to.be.true; + expect(verified).to.equal(true, "invalid merkle branch"); const lodestarProof = new Tree(state.node).getProof({ gindex: specTestProof.leaf_index, @@ -37,7 +32,8 @@ export function merkle(fork: ForkName): void { branch: lodestarProof.witnesses.map(toHexString), }; }, - { + + options: { inputTypes: { state: {type: InputType.SSZ_SNAPPY as const, treeBacked: true as const}, proof: InputType.YAML as const, @@ -50,17 +46,17 @@ export function merkle(fork: ForkName): void { expectFunc: (testCase, expected, actual) => { expect(actual).to.be.deep.equal(expected, "incorrect proof"); }, - } - ); + }, + }; +}; - interface IMerkleTestCase extends IBaseSpecTest { - state: BeaconStateAllForks; - proof: IProof; - } +type MerkleTestCase = { + state: BeaconStateAllForks; + proof: IProof; +}; - interface IProof { - leaf: string; - leaf_index: bigint; - branch: string[]; - } +interface IProof { + leaf: string; + leaf_index: bigint; + branch: string[]; } diff --git a/packages/lodestar/test/spec/presets/operations.ts b/packages/lodestar/test/spec/presets/operations.ts new file mode 100644 index 00000000000..27580cc8e38 --- /dev/null +++ b/packages/lodestar/test/spec/presets/operations.ts @@ -0,0 +1,188 @@ +import { + allForks, + altair, + BeaconStateAllForks, + bellatrix, + CachedBeaconStateAllForks, + CachedBeaconStateAltair, + CachedBeaconStateBellatrix, + CachedBeaconStatePhase0, + phase0, +} from "@chainsafe/lodestar-beacon-state-transition"; +// eslint-disable-next-line no-restricted-imports +import {processExecutionPayload} from "@chainsafe/lodestar-beacon-state-transition/lib/bellatrix/block/processExecutionPayload"; +import {ssz} from "@chainsafe/lodestar-types"; +import {ForkName} from "@chainsafe/lodestar-params"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState"; +import {getConfig} from "../utils/getConfig"; +import {BaseSpecTest, shouldVerify, TestRunnerFn} from "../utils/types"; +import {InputType} from "@chainsafe/lodestar-spec-test-util"; + +/* eslint-disable @typescript-eslint/naming-convention */ + +// Define above to re-use in sync_aggregate and sync_aggregate_random +const sync_aggregate: BlockProcessFn = ( + state, + testCase: {sync_aggregate: altair.SyncAggregate} +) => { + const block = ssz.altair.BeaconBlock.defaultValue(); + + // processSyncAggregate() needs the full block to get the slot + block.slot = state.slot; + block.body.syncAggregate = ssz.altair.SyncAggregate.toViewDU(testCase["sync_aggregate"]); + + altair.processSyncAggregate(state, block); +}; + +const operationFnsPhase0: Record> = { + attestation: (state, testCase: {attestation: phase0.Attestation}) => { + phase0.processAttestation(state, testCase.attestation); + }, + + attester_slashing: (state, testCase: BaseSpecTest & {attester_slashing: phase0.AttesterSlashing}) => { + phase0.processAttesterSlashing(state, testCase.attester_slashing, shouldVerify(testCase)); + }, + + block_header: (state, testCase: {block: phase0.BeaconBlock}) => { + allForks.processBlockHeader(state, testCase.block); + }, + + deposit: (state, testCase: {deposit: phase0.Deposit}) => { + phase0.processDeposit(state, testCase.deposit); + }, + + proposer_slashing: (state, testCase: {proposer_slashing: phase0.ProposerSlashing}) => { + phase0.processProposerSlashing(state, testCase.proposer_slashing); + }, + + voluntary_exit: (state, testCase: {voluntary_exit: phase0.SignedVoluntaryExit}) => { + phase0.processVoluntaryExit(state, testCase.voluntary_exit); + }, +}; + +const operationFnsAltair: Record> = { + attestation: (state, testCase: {attestation: phase0.Attestation}) => { + altair.processAttestations(state, [testCase.attestation]); + }, + + attester_slashing: (state, testCase: BaseSpecTest & {attester_slashing: phase0.AttesterSlashing}) => { + altair.processAttesterSlashing(state, testCase.attester_slashing, shouldVerify(testCase)); + }, + + block_header: (state, testCase: {block: altair.BeaconBlock}) => { + allForks.processBlockHeader(state, testCase.block); + }, + + deposit: (state, testCase: {deposit: phase0.Deposit}) => { + altair.processDeposit(state, testCase.deposit); + }, + + proposer_slashing: (state, testCase: {proposer_slashing: phase0.ProposerSlashing}) => { + altair.processProposerSlashing(state, testCase.proposer_slashing); + }, + + sync_aggregate, + sync_aggregate_random: sync_aggregate, + + voluntary_exit: (state, testCase: {voluntary_exit: phase0.SignedVoluntaryExit}) => { + altair.processVoluntaryExit(state, testCase.voluntary_exit); + }, +}; + +const operationFnsBellatrix: Record> = { + attestation: (state, testCase: {attestation: phase0.Attestation}) => { + altair.processAttestations((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, [testCase.attestation]); + }, + + attester_slashing: (state, testCase: BaseSpecTest & {attester_slashing: phase0.AttesterSlashing}) => { + bellatrix.processAttesterSlashing(state, testCase.attester_slashing, shouldVerify(testCase)); + }, + + block_header: (state, testCase: {block: altair.BeaconBlock}) => { + allForks.processBlockHeader(state, testCase.block); + }, + + deposit: (state, testCase: {deposit: phase0.Deposit}) => { + altair.processDeposit((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, testCase.deposit); + }, + + proposer_slashing: (state, testCase: {proposer_slashing: phase0.ProposerSlashing}) => { + bellatrix.processProposerSlashing(state, testCase.proposer_slashing); + }, + + sync_aggregate, + sync_aggregate_random: sync_aggregate, + + voluntary_exit: (state, testCase: {voluntary_exit: phase0.SignedVoluntaryExit}) => { + altair.processVoluntaryExit( + (state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, + testCase.voluntary_exit + ); + }, + + execution_payload: ( + state, + testCase: {execution_payload: bellatrix.ExecutionPayload; execution: {execution_valid: boolean}} + ) => { + processExecutionPayload( + (state as CachedBeaconStateAllForks) as CachedBeaconStateBellatrix, + testCase.execution_payload, + {notifyNewPayload: () => testCase.execution.execution_valid} + ); + }, +}; + +const epochProcessFnByFork: Record>> = { + [ForkName.phase0]: operationFnsPhase0, + [ForkName.altair]: operationFnsAltair, + [ForkName.bellatrix]: operationFnsBellatrix, +}; + +export type BlockProcessFn = (state: T, testCase: any) => void; + +export type OperationsTestCase = { + meta?: {bls_setting?: bigint}; + pre: BeaconStateAllForks; + post: BeaconStateAllForks; + execution: {execution_valid: boolean}; +}; + +export const operations: TestRunnerFn = (fork, testName) => { + const operationFn = epochProcessFnByFork[fork][testName]; + if (operationFn === undefined) { + throw Error(`No operationFn for ${testName}`); + } + + return { + testFunction: (testcase) => { + const state = testcase.pre.clone(); + const cachedState = createCachedBeaconStateTest(state, getConfig(fork)); + operationFn(cachedState, testcase); + state.commit(); + return state; + }, + options: { + inputTypes: {...inputTypeSszTreeViewDU, execution: InputType.YAML}, + sszTypes: { + pre: ssz[fork].BeaconState, + post: ssz[fork].BeaconState, + attestation: ssz.phase0.Attestation, + attester_slashing: ssz.phase0.AttesterSlashing, + block: ssz[fork].BeaconBlock, + deposit: ssz.phase0.Deposit, + proposer_slashing: ssz.phase0.ProposerSlashing, + voluntary_exit: ssz.phase0.SignedVoluntaryExit, + // Altair + sync_aggregate: ssz.altair.SyncAggregate, + // Bellatrix + execution_payload: ssz.bellatrix.ExecutionPayload, + }, + shouldError: (testCase) => testCase.post === undefined, + getExpected: (testCase) => testCase.post, + expectFunc: (testCase, expected, actual) => { + expectEqualBeaconState(fork, expected, actual); + }, + }, + }; +}; diff --git a/packages/lodestar/test/spec/presets/rewards.ts b/packages/lodestar/test/spec/presets/rewards.ts new file mode 100644 index 00000000000..88909b94ec6 --- /dev/null +++ b/packages/lodestar/test/spec/presets/rewards.ts @@ -0,0 +1,137 @@ +import {expect} from "chai"; +import { + altair, + phase0, + CachedBeaconStatePhase0, + BeaconStateAllForks, + BeaconStateAltair, + beforeProcessEpoch, +} from "@chainsafe/lodestar-beacon-state-transition"; +import {ForkName} from "@chainsafe/lodestar-params"; +import {VectorCompositeType} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState"; +import {getConfig} from "../utils/getConfig"; +import {TestRunnerFn} from "../utils/types"; + +/* eslint-disable @typescript-eslint/naming-convention */ + +export const rewards: TestRunnerFn = (fork, testName, testSuite) => { + switch (fork) { + case ForkName.phase0: + return rewardsPhase0(fork, testName, testSuite); + default: + return rewardsAltair(fork, testName, testSuite); + } +}; + +const deltasType = new VectorCompositeType(ssz.phase0.Balances, 2); + +const rewardsPhase0: TestRunnerFn = (fork: ForkName) => { + return { + testFunction: (testcase) => { + const config = getConfig(fork); + const wrappedState = createCachedBeaconStateTest(testcase.pre, config); + const epochProcess = beforeProcessEpoch(wrappedState); + return phase0.getAttestationDeltas(wrappedState as CachedBeaconStatePhase0, epochProcess); + }, + options: { + inputTypes: inputTypeSszTreeViewDU, + sszTypes: { + pre: ssz[fork].BeaconState, + source_deltas: deltasType, + target_deltas: deltasType, + head_deltas: deltasType, + inclusion_delay_deltas: deltasType, + inactivity_penalty_deltas: deltasType, + }, + timeout: 100000000, + getExpected: (testCase) => + sumDeltas([ + testCase.source_deltas, + testCase.target_deltas, + testCase.head_deltas, + testCase.inclusion_delay_deltas, + testCase.inactivity_penalty_deltas, + ]), + expectFunc: (testCase, expected, actual) => { + expect(actual).to.deep.equal(expected); + }, + }, + }; +}; + +const rewardsAltair: TestRunnerFn = (fork) => { + return { + testFunction: (testcase) => { + const config = getConfig(fork); + const state = createCachedBeaconStateTest(testcase.pre as BeaconStateAltair, config); + const epochProcess = beforeProcessEpoch(state); + // To debug this test and get granular results you can tweak inputs to get more granular results + // + // TIMELY_HEAD_FLAG_INDEX -> FLAG_PREV_HEAD_ATTESTER_OR_UNSLASHED + // TIMELY_SOURCE_FLAG_INDEX -> FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED + // TIMELY_TARGET_FLAG_INDEX -> FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED + // + // - To get head_deltas set TIMELY_SOURCE_FLAG_INDEX | TIMELY_TARGET_FLAG_INDEX to false + // - To get source_deltas set TIMELY_HEAD_FLAG_INDEX | TIMELY_TARGET_FLAG_INDEX to false + // - To get target_deltas set TIMELY_HEAD_FLAG_INDEX | TIMELY_SOURCE_FLAG_INDEX to false + // + set all inactivityScores to zero + // - To get inactivity_penalty_deltas set TIMELY_HEAD_FLAG_INDEX | TIMELY_SOURCE_FLAG_INDEX to false + // + set PARTICIPATION_FLAG_WEIGHTS[TIMELY_TARGET_FLAG_INDEX] to zero + return altair.getRewardsAndPenalties(state, epochProcess); + }, + options: { + inputTypes: inputTypeSszTreeViewDU, + sszTypes: { + pre: ssz[fork].BeaconState, + head_deltas: deltasType, + source_deltas: deltasType, + target_deltas: deltasType, + inactivity_penalty_deltas: deltasType, + }, + getExpected: (testCase) => + sumDeltas([ + testCase.head_deltas, + testCase.source_deltas, + testCase.target_deltas, + testCase.inactivity_penalty_deltas, + ]), + expectFunc: (testCase, expected, actual) => { + expect(actual).to.deep.equal(expected); + }, + }, + }; +}; + +type Deltas = [number[], number[]]; + +type RewardTestCasePhase0 = { + pre: BeaconStateAllForks; + source_deltas: Deltas; + target_deltas: Deltas; + head_deltas: Deltas; + inclusion_delay_deltas: Deltas; + inactivity_penalty_deltas: Deltas; +}; + +type RewardTestCaseAltair = { + pre: BeaconStateAllForks; + head_deltas: Deltas; + source_deltas: Deltas; + target_deltas: Deltas; + inactivity_penalty_deltas: Deltas; +}; + +function sumDeltas(deltasArr: Deltas[]): Deltas { + const totalDeltas: Deltas = [[], []]; + for (const deltas of deltasArr) { + for (const n of [0, 1]) { + for (let i = 0; i < deltas[n].length; i++) { + totalDeltas[n][i] = (totalDeltas[n][i] ?? 0) + deltas[n][i]; + } + } + } + return totalDeltas; +} diff --git a/packages/lodestar/test/spec/allForks/sanity.ts b/packages/lodestar/test/spec/presets/sanity.ts similarity index 70% rename from packages/lodestar/test/spec/allForks/sanity.ts rename to packages/lodestar/test/spec/presets/sanity.ts index b7c689b598c..41bc93f6968 100644 --- a/packages/lodestar/test/spec/allForks/sanity.ts +++ b/packages/lodestar/test/spec/presets/sanity.ts @@ -1,27 +1,29 @@ -import {join} from "node:path"; -import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; +import {InputType} from "@chainsafe/lodestar-spec-test-util"; import {allForks, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {bellatrix, ssz} from "@chainsafe/lodestar-types"; -import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; +import {ForkName} from "@chainsafe/lodestar-params"; import {bnToNum} from "@chainsafe/lodestar-utils"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; -import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest, shouldVerify} from "../type"; -import {getConfig} from "./util"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState"; +import {shouldVerify, TestRunnerFn} from "../utils/types"; +import {getConfig} from "../utils/getConfig"; /* eslint-disable @typescript-eslint/naming-convention */ -export function sanity(fork: ForkName): void { - sanitySlot(fork); - sanityBlock(fork, `/tests/${ACTIVE_PRESET}/${fork}/sanity/blocks/pyspec_tests`); -} +export const sanity: TestRunnerFn = (fork, testName, testSuite) => { + switch (testName) { + case "slots": + return sanitySlots(fork, testName, testSuite); + case "blocks": + return sanityBlocks(fork, testName, testSuite); + default: + throw Error(`Unknown sanity test ${testName}`); + } +}; -export function sanitySlot(fork: ForkName): void { - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/sanity/slots`, - join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/sanity/slots/pyspec_tests`), - (testcase) => { +const sanitySlots: TestRunnerFn = (fork) => { + return { + testFunction: (testcase) => { const stateTB = testcase.pre.clone(); const state = createCachedBeaconStateTest(stateTB, getConfig(fork)); const postState = allForks.processSlots(state, state.slot + bnToNum(testcase.slots)); @@ -29,7 +31,7 @@ export function sanitySlot(fork: ForkName): void { postState.commit(); return postState; }, - { + options: { inputTypes: {...inputTypeSszTreeViewDU, slots: InputType.YAML}, sszTypes: { pre: ssz[fork].BeaconState, @@ -41,15 +43,13 @@ export function sanitySlot(fork: ForkName): void { expectFunc: (testCase, expected, actual) => { expectEqualBeaconState(fork, expected, actual); }, - } - ); -} + }, + }; +}; -export function sanityBlock(fork: ForkName, testPath: string): void { - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/sanity/blocks`, - join(SPEC_TEST_LOCATION, testPath), - (testcase) => { +export const sanityBlocks: TestRunnerFn = (fork) => { + return { + testFunction: (testcase) => { const stateTB = testcase.pre; let wrappedState = createCachedBeaconStateTest(stateTB, getConfig(fork)); const verify = shouldVerify(testcase); @@ -63,7 +63,7 @@ export function sanityBlock(fork: ForkName, testPath: string): void { } return wrappedState; }, - { + options: { inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, @@ -76,9 +76,9 @@ export function sanityBlock(fork: ForkName, testPath: string): void { expectFunc: (testCase, expected, actual) => { expectEqualBeaconState(fork, expected, actual); }, - } - ); -} + }, + }; +}; type BlocksSZZTypeMapping = Record; @@ -90,7 +90,7 @@ export function generateBlocksSZZTypeMapping(fork: ForkName, n: number): BlocksS return blocksMapping; } -interface IBlockSanityTestCase extends IBaseSpecTest { +type SanityBlocksTestCase = { [k: string]: allForks.SignedBeaconBlock | unknown | null | undefined; meta: { blocks_count: number; @@ -98,10 +98,10 @@ interface IBlockSanityTestCase extends IBaseSpecTest { }; pre: BeaconStateAllForks; post: BeaconStateAllForks; -} +}; -interface IProcessSlotsTestCase extends IBaseSpecTest { +type SanitySlotsTestCase = { pre: BeaconStateAllForks; post?: BeaconStateAllForks; slots: bigint; -} +}; diff --git a/packages/lodestar/test/spec/presets/shuffling.ts b/packages/lodestar/test/spec/presets/shuffling.ts new file mode 100644 index 00000000000..83f7b73f593 --- /dev/null +++ b/packages/lodestar/test/spec/presets/shuffling.ts @@ -0,0 +1,28 @@ +import {unshuffleList} from "@chainsafe/lodestar-beacon-state-transition"; +import {InputType} from "@chainsafe/lodestar-spec-test-util"; +import {bnToNum, fromHex} from "@chainsafe/lodestar-utils"; +import {TestRunnerFn} from "../utils/types"; + +export const shuffling: TestRunnerFn = () => { + return { + testFunction: (testcase) => { + const seed = fromHex(testcase.mapping.seed); + const output = Array.from({length: bnToNum(testcase.mapping.count)}, (_, i) => i); + unshuffleList(output, seed); + return output; + }, + options: { + inputTypes: {mapping: InputType.YAML}, + timeout: 10000, + getExpected: (testCase) => testCase.mapping.mapping.map((value) => bnToNum(value)), + }, + }; +}; + +type ShufflingTestCase = { + mapping: { + seed: string; + count: bigint; + mapping: bigint[]; + }; +}; diff --git a/packages/lodestar/test/spec/presets/ssz_static.ts b/packages/lodestar/test/spec/presets/ssz_static.ts new file mode 100644 index 00000000000..354a4310940 --- /dev/null +++ b/packages/lodestar/test/spec/presets/ssz_static.ts @@ -0,0 +1,58 @@ +import fs from "node:fs"; +import path from "node:path"; +import {ssz} from "@chainsafe/lodestar-types"; +import {Type} from "@chainsafe/ssz"; +import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; +import {replaceUintTypeWithUintBigintType} from "../utils/replaceUintTypeWithUintBigintType"; +import {parseSszStaticTestcase} from "../utils/sszTestCaseParser"; +import {runValidSszTest} from "../utils/runValidSszTest"; + +// ssz_static +// | Attestation +// | ssz_nil +// | case_0 +// | roots.yaml +// | serialized.ssz_snappy +// | value.yaml +// +// Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md + +/* eslint-disable + @typescript-eslint/naming-convention, + @typescript-eslint/no-unsafe-assignment, + @typescript-eslint/no-unsafe-call, + @typescript-eslint/no-unsafe-member-access, + no-console +*/ + +// eslint-disable-next-line +type Types = Record>; + +// Mapping of sszGeneric() fn arguments to the path in spec tests +// +// / config / fork / test runner / test handler / test suite / test case +// +// tests / mainnet / altair / ssz_static / Validator / ssz_random / case_0/roots.yaml +// + +export const sszStatic = (fork: ForkName, typeName: string, testSuite: string, testSuiteDirpath: string): void => { + /* eslint-disable @typescript-eslint/strict-boolean-expressions */ + const sszType = (ssz[fork] as Types)[typeName] || (ssz.altair as Types)[typeName] || (ssz.phase0 as Types)[typeName]; + if (!sszType) { + throw Error(`No type for ${typeName}`); + } + + const sszTypeNoUint = replaceUintTypeWithUintBigintType(sszType); + + for (const testCase of fs.readdirSync(testSuiteDirpath)) { + it(testCase, function () { + // Mainnet must deal with big full states and hash each one multiple times + if (ACTIVE_PRESET === "mainnet") { + this.timeout(30 * 1000); + } + + const testData = parseSszStaticTestcase(path.join(testSuiteDirpath, testCase)); + runValidSszTest(sszTypeNoUint, testData); + }); + } +}; diff --git a/packages/lodestar/test/spec/allForks/transition.ts b/packages/lodestar/test/spec/presets/transition.ts similarity index 58% rename from packages/lodestar/test/spec/allForks/transition.ts rename to packages/lodestar/test/spec/presets/transition.ts index 8b45a45d135..8f3e1c0dc46 100644 --- a/packages/lodestar/test/spec/allForks/transition.ts +++ b/packages/lodestar/test/spec/presets/transition.ts @@ -1,27 +1,45 @@ -import {join} from "node:path"; import {allForks, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {ssz} from "@chainsafe/lodestar-types"; -import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import {createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; -import {ForkName, ACTIVE_PRESET} from "@chainsafe/lodestar-params"; -import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {IBaseSpecTest} from "../type"; -import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; +import {ForkName} from "@chainsafe/lodestar-params"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState"; import {bnToNum} from "@chainsafe/lodestar-utils"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {TestRunnerFn} from "../utils/types"; +import {config} from "@chainsafe/lodestar-config/default"; +import {getPreviousFork} from "./fork"; -export function transition( - forkConfig: (forkEpoch: number) => Partial, - pre: ForkName, - fork: Exclude -): void { - describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/transition`, - join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/transition/core/pyspec_tests`), - (testcase) => { +export const transition: TestRunnerFn = (forkNext) => { + if (forkNext === ForkName.phase0) { + throw Error("fork phase0 not supported"); + } + + const forkPrev = getPreviousFork(config, forkNext); + + /** + * https://github.com/ethereum/eth2.0-specs/tree/v1.1.0-alpha.5/tests/formats/transition + */ + function generateBlocksSZZTypeMapping(meta: TransitionTestCase["meta"]): BlocksSZZTypeMapping { + if (meta === undefined) { + throw new Error("No meta data found"); + } + const blocksMapping: BlocksSZZTypeMapping = {}; + // The fork_block is the index in the test data of the last block of the initial fork. + for (let i = 0; i < meta.blocks_count; i++) { + blocksMapping[`blocks_${i}`] = + i <= meta.fork_block ? ssz[forkPrev].SignedBeaconBlock : ssz[forkNext].SignedBeaconBlock; + } + return blocksMapping; + } + + return { + testFunction: (testcase) => { const meta = testcase.meta; + // testConfig is used here to load forkEpoch from meta.yaml - const testConfig = createIChainForkConfig(forkConfig(bnToNum(meta.fork_epoch))); + const forkEpoch = bnToNum(meta.fork_epoch); + const testConfig = createIChainForkConfig(getTransitionConfig(forkNext, forkEpoch)); + let state = createCachedBeaconStateTest(testcase.pre, testConfig); for (let i = 0; i < meta.blocks_count; i++) { const signedBlock = testcase[`blocks_${i}`] as allForks.SignedBeaconBlock; @@ -33,12 +51,12 @@ export function transition( } return state; }, - { + options: { inputTypes: inputTypeSszTreeViewDU, - getSszTypes: (meta: ITransitionTestCase["meta"]) => { + getSszTypes: (meta: TransitionTestCase["meta"]) => { return { - pre: ssz[pre].BeaconState, - post: ssz[fork].BeaconState, + pre: ssz[forkPrev].BeaconState, + post: ssz[forkNext].BeaconState, ...generateBlocksSZZTypeMapping(meta), }; }, @@ -46,31 +64,28 @@ export function transition( timeout: 10000, getExpected: (testCase) => testCase.post, expectFunc: (testCase, expected, actual) => { - expectEqualBeaconState(fork, expected, actual); + expectEqualBeaconState(forkNext, expected, actual); }, - } - ); + }, + }; +}; - /** - * https://github.com/ethereum/eth2.0-specs/tree/v1.1.0-alpha.5/tests/formats/transition - */ - function generateBlocksSZZTypeMapping(meta: ITransitionTestCase["meta"]): BlocksSZZTypeMapping { - if (meta === undefined) { - throw new Error("No meta data found"); - } - const blocksMapping: BlocksSZZTypeMapping = {}; - // The fork_block is the index in the test data of the last block of the initial fork. - for (let i = 0; i < meta.blocks_count; i++) { - blocksMapping[`blocks_${i}`] = i <= meta.fork_block ? ssz[pre].SignedBeaconBlock : ssz[fork].SignedBeaconBlock; - } - return blocksMapping; +/* eslint-disable @typescript-eslint/naming-convention */ + +function getTransitionConfig(fork: ForkName, forkEpoch: number): Partial { + switch (fork) { + case ForkName.phase0: + throw Error("phase0 not allowed"); + case ForkName.altair: + return {ALTAIR_FORK_EPOCH: forkEpoch}; + case ForkName.bellatrix: + return {ALTAIR_FORK_EPOCH: 0, BELLATRIX_FORK_EPOCH: forkEpoch}; } } type BlocksSZZTypeMapping = Record; -/* eslint-disable @typescript-eslint/naming-convention */ -interface ITransitionTestCase extends IBaseSpecTest { +type TransitionTestCase = { [k: string]: allForks.SignedBeaconBlock | unknown | null | undefined; meta: { post_fork: ForkName; @@ -81,4 +96,4 @@ interface ITransitionTestCase extends IBaseSpecTest { }; pre: BeaconStateAllForks; post: BeaconStateAllForks; -} +}; diff --git a/packages/lodestar/test/spec/ssz/type.ts b/packages/lodestar/test/spec/ssz/type.ts deleted file mode 100644 index f9e591ff5a3..00000000000 --- a/packages/lodestar/test/spec/ssz/type.ts +++ /dev/null @@ -1,11 +0,0 @@ -import {IBaseSpecTest} from "../type"; - -export interface IBaseSSZStaticTestCase extends IBaseSpecTest { - roots: { - root: string; - }; - serialized: T; - // eslint-disable-next-line @typescript-eslint/naming-convention - serialized_raw: Uint8Array; - value: T; -} diff --git a/packages/lodestar/test/spec/type.ts b/packages/lodestar/test/spec/type.ts deleted file mode 100644 index c03dc7a2616..00000000000 --- a/packages/lodestar/test/spec/type.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ - -export interface IBaseSpecTest { - meta?: { - bls_setting?: bigint; - }; -} - -export function shouldVerify(testCase: IBaseSpecTest): boolean { - return testCase.meta?.bls_setting === BigInt(1); -} diff --git a/packages/lodestar/test/spec/util.ts b/packages/lodestar/test/spec/utils/expectEqualBeaconState.ts similarity index 100% rename from packages/lodestar/test/spec/util.ts rename to packages/lodestar/test/spec/utils/expectEqualBeaconState.ts diff --git a/packages/lodestar/test/spec/utils/getConfig.ts b/packages/lodestar/test/spec/utils/getConfig.ts new file mode 100644 index 00000000000..1be8cdec55d --- /dev/null +++ b/packages/lodestar/test/spec/utils/getConfig.ts @@ -0,0 +1,22 @@ +import {ForkName} from "@chainsafe/lodestar-params"; +import {config} from "@chainsafe/lodestar-config/default"; +import {IChainForkConfig, createIChainForkConfig} from "@chainsafe/lodestar-config"; + +/* eslint-disable @typescript-eslint/naming-convention */ + +export function getConfig(fork: ForkName): IChainForkConfig { + switch (fork) { + case ForkName.phase0: + return config; + case ForkName.altair: + return createIChainForkConfig({ALTAIR_FORK_EPOCH: 0}); + case ForkName.bellatrix: + return createIChainForkConfig({ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + TERMINAL_TOTAL_DIFFICULTY: BigInt( + "115792089237316195423570985008687907853269984665640564039457584007913129638912" + ), + }); + } +} diff --git a/packages/lodestar/test/spec/utils/specTestIterator.ts b/packages/lodestar/test/spec/utils/specTestIterator.ts new file mode 100644 index 00000000000..78cf1bb90dc --- /dev/null +++ b/packages/lodestar/test/spec/utils/specTestIterator.ts @@ -0,0 +1,91 @@ +import {ForkName} from "@chainsafe/lodestar-params"; +import fs from "node:fs"; +import path from "node:path"; +import {SPEC_TEST_LOCATION} from "../specTestVersioning"; +import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; +import {RunnerType, TestRunner} from "./types"; +import {expect} from "chai"; + +const specTestsTestPath = path.join(SPEC_TEST_LOCATION, "tests"); +const ARTIFACT_FILENAMES = new Set(["._.DS_Store", ".DS_Store"]); + +/** + * This helper ensures that strictly all tests are run. There's no hardcoded value beyond "config". + * Any additional unknown fork, testRunner, testHandler, or testSuite will result in an error. + * + * File path structure: + * ``` + * tests/ + * / [general, mainnet, minimal] + * / [phase0, altair, bellatrix] + * / [bls, ssz_static, fork] + * / ... + * / + * / + * ``` + * + * Examples + * ``` + * / config / fork / test runner / test handler / test suite / test case + * + * tests / general / phase0 / bls / aggregate / small / aggregate_na_signatures/data.yaml + * tests / general / phase0 / ssz_generic / basic_vector / valid / vec_bool_1_max/meta.yaml + * tests / mainnet / altair / ssz_static / Validator / ssz_random / case_0/roots.yaml + * tests / mainnet / altair / fork / fork / pyspec_tests / altair_fork_random_0/meta.yaml + * ``` + * Ref: https://github.com/ethereum/consensus-specs/tree/dev/tests/formats#test-structure + */ +export function specTestIterator(configName: string, testRunners: Record): void { + // Check no unknown directory at the top level + it("Check top level directories", () => { + expect(readdirSyncSpec(specTestsTestPath).sort()).to.deep.equal( + ["general", "mainnet", "minimal"], + "Unknown top level directories" + ); + }); + + const configDirpath = path.join(specTestsTestPath, configName); + for (const forkStr of readdirSyncSpec(configDirpath)) { + const fork = ForkName[forkStr as ForkName]; + if (fork === undefined) { + throw Error(`Unknown fork ${forkStr}`); + } + + const forkDirpath = path.join(configDirpath, fork); + for (const testRunnerName of readdirSyncSpec(forkDirpath)) { + const testRunnerDirpath = path.join(forkDirpath, testRunnerName); + + const testRunner = testRunners[testRunnerName]; + if (testRunner === undefined) { + throw Error(`No test runner for ${testRunnerName}`); + } + + for (const testHandler of readdirSyncSpec(testRunnerDirpath)) { + const testHandlerDirpath = path.join(testRunnerDirpath, testHandler); + for (const testSuite of readdirSyncSpec(testHandlerDirpath)) { + const testId = `${configName}/${fork}/${testRunnerName}/${testHandler}/${testSuite}`; + const testSuiteDirpath = path.join(testHandlerDirpath, testSuite); + + // Specific logic for ssz_static since it has one extra level of directories + if (testRunner.type === RunnerType.custom) { + describe(testId, () => { + testRunner.fn(fork, testHandler, testSuite, testSuiteDirpath); + }); + } + + // Generic testRunner + else { + const {testFunction, options} = testRunner.fn(fork, testHandler, testSuite); + + describeDirectorySpecTest(testId, testSuiteDirpath, testFunction, options); + } + } + } + } + } +} + +function readdirSyncSpec(dirpath: string): string[] { + const files = fs.readdirSync(dirpath); + return files.filter((file) => !ARTIFACT_FILENAMES.has(file)); +} diff --git a/packages/lodestar/test/spec/utils/types.ts b/packages/lodestar/test/spec/utils/types.ts new file mode 100644 index 00000000000..d8664ebcae5 --- /dev/null +++ b/packages/lodestar/test/spec/utils/types.ts @@ -0,0 +1,39 @@ +import {ForkName} from "@chainsafe/lodestar-params"; +import {ISpecTestOptions} from "@chainsafe/lodestar-spec-test-util"; + +export enum RunnerType { + custom, + default, +} + +export type TestRunnerFn = ( + fork: ForkName, + testHandler: string, + testSuite: string +) => { + testFunction: (testCase: TestCase, directoryName: string) => Result; + options: Partial>; +}; + +export type TestRunnerCustom = ( + fork: ForkName, + testHandler: string, + testSuite: string, + testSuiteDirpath: string +) => void; + +export type TestRunner = + | {type: RunnerType.default; fn: TestRunnerFn} + | {type: RunnerType.custom; fn: TestRunnerCustom}; + +/* eslint-disable @typescript-eslint/naming-convention */ + +export type BaseSpecTest = { + meta?: { + bls_setting?: bigint; + }; +}; + +export function shouldVerify(testCase: BaseSpecTest): boolean { + return testCase.meta?.bls_setting === BigInt(1); +}