diff --git a/package.json b/package.json index 4ee9466385d..d39617f3397 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,8 @@ "lint": "lerna run lint --no-bail", "check-types": "lerna run check-types --no-bail", "coverage": "lerna run coverage --no-bail", - "test:unit": "lerna run test:unit --no-bail", - "test:e2e": "lerna run test:e2e --no-bail", + "test:unit": "lerna run test:unit --no-bail --concurrency 1", + "test:e2e": "lerna run test:e2e --no-bail --concurrency 1", "test:e2e:sim": "lerna run test:e2e:sim --no-bail", "test:spec-min": "lerna run test:spec-min --no-bail", "test:spec-fast": "lerna run test:spec-fast --no-bail", diff --git a/packages/api/package.json b/packages/api/package.json index ebd6dcd3b8c..4ceb2bc7d95 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -40,8 +40,8 @@ "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/persistent-merkle-tree": "^0.3.7", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/persistent-merkle-tree": "^0.4.0", + "@chainsafe/ssz": "^0.9.0", "cross-fetch": "^3.1.4", "eventsource": "^1.1.0", "qs": "^6.10.1" diff --git a/packages/api/src/client/utils/client.ts b/packages/api/src/client/utils/client.ts index f3d2efbc081..3069547bd03 100644 --- a/packages/api/src/client/utils/client.ts +++ b/packages/api/src/client/utils/client.ts @@ -1,4 +1,3 @@ -import {Json} from "@chainsafe/ssz"; import {mapValues} from "@chainsafe/lodestar-utils"; import {FetchOpts, IHttpClient} from "./httpClient"; import {compileRouteUrlFormater} from "../../utils/urlFormat"; @@ -8,7 +7,6 @@ import { RouteGeneric, ReturnTypes, TypeJson, - jsonOpts, ReqSerializer, ReqSerializers, RoutesData, @@ -69,10 +67,10 @@ export function generateGenericJsonClient< const returnType = returnTypes[routeKey as keyof ReturnTypes] as TypeJson | null; return async function request(...args: Parameters): Promise { - const res = await fetchFn.json(fetchOptsSerializer(...args)); + const res = await fetchFn.json(fetchOptsSerializer(...args)); if (returnType) { // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return returnType.fromJson(res, jsonOpts) as ReturnType; + return returnType.fromJson(res) as ReturnType; } }; }) as Api; diff --git a/packages/api/src/keymanager/routes.ts b/packages/api/src/keymanager/routes.ts index 55288fb5ffc..a278145c436 100644 --- a/packages/api/src/keymanager/routes.ts +++ b/packages/api/src/keymanager/routes.ts @@ -154,8 +154,8 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { return { - listKeys: jsonType(), - importKeystores: jsonType(), - deleteKeystores: jsonType(), + listKeys: jsonType("camel"), + importKeystores: jsonType("camel"), + deleteKeystores: jsonType("camel"), }; } diff --git a/packages/api/src/routes/beacon/block.ts b/packages/api/src/routes/beacon/block.ts index ce6c3064c1f..b71afda8a87 100644 --- a/packages/api/src/routes/beacon/block.ts +++ b/packages/api/src/routes/beacon/block.ts @@ -1,4 +1,4 @@ -import {ContainerType, Json} from "@chainsafe/ssz"; +import {ContainerType} from "@chainsafe/ssz"; import {ForkName} from "@chainsafe/lodestar-params"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {phase0, allForks, Slot, Root, ssz} from "@chainsafe/lodestar-types"; @@ -114,7 +114,7 @@ export type ReqTypes = { getBlockHeader: BlockIdOnlyReq; getBlockHeaders: {query: {slot?: number; parent_root?: string}}; getBlockRoot: BlockIdOnlyReq; - publishBlock: {body: Json}; + publishBlock: {body: unknown}; }; export function getReqSerializers(config: IChainForkConfig): ReqSerializers { @@ -124,13 +124,13 @@ export function getReqSerializers(config: IChainForkConfig): ReqSerializers => + // Compute block type from JSON payload. See https://github.com/ethereum/eth2.0-APIs/pull/142 + const getSignedBeaconBlockType = (data: allForks.SignedBeaconBlock): allForks.AllForksSSZTypes["SignedBeaconBlock"] => config.getForkTypes(data.message.slot).SignedBeaconBlock; + const AllForksSignedBeaconBlock: TypeJson = { - toJson: (data, opts) => getSignedBeaconBlockType(data).toJson(data, opts), - fromJson: (data, opts) => - getSignedBeaconBlockType((data as unknown) as allForks.SignedBeaconBlock).fromJson(data, opts), + toJson: (data) => getSignedBeaconBlockType(data).toJson(data), + fromJson: (data) => getSignedBeaconBlockType((data as unknown) as allForks.SignedBeaconBlock).fromJson(data), }; return { @@ -149,14 +149,10 @@ export function getReqSerializers(config: IChainForkConfig): ReqSerializers { - const BeaconHeaderResType = new ContainerType({ - fields: { - root: ssz.Root, - canonical: ssz.Boolean, - header: ssz.phase0.SignedBeaconBlockHeader, - }, - //from beacon apis - expectedCase: "notransform", + const BeaconHeaderResType = new ContainerType({ + root: ssz.Root, + canonical: ssz.Boolean, + header: ssz.phase0.SignedBeaconBlockHeader, }); return { diff --git a/packages/api/src/routes/beacon/pool.ts b/packages/api/src/routes/beacon/pool.ts index fbe8e16cccf..3cf8c787b49 100644 --- a/packages/api/src/routes/beacon/pool.ts +++ b/packages/api/src/routes/beacon/pool.ts @@ -1,5 +1,4 @@ import {phase0, altair, CommitteeIndex, Slot, ssz} from "@chainsafe/lodestar-types"; -import {Json} from "@chainsafe/ssz"; import { RoutesData, ReturnTypes, @@ -122,11 +121,11 @@ export type ReqTypes = { getPoolAttesterSlashings: ReqEmpty; getPoolProposerSlashings: ReqEmpty; getPoolVoluntaryExits: ReqEmpty; - submitPoolAttestations: {body: Json}; - submitPoolAttesterSlashing: {body: Json}; - submitPoolProposerSlashing: {body: Json}; - submitPoolVoluntaryExit: {body: Json}; - submitPoolSyncCommitteeSignatures: {body: Json}; + submitPoolAttestations: {body: unknown}; + submitPoolAttesterSlashing: {body: unknown}; + submitPoolProposerSlashing: {body: unknown}; + submitPoolVoluntaryExit: {body: unknown}; + submitPoolSyncCommitteeSignatures: {body: unknown}; }; export function getReqSerializers(): ReqSerializers { diff --git a/packages/api/src/routes/beacon/state.ts b/packages/api/src/routes/beacon/state.ts index 24e6fcfe7be..7cf113565ad 100644 --- a/packages/api/src/routes/beacon/state.ts +++ b/packages/api/src/routes/beacon/state.ts @@ -217,54 +217,49 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { - const FinalityCheckpoints = new ContainerType({ - fields: { + const FinalityCheckpoints = new ContainerType( + { previousJustified: ssz.phase0.Checkpoint, currentJustified: ssz.phase0.Checkpoint, finalized: ssz.phase0.Checkpoint, }, - // From beacon apis - casingMap: { - previousJustified: "previous_justified", - currentJustified: "current_justified", - finalized: "finalized", - }, - }); + {jsonCase: "eth2"} + ); - const ValidatorResponse = new ContainerType({ - fields: { + const ValidatorResponse = new ContainerType( + { index: ssz.ValidatorIndex, - balance: ssz.Number64, + balance: ssz.UintNum64, status: new StringType(), validator: ssz.phase0.Validator, }, - // From beacon apis - expectedCase: "notransform", - }); + {jsonCase: "eth2"} + ); - const ValidatorBalance = new ContainerType({ - fields: { + const ValidatorBalance = new ContainerType( + { index: ssz.ValidatorIndex, - balance: ssz.Number64, + balance: ssz.UintNum64, }, - // From beacon apis - expectedCase: "notransform", - }); + {jsonCase: "eth2"} + ); - const EpochCommitteeResponse = new ContainerType({ - fields: { + const EpochCommitteeResponse = new ContainerType( + { index: ssz.CommitteeIndex, slot: ssz.Slot, validators: ssz.phase0.CommitteeIndices, }, - }); + {jsonCase: "eth2"} + ); - const EpochSyncCommitteesResponse = new ContainerType({ - fields: { + const EpochSyncCommitteesResponse = new ContainerType( + { validators: ArrayOf(ssz.ValidatorIndex), validatorAggregates: ArrayOf(ssz.ValidatorIndex), }, - }); + {jsonCase: "eth2"} + ); return { getStateRoot: ContainerData(ssz.Root), diff --git a/packages/api/src/routes/config.ts b/packages/api/src/routes/config.ts index 82ff9323810..eee2d632fcd 100644 --- a/packages/api/src/routes/config.ts +++ b/packages/api/src/routes/config.ts @@ -1,6 +1,6 @@ import {BeaconPreset} from "@chainsafe/lodestar-params"; import {IChainConfig} from "@chainsafe/lodestar-config"; -import {Bytes32, Number64, phase0, ssz} from "@chainsafe/lodestar-types"; +import {Bytes32, UintNum64, phase0, ssz} from "@chainsafe/lodestar-types"; import {mapValues} from "@chainsafe/lodestar-utils"; import {ByteVectorType, ContainerType} from "@chainsafe/ssz"; import {ArrayOf, ContainerData, ReqEmpty, reqEmpty, ReturnTypes, ReqSerializers, RoutesData, sameType} from "../utils"; @@ -8,7 +8,7 @@ import {ArrayOf, ContainerData, ReqEmpty, reqEmpty, ReturnTypes, ReqSerializers, // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes export type DepositContract = { - chainId: Number64; + chainId: UintNum64; address: Bytes32; }; @@ -57,17 +57,13 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { - const DepositContract = new ContainerType({ - fields: { - chainId: ssz.Number64, - address: new ByteVectorType({length: 20}), + const DepositContract = new ContainerType( + { + chainId: ssz.UintNum64, + address: new ByteVectorType(20), }, - // From beacon apis - casingMap: { - chainId: "chain_id", - address: "address", - }, - }); + {jsonCase: "eth2"} + ); return { getDepositContract: ContainerData(DepositContract), diff --git a/packages/api/src/routes/debug.ts b/packages/api/src/routes/debug.ts index ae9783329da..e4d8748e9b2 100644 --- a/packages/api/src/routes/debug.ts +++ b/packages/api/src/routes/debug.ts @@ -115,14 +115,13 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { const stringType = new StringType(); - const SlotRoot = new ContainerType({ - fields: { + const SlotRoot = new ContainerType( + { slot: ssz.Slot, root: stringType, }, - // From beacon apis - expectedCase: "notransform", - }); + {jsonCase: "eth2"} + ); return { getHeads: ContainerData(ArrayOf(SlotRoot)), diff --git a/packages/api/src/routes/events.ts b/packages/api/src/routes/events.ts index ac4e58511d3..07b7ba02b4b 100644 --- a/packages/api/src/routes/events.ts +++ b/packages/api/src/routes/events.ts @@ -1,6 +1,6 @@ -import {Epoch, Number64, phase0, Slot, ssz, StringType, RootHex, altair} from "@chainsafe/lodestar-types"; -import {ContainerType, Json, Type} from "@chainsafe/ssz"; -import {jsonOpts, RouteDef, TypeJson} from "../utils"; +import {Epoch, phase0, Slot, ssz, StringType, RootHex, altair, UintNum64} from "@chainsafe/lodestar-types"; +import {ContainerType, Type} from "@chainsafe/ssz"; +import {RouteDef, TypeJson} from "../utils"; // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes @@ -46,7 +46,7 @@ export type EventData = { [EventType.finalizedCheckpoint]: {block: RootHex; state: RootHex; epoch: Epoch}; [EventType.chainReorg]: { slot: Slot; - depth: Number64; + depth: UintNum64; oldHeadBlock: RootHex; newHeadBlock: RootHex; oldHeadState: RootHex; @@ -87,8 +87,8 @@ export type ReqTypes = { export function getTypeByEvent(): {[K in EventType]: Type} { const stringType = new StringType(); return { - [EventType.head]: new ContainerType({ - fields: { + [EventType.head]: new ContainerType( + { slot: ssz.Slot, block: stringType, state: stringType, @@ -96,71 +96,49 @@ export function getTypeByEvent(): {[K in EventType]: Type} { previousDutyDependentRoot: stringType, currentDutyDependentRoot: stringType, }, - // From beacon apis eventstream - casingMap: { - slot: "slot", - block: "block", - state: "state", - epochTransition: "epoch_transition", - previousDutyDependentRoot: "previous_duty_dependent_root", - currentDutyDependentRoot: "current_duty_dependent_root", - }, - }), + {jsonCase: "eth2"} + ), - [EventType.block]: new ContainerType({ - fields: { + [EventType.block]: new ContainerType( + { slot: ssz.Slot, block: stringType, }, - // From beacon apis eventstream - expectedCase: "notransform", - }), + {jsonCase: "eth2"} + ), [EventType.attestation]: ssz.phase0.Attestation, [EventType.voluntaryExit]: ssz.phase0.SignedVoluntaryExit, - [EventType.finalizedCheckpoint]: new ContainerType({ - fields: { + [EventType.finalizedCheckpoint]: new ContainerType( + { block: stringType, state: stringType, epoch: ssz.Epoch, }, - // From beacon apis eventstream - expectedCase: "notransform", - }), + {jsonCase: "eth2"} + ), - [EventType.chainReorg]: new ContainerType({ - fields: { + [EventType.chainReorg]: new ContainerType( + { slot: ssz.Slot, - depth: ssz.Number64, + depth: ssz.UintNum64, oldHeadBlock: stringType, newHeadBlock: stringType, oldHeadState: stringType, newHeadState: stringType, epoch: ssz.Epoch, }, - // From beacon apis eventstream - casingMap: { - slot: "slot", - depth: "depth", - oldHeadBlock: "old_head_block", - newHeadBlock: "new_head_block", - oldHeadState: "old_head_state", - newHeadState: "new_head_state", - epoch: "epoch", - }, - }), + {jsonCase: "eth2"} + ), - [EventType.lightclientHeaderUpdate]: new ContainerType({ - fields: { + [EventType.lightclientHeaderUpdate]: new ContainerType( + { syncAggregate: ssz.altair.SyncAggregate, attestedHeader: ssz.phase0.BeaconBlockHeader, }, - casingMap: { - syncAggregate: "sync_aggregate", - attestedHeader: "attested_header", - }, - }), + {jsonCase: "eth2"} + ), }; } @@ -169,13 +147,13 @@ export function getEventSerdes() { const typeByEvent = getTypeByEvent(); return { - toJson: (event: BeaconEvent): Json => { + toJson: (event: BeaconEvent): unknown => { const eventType = typeByEvent[event.type] as TypeJson; - return eventType.toJson(event.message, jsonOpts); + return eventType.toJson(event.message); }, - fromJson: (type: EventType, data: Json): BeaconEvent["message"] => { + fromJson: (type: EventType, data: unknown): BeaconEvent["message"] => { const eventType = typeByEvent[type] as TypeJson; - return eventType.fromJson(data, jsonOpts); + return eventType.fromJson(data); }, }; } diff --git a/packages/api/src/routes/lightclient.ts b/packages/api/src/routes/lightclient.ts index 39efffb7f90..52d292f3c64 100644 --- a/packages/api/src/routes/lightclient.ts +++ b/packages/api/src/routes/lightclient.ts @@ -1,4 +1,4 @@ -import {ContainerType, Path, VectorType} from "@chainsafe/ssz"; +import {ContainerType, JsonPath, VectorCompositeType} from "@chainsafe/ssz"; import {Proof} from "@chainsafe/persistent-merkle-tree"; import {altair, phase0, ssz, SyncPeriod} from "@chainsafe/lodestar-types"; import { @@ -29,10 +29,10 @@ export type LightclientSnapshotWithProof = { export type Api = { /** - * Returns a multiproof of `paths` at the requested `stateId`. + * Returns a multiproof of `jsonPaths` at the requested `stateId`. * The requested `stateId` may not be available. Regular nodes only keep recent states in memory. */ - getStateProof(stateId: string, paths: Path[]): Promise<{data: Proof}>; + getStateProof(stateId: string, jsonPaths: JsonPath[]): Promise<{data: Proof}>; /** * Returns an array of best updates in the requested periods within the inclusive range `from` - `to`. * Best is defined by (in order of priority): @@ -96,30 +96,22 @@ export function getReqSerializers(): ReqSerializers { } export function getReturnTypes(): ReturnTypes { - const lightclientSnapshotWithProofType = new ContainerType({ - fields: { + const lightclientSnapshotWithProofType = new ContainerType( + { header: ssz.phase0.BeaconBlockHeader, currentSyncCommittee: ssz.altair.SyncCommittee, - currentSyncCommitteeBranch: new VectorType({elementType: ssz.Root, length: 5}), + currentSyncCommitteeBranch: new VectorCompositeType(ssz.Root, 5), }, - // Custom type, not in the consensus specs - casingMap: { - header: "header", - currentSyncCommittee: "current_sync_committee", - currentSyncCommitteeBranch: "current_sync_committee_branch", - }, - }); + {jsonCase: "eth2"} + ); - const lightclientHeaderUpdate = new ContainerType({ - fields: { + const lightclientHeaderUpdate = new ContainerType( + { syncAggregate: ssz.altair.SyncAggregate, attestedHeader: ssz.phase0.BeaconBlockHeader, }, - casingMap: { - syncAggregate: "sync_aggregate", - attestedHeader: "attested_header", - }, - }); + {jsonCase: "eth2"} + ); return { // Just sent the proof JSON as-is diff --git a/packages/api/src/routes/lodestar.ts b/packages/api/src/routes/lodestar.ts index c855d5207ef..79063e337b5 100644 --- a/packages/api/src/routes/lodestar.ts +++ b/packages/api/src/routes/lodestar.ts @@ -1,16 +1,5 @@ -import {Epoch, RootHex, Slot, ssz, StringType} from "@chainsafe/lodestar-types"; -import {ByteVectorType, ContainerType, Json} from "@chainsafe/ssz"; -import { - jsonType, - ReqEmpty, - reqEmpty, - ReturnTypes, - ReqSerializers, - RoutesData, - sameType, - Schema, - ArrayOf, -} from "../utils"; +import {Epoch, RootHex, Slot} from "@chainsafe/lodestar-types"; +import {jsonType, ReqEmpty, reqEmpty, ReturnTypes, ReqSerializers, RoutesData, sameType, Schema} from "../utils"; import {FilterGetPeers, NodePeer, PeerDirection, PeerState} from "./node"; // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes @@ -35,7 +24,7 @@ export type GossipQueueItem = { export type RegenQueueItem = { key: string; - args: Json; + args: unknown; addedTimeMs: number; }; @@ -174,34 +163,17 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { - const stringType = new StringType(); - const GossipQueueItem = new ContainerType({ - fields: { - topic: stringType, - receivedFrom: stringType, - data: new ByteVectorType({length: 256}), - addedTimeMs: ssz.Slot, - }, - // Custom type, not in the consensus specs - casingMap: { - topic: "topic", - receivedFrom: "received_from", - data: "data", - addedTimeMs: "added_time_ms", - }, - }); - return { getWtfNode: sameType(), writeHeapdump: sameType(), getLatestWeakSubjectivityCheckpointEpoch: sameType(), - getSyncChainsDebugState: jsonType(), - getGossipQueueItems: ArrayOf(GossipQueueItem), - getRegenQueueItems: jsonType(), - getBlockProcessorQueueItems: jsonType(), - getStateCacheItems: jsonType(), - getCheckpointStateCacheItems: jsonType(), - getPeers: jsonType(), - discv5GetKadValues: jsonType(), + getSyncChainsDebugState: jsonType("camel"), + getGossipQueueItems: jsonType("camel"), + getRegenQueueItems: jsonType("camel"), + getBlockProcessorQueueItems: jsonType("camel"), + getStateCacheItems: jsonType("camel"), + getCheckpointStateCacheItems: jsonType("camel"), + getPeers: jsonType("camel"), + discv5GetKadValues: jsonType("camel"), }; } diff --git a/packages/api/src/routes/node.ts b/packages/api/src/routes/node.ts index d217f02d182..75be39556b4 100644 --- a/packages/api/src/routes/node.ts +++ b/packages/api/src/routes/node.ts @@ -162,23 +162,16 @@ export function getReqSerializers(): ReqSerializers { /* eslint-disable @typescript-eslint/naming-convention */ export function getReturnTypes(): ReturnTypes { const stringType = new StringType(); - const NetworkIdentity = new ContainerType({ - fields: { + const NetworkIdentity = new ContainerType( + { peerId: stringType, enr: stringType, p2pAddresses: ArrayOf(stringType), discoveryAddresses: ArrayOf(stringType), metadata: ssz.altair.Metadata, }, - // From beacon apis - casingMap: { - peerId: "peer_id", - enr: "enr", - p2pAddresses: "p2p_addresses", - discoveryAddresses: "discovery_addresses", - metadata: "metadata", - }, - }); + {jsonCase: "eth2"} + ); return { // @@ -187,11 +180,11 @@ export function getReturnTypes(): ReturnTypes { getNetworkIdentity: ContainerData(NetworkIdentity), // All these types don't contain any BigInt nor Buffer instances. // Use jsonType() to translate the casing in a generic way. - getPeers: jsonType(), - getPeer: jsonType(), - getPeerCount: jsonType(), - getNodeVersion: jsonType(), - getSyncingStatus: jsonType(), + getPeers: jsonType("camel"), + getPeer: jsonType("camel"), + getPeerCount: jsonType("camel"), + getNodeVersion: jsonType("camel"), + getSyncingStatus: jsonType("camel"), getHealth: sameType(), }; } diff --git a/packages/api/src/routes/validator.ts b/packages/api/src/routes/validator.ts index c5d0cd4e9dd..cf7c2e481d7 100644 --- a/packages/api/src/routes/validator.ts +++ b/packages/api/src/routes/validator.ts @@ -1,4 +1,4 @@ -import {ContainerType, fromHexString, Json, toHexString, Type} from "@chainsafe/ssz"; +import {ContainerType, fromHexString, toHexString, Type} from "@chainsafe/ssz"; import {ForkName} from "@chainsafe/lodestar-params"; import { allForks, @@ -7,11 +7,11 @@ import { BLSSignature, CommitteeIndex, Epoch, - Number64, phase0, Root, Slot, ssz, + UintNum64, ValidatorIndex, } from "@chainsafe/lodestar-types"; import { @@ -57,11 +57,11 @@ export type AttesterDuty = { validatorIndex: ValidatorIndex; committeeIndex: CommitteeIndex; // Number of validators in committee - committeeLength: Number64; + committeeLength: UintNum64; // Number of committees at the provided slot - committeesAtSlot: Number64; + committeesAtSlot: UintNum64; // Index of validator in committee - validatorCommitteeIndex: Number64; + validatorCommitteeIndex: UintNum64; // The slot at which the validator must attest. slot: Slot; }; @@ -227,44 +227,32 @@ export type ReqTypes = { produceAttestationData: {query: {slot: number; committee_index: number}}; produceSyncCommitteeContribution: {query: {slot: number; subcommittee_index: number; beacon_block_root: string}}; getAggregatedAttestation: {query: {attestation_data_root: string; slot: number}}; - publishAggregateAndProofs: {body: Json}; - publishContributionAndProofs: {body: Json}; - prepareBeaconCommitteeSubnet: {body: Json}; - prepareSyncCommitteeSubnets: {body: Json}; + publishAggregateAndProofs: {body: unknown}; + publishContributionAndProofs: {body: unknown}; + prepareBeaconCommitteeSubnet: {body: unknown}; + prepareSyncCommitteeSubnets: {body: unknown}; }; export function getReqSerializers(): ReqSerializers { - const BeaconCommitteeSubscription = new ContainerType({ - fields: { + const BeaconCommitteeSubscription = new ContainerType( + { validatorIndex: ssz.ValidatorIndex, committeeIndex: ssz.CommitteeIndex, committeesAtSlot: ssz.Slot, slot: ssz.Slot, isAggregator: ssz.Boolean, }, - // From beacon apis - casingMap: { - validatorIndex: "validator_index", - committeeIndex: "committee_index", - committeesAtSlot: "committees_at_slot", - slot: "slot", - isAggregator: "is_aggregator", - }, - }); + {jsonCase: "eth2"} + ); - const SyncCommitteeSubscription = new ContainerType({ - fields: { + const SyncCommitteeSubscription = new ContainerType( + { validatorIndex: ssz.ValidatorIndex, syncCommitteeIndices: ArrayOf(ssz.CommitteeIndex), untilEpoch: ssz.Epoch, }, - // From beacon apis - casingMap: { - validatorIndex: "validator_index", - syncCommitteeIndices: "sync_committee_indices", - untilEpoch: "until_epoch", - }, - }); + {jsonCase: "eth2"} + ); const produceBlock: ReqSerializers["produceBlock"] = { writeReq: (slot, randaoReveal, grafitti) => ({ @@ -346,58 +334,39 @@ export function getReqSerializers(): ReqSerializers { } export function getReturnTypes(): ReturnTypes { - const WithDependentRoot = (dataType: Type): ContainerType<{data: T; dependentRoot: Root}> => - new ContainerType({fields: {data: dataType, dependentRoot: ssz.Root}}); + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type + const WithDependentRoot = (dataType: Type) => new ContainerType({data: dataType, dependentRoot: ssz.Root}); - const AttesterDuty = new ContainerType({ - fields: { + const AttesterDuty = new ContainerType( + { pubkey: ssz.BLSPubkey, validatorIndex: ssz.ValidatorIndex, committeeIndex: ssz.CommitteeIndex, - committeeLength: ssz.Number64, - committeesAtSlot: ssz.Number64, - validatorCommitteeIndex: ssz.Number64, + committeeLength: ssz.UintNum64, + committeesAtSlot: ssz.UintNum64, + validatorCommitteeIndex: ssz.UintNum64, slot: ssz.Slot, }, - // From beacon apis - casingMap: { - pubkey: "pubkey", - validatorIndex: "validator_index", - committeeIndex: "committee_index", - committeeLength: "committee_length", - committeesAtSlot: "committees_at_slot", - validatorCommitteeIndex: "validator_committee_index", - slot: "slot", - }, - }); + {jsonCase: "eth2"} + ); - const ProposerDuty = new ContainerType({ - fields: { + const ProposerDuty = new ContainerType( + { slot: ssz.Slot, validatorIndex: ssz.ValidatorIndex, pubkey: ssz.BLSPubkey, }, - // From beacon apis - casingMap: { - slot: "slot", - validatorIndex: "validator_index", - pubkey: "pubkey", - }, - }); + {jsonCase: "eth2"} + ); - const SyncDuty = new ContainerType({ - fields: { + const SyncDuty = new ContainerType( + { pubkey: ssz.BLSPubkey, validatorIndex: ssz.ValidatorIndex, - validatorSyncCommitteeIndices: ArrayOf(ssz.Number64), - }, - // From beacon apis - casingMap: { - pubkey: "pubkey", - validatorIndex: "validator_index", - validatorSyncCommitteeIndices: "validator_sync_committee_indices", + validatorSyncCommitteeIndices: ArrayOf(ssz.UintNum64), }, - }); + {jsonCase: "eth2"} + ); return { getAttesterDuties: WithDependentRoot(ArrayOf(AttesterDuty)), diff --git a/packages/api/src/server/debug.ts b/packages/api/src/server/debug.ts index 619d07741cc..2388ea55710 100644 --- a/packages/api/src/server/debug.ts +++ b/packages/api/src/server/debug.ts @@ -1,5 +1,4 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {jsonOpts} from "../utils"; import {ServerRoutes, getGenericJsonServer} from "./utils"; import {Api, ReqTypes, routesData, getReturnTypes, getReqSerializers} from "../routes/debug"; @@ -25,7 +24,7 @@ export function getRoutes(config: IChainForkConfig, api: Api): ServerRoutes { + handler: async function handler(req: ReqGeneric): Promise { const args: any[] = routeSerdes.parseReq(req as ReqTypes[keyof Api]); const data = (await api[routeKey](...args)) as Resolves; if (returnType) { - return returnType.toJson(data, jsonOpts); + return returnType.toJson(data); } else { return {}; } diff --git a/packages/api/src/utils/serdes.ts b/packages/api/src/utils/serdes.ts index 0f35f11feca..c7c419eac4c 100644 --- a/packages/api/src/utils/serdes.ts +++ b/packages/api/src/utils/serdes.ts @@ -1,11 +1,11 @@ -import {Path} from "@chainsafe/ssz"; +import {JsonPath} from "@chainsafe/ssz"; /** * Serialize proof path to JSON. * @param paths `[["finalized_checkpoint", 0, "root", 12000]]` * @returns `['["finalized_checkpoint",0,"root",12000]']` */ -export function querySerializeProofPathsArr(paths: Path[]): string[] { +export function querySerializeProofPathsArr(paths: JsonPath[]): string[] { return paths.map((path) => JSON.stringify(path)); } @@ -14,11 +14,11 @@ export function querySerializeProofPathsArr(paths: Path[]): string[] { * @param pathStrs `['["finalized_checkpoint",0,"root",12000]']` * @returns `[["finalized_checkpoint", 0, "root", 12000]]` */ -export function queryParseProofPathsArr(pathStrs: string | string[]): Path[] { +export function queryParseProofPathsArr(pathStrs: string | string[]): JsonPath[] { if (Array.isArray(pathStrs)) { return pathStrs.map((pathStr) => queryParseProofPaths(pathStr)); } else { - return [queryParseProofPaths(pathStrs) as Path]; + return [queryParseProofPaths(pathStrs) as JsonPath]; } } @@ -27,8 +27,8 @@ export function queryParseProofPathsArr(pathStrs: string | string[]): Path[] { * @param pathStr `'["finalized_checkpoint",0,"root",12000]'` * @returns `["finalized_checkpoint", 0, "root", 12000]` */ -export function queryParseProofPaths(pathStr: string): Path { - const path = JSON.parse(pathStr) as Path; +export function queryParseProofPaths(pathStr: string): JsonPath { + const path = JSON.parse(pathStr) as JsonPath; if (!Array.isArray(path)) { throw Error("Proof pathStr is not an array"); diff --git a/packages/api/src/utils/types.ts b/packages/api/src/utils/types.ts index 4c26ddcf8d1..bb39caed5f5 100644 --- a/packages/api/src/utils/types.ts +++ b/packages/api/src/utils/types.ts @@ -1,4 +1,4 @@ -import {IJsonOptions, Json, ListType, Type} from "@chainsafe/ssz"; +import {isBasicType, ListBasicType, Type, isCompositeType, ListCompositeType, ArrayType} from "@chainsafe/ssz"; import {ForkName} from "@chainsafe/lodestar-params"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {objectToExpectedCase} from "@chainsafe/lodestar-utils"; @@ -8,10 +8,8 @@ import {Schema, SchemaDefinition} from "./schema"; /* eslint-disable @typescript-eslint/naming-convention, @typescript-eslint/no-explicit-any */ -/** All JSON must be sent in snake case */ -export const jsonOpts = {case: "snake" as const}; /** All JSON inside the JS code must be camel case */ -export const codeCase = "camel" as const; +const codeCase = "camel" as const; export type RouteGroupDefinition< Api extends Record, @@ -42,8 +40,8 @@ type ThenArg = T extends PromiseLike ? U : T; export type Resolves any> = ThenArg>; export type TypeJson = { - toJson(val: T, opts?: IJsonOptions): Json; - fromJson(json: Json, opts?: IJsonOptions): T; + toJson(val: T): unknown; + fromJson(json: unknown): T; }; // @@ -94,15 +92,21 @@ export const reqEmpty: ReqSerializer<() => void, ReqEmpty> = { export const reqOnlyBody = ( type: TypeJson, bodySchema: Schema -): ReqGenArg<(arg: T) => Promise, {body: Json}> => ({ - writeReq: (items) => ({body: type.toJson(items, jsonOpts)}), - parseReq: ({body}) => [type.fromJson(body, jsonOpts)], +): ReqGenArg<(arg: T) => Promise, {body: unknown}> => ({ + writeReq: (items) => ({body: type.toJson(items)}), + parseReq: ({body}) => [type.fromJson(body)], schema: {body: bodySchema}, }); /** SSZ factory helper + typed. limit = 1e6 as a big enough random number */ -export function ArrayOf(elementType: Type, limit = 1e6): ListType { - return new ListType({elementType, limit}); +export function ArrayOf(elementType: Type): ArrayType, unknown, unknown> { + if (isCompositeType(elementType)) { + return (new ListCompositeType(elementType, Infinity) as unknown) as ArrayType, unknown, unknown>; + } else if (isBasicType(elementType)) { + return (new ListBasicType(elementType, Infinity) as unknown) as ArrayType, unknown, unknown>; + } else { + throw Error(`Unknown type ${elementType.typeName}`); + } } /** @@ -113,12 +117,12 @@ export function ArrayOf(elementType: Type, limit = 1e6): ListType { */ export function ContainerData(dataType: TypeJson): TypeJson<{data: T}> { return { - toJson: ({data}, opts) => ({ - data: dataType.toJson(data, opts), + toJson: ({data}) => ({ + data: dataType.toJson(data), }), - fromJson: ({data}: {data: Json}, opts) => { + fromJson: ({data}: {data: unknown}) => { return { - data: dataType.fromJson(data, opts), + data: dataType.fromJson(data), }; }, }; @@ -133,26 +137,30 @@ export function ContainerData(dataType: TypeJson): TypeJson<{data: T}> { */ export function WithVersion(getType: (fork: ForkName) => TypeJson): TypeJson<{data: T; version: ForkName}> { return { - toJson: ({data, version}, opts) => ({ - data: getType(version || ForkName.phase0).toJson(data, opts), + toJson: ({data, version}) => ({ + data: getType(version || ForkName.phase0).toJson(data), version, }), - fromJson: ({data, version}: {data: Json; version: string}, opts) => { + fromJson: ({data, version}: {data: unknown; version: string}) => { // Un-safe external data, validate version is known ForkName value if (!ForkName[version as ForkName]) throw Error(`Invalid version ${version}`); return { - data: getType(version as ForkName).fromJson(data, opts), + data: getType(version as ForkName).fromJson(data), version: version as ForkName, }; }, }; } +type JsonCase = "snake" | "constant" | "camel" | "param" | "header" | "pascal" | "dot" | "notransform"; + /** Helper to only translate casing */ -export function jsonType | Record[]>(): TypeJson { +export function jsonType | Record[]>( + jsonCase: JsonCase +): TypeJson { return { - toJson: (val, opts) => objectToExpectedCase(val, opts?.case) as Json, + toJson: (val) => objectToExpectedCase(val, jsonCase) as unknown, fromJson: (json) => objectToExpectedCase(json as Record, codeCase) as T, }; } @@ -160,7 +168,7 @@ export function jsonType | Record(): TypeJson { return { - toJson: (val) => (val as unknown) as Json, + toJson: (val) => val as unknown, fromJson: (json) => (json as unknown) as T, }; } diff --git a/packages/beacon-state-transition/README.md b/packages/beacon-state-transition/README.md index 691101aa311..f135df9ec4d 100644 --- a/packages/beacon-state-transition/README.md +++ b/packages/beacon-state-transition/README.md @@ -18,17 +18,13 @@ import {generateEmptySignedBlock} from "../test/utils/block"; import {generateState} from "../test/utils/state"; // dummy test state -const state: CachedBeaconStateAllForks = generateState() as CachedBeaconStateAllForks; +const preState: CachedBeaconStateAllForks = generateState() as CachedBeaconStateAllForks; // dummy test block const block: allForks.SignedBeaconBlock = generateEmptySignedBlock(); -let postStateContext: allForks.BeaconState; -try { - postStateContext = allForks.stateTransition(state, block); -} catch (e) { - console.log(e); -} +// Run state transition on block +const postState = allForks.stateTransition(preState, block); ``` ## License diff --git a/packages/beacon-state-transition/package.json b/packages/beacon-state-transition/package.json index 1f088f522dc..4ea2f29c616 100644 --- a/packages/beacon-state-transition/package.json +++ b/packages/beacon-state-transition/package.json @@ -35,14 +35,15 @@ }, "types": "lib/index.d.ts", "dependencies": { + "@chainsafe/as-sha256": "^0.3.0", "@chainsafe/bls": "6.0.3", "@chainsafe/lodestar-config": "^0.36.0", "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/persistent-merkle-tree": "^0.3.7", + "@chainsafe/persistent-merkle-tree": "^0.4.0", "@chainsafe/persistent-ts": "^0.19.1", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "^0.9.0", "bigint-buffer": "^1.1.5", "buffer-xor": "^2.0.2" }, diff --git a/packages/beacon-state-transition/src/allForks/block/initiateValidatorExit.ts b/packages/beacon-state-transition/src/allForks/block/initiateValidatorExit.ts index b4ddc9ff8e3..e2a1411e303 100644 --- a/packages/beacon-state-transition/src/allForks/block/initiateValidatorExit.ts +++ b/packages/beacon-state-transition/src/allForks/block/initiateValidatorExit.ts @@ -6,7 +6,7 @@ import {CachedBeaconStateAllForks} from "../../types"; * Initiate the exit of the validator with index ``index``. * * NOTE: This function takes a `validator` as argument instead of the validator index. - * SSZ TreeBacked have a dangerous edge case that may break the code here in a non-obvious way. + * SSZ TreeViews have a dangerous edge case that may break the code here in a non-obvious way. * When running `state.validators[i]` you get a SubTree of that validator with a hook to the state. * Then, when a property of `validator` is set it propagates the changes upwards to the parent tree up to the state. * This means that `validator` will propagate its new state along with the current state of its parent tree up to diff --git a/packages/beacon-state-transition/src/allForks/block/isValidIndexedAttestation.ts b/packages/beacon-state-transition/src/allForks/block/isValidIndexedAttestation.ts index 0a75ee82401..9a6eeb7d891 100644 --- a/packages/beacon-state-transition/src/allForks/block/isValidIndexedAttestation.ts +++ b/packages/beacon-state-transition/src/allForks/block/isValidIndexedAttestation.ts @@ -1,4 +1,3 @@ -import {readonlyValues} from "@chainsafe/ssz"; import {MAX_VALIDATORS_PER_COMMITTEE} from "@chainsafe/lodestar-params"; import {phase0} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAllForks} from "../../types"; @@ -12,7 +11,7 @@ export function isValidIndexedAttestation( indexedAttestation: phase0.IndexedAttestation, verifySignature = true ): boolean { - const indices = Array.from(readonlyValues(indexedAttestation.attestingIndices)); + const indices = indexedAttestation.attestingIndices; // verify max number of indices if (!(indices.length > 0 && indices.length <= MAX_VALIDATORS_PER_COMMITTEE)) { diff --git a/packages/beacon-state-transition/src/allForks/block/processAttesterSlashing.ts b/packages/beacon-state-transition/src/allForks/block/processAttesterSlashing.ts index 981a1d5744a..cf513a60ab4 100644 --- a/packages/beacon-state-transition/src/allForks/block/processAttesterSlashing.ts +++ b/packages/beacon-state-transition/src/allForks/block/processAttesterSlashing.ts @@ -18,7 +18,7 @@ export function processAttesterSlashing( attesterSlashing: phase0.AttesterSlashing, verifySignatures = true ): void { - assertValidAttesterSlashing(state as CachedBeaconStateAllForks, attesterSlashing, verifySignatures); + assertValidAttesterSlashing(state, attesterSlashing, verifySignatures); const intersectingIndices = getAttesterSlashableIndices(attesterSlashing); @@ -26,7 +26,7 @@ export function processAttesterSlashing( const validators = state.validators; // Get the validators sub tree once for all indices // Spec requires to sort indexes beforehand for (const index of intersectingIndices.sort((a, b) => a - b)) { - if (isSlashableValidator(validators[index], state.epochCtx.currentShuffling.epoch)) { + if (isSlashableValidator(validators.get(index), state.epochCtx.currentShuffling.epoch)) { slashValidatorAllForks(fork, state, index); slashedAny = true; } diff --git a/packages/beacon-state-transition/src/allForks/block/processBlockHeader.ts b/packages/beacon-state-transition/src/allForks/block/processBlockHeader.ts index 125d17120a9..77fcdfd8c27 100644 --- a/packages/beacon-state-transition/src/allForks/block/processBlockHeader.ts +++ b/packages/beacon-state-transition/src/allForks/block/processBlockHeader.ts @@ -1,4 +1,4 @@ -import {toHexString} from "@chainsafe/ssz"; +import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; import {allForks, ssz} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAllForks} from "../../types"; import {ZERO_HASH} from "../../constants"; @@ -22,7 +22,7 @@ export function processBlockHeader(state: CachedBeaconStateAllForks, block: allF ); } // verify that proposer index is the correct index - const proposerIndex = state.getBeaconProposer(slot); + const proposerIndex = state.epochCtx.getBeaconProposer(slot); if (block.proposerIndex !== proposerIndex) { throw new Error( `Block proposer index does not match state proposer index blockProposerIndex=${block.proposerIndex} stateProposerIndex=${proposerIndex}` @@ -31,22 +31,23 @@ export function processBlockHeader(state: CachedBeaconStateAllForks, block: allF const types = state.config.getForkTypes(slot); // verify that the parent matches - if (!ssz.Root.equals(block.parentRoot, ssz.phase0.BeaconBlockHeader.hashTreeRoot(state.latestBlockHeader))) { + if (!byteArrayEquals(block.parentRoot, ssz.phase0.BeaconBlockHeader.hashTreeRoot(state.latestBlockHeader))) { throw new Error( `Block parent root ${toHexString(block.parentRoot)} does not match state latest block, block slot=${slot}` ); } + // cache current block as the new latest block - state.latestBlockHeader = { + state.latestBlockHeader = ssz.phase0.BeaconBlockHeader.toViewDU({ slot: slot, proposerIndex: block.proposerIndex, parentRoot: block.parentRoot, stateRoot: ZERO_HASH, bodyRoot: types.BeaconBlockBody.hashTreeRoot(block.body), - }; + }); // verify proposer is not slashed. Only once per block, may use the slower read from tree - if (state.validators[proposerIndex].slashed) { + if (state.validators.get(proposerIndex).slashed) { throw new Error("Block proposer is slashed"); } } diff --git a/packages/beacon-state-transition/src/allForks/block/processDeposit.ts b/packages/beacon-state-transition/src/allForks/block/processDeposit.ts index b865b857818..eb8b57c67fc 100644 --- a/packages/beacon-state-transition/src/allForks/block/processDeposit.ts +++ b/packages/beacon-state-transition/src/allForks/block/processDeposit.ts @@ -22,14 +22,15 @@ import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../types"; */ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, deposit: phase0.Deposit): void { const {config, validators, epochCtx} = state; + // verify the merkle branch if ( !verifyMerkleBranch( ssz.phase0.DepositData.hashTreeRoot(deposit.data), - Array.from(deposit.proof).map((p) => p.valueOf() as Uint8Array), + deposit.proof, DEPOSIT_CONTRACT_TREE_DEPTH + 1, state.eth1DepositIndex, - state.eth1Data.depositRoot.valueOf() as Uint8Array + state.eth1Data.depositRoot ) ) { throw new Error("Deposit has invalid merkle proof"); @@ -38,7 +39,7 @@ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, // deposits must be processed in order state.eth1DepositIndex += 1; - const pubkey = deposit.data.pubkey.valueOf() as Uint8Array; // Drop tree + const pubkey = deposit.data.pubkey; // Drop tree const amount = deposit.data.amount; const cachedIndex = epochCtx.pubkey2index.get(pubkey); if (cachedIndex === undefined || !Number.isSafeInteger(cachedIndex) || cachedIndex >= validators.length) { @@ -54,7 +55,7 @@ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, try { // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed const publicKey = bls.PublicKey.fromBytes(pubkey, CoordType.affine, true); - const signature = bls.Signature.fromBytes(deposit.data.signature.valueOf() as Uint8Array, CoordType.affine, true); + const signature = bls.Signature.fromBytes(deposit.data.signature, CoordType.affine, true); if (!signature.verify(publicKey, signingRoot)) { return; } @@ -64,17 +65,19 @@ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, // add validator and balance entries const effectiveBalance = Math.min(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); - validators.push({ - pubkey, - withdrawalCredentials: deposit.data.withdrawalCredentials.valueOf() as Uint8Array, // Drop tree - activationEligibilityEpoch: FAR_FUTURE_EPOCH, - activationEpoch: FAR_FUTURE_EPOCH, - exitEpoch: FAR_FUTURE_EPOCH, - withdrawableEpoch: FAR_FUTURE_EPOCH, - effectiveBalance, - slashed: false, - }); - state.balanceList.push(Number(amount)); + validators.push( + ssz.phase0.Validator.toViewDU({ + pubkey, + withdrawalCredentials: deposit.data.withdrawalCredentials, + activationEligibilityEpoch: FAR_FUTURE_EPOCH, + activationEpoch: FAR_FUTURE_EPOCH, + exitEpoch: FAR_FUTURE_EPOCH, + withdrawableEpoch: FAR_FUTURE_EPOCH, + effectiveBalance, + slashed: false, + }) + ); + state.balances.push(amount); const validatorIndex = validators.length - 1; // Updating here is better than updating at once on epoch transition @@ -86,16 +89,18 @@ export function processDeposit(fork: ForkName, state: CachedBeaconStateAllForks, // now that there is a new validator, update the epoch context with the new pubkey epochCtx.addPubkey(validatorIndex, pubkey); - // add participation caches - state.previousEpochParticipation.push(0); - state.currentEpochParticipation.push(0); - - // Forks: altair, bellatrix, and future + // Only after altair: if (fork !== ForkName.phase0) { - (state as CachedBeaconStateAltair).inactivityScores.push(0); + const stateAltair = state as CachedBeaconStateAltair; + + stateAltair.inactivityScores.push(0); + + // add participation caches + stateAltair.previousEpochParticipation.push(0); + stateAltair.currentEpochParticipation.push(0); } } else { // increase balance by deposit amount - increaseBalance(state, cachedIndex, Number(amount)); + increaseBalance(state, cachedIndex, amount); } } diff --git a/packages/beacon-state-transition/src/allForks/block/processEth1Data.ts b/packages/beacon-state-transition/src/allForks/block/processEth1Data.ts index dca8351e340..1ec8407f713 100644 --- a/packages/beacon-state-transition/src/allForks/block/processEth1Data.ts +++ b/packages/beacon-state-transition/src/allForks/block/processEth1Data.ts @@ -1,8 +1,8 @@ import {EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {Node} from "@chainsafe/persistent-merkle-tree"; -import {readonlyValues, TreeBacked} from "@chainsafe/ssz"; -import {CachedBeaconStateAllForks} from "../../types"; +import {CompositeViewDU} from "@chainsafe/ssz"; +import {BeaconStateAllForks, CachedBeaconStateAllForks} from "../../types"; /** * Store vote counts for every eth-execution block that has votes; if any eth-execution block wins majority support within a 1024-slot @@ -12,24 +12,24 @@ import {CachedBeaconStateAllForks} from "../../types"; * - Best case: Vote is already decided, zero work. See becomesNewEth1Data conditions * - Worst case: 1023 votes and no majority vote yet. */ -export function processEth1Data(state: CachedBeaconStateAllForks, body: allForks.BeaconBlockBody): void { +export function processEth1Data(state: CachedBeaconStateAllForks, eth1Data: phase0.Eth1Data): void { // Convert to view first to hash once and compare hashes - const eth1DataView = ssz.phase0.Eth1Data.createTreeBackedFromStruct(body.eth1Data); + const eth1DataView = ssz.phase0.Eth1Data.toViewDU(eth1Data); if (becomesNewEth1Data(state, eth1DataView)) { state.eth1Data = eth1DataView; } - state.eth1DataVotes.push(body.eth1Data); + state.eth1DataVotes.push(eth1DataView); } /** - * Returns `newEth1Data` if adding the given `eth1Data` to `state.eth1DataVotes` would + * Returns true if adding the given `eth1Data` to `state.eth1DataVotes` would * result in a change to `state.eth1Data`. */ export function becomesNewEth1Data( - state: CachedBeaconStateAllForks, - newEth1Data: TreeBacked + state: BeaconStateAllForks, + newEth1Data: CompositeViewDU ): boolean { const SLOTS_PER_ETH1_VOTING_PERIOD = EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH; @@ -39,7 +39,7 @@ export function becomesNewEth1Data( } // Nothing to do if the state already has this as eth1data (happens a lot after majority vote is in) - if (ssz.phase0.Eth1Data.equals(state.eth1Data, newEth1Data)) { + if (isEqualEth1DataView(state.eth1Data, newEth1Data)) { return false; } @@ -48,7 +48,7 @@ export function becomesNewEth1Data( // Then isEqualEth1DataView compares cached roots (HashObject as of Jan 2022) which is much cheaper // than doing structural equality, which requires tree -> value conversions let sameVotesCount = 0; - const eth1DataVotes = Array.from(readonlyValues(state.eth1DataVotes)) as TreeBacked[]; + const eth1DataVotes = state.eth1DataVotes.getAllReadonly(); for (let i = 0; i < eth1DataVotes.length; i++) { if (isEqualEth1DataView(eth1DataVotes[i], newEth1Data)) { sameVotesCount++; @@ -63,8 +63,11 @@ export function becomesNewEth1Data( } } -function isEqualEth1DataView(eth1DataA: TreeBacked, eth1DataB: TreeBacked): boolean { - return isEqualNode(eth1DataA.tree.rootNode, eth1DataB.tree.rootNode); +function isEqualEth1DataView( + eth1DataA: CompositeViewDU, + eth1DataB: CompositeViewDU +): boolean { + return isEqualNode(eth1DataA.node, eth1DataB.node); } // TODO: Upstream to persistent-merkle-tree diff --git a/packages/beacon-state-transition/src/allForks/block/processProposerSlashing.ts b/packages/beacon-state-transition/src/allForks/block/processProposerSlashing.ts index d9fa8d5f98d..4698b1c3780 100644 --- a/packages/beacon-state-transition/src/allForks/block/processProposerSlashing.ts +++ b/packages/beacon-state-transition/src/allForks/block/processProposerSlashing.ts @@ -18,7 +18,7 @@ export function processProposerSlashing( proposerSlashing: phase0.ProposerSlashing, verifySignatures = true ): void { - assertValidProposerSlashing(state as CachedBeaconStateAllForks, proposerSlashing, verifySignatures); + assertValidProposerSlashing(state, proposerSlashing, verifySignatures); slashValidatorAllForks(fork, state, proposerSlashing.signedHeader1.message.proposerIndex); } @@ -51,18 +51,16 @@ export function assertValidProposerSlashing( } // verify the proposer is slashable - const proposer = state.validators[header1.proposerIndex]; + const proposer = state.validators.get(header1.proposerIndex); if (!isSlashableValidator(proposer, epochCtx.currentShuffling.epoch)) { throw new Error("ProposerSlashing proposer is not slashable"); } // verify signatures if (verifySignatures) { - for (const [i, signatureSet] of getProposerSlashingSignatureSets( - state as CachedBeaconStateAllForks, - proposerSlashing - ).entries()) { - if (!verifySignatureSet(signatureSet)) { + const signatureSets = getProposerSlashingSignatureSets(state, proposerSlashing); + for (let i = 0; i < signatureSets.length; i++) { + if (!verifySignatureSet(signatureSets[i])) { throw new Error(`ProposerSlashing header${i + 1} signature invalid`); } } diff --git a/packages/beacon-state-transition/src/allForks/block/processRandao.ts b/packages/beacon-state-transition/src/allForks/block/processRandao.ts index f1c57ea6504..9696ad53ae0 100644 --- a/packages/beacon-state-transition/src/allForks/block/processRandao.ts +++ b/packages/beacon-state-transition/src/allForks/block/processRandao.ts @@ -1,5 +1,5 @@ import xor from "buffer-xor"; -import {hash} from "@chainsafe/ssz"; +import {digest} from "@chainsafe/as-sha256"; import {allForks} from "@chainsafe/lodestar-types"; import {getRandaoMix} from "../../util"; import {verifyRandaoSignature} from "../signatureSets"; @@ -18,18 +18,16 @@ export function processRandao( ): void { const {epochCtx} = state; const epoch = epochCtx.currentShuffling.epoch; - const randaoReveal = block.body.randaoReveal.valueOf() as Uint8Array; + const randaoReveal = block.body.randaoReveal; // verify RANDAO reveal if (verifySignature) { - if (!verifyRandaoSignature(state as CachedBeaconStateAllForks, block)) { + if (!verifyRandaoSignature(state, block)) { throw new Error("RANDAO reveal is an invalid signature"); } } // mix in RANDAO reveal - state.randaoMixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = xor( - Buffer.from(getRandaoMix(state, epoch) as Uint8Array), - Buffer.from(hash(randaoReveal)) - ); + const randaoMix = xor(Buffer.from(getRandaoMix(state, epoch) as Uint8Array), Buffer.from(digest(randaoReveal))); + state.randaoMixes.set(epoch % EPOCHS_PER_HISTORICAL_VECTOR, randaoMix); } diff --git a/packages/beacon-state-transition/src/allForks/block/processVoluntaryExit.ts b/packages/beacon-state-transition/src/allForks/block/processVoluntaryExit.ts index cab6625c8f7..5b52a7ed36b 100644 --- a/packages/beacon-state-transition/src/allForks/block/processVoluntaryExit.ts +++ b/packages/beacon-state-transition/src/allForks/block/processVoluntaryExit.ts @@ -15,12 +15,12 @@ export function processVoluntaryExitAllForks( signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature = true ): void { - if (!isValidVoluntaryExit(state as CachedBeaconStateAllForks, signedVoluntaryExit, verifySignature)) { + if (!isValidVoluntaryExit(state, signedVoluntaryExit, verifySignature)) { throw Error("Invalid voluntary exit"); } - const validator = state.validators[signedVoluntaryExit.message.validatorIndex]; - initiateValidatorExit(state as CachedBeaconStateAllForks, validator); + const validator = state.validators.get(signedVoluntaryExit.message.validatorIndex); + initiateValidatorExit(state, validator); } export function isValidVoluntaryExit( @@ -30,7 +30,7 @@ export function isValidVoluntaryExit( ): boolean { const {config, epochCtx} = state; const voluntaryExit = signedVoluntaryExit.message; - const validator = state.validators[voluntaryExit.validatorIndex]; + const validator = state.validators.get(voluntaryExit.validatorIndex); const currentEpoch = epochCtx.currentShuffling.epoch; return ( @@ -43,6 +43,6 @@ export function isValidVoluntaryExit( // verify the validator had been active long enough currentEpoch >= validator.activationEpoch + config.SHARD_COMMITTEE_PERIOD && // verify signature - (!verifySignature || verifyVoluntaryExitSignature(state as CachedBeaconStateAllForks, signedVoluntaryExit)) + (!verifySignature || verifyVoluntaryExitSignature(state, signedVoluntaryExit)) ); } diff --git a/packages/beacon-state-transition/src/allForks/block/slashValidator.ts b/packages/beacon-state-transition/src/allForks/block/slashValidator.ts index 49043e585fc..96379cde46e 100644 --- a/packages/beacon-state-transition/src/allForks/block/slashValidator.ts +++ b/packages/beacon-state-transition/src/allForks/block/slashValidator.ts @@ -23,17 +23,18 @@ export function slashValidatorAllForks( ): void { const {epochCtx} = state; const epoch = epochCtx.currentShuffling.epoch; - const validator = state.validators[slashedIndex]; + const validator = state.validators.get(slashedIndex); // TODO: Bellatrix initiateValidatorExit validators.update() with the one below - initiateValidatorExit(state as CachedBeaconStateAllForks, validator); + initiateValidatorExit(state, validator); validator.slashed = true; validator.withdrawableEpoch = Math.max(validator.withdrawableEpoch, epoch + EPOCHS_PER_SLASHINGS_VECTOR); const {effectiveBalance} = validator; // TODO: could state.slashings be number? - state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += BigInt(effectiveBalance); + const slashingIndex = epoch % EPOCHS_PER_SLASHINGS_VECTOR; + state.slashings.set(slashingIndex, state.slashings.get(slashingIndex) + BigInt(effectiveBalance)); const minSlashingPenaltyQuotient = fork === ForkName.phase0 diff --git a/packages/beacon-state-transition/src/allForks/epoch/processEffectiveBalanceUpdates.ts b/packages/beacon-state-transition/src/allForks/epoch/processEffectiveBalanceUpdates.ts index adbf9e816d8..910a172eba3 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processEffectiveBalanceUpdates.ts @@ -26,16 +26,18 @@ export function processEffectiveBalanceUpdates(state: CachedBeaconStateAllForks, let nextEpochTotalActiveBalanceByIncrement = 0; // update effective balances with hysteresis - if (!epochProcess.balances) { - // only do this for genesis epoch, or spec test - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - epochProcess.balances = Array.from({length: state.balanceList.length}, (_, i) => state.balanceList.get(i)!); - } - for (let i = 0, len = epochProcess.balances.length; i < len; i++) { - const balance = epochProcess.balances[i]; + // epochProcess.balances is set in processRewardsAndPenalties(), so it's recycled here for performance. + // It defaults to `state.balances.getAll()` to make Typescript happy and for spec tests + const balances = epochProcess.balances ?? state.balances.getAll(); + + for (let i = 0, len = balances.length; i < len; i++) { + const balance = balances[i]; + + // PERF: It's faster to access to get() every single element (4ms) than to convert to regular array then loop (9ms) let effectiveBalanceIncrement = effectiveBalanceIncrements[i]; let effectiveBalance = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT; + if ( // Too big effectiveBalance > balance + DOWNWARD_THRESHOLD || @@ -44,13 +46,14 @@ export function processEffectiveBalanceUpdates(state: CachedBeaconStateAllForks, ) { effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); // Update the state tree - validators[i].effectiveBalance = effectiveBalance; - // Also update the fast cached version // Should happen rarely, so it's fine to update the tree - // TODO: Update all in batch after this loop + validators.get(i).effectiveBalance = effectiveBalance; + // Also update the fast cached version effectiveBalanceIncrement = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); effectiveBalanceIncrements[i] = effectiveBalanceIncrement; } + + // TODO: Do this in afterEpochProcess, looping a Uint8Array should be very cheap if (epochProcess.isActiveNextEpoch[i]) { // We track nextEpochTotalActiveBalanceByIncrement as ETH to fit total network balance in a JS number (53 bits) nextEpochTotalActiveBalanceByIncrement += effectiveBalanceIncrement; diff --git a/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts b/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts index fd8d11a945d..3998e3aef0e 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processEth1DataReset.ts @@ -1,6 +1,5 @@ import {EPOCHS_PER_ETH1_VOTING_PERIOD} from "@chainsafe/lodestar-params"; -import {phase0} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; import {EpochProcess, CachedBeaconStateAllForks} from "../../types"; /** @@ -13,6 +12,6 @@ export function processEth1DataReset(state: CachedBeaconStateAllForks, epochProc // reset eth1 data votes if (nextEpoch % EPOCHS_PER_ETH1_VOTING_PERIOD === 0) { - state.eth1DataVotes = ([] as phase0.Eth1Data[]) as List; + state.eth1DataVotes = ssz.phase0.Eth1DataVotes.defaultViewDU(); } } diff --git a/packages/beacon-state-transition/src/allForks/epoch/processHistoricalRootsUpdate.ts b/packages/beacon-state-transition/src/allForks/epoch/processHistoricalRootsUpdate.ts index 1c32db99272..459ebc49f80 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processHistoricalRootsUpdate.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processHistoricalRootsUpdate.ts @@ -14,9 +14,11 @@ export function processHistoricalRootsUpdate(state: CachedBeaconStateAllForks, e // set historical root accumulator if (nextEpoch % intDiv(SLOTS_PER_HISTORICAL_ROOT, SLOTS_PER_EPOCH) === 0) { state.historicalRoots.push( - ssz.phase0.HistoricalBatch.hashTreeRoot({ - blockRoots: state.blockRoots, - stateRoots: state.stateRoots, + // HistoricalBatchRoots = Non-spec'ed helper type to allow efficient hashing in epoch transition. + // This type is like a 'Header' of HistoricalBatch where its fields are hashed. + ssz.phase0.HistoricalBatchRoots.hashTreeRoot({ + blockRoots: state.blockRoots.hashTreeRoot(), + stateRoots: state.stateRoots.hashTreeRoot(), }) ); } diff --git a/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts b/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts index e2b8721e49e..2d0dd10e7d2 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processJustificationAndFinalization.ts @@ -1,5 +1,6 @@ import {GENESIS_EPOCH} from "@chainsafe/lodestar-params"; - +import {BitArray} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; import {getBlockRoot} from "../../util"; import {CachedBeaconStateAllForks, EpochProcess} from "../../types"; @@ -26,27 +27,32 @@ export function processJustificationAndFinalization( // Process justifications state.previousJustifiedCheckpoint = state.currentJustifiedCheckpoint; - const bits = state.justificationBits; + const bits = state.justificationBits.toBoolArray(); + + // Rotate bits for (let i = bits.length - 1; i >= 1; i--) { bits[i] = bits[i - 1]; } bits[0] = false; if (epochProcess.prevEpochUnslashedStake.targetStakeByIncrement * 3 >= epochProcess.totalActiveStakeByIncrement * 2) { - state.currentJustifiedCheckpoint = { + state.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ epoch: previousEpoch, root: getBlockRoot(state, previousEpoch), - }; + }); bits[1] = true; } if (epochProcess.currEpochUnslashedTargetStakeByIncrement * 3 >= epochProcess.totalActiveStakeByIncrement * 2) { - state.currentJustifiedCheckpoint = { + state.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ epoch: currentEpoch, root: getBlockRoot(state, currentEpoch), - }; + }); bits[0] = true; } - state.justificationBits = bits; + + state.justificationBits = ssz.phase0.JustificationBits.toViewDU(BitArray.fromBoolArray(bits)); + + // TODO: Consider rendering bits as array of boolean for faster repeated access here // Process finalizations // The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source diff --git a/packages/beacon-state-transition/src/allForks/epoch/processRandaoMixesReset.ts b/packages/beacon-state-transition/src/allForks/epoch/processRandaoMixesReset.ts index 5ea10de0241..40f47f0437e 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processRandaoMixesReset.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processRandaoMixesReset.ts @@ -1,5 +1,4 @@ import {EPOCHS_PER_HISTORICAL_VECTOR} from "@chainsafe/lodestar-params"; -import {getRandaoMix} from "../../util"; import {EpochProcess, CachedBeaconStateAllForks} from "../../types"; /** @@ -12,5 +11,8 @@ export function processRandaoMixesReset(state: CachedBeaconStateAllForks, epochP const nextEpoch = currentEpoch + 1; // set randao mix - state.randaoMixes[nextEpoch % EPOCHS_PER_HISTORICAL_VECTOR] = getRandaoMix(state, currentEpoch); + state.randaoMixes.set( + nextEpoch % EPOCHS_PER_HISTORICAL_VECTOR, + state.randaoMixes.get(currentEpoch % EPOCHS_PER_HISTORICAL_VECTOR) + ); } diff --git a/packages/beacon-state-transition/src/allForks/epoch/processRegistryUpdates.ts b/packages/beacon-state-transition/src/allForks/epoch/processRegistryUpdates.ts index 5da49036252..4825bf133a1 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processRegistryUpdates.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processRegistryUpdates.ts @@ -28,18 +28,18 @@ export function processRegistryUpdates(state: CachedBeaconStateAllForks, epochPr for (const index of epochProcess.indicesToEject) { // set validator exit epoch and withdrawable epoch // TODO: Figure out a way to quickly set properties on the validators tree - initiateValidatorExit(state, validators[index]); + initiateValidatorExit(state, validators.get(index)); } // set new activation eligibilities for (const index of epochProcess.indicesEligibleForActivationQueue) { - validators[index].activationEligibilityEpoch = epochCtx.currentShuffling.epoch + 1; + validators.get(index).activationEligibilityEpoch = epochCtx.currentShuffling.epoch + 1; } const finalityEpoch = state.finalizedCheckpoint.epoch; // dequeue validators for activation up to churn limit for (const index of epochProcess.indicesEligibleForActivation.slice(0, epochCtx.churnLimit)) { - const validator = validators[index]; + const validator = validators.get(index); // placement in queue is finalized if (validator.activationEligibilityEpoch > finalityEpoch) { // remaining validators all have an activationEligibilityEpoch that is higher anyway, break early diff --git a/packages/beacon-state-transition/src/allForks/epoch/processRewardsAndPenalties.ts b/packages/beacon-state-transition/src/allForks/epoch/processRewardsAndPenalties.ts index e4cc7076b8c..2db619628aa 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processRewardsAndPenalties.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processRewardsAndPenalties.ts @@ -1,17 +1,18 @@ -import {CachedBeaconStateAltair, CachedBeaconStatePhase0, CachedBeaconStateAllForks, EpochProcess} from "../../types"; import {ForkName, GENESIS_EPOCH} from "@chainsafe/lodestar-params"; +import {ssz} from "@chainsafe/lodestar-types"; +import {EpochProcess} from "../../cache/epochProcess"; import {getAttestationDeltas as getAttestationDeltasPhase0} from "../../phase0/epoch/getAttestationDeltas"; import {getRewardsAndPenalties as getRewardsPenaltiesAltair} from "../../altair/epoch/getRewardsAndPenalties"; +import {CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../../cache/stateCache"; /** * Iterate over all validator and compute rewards and penalties to apply to balances. * - * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags - * are true, worst case: FLAG_UNSLASHED + FLAG_ELIGIBLE_ATTESTER + FLAG_PREV_* + * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags are set */ -export function processRewardsAndPenaltiesAllForks( +export function processRewardsAndPenaltiesAllForks( fork: ForkName, - state: T, + state: CachedBeaconStateAllForks, epochProcess: EpochProcess ): void { // No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch @@ -24,13 +25,17 @@ export function processRewardsAndPenaltiesAllForks a + b, BigInt(0)); + + // TODO: Could totalSlashings be number? + // TODO: Could totalSlashing be cached? + let totalSlashings = BigInt(0); + const slashings = state.slashings.getAll(); + for (let i = 0; i < slashings.length; i++) { + totalSlashings += slashings[i]; + } const proportionalSlashingMultiplier = fork === ForkName.phase0 diff --git a/packages/beacon-state-transition/src/allForks/epoch/processSlashingsReset.ts b/packages/beacon-state-transition/src/allForks/epoch/processSlashingsReset.ts index 6bccd3cb87c..c25f9a5f4a7 100644 --- a/packages/beacon-state-transition/src/allForks/epoch/processSlashingsReset.ts +++ b/packages/beacon-state-transition/src/allForks/epoch/processSlashingsReset.ts @@ -10,5 +10,5 @@ export function processSlashingsReset(state: CachedBeaconStateAllForks, epochPro const nextEpoch = epochProcess.currentEpoch + 1; // reset slashings - state.slashings[nextEpoch % EPOCHS_PER_SLASHINGS_VECTOR] = BigInt(0); + state.slashings.set(nextEpoch % EPOCHS_PER_SLASHINGS_VECTOR, BigInt(0)); } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts b/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts index 877f0c44902..cd6952c683a 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/attesterSlashings.ts @@ -1,4 +1,3 @@ -import {readonlyValues} from "@chainsafe/ssz"; import {allForks, phase0} from "@chainsafe/lodestar-types"; import {ISignatureSet} from "../../util"; import {CachedBeaconStateAllForks} from "../../types"; @@ -19,7 +18,7 @@ export function getAttesterSlashingsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - return Array.from(readonlyValues(signedBlock.message.body.attesterSlashings), (attesterSlashing) => - getAttesterSlashingSignatureSets(state, attesterSlashing) - ).flat(1); + return signedBlock.message.body.attesterSlashings + .map((attesterSlashing) => getAttesterSlashingSignatureSets(state, attesterSlashing)) + .flat(1); } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts b/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts index e50fe82f848..eeaaf4fa531 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/indexedAttestation.ts @@ -1,6 +1,5 @@ import {DOMAIN_BEACON_ATTESTER} from "@chainsafe/lodestar-params"; import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; -import {readonlyValues} from "@chainsafe/ssz"; import { computeSigningRoot, computeStartSlotAtEpoch, @@ -31,7 +30,7 @@ export function getAttestationWithIndicesSignatureSet( type: SignatureSetType.aggregate, pubkeys: indices.map((i) => epochCtx.index2pubkey[i]), signingRoot: computeSigningRoot(ssz.phase0.AttestationData, attestation.data, domain), - signature: attestation.signature.valueOf() as Uint8Array, + signature: attestation.signature, }; } @@ -43,7 +42,7 @@ export function getIndexedAttestationSignatureSet( return getAttestationWithIndicesSignatureSet( state, indexedAttestation, - indices ?? Array.from(readonlyValues(indexedAttestation.attestingIndices)) + indices ?? indexedAttestation.attestingIndices ); } @@ -51,7 +50,7 @@ export function getAttestationsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - return Array.from(readonlyValues(signedBlock.message.body.attestations), (attestation) => - getIndexedAttestationSignatureSet(state, state.getIndexedAttestation(attestation)) + return signedBlock.message.body.attestations.map((attestation) => + getIndexedAttestationSignatureSet(state, state.epochCtx.getIndexedAttestation(attestation)) ); } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/proposer.ts b/packages/beacon-state-transition/src/allForks/signatureSets/proposer.ts index 976fb3966c4..7a4d5b6edd3 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/proposer.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/proposer.ts @@ -27,6 +27,6 @@ export function getProposerSignatureSet( signedBlock.message, domain ), - signature: signedBlock.signature.valueOf() as Uint8Array, + signature: signedBlock.signature, }; } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts b/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts index e2e2feac58d..8a939383ac5 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/proposerSlashings.ts @@ -1,5 +1,4 @@ import {DOMAIN_BEACON_PROPOSER} from "@chainsafe/lodestar-params"; -import {readonlyValues} from "@chainsafe/ssz"; import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; import {computeSigningRoot, ISignatureSet, SignatureSetType} from "../../util"; import {CachedBeaconStateAllForks} from "../../types"; @@ -23,7 +22,7 @@ export function getProposerSlashingSignatureSets( type: SignatureSetType.single, pubkey, signingRoot: computeSigningRoot(beaconBlockHeaderType, signedHeader.message, domain), - signature: signedHeader.signature.valueOf() as Uint8Array, + signature: signedHeader.signature, }; } ); @@ -33,7 +32,7 @@ export function getProposerSlashingsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - return Array.from(readonlyValues(signedBlock.message.body.proposerSlashings), (proposerSlashing) => - getProposerSlashingSignatureSets(state, proposerSlashing) - ).flat(1); + return signedBlock.message.body.proposerSlashings + .map((proposerSlashing) => getProposerSlashingSignatureSets(state, proposerSlashing)) + .flat(1); } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/randao.ts b/packages/beacon-state-transition/src/allForks/signatureSets/randao.ts index 3432e1ac35c..fd1f28ddbf7 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/randao.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/randao.ts @@ -23,6 +23,6 @@ export function getRandaoRevealSignatureSet( type: SignatureSetType.single, pubkey: epochCtx.index2pubkey[block.proposerIndex], signingRoot: computeSigningRoot(ssz.Epoch, epoch, domain), - signature: block.body.randaoReveal.valueOf() as Uint8Array, + signature: block.body.randaoReveal, }; } diff --git a/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts b/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts index b4939f54f50..8be8f053eea 100644 --- a/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts +++ b/packages/beacon-state-transition/src/allForks/signatureSets/voluntaryExits.ts @@ -1,5 +1,4 @@ import {DOMAIN_VOLUNTARY_EXIT} from "@chainsafe/lodestar-params"; -import {readonlyValues} from "@chainsafe/ssz"; import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; import { computeSigningRoot, @@ -32,7 +31,7 @@ export function getVoluntaryExitSignatureSet( type: SignatureSetType.single, pubkey: epochCtx.index2pubkey[signedVoluntaryExit.message.validatorIndex], signingRoot: computeSigningRoot(ssz.phase0.VoluntaryExit, signedVoluntaryExit.message, domain), - signature: signedVoluntaryExit.signature.valueOf() as Uint8Array, + signature: signedVoluntaryExit.signature, }; } @@ -40,7 +39,7 @@ export function getVoluntaryExitsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: allForks.SignedBeaconBlock ): ISignatureSet[] { - return Array.from(readonlyValues(signedBlock.message.body.voluntaryExits), (voluntaryExit) => + return signedBlock.message.body.voluntaryExits.map((voluntaryExit) => getVoluntaryExitSignatureSet(state, voluntaryExit) ); } diff --git a/packages/beacon-state-transition/src/allForks/slot/index.ts b/packages/beacon-state-transition/src/allForks/slot/index.ts index a30cad8e5ed..3ea0ddbf2c5 100644 --- a/packages/beacon-state-transition/src/allForks/slot/index.ts +++ b/packages/beacon-state-transition/src/allForks/slot/index.ts @@ -1,5 +1,5 @@ -import {ssz} from "@chainsafe/lodestar-types"; import {SLOTS_PER_HISTORICAL_ROOT} from "@chainsafe/lodestar-params"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks} from "../../types"; import {ZERO_HASH} from "../../constants"; @@ -7,19 +7,18 @@ import {ZERO_HASH} from "../../constants"; * Dial state to next slot. Common for all forks */ export function processSlot(state: CachedBeaconStateAllForks): void { - const {config} = state; - const types = config.getForkTypes(state.slot); - // Cache state root - const previousStateRoot = types.BeaconState.hashTreeRoot(state); - state.stateRoots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previousStateRoot; + // Note: .hashTreeRoot() automatically commits() pending changes + const previousStateRoot = state.hashTreeRoot(); + state.stateRoots.set(state.slot % SLOTS_PER_HISTORICAL_ROOT, previousStateRoot); // Cache latest block header state root - if (ssz.Root.equals(state.latestBlockHeader.stateRoot, ZERO_HASH)) { + if (byteArrayEquals(state.latestBlockHeader.stateRoot, ZERO_HASH)) { state.latestBlockHeader.stateRoot = previousStateRoot; } // Cache block root - const previousBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(state.latestBlockHeader); - state.blockRoots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previousBlockRoot; + // Note: .hashTreeRoot() automatically commits() pending changes + const previousBlockRoot = state.latestBlockHeader.hashTreeRoot(); + state.blockRoots.set(state.slot % SLOTS_PER_HISTORICAL_ROOT, previousBlockRoot); } diff --git a/packages/beacon-state-transition/src/allForks/stateTransition.ts b/packages/beacon-state-transition/src/allForks/stateTransition.ts index 1fd120a075b..ebcd769a970 100644 --- a/packages/beacon-state-transition/src/allForks/stateTransition.ts +++ b/packages/beacon-state-transition/src/allForks/stateTransition.ts @@ -54,8 +54,8 @@ export function stateTransition( let postState = state.clone(); - // Turn caches into a data-structure optimized for fast writes - postState.setStateCachesAsTransient(); + // State is already a ViewDU, which won't commit changes. Equivalent to .setStateCachesAsTransient() + // postState.setStateCachesAsTransient(); // Process slots (including those with no blocks) since block. // Includes state upgrades @@ -71,20 +71,21 @@ export function stateTransition( // Process block processBlock(postState, block, options, metrics); + // Apply changes to state, must do before hashing. Note: .hashTreeRoot() automatically commits() too + postState.commit(); + // Verify state root if (verifyStateRoot) { - if (!ssz.Root.equals(block.stateRoot, postState.tree.root)) { + const stateRoot = postState.hashTreeRoot(); + if (!ssz.Root.equals(block.stateRoot, stateRoot)) { throw new Error( `Invalid state root at slot ${block.slot}, expected=${toHexString(block.stateRoot)}, actual=${toHexString( - postState.tree.root + stateRoot )}` ); } } - // Turn caches into a data-structure optimized for hashing and structural sharing - postState.setStateCachesAsPersistent(); - return postState; } @@ -122,13 +123,13 @@ export function processSlots( ): CachedBeaconStateAllForks { let postState = state.clone(); - // Turn caches into a data-structure optimized for fast writes - postState.setStateCachesAsTransient(); + // State is already a ViewDU, which won't commit changes. Equivalent to .setStateCachesAsTransient() + // postState.setStateCachesAsTransient(); postState = processSlotsWithTransientCache(postState, slot, metrics); - // Turn caches into a data-structure optimized for hashing and structural sharing - postState.setStateCachesAsPersistent(); + // Apply changes to state, must do before hashing + postState.commit(); return postState; } diff --git a/packages/beacon-state-transition/src/altair/block/index.ts b/packages/beacon-state-transition/src/altair/block/index.ts index b62d1d6260f..8cf54be8cc0 100644 --- a/packages/beacon-state-transition/src/altair/block/index.ts +++ b/packages/beacon-state-transition/src/altair/block/index.ts @@ -1,6 +1,6 @@ import {altair} from "@chainsafe/lodestar-types"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processBlockHeader, processEth1Data, processRandao} from "../../allForks/block"; import {processOperations} from "./processOperations"; import {processAttestations, RootCache} from "./processAttestation"; @@ -22,9 +22,9 @@ export { }; export function processBlock(state: CachedBeaconStateAltair, block: altair.BeaconBlock, verifySignatures = true): void { - processBlockHeader(state as CachedBeaconStateAllForks, block); - processRandao(state as CachedBeaconStateAllForks, block, verifySignatures); - processEth1Data(state as CachedBeaconStateAllForks, block.body); + processBlockHeader(state, block); + processRandao(state, block, verifySignatures); + processEth1Data(state, block.body.eth1Data); processOperations(state, block.body, verifySignatures); processSyncAggregate(state, block, verifySignatures); } diff --git a/packages/beacon-state-transition/src/altair/block/processAttestation.ts b/packages/beacon-state-transition/src/altair/block/processAttestation.ts index 7871b1ab876..bdf235d3dc1 100644 --- a/packages/beacon-state-transition/src/altair/block/processAttestation.ts +++ b/packages/beacon-state-transition/src/altair/block/processAttestation.ts @@ -1,8 +1,9 @@ -import {Epoch, ParticipationFlags, phase0, Root, Slot, ssz} from "@chainsafe/lodestar-types"; +import {Epoch, phase0, Root, Slot} from "@chainsafe/lodestar-types"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {intSqrt} from "@chainsafe/lodestar-utils"; import {getBlockRoot, getBlockRootAtSlot, increaseBalance, verifySignatureSet} from "../../util"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochContext} from "../../types"; +import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; import { MIN_ATTESTATION_INCLUSION_DELAY, PROPOSER_WEIGHT, @@ -17,7 +18,6 @@ import { } from "@chainsafe/lodestar-params"; import {checkpointToStr, validateAttestation} from "../../phase0/block/processAttestation"; import {getAttestationWithIndicesSignatureSet} from "../../allForks"; -import {CachedEpochParticipation} from "../../cache/cachedEpochParticipation"; const PROPOSER_REWARD_DOMINATOR = ((WEIGHT_DENOMINATOR - PROPOSER_WEIGHT) * WEIGHT_DENOMINATOR) / PROPOSER_WEIGHT; @@ -34,29 +34,24 @@ export function processAttestations( const {epochCtx} = state; const {effectiveBalanceIncrements} = epochCtx; const stateSlot = state.slot; - const rootCache = new RootCache(state as CachedBeaconStateAllForks); + const rootCache = new RootCache(state); // Process all attestations first and then increase the balance of the proposer once let proposerReward = 0; - const previousEpochStatuses = new Map(); - const currentEpochStatuses = new Map(); for (const attestation of attestations) { const data = attestation.data; - validateAttestation(state as CachedBeaconStateAllForks, attestation); + validateAttestation(state, attestation); // Retrieve the validator indices from the attestation participation bitfield - const attestingIndices = epochCtx.getAttestingIndices(data, attestation.aggregationBits); + const committeeIndices = epochCtx.getBeaconCommittee(data.slot, data.index); + const attestingIndices = attestation.aggregationBits.intersectValues(committeeIndices); // this check is done last because its the most expensive (if signature verification is toggled on) // TODO: Why should we verify an indexed attestation that we just created? If it's just for the signature // we can verify only that and nothing else. if (verifySignature) { - const sigSet = getAttestationWithIndicesSignatureSet( - state as CachedBeaconStateAllForks, - attestation, - attestingIndices - ); + const sigSet = getAttestationWithIndicesSignatureSet(state, attestation, attestingIndices); if (!verifySignatureSet(sigSet)) { throw new Error("Attestation signature is not valid"); } @@ -66,23 +61,21 @@ export function processAttestations( data.target.epoch === epochCtx.currentShuffling.epoch ? state.currentEpochParticipation : state.previousEpochParticipation; - const epochStatuses = - data.target.epoch === epochCtx.currentShuffling.epoch ? currentEpochStatuses : previousEpochStatuses; - const flagsAttestation = getAttestationParticipationStatus(data, stateSlot - data.slot, rootCache, epochCtx); + const flagsAttestation = getAttestationParticipationStatus(data, stateSlot - data.slot, epochCtx.epoch, rootCache); // For each participant, update their participation // In epoch processing, this participation info is used to calculate balance updates let totalBalanceIncrementsWithWeight = 0; for (const index of attestingIndices) { - const flags = epochStatuses.get(index) ?? (epochParticipation.get(index) as ParticipationFlags); - // Merge (OR) `flagsAttestation` (new flags) with `flags` (current flags) - const newFlags = flagsAttestation | flags; + const flags = epochParticipation.get(index); // For normal block, > 90% of attestations belong to current epoch // At epoch boundary, 100% of attestations belong to previous epoch // so we want to update the participation flag tree in batch - epochStatuses.set(index, newFlags); + + // Note ParticipationFlags type uses option {setBitwiseOR: true}, .set() does a |= operation + epochParticipation.set(index, flagsAttestation); // epochParticipation.setStatus(index, newStatus); // Returns flags that are NOT set before (~ bitwise NOT) AND are set after @@ -103,19 +96,9 @@ export function processAttestations( // Do the discrete math inside the loop to ensure a deterministic result const totalIncrements = totalBalanceIncrementsWithWeight; - const proposerRewardNumerator = totalIncrements * state.baseRewardPerIncrement; + const proposerRewardNumerator = totalIncrements * state.epochCtx.baseRewardPerIncrement; proposerReward += Math.floor(proposerRewardNumerator / PROPOSER_REWARD_DOMINATOR); } - updateEpochParticipants( - previousEpochStatuses, - state.previousEpochParticipation, - epochCtx.previousShuffling.activeIndices.length - ); - updateEpochParticipants( - currentEpochStatuses, - state.currentEpochParticipation, - epochCtx.currentShuffling.activeIndices.length - ); increaseBalance(state, epochCtx.getBeaconProposer(state.slot), proposerReward); } @@ -126,13 +109,11 @@ export function processAttestations( export function getAttestationParticipationStatus( data: phase0.AttestationData, inclusionDelay: number, - rootCache: RootCache, - epochCtx: EpochContext -): ParticipationFlags { + currentEpoch: Epoch, + rootCache: RootCache +): number { const justifiedCheckpoint = - data.target.epoch === epochCtx.currentShuffling.epoch - ? rootCache.currentJustifiedCheckpoint - : rootCache.previousJustifiedCheckpoint; + data.target.epoch === currentEpoch ? rootCache.currentJustifiedCheckpoint : rootCache.previousJustifiedCheckpoint; // The source and target votes are part of the FFG vote, the head vote is part of the fork choice vote // Both are tracked to properly incentivise validators @@ -140,7 +121,7 @@ export function getAttestationParticipationStatus( // The source vote always matches the justified checkpoint (else its invalid) // The target vote should match the most recent checkpoint (eg: the first root of the epoch) // The head vote should match the root at the attestation slot (eg: the root at data.slot) - const isMatchingSource = ssz.phase0.Checkpoint.equals(data.source, justifiedCheckpoint); + const isMatchingSource = checkpointValueEquals(data.source, justifiedCheckpoint); if (!isMatchingSource) { throw new Error( `Attestation source does not equal justified checkpoint: source=${checkpointToStr( @@ -149,11 +130,11 @@ export function getAttestationParticipationStatus( ); } - const isMatchingTarget = ssz.Root.equals(data.target.root, rootCache.getBlockRoot(data.target.epoch)); + const isMatchingTarget = byteArrayEquals(data.target.root, rootCache.getBlockRoot(data.target.epoch)); // a timely head is only be set if the target is _also_ matching const isMatchingHead = - isMatchingTarget && ssz.Root.equals(data.beaconBlockRoot, rootCache.getBlockRootAtSlot(data.slot)); + isMatchingTarget && byteArrayEquals(data.beaconBlockRoot, rootCache.getBlockRootAtSlot(data.slot)); let flags = 0; if (isMatchingSource && inclusionDelay <= intSqrt(SLOTS_PER_EPOCH)) flags |= TIMELY_SOURCE; @@ -163,6 +144,10 @@ export function getAttestationParticipationStatus( return flags; } +export function checkpointValueEquals(cp1: phase0.Checkpoint, cp2: phase0.Checkpoint): boolean { + return cp1.epoch === cp2.epoch && byteArrayEquals(cp1.root, cp2.root); +} + /** * Cache to prevent accessing the state tree to fetch block roots repeteadly. * In normal network conditions the same root is read multiple times, specially the target. @@ -196,23 +181,3 @@ export class RootCache { return root; } } - -export function updateEpochParticipants( - epochStatuses: Map, - epochParticipation: CachedEpochParticipation, - numActiveValidators: number -): void { - // all active validators are attesters, there are 32 slots per epoch - // if 1/2 of that or more are updated flags, do that in batch, see the benchmark for more details - if (epochStatuses.size >= numActiveValidators / (2 * SLOTS_PER_EPOCH)) { - const transientVector = epochParticipation.persistent.asTransient(); - for (const [index, flags] of epochStatuses.entries()) { - transientVector.set(index, flags); - } - epochParticipation.updateAllStatus(transientVector.vector); - } else { - for (const [index, flags] of epochStatuses.entries()) { - epochParticipation.set(index, flags); - } - } -} diff --git a/packages/beacon-state-transition/src/altair/block/processAttesterSlashing.ts b/packages/beacon-state-transition/src/altair/block/processAttesterSlashing.ts index 5459a51575d..a840f07e513 100644 --- a/packages/beacon-state-transition/src/altair/block/processAttesterSlashing.ts +++ b/packages/beacon-state-transition/src/altair/block/processAttesterSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processAttesterSlashing as processAttesterSlashingAllForks} from "../../allForks/block"; export function processAttesterSlashing( @@ -8,10 +8,5 @@ export function processAttesterSlashing( attesterSlashing: phase0.AttesterSlashing, verifySignatures = true ): void { - processAttesterSlashingAllForks( - ForkName.altair, - state as CachedBeaconStateAllForks, - attesterSlashing, - verifySignatures - ); + processAttesterSlashingAllForks(ForkName.altair, state, attesterSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/altair/block/processDeposit.ts b/packages/beacon-state-transition/src/altair/block/processDeposit.ts index 59c42232924..967b1640e30 100644 --- a/packages/beacon-state-transition/src/altair/block/processDeposit.ts +++ b/packages/beacon-state-transition/src/altair/block/processDeposit.ts @@ -1,9 +1,9 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processDeposit as processDepositAllForks} from "../../allForks/block"; export function processDeposit(state: CachedBeaconStateAltair, deposit: phase0.Deposit): void { - processDepositAllForks(ForkName.altair, state as CachedBeaconStateAllForks, deposit); + processDepositAllForks(ForkName.altair, state, deposit); } diff --git a/packages/beacon-state-transition/src/altair/block/processOperations.ts b/packages/beacon-state-transition/src/altair/block/processOperations.ts index 37151f52f4d..6fe79034edf 100644 --- a/packages/beacon-state-transition/src/altair/block/processOperations.ts +++ b/packages/beacon-state-transition/src/altair/block/processOperations.ts @@ -1,4 +1,3 @@ -import {readonlyValues} from "@chainsafe/ssz"; import {altair} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAltair} from "../../types"; @@ -22,19 +21,19 @@ export function processOperations( ); } - for (const proposerSlashing of readonlyValues(body.proposerSlashings)) { + for (const proposerSlashing of body.proposerSlashings) { processProposerSlashing(state, proposerSlashing, verifySignatures); } - for (const attesterSlashing of readonlyValues(body.attesterSlashings)) { + for (const attesterSlashing of body.attesterSlashings) { processAttesterSlashing(state, attesterSlashing, verifySignatures); } - processAttestations(state, Array.from(readonlyValues(body.attestations)), verifySignatures); + processAttestations(state, body.attestations, verifySignatures); - for (const deposit of readonlyValues(body.deposits)) { + for (const deposit of body.deposits) { processDeposit(state, deposit); } - for (const voluntaryExit of readonlyValues(body.voluntaryExits)) { + for (const voluntaryExit of body.voluntaryExits) { processVoluntaryExit(state, voluntaryExit, verifySignatures); } } diff --git a/packages/beacon-state-transition/src/altair/block/processProposerSlashing.ts b/packages/beacon-state-transition/src/altair/block/processProposerSlashing.ts index 87ff108168a..40f5d8aa7d3 100644 --- a/packages/beacon-state-transition/src/altair/block/processProposerSlashing.ts +++ b/packages/beacon-state-transition/src/altair/block/processProposerSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processProposerSlashing as processProposerSlashingAllForks} from "../../allForks/block"; export function processProposerSlashing( @@ -8,10 +8,5 @@ export function processProposerSlashing( proposerSlashing: phase0.ProposerSlashing, verifySignatures = true ): void { - processProposerSlashingAllForks( - ForkName.altair, - state as CachedBeaconStateAllForks, - proposerSlashing, - verifySignatures - ); + processProposerSlashingAllForks(ForkName.altair, state, proposerSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts b/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts index 70b50df8f56..3164849864d 100644 --- a/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts +++ b/packages/beacon-state-transition/src/altair/block/processSyncCommittee.ts @@ -1,25 +1,21 @@ -import {altair, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {altair, ssz} from "@chainsafe/lodestar-types"; import {DOMAIN_SYNC_COMMITTEE} from "@chainsafe/lodestar-params"; +import {byteArrayEquals} from "@chainsafe/ssz"; -import { - computeSigningRoot, - getBlockRootAtSlot, - ISignatureSet, - SignatureSetType, - verifySignatureSet, - zipAllIndexesSyncCommitteeBits, - zipIndexesSyncCommitteeBits, -} from "../../util"; -import {CachedBeaconStateAltair} from "../../types"; +import {computeSigningRoot, getBlockRootAtSlot, ISignatureSet, SignatureSetType, verifySignatureSet} from "../../util"; +import {CachedBeaconStateAllForks} from "../../types"; import {G2_POINT_AT_INFINITY} from "../../constants"; +import {getUnparticipantValues} from "../../util/array"; export function processSyncAggregate( - state: CachedBeaconStateAltair, + state: CachedBeaconStateAllForks, block: altair.BeaconBlock, verifySignatures = true ): void { const {syncParticipantReward, syncProposerReward} = state.epochCtx; - const [participantIndices, unparticipantIndices] = getParticipantInfo(state, block.body.syncAggregate); + const committeeIndices = state.epochCtx.currentSyncCommitteeIndexed.validatorIndices; + const participantIndices = block.body.syncAggregate.syncCommitteeBits.intersectValues(committeeIndices); + const unparticipantIndices = getUnparticipantValues(participantIndices, committeeIndices); // different from the spec but not sure how to get through signature verification for default/empty SyncAggregate in the spec test if (verifySignatures) { @@ -30,27 +26,33 @@ export function processSyncAggregate( throw Error("Sync committee signature invalid"); } } - const deltaByIndex = new Map(); + + const balances = state.balances; + + // Proposer reward const proposerIndex = state.epochCtx.getBeaconProposer(state.slot); - for (const participantIndex of participantIndices) { - accumulateDelta(deltaByIndex, participantIndex, syncParticipantReward); + balances.set(proposerIndex, balances.get(proposerIndex) + syncProposerReward * participantIndices.length); + + // Positive rewards for participants + for (const index of participantIndices) { + balances.set(index, balances.get(index) + syncParticipantReward); } - accumulateDelta(deltaByIndex, proposerIndex, syncProposerReward * participantIndices.length); - for (const unparticipantIndex of unparticipantIndices) { - accumulateDelta(deltaByIndex, unparticipantIndex, -syncParticipantReward); + + // Negative rewards for non participants + for (const index of unparticipantIndices) { + balances.set(index, balances.get(index) - syncParticipantReward); } - state.balanceList.applyDeltaInBatch(deltaByIndex); } export function getSyncCommitteeSignatureSet( - state: CachedBeaconStateAltair, + state: CachedBeaconStateAllForks, block: altair.BeaconBlock, /** Optional parameter to prevent computing it twice */ participantIndices?: number[] ): ISignatureSet | null { const {epochCtx} = state; const {syncAggregate} = block.body; - const signature = syncAggregate.syncCommitteeSignature.valueOf() as Uint8Array; + const signature = syncAggregate.syncCommitteeSignature; // The spec uses the state to get the previous slot // ```python @@ -67,14 +69,15 @@ export function getSyncCommitteeSignatureSet( const rootSigned = getBlockRootAtSlot(state, previousSlot); if (!participantIndices) { - participantIndices = getParticipantIndices(state, syncAggregate); + const committeeIndices = state.epochCtx.currentSyncCommitteeIndexed.validatorIndices; + participantIndices = syncAggregate.syncCommitteeBits.intersectValues(committeeIndices); } // When there's no participation we consider the signature valid and just ignore it if (participantIndices.length === 0) { // Must set signature as G2_POINT_AT_INFINITY when participating bits are empty - // https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/altair/bls.md#eth_fast_aggregate_verify - if (ssz.BLSSignature.equals(signature, G2_POINT_AT_INFINITY)) { + // https://github.com/ethereum/eth2.0-specs/blob/30f2a076377264677e27324a8c3c78c590ae5e20/specs/altair/bls.md#eth2_fast_aggregate_verify + if (byteArrayEquals(signature, G2_POINT_AT_INFINITY)) { return null; } else { throw Error("Empty sync committee signature is not infinity"); @@ -90,24 +93,3 @@ export function getSyncCommitteeSignatureSet( signature, }; } - -/** Get participant indices for a sync committee. */ -function getParticipantIndices(state: CachedBeaconStateAltair, syncAggregate: altair.SyncAggregate): number[] { - const committeeIndices = state.epochCtx.currentSyncCommitteeIndexed.validatorIndices; - return zipIndexesSyncCommitteeBits(committeeIndices, syncAggregate.syncCommitteeBits); -} - -/** Return [0] as participant indices and [1] as unparticipant indices for a sync committee. */ -function getParticipantInfo(state: CachedBeaconStateAltair, syncAggregate: altair.SyncAggregate): [number[], number[]] { - const committeeIndices = state.epochCtx.currentSyncCommitteeIndexed.validatorIndices; - return zipAllIndexesSyncCommitteeBits(committeeIndices, syncAggregate.syncCommitteeBits); -} - -function accumulateDelta(deltaByIndex: Map, index: ValidatorIndex, delta: number): void { - const existingDelta = deltaByIndex.get(index); - if (existingDelta === undefined) { - deltaByIndex.set(index, delta); - } else { - deltaByIndex.set(index, delta + existingDelta); - } -} diff --git a/packages/beacon-state-transition/src/altair/block/processVoluntaryExit.ts b/packages/beacon-state-transition/src/altair/block/processVoluntaryExit.ts index 1d2184b05f4..6d8fd35c328 100644 --- a/packages/beacon-state-transition/src/altair/block/processVoluntaryExit.ts +++ b/packages/beacon-state-transition/src/altair/block/processVoluntaryExit.ts @@ -1,5 +1,5 @@ import {phase0} from "@chainsafe/lodestar-types"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateAltair} from "../../types"; import {processVoluntaryExitAllForks} from "../../allForks/block"; export function processVoluntaryExit( @@ -7,5 +7,5 @@ export function processVoluntaryExit( signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature = true ): void { - processVoluntaryExitAllForks(state as CachedBeaconStateAllForks, signedVoluntaryExit, verifySignature); + processVoluntaryExitAllForks(state, signedVoluntaryExit, verifySignature); } diff --git a/packages/beacon-state-transition/src/altair/epoch/getRewardsAndPenalties.ts b/packages/beacon-state-transition/src/altair/epoch/getRewardsAndPenalties.ts index b5337991579..9ec87ad3a1b 100644 --- a/packages/beacon-state-transition/src/altair/epoch/getRewardsAndPenalties.ts +++ b/packages/beacon-state-transition/src/altair/epoch/getRewardsAndPenalties.ts @@ -9,12 +9,12 @@ import { WEIGHT_DENOMINATOR, ForkName, } from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStateAltair, EpochProcess} from "../../types"; import { FLAG_ELIGIBLE_ATTESTER, - FLAG_PREV_HEAD_ATTESTER_OR_UNSLASHED, - FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED, - FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED, + FLAG_PREV_HEAD_ATTESTER_UNSLASHED, + FLAG_PREV_SOURCE_ATTESTER_UNSLASHED, + FLAG_PREV_TARGET_ATTESTER_UNSLASHED, hasMarkers, } from "../../util/attesterStatus"; import {isInInactivityLeak, newZeroedArray} from "../../util"; @@ -48,7 +48,7 @@ export function getRewardsAndPenalties(state: CachedBeaconStateAltair, process: const rewards = newZeroedArray(validatorCount); const penalties = newZeroedArray(validatorCount); - const isInInactivityLeakBn = isInInactivityLeak(state as CachedBeaconStateAllForks); + const isInInactivityLeakBn = isInInactivityLeak(state); // effectiveBalance is multiple of EFFECTIVE_BALANCE_INCREMENT and less than MAX_EFFECTIVE_BALANCE // so there are limited values of them like 32, 31, 30 const rewardPenaltyItemCache = new Map(); @@ -100,31 +100,35 @@ export function getRewardsAndPenalties(state: CachedBeaconStateAltair, process: } = rewardPenaltyItem; // same logic to getFlagIndexDeltas - if (hasMarkers(status.flags, FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED)) { + if (hasMarkers(status.flags, FLAG_PREV_SOURCE_ATTESTER_UNSLASHED)) { if (!isInInactivityLeakBn) { rewards[i] += timelySourceReward; } } else { penalties[i] += timelySourcePenalty; } - if (hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED)) { + + if (hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) { if (!isInInactivityLeakBn) { rewards[i] += timelyTargetReward; } } else { penalties[i] += timelyTargetPenalty; } - if (hasMarkers(status.flags, FLAG_PREV_HEAD_ATTESTER_OR_UNSLASHED)) { + + if (hasMarkers(status.flags, FLAG_PREV_HEAD_ATTESTER_UNSLASHED)) { if (!isInInactivityLeakBn) { rewards[i] += timelyHeadReward; } } + // Same logic to getInactivityPenaltyDeltas // TODO: if we have limited value in inactivityScores we can provide a cache too - if (!hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED)) { - const penaltyNumerator = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT * state.inactivityScores[i]; + if (!hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) { + const penaltyNumerator = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT * state.inactivityScores.get(i); penalties[i] += Math.floor(penaltyNumerator / penaltyDenominator); } } + return [rewards, penalties]; } diff --git a/packages/beacon-state-transition/src/altair/epoch/index.ts b/packages/beacon-state-transition/src/altair/epoch/index.ts index 946e616bc6e..1623f86d633 100644 --- a/packages/beacon-state-transition/src/altair/epoch/index.ts +++ b/packages/beacon-state-transition/src/altair/epoch/index.ts @@ -1,4 +1,4 @@ -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStateAltair, EpochProcess} from "../../types"; import { processJustificationAndFinalization, processRegistryUpdates, @@ -26,16 +26,16 @@ export { }; export function processEpoch(state: CachedBeaconStateAltair, epochProcess: EpochProcess): void { - processJustificationAndFinalization(state as CachedBeaconStateAllForks, epochProcess); + processJustificationAndFinalization(state, epochProcess); processInactivityUpdates(state, epochProcess); processRewardsAndPenalties(state, epochProcess); - processRegistryUpdates(state as CachedBeaconStateAllForks, epochProcess); + processRegistryUpdates(state, epochProcess); processSlashings(state, epochProcess); - processEth1DataReset(state as CachedBeaconStateAllForks, epochProcess); - processEffectiveBalanceUpdates(state as CachedBeaconStateAllForks, epochProcess); - processSlashingsReset(state as CachedBeaconStateAllForks, epochProcess); - processRandaoMixesReset(state as CachedBeaconStateAllForks, epochProcess); - processHistoricalRootsUpdate(state as CachedBeaconStateAllForks, epochProcess); + processEth1DataReset(state, epochProcess); + processEffectiveBalanceUpdates(state, epochProcess); + processSlashingsReset(state, epochProcess); + processRandaoMixesReset(state, epochProcess); + processHistoricalRootsUpdate(state, epochProcess); processParticipationFlagUpdates(state); processSyncCommitteeUpdates(state); } diff --git a/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts b/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts index 7e55339e4bc..adec4cfdaf6 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processInactivityUpdates.ts @@ -1,6 +1,5 @@ import {GENESIS_EPOCH} from "@chainsafe/lodestar-params"; -import {Number64} from "@chainsafe/lodestar-types"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStateAltair, EpochProcess} from "../../types"; import * as attesterStatusUtil from "../../util/attesterStatus"; import {isInInactivityLeak} from "../../util"; @@ -19,33 +18,36 @@ import {isInInactivityLeak} from "../../util"; * TODO: Compute from altair testnet inactivityScores updates on average */ export function processInactivityUpdates(state: CachedBeaconStateAltair, epochProcess: EpochProcess): void { - if (state.currentShuffling.epoch === GENESIS_EPOCH) { + if (state.epochCtx.epoch === GENESIS_EPOCH) { return; } + const {config, inactivityScores} = state; const {INACTIVITY_SCORE_BIAS, INACTIVITY_SCORE_RECOVERY_RATE} = config; - const {statuses} = epochProcess; - const inActivityLeak = isInInactivityLeak(state as CachedBeaconStateAllForks); + const {statuses, eligibleValidatorIndices} = epochProcess; + const inActivityLeak = isInInactivityLeak(state); // this avoids importing FLAG_ELIGIBLE_ATTESTER inside the for loop, check the compiled code - const {FLAG_ELIGIBLE_ATTESTER, FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED, hasMarkers} = attesterStatusUtil; - const newValues = new Map(); - inactivityScores.forEach(function processInactivityScore(inactivityScore, i) { + const {FLAG_PREV_TARGET_ATTESTER_UNSLASHED, hasMarkers} = attesterStatusUtil; + + const inactivityScoresArr = inactivityScores.getAll(); + + for (let j = 0; j < eligibleValidatorIndices.length; j++) { + const i = eligibleValidatorIndices[j]; const status = statuses[i]; - if (hasMarkers(status.flags, FLAG_ELIGIBLE_ATTESTER)) { - const prevInactivityScore = inactivityScore; - if (hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED)) { - inactivityScore -= Math.min(1, inactivityScore); - } else { - inactivityScore += Number(INACTIVITY_SCORE_BIAS); - } - if (!inActivityLeak) { - inactivityScore -= Math.min(Number(INACTIVITY_SCORE_RECOVERY_RATE), inactivityScore); - } - if (inactivityScore !== prevInactivityScore) { - newValues.set(i, inactivityScore); - } + let inactivityScore = inactivityScoresArr[i]; + + const prevInactivityScore = inactivityScore; + if (hasMarkers(status.flags, FLAG_PREV_TARGET_ATTESTER_UNSLASHED)) { + inactivityScore -= Math.min(1, inactivityScore); + } else { + inactivityScore += INACTIVITY_SCORE_BIAS; } - }); - inactivityScores.setMultiple(newValues); + if (!inActivityLeak) { + inactivityScore -= Math.min(INACTIVITY_SCORE_RECOVERY_RATE, inactivityScore); + } + if (inactivityScore !== prevInactivityScore) { + inactivityScores.set(i, inactivityScore); + } + } } diff --git a/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts b/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts index 9d5dd1dfbf6..3c08322d8fd 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processParticipationFlagUpdates.ts @@ -1,5 +1,5 @@ -import {PersistentVector} from "@chainsafe/persistent-ts"; -import {newFilledArray} from "../../util/array"; +import {zeroNode} from "@chainsafe/persistent-merkle-tree"; +import {ssz} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAltair} from "../../types"; /** @@ -10,6 +10,18 @@ import {CachedBeaconStateAltair} from "../../types"; * trees completely. */ export function processParticipationFlagUpdates(state: CachedBeaconStateAltair): void { - state.previousEpochParticipation.updateAllStatus(state.currentEpochParticipation.persistent.vector); - state.currentEpochParticipation.updateAllStatus(PersistentVector.from(newFilledArray(state.validators.length, 0))); + // Set view and tree from currentEpochParticipation to previousEpochParticipation + state.previousEpochParticipation = state.currentEpochParticipation; + + // We need to replace the node of currentEpochParticipation with a node that represents and empty list of some length. + // SSZ represents a list as = new BranchNode(chunksNode, lengthNode). + // Since the chunks represent all zero'ed data we can re-use the pre-compouted zeroNode at chunkDepth to skip any + // data transformation and create the required tree almost for free. + const currentEpochParticipationNode = ssz.altair.EpochParticipation.tree_setChunksNode( + state.currentEpochParticipation.node, + zeroNode(ssz.altair.EpochParticipation.chunkDepth), + state.currentEpochParticipation.length + ); + + state.currentEpochParticipation = ssz.altair.EpochParticipation.getViewDU(currentEpochParticipationNode); } diff --git a/packages/beacon-state-transition/src/altair/epoch/processRewardsAndPenalties.ts b/packages/beacon-state-transition/src/altair/epoch/processRewardsAndPenalties.ts index ee3c764955a..f75ec2eab25 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processRewardsAndPenalties.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processRewardsAndPenalties.ts @@ -1,13 +1,13 @@ -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; import {ForkName} from "@chainsafe/lodestar-params"; import {processRewardsAndPenaltiesAllForks} from "../../allForks/epoch/processRewardsAndPenalties"; +import {EpochProcess} from "../../cache/epochProcess"; +import {CachedBeaconStateAltair} from "../../types"; /** * Iterate over all validator and compute rewards and penalties to apply to balances. * - * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags - * are true, worst case: FLAG_UNSLASHED + FLAG_ELIGIBLE_ATTESTER + FLAG_PREV_* + * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags are set */ export function processRewardsAndPenalties(state: CachedBeaconStateAltair, epochProcess: EpochProcess): void { - processRewardsAndPenaltiesAllForks(ForkName.altair, state as CachedBeaconStateAllForks, epochProcess); + processRewardsAndPenaltiesAllForks(ForkName.altair, state, epochProcess); } diff --git a/packages/beacon-state-transition/src/altair/epoch/processSlashings.ts b/packages/beacon-state-transition/src/altair/epoch/processSlashings.ts index d30ff90a901..0a4b1307019 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processSlashings.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processSlashings.ts @@ -1,7 +1,7 @@ import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStateAltair, EpochProcess} from "../../types"; import {processSlashingsAllForks} from "../../allForks/epoch/processSlashings"; export function processSlashings(state: CachedBeaconStateAltair, epochProcess: EpochProcess): void { - processSlashingsAllForks(ForkName.altair, state as CachedBeaconStateAllForks, epochProcess); + processSlashingsAllForks(ForkName.altair, state, epochProcess); } diff --git a/packages/beacon-state-transition/src/altair/epoch/processSyncCommitteeUpdates.ts b/packages/beacon-state-transition/src/altair/epoch/processSyncCommitteeUpdates.ts index 4c8d8abe43d..6f662900a74 100644 --- a/packages/beacon-state-transition/src/altair/epoch/processSyncCommitteeUpdates.ts +++ b/packages/beacon-state-transition/src/altair/epoch/processSyncCommitteeUpdates.ts @@ -1,7 +1,8 @@ -import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@chainsafe/lodestar-params"; import {aggregatePublicKeys} from "@chainsafe/bls"; -import {CachedBeaconStateAltair} from "../../types"; +import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@chainsafe/lodestar-params"; +import {ssz} from "@chainsafe/lodestar-types"; import {getNextSyncCommitteeIndices} from "../../util/seed"; +import {CachedBeaconStateAltair} from "../../types"; /** * Rotate nextSyncCommittee to currentSyncCommittee if sync committee period is over. @@ -23,17 +24,16 @@ export function processSyncCommitteeUpdates(state: CachedBeaconStateAltair): voi ); // Using the index2pubkey cache is slower because it needs the serialized pubkey. - const nextSyncCommitteePubkeys = nextSyncCommitteeIndices.map( - (index) => state.validators[index].pubkey.valueOf() as Uint8Array - ); + const nextSyncCommitteePubkeys = nextSyncCommitteeIndices.map((index) => state.validators.get(index).pubkey); // Rotate syncCommittee in state state.currentSyncCommittee = state.nextSyncCommittee; - state.nextSyncCommittee = { + state.nextSyncCommittee = ssz.altair.SyncCommittee.toViewDU({ pubkeys: nextSyncCommitteePubkeys, aggregatePubkey: aggregatePublicKeys(nextSyncCommitteePubkeys), - }; + }); + // Rotate syncCommittee cache state.epochCtx.rotateSyncCommitteeIndexed(nextSyncCommitteeIndices); } } diff --git a/packages/beacon-state-transition/src/altair/upgradeState.ts b/packages/beacon-state-transition/src/altair/upgradeState.ts index 3a9f9a0a3ef..d593cd2ac24 100644 --- a/packages/beacon-state-transition/src/altair/upgradeState.ts +++ b/packages/beacon-state-transition/src/altair/upgradeState.ts @@ -1,61 +1,125 @@ -import {altair, ParticipationFlags, phase0, ssz, Uint8} from "@chainsafe/lodestar-types"; -import {CachedBeaconStatePhase0, CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../types"; -import {createCachedBeaconState} from "../cache/cachedBeaconState"; +import {ssz} from "@chainsafe/lodestar-types"; +import {CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../types"; import {newZeroedArray} from "../util"; -import {List, TreeBacked} from "@chainsafe/ssz"; -import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {getAttestationParticipationStatus, RootCache} from "./block/processAttestation"; import {getNextSyncCommittee} from "../util/syncCommittee"; +import {CompositeViewDU} from "@chainsafe/ssz"; +import {getCachedBeaconState} from "../cache/stateCache"; /** * Upgrade a state from phase0 to altair. */ -export function upgradeState(state: CachedBeaconStatePhase0): CachedBeaconStateAltair { - const {config} = state; - const pendingAttesations = Array.from(state.previousEpochAttestations); - const postTreeBackedState = upgradeTreeBackedState(config, state); - const postState = createCachedBeaconState(config, postTreeBackedState); - translateParticipation(postState, pendingAttesations); - return postState; -} +export function upgradeState(statePhase0: CachedBeaconStatePhase0): CachedBeaconStateAltair { + const {config} = statePhase0; + + // Get underlying node and cast phase0 tree to altair tree + // + // A phase0 BeaconState tree can be safely casted to an altair BeaconState tree because: + // - Deprecated fields are replaced by new fields at the exact same indexes + // - All new fields are appended at the end + // + // So by just setting all new fields to some value, all the old nodes are dropped + // + // phase0 | op | altair + // ----------------------------- | ---- | ------------ + // genesis_time | - | genesis_time + // genesis_validators_root | - | genesis_validators_root + // slot | - | slot + // fork | - | fork + // latest_block_header | - | latest_block_header + // block_roots | - | block_roots + // state_roots | - | state_roots + // historical_roots | - | historical_roots + // eth1_data | - | eth1_data + // eth1_data_votes | - | eth1_data_votes + // eth1_deposit_index | - | eth1_deposit_index + // validators | - | validators + // balances | - | balances + // randao_mixes | - | randao_mixes + // slashings | - | slashings + // previous_epoch_attestations | diff | previous_epoch_participation + // current_epoch_attestations | diff | current_epoch_participation + // justification_bits | - | justification_bits + // previous_justified_checkpoint | - | previous_justified_checkpoint + // current_justified_checkpoint | - | current_justified_checkpoint + // finalized_checkpoint | - | finalized_checkpoint + // - | new | inactivity_scores + // - | new | current_sync_committee + // - | new | next_sync_committee + + const statePhase0Node = ssz.phase0.BeaconState.commitViewDU(statePhase0); + const stateAltairView = ssz.altair.BeaconState.getViewDU(statePhase0Node); + // Attach existing BeaconStateCache from statePhase0 to new stateAltairView object + const stateAltair = getCachedBeaconState(stateAltairView, statePhase0); -function upgradeTreeBackedState(config: IBeaconConfig, state: CachedBeaconStatePhase0): TreeBacked { - const nextEpochActiveIndices = state.nextShuffling.activeIndices; - const stateTB = ssz.phase0.BeaconState.createTreeBacked(state.tree); - const validatorCount = stateTB.validators.length; - const epoch = state.currentShuffling.epoch; - // TODO: Does this preserve the hashing cache? In altair devnets memory spikes on the fork transition - const postState = ssz.altair.BeaconState.createTreeBacked(stateTB.tree); - postState.fork = { - previousVersion: stateTB.fork.currentVersion, + stateAltair.fork = ssz.phase0.Fork.toViewDU({ + previousVersion: statePhase0.fork.currentVersion, currentVersion: config.ALTAIR_FORK_VERSION, - epoch, - }; - postState.previousEpochParticipation = newZeroedArray(validatorCount) as List; - postState.currentEpochParticipation = newZeroedArray(validatorCount) as List; - postState.inactivityScores = newZeroedArray(validatorCount) as List; - const syncCommittee = getNextSyncCommittee(state, nextEpochActiveIndices, state.epochCtx.effectiveBalanceIncrements); - postState.currentSyncCommittee = syncCommittee; - postState.nextSyncCommittee = syncCommittee; - return postState; + epoch: statePhase0.epochCtx.epoch, + }); + + const validatorCount = statePhase0.validators.length; + const emptyEpochParticipationView = ssz.altair.EpochParticipation.toViewDU(newZeroedArray(validatorCount)); + const emptyEpochParticipationNode = ssz.altair.EpochParticipation.commitViewDU(emptyEpochParticipationView); + stateAltair.previousEpochParticipation = emptyEpochParticipationView; + // Cloned instance with same immutable Node + stateAltair.currentEpochParticipation = ssz.altair.EpochParticipation.getViewDU(emptyEpochParticipationNode); + + stateAltair.inactivityScores = ssz.altair.InactivityScores.toViewDU(newZeroedArray(validatorCount)); + + const {syncCommittee, indices} = getNextSyncCommittee( + stateAltair, + stateAltair.epochCtx.nextShuffling.activeIndices, + stateAltair.epochCtx.effectiveBalanceIncrements + ); + const syncCommitteeView = ssz.altair.SyncCommittee.toViewDU(syncCommittee); + + stateAltair.currentSyncCommittee = syncCommitteeView; + stateAltair.nextSyncCommittee = syncCommitteeView; + stateAltair.epochCtx.setSyncCommitteesIndexed(indices); + + const pendingAttesations = statePhase0.previousEpochAttestations; + translateParticipation(stateAltair, pendingAttesations); + + // Commit new added fields ViewDU to the root node + stateAltair.commit(); + // Clear cache to ensure the cache of phase0 fields is not used by new altair fields + // [15] previous_epoch_attestations -> previous_epoch_participation + // [16] current_epoch_attestations -> current_epoch_participation + // + // TODO: This could only drop the caches of index 15,16. However this would couple this code tightly with SSZ ViewDU + // internals. If the cache is not cleared, consuming the ViewDU instance could break in strange ways. + stateAltair["clearCache"](); + + return stateAltair; } /** * Translate_participation in https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/altair/fork.md */ -function translateParticipation(state: CachedBeaconStateAltair, pendingAttesations: phase0.PendingAttestation[]): void { +function translateParticipation( + state: CachedBeaconStateAltair, + pendingAttesations: CompositeViewDU +): void { const {epochCtx} = state; - const rootCache = new RootCache(state as CachedBeaconStateAllForks); + const rootCache = new RootCache(state); const epochParticipation = state.previousEpochParticipation; - for (const attestation of pendingAttesations) { + + for (const attestation of pendingAttesations.getAllReadonly()) { const data = attestation.data; - const flagsAttestation = getAttestationParticipationStatus(data, attestation.inclusionDelay, rootCache, epochCtx); + const attestationFlags = getAttestationParticipationStatus( + data, + attestation.inclusionDelay, + epochCtx.epoch, + rootCache + ); + + const committeeIndices = epochCtx.getBeaconCommittee(data.slot, data.index); + const attestingIndices = attestation.aggregationBits.intersectValues(committeeIndices); - const attestingIndices = state.getAttestingIndices(data, attestation.aggregationBits); for (const index of attestingIndices) { - const flags = epochParticipation.get(index) as ParticipationFlags; - // Merge (OR) `flagsAttestation` (new flags) with `flags` (current flags) - epochParticipation.set(index, flags | flagsAttestation); + // ParticipationFlags type uses option {setBitwiseOR: true}, .set() does a |= operation + epochParticipation.set(index, attestationFlags); } } } diff --git a/packages/beacon-state-transition/src/bellatrix/block/index.ts b/packages/beacon-state-transition/src/bellatrix/block/index.ts index bdb21ff47ba..8c7e3d720c0 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/index.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/index.ts @@ -1,6 +1,6 @@ import {bellatrix} from "@chainsafe/lodestar-types"; -import {CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateBellatrix} from "../../types"; import {processBlockHeader, processEth1Data, processRandao} from "../../allForks/block"; import {processOperations} from "./processOperations"; import {processSyncAggregate} from "../../altair/block/processSyncCommittee"; @@ -18,15 +18,15 @@ export function processBlock( verifySignatures = true, executionEngine: ExecutionEngine | null ): void { - processBlockHeader(state as CachedBeaconStateAllForks, block); + processBlockHeader(state, block); // The call to the process_execution_payload must happen before the call to the process_randao as the former depends // on the randao_mix computed with the reveal of the previous block. if (isExecutionEnabled(state, block.body)) { processExecutionPayload(state, block.body.executionPayload, executionEngine); } - processRandao(state as CachedBeaconStateAllForks, block, verifySignatures); - processEth1Data(state as CachedBeaconStateAllForks, block.body); + processRandao(state, block, verifySignatures); + processEth1Data(state, block.body.eth1Data); processOperations(state, block.body, verifySignatures); - processSyncAggregate((state as unknown) as CachedBeaconStateAltair, block, verifySignatures); + processSyncAggregate(state, block, verifySignatures); } diff --git a/packages/beacon-state-transition/src/bellatrix/block/processAttesterSlashing.ts b/packages/beacon-state-transition/src/bellatrix/block/processAttesterSlashing.ts index 7a04b955ba0..bb6a0b3a7b2 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processAttesterSlashing.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processAttesterSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateBellatrix, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateBellatrix} from "../../types"; import {processAttesterSlashing as processAttesterSlashingAllForks} from "../../allForks/block"; export function processAttesterSlashing( @@ -8,10 +8,5 @@ export function processAttesterSlashing( attesterSlashing: phase0.AttesterSlashing, verifySignatures = true ): void { - processAttesterSlashingAllForks( - ForkName.bellatrix, - state as CachedBeaconStateAllForks, - attesterSlashing, - verifySignatures - ); + processAttesterSlashingAllForks(ForkName.bellatrix, state, attesterSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts b/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts index 8d2e251ab4b..40c0cdfa841 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processExecutionPayload.ts @@ -1,5 +1,5 @@ import {bellatrix, ssz} from "@chainsafe/lodestar-types"; -import {byteArrayEquals, List, toHexString} from "@chainsafe/ssz"; +import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; import {CachedBeaconStateBellatrix} from "../../types"; import {getRandaoMix} from "../../util"; import {ExecutionEngine} from "../executionEngine"; @@ -24,7 +24,7 @@ export function processExecutionPayload( } // Verify random - const expectedRandom = getRandaoMix(state, state.currentShuffling.epoch); + const expectedRandom = getRandaoMix(state, state.epochCtx.epoch); if (!byteArrayEquals(payload.prevRandao as Uint8Array, expectedRandom as Uint8Array)) { throw Error( `Invalid execution payload random ${toHexString(payload.prevRandao)} expected=${toHexString(expectedRandom)}` @@ -51,7 +51,7 @@ export function processExecutionPayload( } // Cache execution payload header - state.latestExecutionPayloadHeader = { + state.latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.toViewDU({ parentHash: payload.parentHash, feeRecipient: payload.feeRecipient, stateRoot: payload.stateRoot, @@ -65,6 +65,6 @@ export function processExecutionPayload( extraData: payload.extraData, baseFeePerGas: payload.baseFeePerGas, blockHash: payload.blockHash, - transactionsRoot: ssz.bellatrix.Transactions.hashTreeRoot(payload.transactions as List), - }; + transactionsRoot: ssz.bellatrix.Transactions.hashTreeRoot(payload.transactions), + }); } diff --git a/packages/beacon-state-transition/src/bellatrix/block/processOperations.ts b/packages/beacon-state-transition/src/bellatrix/block/processOperations.ts index 53fbc8cbd95..922dee0e63a 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processOperations.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processOperations.ts @@ -1,13 +1,12 @@ -import {readonlyValues} from "@chainsafe/ssz"; import {bellatrix} from "@chainsafe/lodestar-types"; +import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateBellatrix, CachedBeaconStateAltair, CachedBeaconStateAllForks} from "../../types"; import {processProposerSlashing} from "./processProposerSlashing"; import {processAttesterSlashing} from "./processAttesterSlashing"; import {processAttestations} from "../../altair/block/processAttestation"; import {processDeposit} from "../../altair/block/processDeposit"; import {processVoluntaryExit} from "../../altair/block/processVoluntaryExit"; -import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; export function processOperations( state: CachedBeaconStateBellatrix, @@ -22,23 +21,23 @@ export function processOperations( ); } - for (const proposerSlashing of readonlyValues(body.proposerSlashings)) { + for (const proposerSlashing of body.proposerSlashings) { processProposerSlashing(state, proposerSlashing, verifySignatures); } - for (const attesterSlashing of readonlyValues(body.attesterSlashings)) { + for (const attesterSlashing of body.attesterSlashings) { processAttesterSlashing(state, attesterSlashing, verifySignatures); } processAttestations( (state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, - Array.from(readonlyValues(body.attestations)), + body.attestations, verifySignatures ); - for (const deposit of readonlyValues(body.deposits)) { + for (const deposit of body.deposits) { processDeposit((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, deposit); } - for (const voluntaryExit of readonlyValues(body.voluntaryExits)) { + for (const voluntaryExit of body.voluntaryExits) { processVoluntaryExit( (state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, voluntaryExit, diff --git a/packages/beacon-state-transition/src/bellatrix/block/processProposerSlashing.ts b/packages/beacon-state-transition/src/bellatrix/block/processProposerSlashing.ts index 8c1aa07cd57..d652d3d83a9 100644 --- a/packages/beacon-state-transition/src/bellatrix/block/processProposerSlashing.ts +++ b/packages/beacon-state-transition/src/bellatrix/block/processProposerSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStateBellatrix, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStateBellatrix} from "../../types"; import {processProposerSlashing as processProposerSlashingAllForks} from "../../allForks/block"; export function processProposerSlashing( @@ -8,10 +8,5 @@ export function processProposerSlashing( proposerSlashing: phase0.ProposerSlashing, verifySignatures = true ): void { - processProposerSlashingAllForks( - ForkName.bellatrix, - state as CachedBeaconStateAllForks, - proposerSlashing, - verifySignatures - ); + processProposerSlashingAllForks(ForkName.bellatrix, state, proposerSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/bellatrix/upgradeState.ts b/packages/beacon-state-transition/src/bellatrix/upgradeState.ts index b717c0b5153..e1703ef3e1e 100644 --- a/packages/beacon-state-transition/src/bellatrix/upgradeState.ts +++ b/packages/beacon-state-transition/src/bellatrix/upgradeState.ts @@ -1,34 +1,63 @@ -import {bellatrix, ssz} from "@chainsafe/lodestar-types"; -import {createCachedBeaconState} from "../cache/cachedBeaconState"; +import {ssz} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAltair, CachedBeaconStateBellatrix} from "../types"; -import {TreeBacked} from "@chainsafe/ssz"; -import {IBeaconConfig} from "@chainsafe/lodestar-config"; +import {getCachedBeaconState} from "../cache/stateCache"; /** * Upgrade a state from altair to bellatrix. */ -export function upgradeState(state: CachedBeaconStateAltair): CachedBeaconStateBellatrix { - const {config} = state; - const postTreeBackedState = upgradeTreeBackedState(config, state); - // TODO: This seems very sub-optimal, review - return createCachedBeaconState(config, postTreeBackedState); -} +export function upgradeState(stateAltair: CachedBeaconStateAltair): CachedBeaconStateBellatrix { + const {config} = stateAltair; + + // Get underlying node and cast altair tree to bellatrix tree + // + // An altair BeaconState tree can be safely casted to a bellatrix BeaconState tree because: + // - All new fields are appended at the end + // + // altair | op | altair + // ----------------------------- | --- | ------------ + // genesis_time | - | genesis_time + // genesis_validators_root | - | genesis_validators_root + // slot | - | slot + // fork | - | fork + // latest_block_header | - | latest_block_header + // block_roots | - | block_roots + // state_roots | - | state_roots + // historical_roots | - | historical_roots + // eth1_data | - | eth1_data + // eth1_data_votes | - | eth1_data_votes + // eth1_deposit_index | - | eth1_deposit_index + // validators | - | validators + // balances | - | balances + // randao_mixes | - | randao_mixes + // slashings | - | slashings + // previous_epoch_participation | - | previous_epoch_participation + // current_epoch_participation | - | current_epoch_participation + // justification_bits | - | justification_bits + // previous_justified_checkpoint | - | previous_justified_checkpoint + // current_justified_checkpoint | - | current_justified_checkpoint + // finalized_checkpoint | - | finalized_checkpoint + // inactivity_scores | - | inactivity_scores + // current_sync_committee | - | current_sync_committee + // next_sync_committee | - | next_sync_committee + // - | new | latest_execution_payload_header -function upgradeTreeBackedState( - config: IBeaconConfig, - state: CachedBeaconStateAltair -): TreeBacked { - const stateTB = ssz.phase0.BeaconState.createTreeBacked(state.tree); + const stateAltairNode = ssz.altair.BeaconState.commitViewDU(stateAltair); + const stateBellatrixView = ssz.bellatrix.BeaconState.getViewDU(stateAltairNode); + // Attach existing BeaconStateCache from stateAltair to new stateBellatrixView object + const stateBellatrix = getCachedBeaconState(stateBellatrixView, stateAltair); - // TODO: Does this preserve the hashing cache? In altair devnets memory spikes on the fork transition - const postState = ssz.bellatrix.BeaconState.createTreeBacked(stateTB.tree); - postState.fork = { - previousVersion: stateTB.fork.currentVersion, + stateBellatrix.fork = ssz.phase0.Fork.toViewDU({ + previousVersion: stateAltair.fork.currentVersion, currentVersion: config.BELLATRIX_FORK_VERSION, - epoch: state.currentShuffling.epoch, - }; + epoch: stateAltair.epochCtx.epoch, + }); + // Execution-layer - postState.latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultTreeBacked(); + stateBellatrix.latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultViewDU(); + + // Commit new added fields ViewDU to the root node + stateBellatrix.commit(); + // No need to clear cache since no index is replaced, only appended at the end - return postState; + return stateBellatrix; } diff --git a/packages/beacon-state-transition/src/bellatrix/utils.ts b/packages/beacon-state-transition/src/bellatrix/utils.ts index 529bffd22b0..6ad4e053329 100644 --- a/packages/beacon-state-transition/src/bellatrix/utils.ts +++ b/packages/beacon-state-transition/src/bellatrix/utils.ts @@ -1,10 +1,16 @@ import {allForks, bellatrix, ssz} from "@chainsafe/lodestar-types"; +import { + BeaconStateBellatrix, + BeaconStateAllForks, + CachedBeaconStateBellatrix, + CachedBeaconStateAllForks, +} from "../types"; /** * Execution enabled = merge is done. * When (A) state has execution data OR (B) block has execution data */ -export function isExecutionEnabled(state: bellatrix.BeaconState, body: bellatrix.BeaconBlockBody): boolean { +export function isExecutionEnabled(state: BeaconStateBellatrix, body: bellatrix.BeaconBlockBody): boolean { return ( isMergeTransitionComplete(state) || !ssz.bellatrix.ExecutionPayload.equals(body.executionPayload, ssz.bellatrix.ExecutionPayload.defaultValue()) @@ -15,7 +21,7 @@ export function isExecutionEnabled(state: bellatrix.BeaconState, body: bellatrix * Merge block is the SINGLE block that transitions from POW to POS. * state has no execution data AND this block has execution data */ -export function isMergeTransitionBlock(state: bellatrix.BeaconState, body: bellatrix.BeaconBlockBody): boolean { +export function isMergeTransitionBlock(state: BeaconStateBellatrix, body: bellatrix.BeaconBlockBody): boolean { return ( !isMergeTransitionComplete(state) && !ssz.bellatrix.ExecutionPayload.equals(body.executionPayload, ssz.bellatrix.ExecutionPayload.defaultValue()) @@ -26,16 +32,22 @@ export function isMergeTransitionBlock(state: bellatrix.BeaconState, body: bella * Merge is complete when the state includes execution layer data: * state.latestExecutionPayloadHeader NOT EMPTY */ -export function isMergeTransitionComplete(state: bellatrix.BeaconState): boolean { +export function isMergeTransitionComplete(state: BeaconStateBellatrix): boolean { return !ssz.bellatrix.ExecutionPayloadHeader.equals( state.latestExecutionPayloadHeader, - ssz.bellatrix.ExecutionPayloadHeader.defaultTreeBacked() + // TODO: Performance + ssz.bellatrix.ExecutionPayloadHeader.defaultValue() ); } /** Type guard for bellatrix.BeaconState */ -export function isBellatrixStateType(state: allForks.BeaconState): state is bellatrix.BeaconState { - return (state as bellatrix.BeaconState).latestExecutionPayloadHeader !== undefined; +export function isBellatrixStateType(state: BeaconStateAllForks): state is BeaconStateBellatrix { + return (state as BeaconStateBellatrix).latestExecutionPayloadHeader !== undefined; +} + +/** Type guard for bellatrix.CachedBeaconState */ +export function isBellatrixCachedStateType(state: CachedBeaconStateAllForks): state is CachedBeaconStateBellatrix { + return (state as CachedBeaconStateBellatrix).latestExecutionPayloadHeader !== undefined; } /** Type guard for bellatrix.BeaconBlockBody */ diff --git a/packages/beacon-state-transition/src/cache/balanceList.ts b/packages/beacon-state-transition/src/cache/balanceList.ts deleted file mode 100644 index 41270266df3..00000000000 --- a/packages/beacon-state-transition/src/cache/balanceList.ts +++ /dev/null @@ -1,67 +0,0 @@ -import {Number64ListType} from "@chainsafe/ssz"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; - -/** - * Manage balances of BeaconState, use this instead of state.balances - */ -export class BalanceList { - tree: Tree; - type: Number64ListType; - - constructor(type: Number64ListType, tree: Tree) { - this.type = type; - this.tree = tree; - } - - get length(): number { - return this.type.tree_getLength(this.tree); - } - - get(index: number): number | undefined { - return this.type.tree_getProperty(this.tree, index) as number | undefined; - } - - set(index: number, value: number): void { - this.type.tree_setProperty(this.tree, index, value); - } - - applyDelta(index: number, delta: number): number { - return this.type.tree_applyDeltaAtIndex(this.tree, index, delta); - } - - applyDeltaInBatch(deltaByIndex: Map): void { - this.type.tree_applyDeltaInBatch(this.tree, deltaByIndex); - } - - /** Return the new balances */ - updateAll(deltas: number[]): number[] { - const [newTree, newBalances] = this.type.tree_newTreeFromDeltas(this.tree, deltas); - this.tree.rootNode = newTree.rootNode; - this.type.tree_setLength(this.tree, newBalances.length); - return newBalances; - } - - push(value: number): number { - return this.type.tree_push(this.tree, value); - } - - pop(): number { - return this.type.tree_pop(this.tree); - } - - *[Symbol.iterator](): Iterator { - for (let i = 0; i < this.length; i++) { - yield this.get(i) as number; - } - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - find(fn: (value: number, index: number, list: this) => boolean): number | undefined { - return; - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - findIndex(fn: (value: number, index: number, list: this) => boolean): number { - return -1; - } -} diff --git a/packages/beacon-state-transition/src/cache/cachedBeaconState.ts b/packages/beacon-state-transition/src/cache/cachedBeaconState.ts deleted file mode 100644 index 8daea5ab50d..00000000000 --- a/packages/beacon-state-transition/src/cache/cachedBeaconState.ts +++ /dev/null @@ -1,355 +0,0 @@ -import { - BasicListType, - CompositeValue, - ContainerType, - isCompositeType, - isTreeBacked, - ITreeBacked, - List, - Number64ListType, - readonlyValues, - TreeBacked, -} from "@chainsafe/ssz"; -import {allForks, altair, Number64, ParticipationFlags} from "@chainsafe/lodestar-types"; -import {createIBeaconConfig, IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; -import {MutableVector} from "@chainsafe/persistent-ts"; -import {EpochContext, EpochContextOpts} from "./epochContext"; -import {BalanceList} from "./balanceList"; -import {CachedEpochParticipation, CachedEpochParticipationProxyHandler} from "./cachedEpochParticipation"; -import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedInactivityScoreList, CachedInactivityScoreListProxyHandler} from "./cachedInactivityScoreList"; -import {newFilledArray} from "../util/array"; - -/** - * `BeaconState` with various caches - * - * Currently contains the following: - * - The full list of network params, ssz types, and fork schedule - * - The ssz type for the state - * - The full merkle tree representation of the state - * - A cache of shufflings, committees, proposers, expanded pubkeys - * - A flat copy of validators (for fast access/iteration) - * - * ### BeaconState data representation tradeoffs: - * - * Requirements of a BeaconState: - * - Block processing and epoch processing be performant to not block the node. This functions requires to iterate over - * very large arrays fast, while doing random mutations or big mutations. After them the state must be hashed. - * Processing times: (ideal / current / maximum) - * - block processing: 20ms / 200ms / 500ms - * - epoch processing: 200ms / 2s / 4s - * - * - BeaconState must be memory efficient. Data should only be represented once in a succint manner. Uint8Arrays are - * expensive, native types are not. - * - BeaconState must be hashed efficiently. Data must be merkelized before hashing so the conversion to merkelized - * must be fast or be done already. It must must persist a hashing cache that should be structurally shared between - * states for memory efficiency. - * - BeaconState raw data changes sparsingly, so it should be structurally shared between states for memory efficiency - * - * Summary of goals: - * - Structurally share data + hashing cache - * - Very fast read and iteration over large arrays - * - Fast bulk writes and somewhat fast single writes - * - Fast merkelization of data for hashing - * - * #### state.validators - * - * 91% of all memory merkelized is state.validators. In normal network conditions state.validators changes rarely. - * However for epoch processing the entire array must be iterated and read. So we need fast reads and slow writes. - * Tradeoffs to achieve that: - * - Represent leaf data with native JS types (deserialized form) - * - Use a single Tree for structurally sharing leaf data + hashing cache - * - Keep only the root cached on leaf nodes - * - Micro-optimizations (TODO): - * - Keep also the root of the node above pubkey and withdrawal creds. Will never change - * - Keep pubkey + withdrawal creds in the same Uint8Array - * - Have a global pubkey + withdrawal creds Uint8Array global cache, like with the index2pubkey cache - */ -export type CachedBeaconState = - // Wrapper object ({validators, clone()}) - BeaconStateContext & - // Epoch Cache - EpochContext & - // SSZ ops - ITreeBacked & - // Beacon State interface - T; - -export function createCachedBeaconState( - chainForkConfig: IChainForkConfig, - state: TreeBacked, - opts?: EpochContextOpts -): CachedBeaconState { - const config = createIBeaconConfig(chainForkConfig, state.genesisValidatorsRoot); - - let cachedPreviousParticipation, cachedCurrentParticipation; - const forkName = config.getForkName(state.slot); - const epochCtx = EpochContext.createFromState(config, state, opts); - let cachedInactivityScores: MutableVector; - if (forkName === ForkName.phase0) { - // TODO: More efficient way of getting the length? - const validatorCount = state.validators.length; - // Can these arrays be zero-ed for phase0? Are they actually used? - cachedPreviousParticipation = MutableVector.from(newFilledArray(validatorCount, 0)); - cachedCurrentParticipation = MutableVector.from(newFilledArray(validatorCount, 0)); - cachedInactivityScores = MutableVector.empty(); - } else { - const altairState = (state as unknown) as TreeBacked; - cachedPreviousParticipation = MutableVector.from( - Array.from(readonlyValues(altairState.previousEpochParticipation)) - ); - cachedCurrentParticipation = MutableVector.from(Array.from(readonlyValues(altairState.currentEpochParticipation))); - cachedInactivityScores = MutableVector.from(readonlyValues(altairState.inactivityScores)); - } - return new Proxy( - new BeaconStateContext( - state.type as ContainerType, - state.tree, - cachedPreviousParticipation, - cachedCurrentParticipation, - cachedInactivityScores, - epochCtx - ), - (CachedBeaconStateProxyHandler as unknown) as ProxyHandler> - ) as CachedBeaconState; -} - -/** - * Cache useful data associated to a specific state. - * Optimize processing speed of block processing + gossip validation while having a low memory cost. - * - * Previously BeaconStateContext included: - * ```ts - * validators: CachedValidatorList & T["validators"]; - * balances: CachedBalanceList & T["balances"]; - * inactivityScores: CachedInactivityScoreList & List; - * ``` - * - * Those caches where removed since they are no strictly necessary to make the epoch transition faster, - * but have a high memory cost. Note that all data was duplicated between the Tree and MutableVector. - * 1. TreeBacked, for efficient hashing - * 2. MutableVector (persistent-ts) with StructBacked validator objects for fast accessing and iteration - * - * ### validators - * state.validators is the heaviest data structure in the state. As TreeBacked, the leafs account for 91% with - * 200_000 validators. It requires ~ 2_000_000 Uint8Array instances with total memory of ~ 400MB. - * However its contents don't change very often. Validators only change when; - * - they first deposit - * - they dip from 32 effective balance to 31 (pretty much only when inactive for very long, or slashed) - * - they activate (once) - * - they exit (once) - * - they get slashed (max once) - * - * ### balances - * The balances array completely changes at the epoch boundary, where almost all the validator balances - * are updated. However it may have tiny changes during block processing if: - * - On a valid deposit - * - Validator gets slashed - * - On altair, the block proposer. Optimized to only happen once per block - * - * ### inactivityScores - * inactivityScores can be changed only: - * - At the epoch transition. It only changes when a validator is offline. So it may change a bit but not - * a lot on normal network conditions. - * - During block processing, when a validator joins a new 0 entry is pushed - * - * RESULT: Don't keep a duplicated structure around always. During block processing just push to the tree. During - * epoch processing some temporary flat structures are computed but dropped after processing the epoch. - */ -export class BeaconStateContext { - config: IBeaconConfig; - /** - * Epoch cache: Caches constant data through the epoch: @see EpochContext - * - Proposer indexes: 32 x Number - * - Shufflings: 3 x $VALIDATOR_COUNT x Number - */ - epochCtx: EpochContext; - /** The BeaconState ssz type */ - type: ContainerType; - /** The original BeaconState as a Tree */ - tree: Tree; - /** - * Returns a BalanceList instance with some convenient methods to work with Tree more efficiently. - * Notice that we want to work with state.balanceList instead of state.balances. - * - * The balances array completely changes at the epoch boundary, where almost all the validator balances - * are updated. However it may have tiny changes during block processing if: - * - On a valid deposit - * - Validator gets slashed? - * - On altair, the block proposer - * - */ - balanceList: BalanceList; - /** - * Returns a Proxy to CachedEpochParticipation - * - * Stores state.previousEpochParticipation in two duplicated forms (both structures are structurally shared): - * 1. TreeBacked, for efficient hashing - * 2. MutableVector (persistent-ts) with each validator participation flags (uint8) in object form - * - * epochParticipation changes continuously through the epoch for each partipation bit of each valid attestation in the state. - * The entire structure is dropped after two epochs. - * - * TODO: Consider representing participation as a uint8 always, and have a fast transformation fuction with precomputed values. - * Here using a Uint8Array is probably the most efficient way of representing this structure. Then we only need a way to get - * and set the values fast to the tree. Maybe batching? - */ - previousEpochParticipation: CachedEpochParticipation & List; - /** Same as previousEpochParticipation */ - currentEpochParticipation: CachedEpochParticipation & List; - /** - * Returns a Proxy to CachedInactivityScoreList - * - * Stores state.inactivityScores in two duplicated forms (both structures are structurally shared): - * 1. TreeBacked, for efficient hashing - * 2. MutableVector (persistent-ts) with a uint64 for each validator - * - * inactivityScores can be changed only: - * - At the epoch transition. It only changes when a validator is offline. So it may change a bit but not - * a lot on normal network conditions. - * - During block processing, when a validator joins a new 0 entry is pushed - * - * TODO: Don't keep a duplicated structure around always. During block processing just push to the tree, - * and maybe batch the changes. Then on process_inactivity_updates() compute the total deltas, and depending - * on the number of changes convert tree to array, apply diff, write to tree again. Or if there are just a few - * changes update the tree directly. - */ - inactivityScores: CachedInactivityScoreList & List; - - constructor( - type: ContainerType, - tree: Tree, - previousEpochParticipationCache: MutableVector, - currentEpochParticipationCache: MutableVector, - inactivityScoresCache: MutableVector, - epochCtx: EpochContext - ) { - this.config = epochCtx.config; - this.type = type; - this.tree = tree; - this.epochCtx = epochCtx; - this.balanceList = new BalanceList( - this.type.fields["balances"] as Number64ListType, - this.type.tree_getProperty(this.tree, "balances") as Tree - ); - this.previousEpochParticipation = (new Proxy( - new CachedEpochParticipation({ - type: this.type.fields["previousEpochParticipation"] as BasicListType>, - tree: this.type.tree_getProperty(this.tree, "previousEpochParticipation") as Tree, - persistent: previousEpochParticipationCache, - }), - CachedEpochParticipationProxyHandler - ) as unknown) as CachedEpochParticipation & List; - this.currentEpochParticipation = (new Proxy( - new CachedEpochParticipation({ - type: this.type.fields["currentEpochParticipation"] as BasicListType>, - tree: this.type.tree_getProperty(this.tree, "currentEpochParticipation") as Tree, - persistent: currentEpochParticipationCache, - }), - CachedEpochParticipationProxyHandler - ) as unknown) as CachedEpochParticipation & List; - this.inactivityScores = (new Proxy( - new CachedInactivityScoreList( - this.type.fields["inactivityScores"] as BasicListType>, - this.type.tree_getProperty(this.tree, "inactivityScores") as Tree, - inactivityScoresCache - ), - CachedInactivityScoreListProxyHandler - ) as unknown) as CachedInactivityScoreList & List; - } - - clone(): CachedBeaconState { - return new Proxy( - new BeaconStateContext( - this.type, - this.tree.clone(), - this.previousEpochParticipation.persistent.clone(), - this.currentEpochParticipation.persistent.clone(), - this.inactivityScores.persistent.clone(), - this.epochCtx.copy() - ), - (CachedBeaconStateProxyHandler as unknown) as ProxyHandler> - ) as CachedBeaconState; - } - - /** - * Toggle all `MutableVector` caches to use `TransientVector` - */ - setStateCachesAsTransient(): void { - this.previousEpochParticipation.persistent.asTransient(); - this.currentEpochParticipation.persistent.asTransient(); - this.inactivityScores.persistent.asTransient(); - } - - /** - * Toggle all `MutableVector` caches to use `PersistentVector` - */ - setStateCachesAsPersistent(): void { - this.previousEpochParticipation.persistent.asPersistent(); - this.currentEpochParticipation.persistent.asPersistent(); - this.inactivityScores.persistent.asPersistent(); - } -} - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const CachedBeaconStateProxyHandler: ProxyHandler> = { - get(target: CachedBeaconState, key: string): unknown { - if (key === "balanceList") { - return target.balanceList; - } else if (key === "previousEpochParticipation") { - return target.previousEpochParticipation; - } else if (key === "currentEpochParticipation") { - return target.currentEpochParticipation; - } else if (key === "inactivityScores") { - return target.inactivityScores; - } else if (target.type.fields[key] !== undefined) { - const propType = target.type.fields[key]; - const propValue = target.type.tree_getProperty(target.tree, key); - if (!isCompositeType(propType)) { - return propValue; - } else { - return propType.createTreeBacked(propValue as Tree); - } - } else if (key in target.epochCtx) { - return target.epochCtx[key as keyof EpochContext]; - } else if (key in target) { - return target[key as keyof CachedBeaconState]; - } else { - const treeBacked = target.type.createTreeBacked(target.tree); - if (key in treeBacked) { - return treeBacked[key as keyof TreeBacked]; - } - } - return undefined; - }, - set(target: CachedBeaconState, key: string, value: unknown): boolean { - if (key === "validators") { - throw new Error("Cannot set validators"); - } else if (key === "balanceList" || key === "balances") { - throw new Error("Cannot set either balanceList or balances"); - } else if (key === "previousEpochParticipation") { - throw new Error("Cannot set previousEpochParticipation"); - } else if (key === "currentEpochParticipation") { - throw new Error("Cannot set currentEpochParticipation"); - } else if (key === "inactivityScores") { - throw new Error("Cannot set inactivityScores"); - } else if (target.type.fields[key] !== undefined) { - const propType = target.type.fields[key]; - if (!isCompositeType(propType)) { - return target.type.tree_setProperty(target.tree, key, value); - } else { - if (isTreeBacked(value)) { - return target.type.tree_setProperty(target.tree, key, value.tree); - } else { - return target.type.tree_setProperty( - target.tree, - key, - propType.struct_convertToTree((value as unknown) as CompositeValue) - ); - } - } - } - return false; - }, -}; diff --git a/packages/beacon-state-transition/src/cache/cachedEpochParticipation.ts b/packages/beacon-state-transition/src/cache/cachedEpochParticipation.ts deleted file mode 100644 index 4126d6643c9..00000000000 --- a/packages/beacon-state-transition/src/cache/cachedEpochParticipation.ts +++ /dev/null @@ -1,119 +0,0 @@ -import {BasicListType, List, TreeBacked} from "@chainsafe/ssz"; -import {ParticipationFlags, Uint8} from "@chainsafe/lodestar-types"; -import {MutableVector, PersistentVector, TransientVector} from "@chainsafe/persistent-ts"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; -import {unsafeUint8ArrayToTree} from "../util/unsafeUint8ArrayToTree"; - -interface ICachedEpochParticipationOpts { - type?: BasicListType>; - tree?: Tree; - persistent: MutableVector; -} - -export class CachedEpochParticipation implements List { - [index: number]: ParticipationFlags; - type?: BasicListType>; - tree?: Tree; - persistent: MutableVector; - - constructor(opts: ICachedEpochParticipationOpts) { - this.type = opts.type; - this.tree = opts.tree; - this.persistent = opts.persistent; - } - - get length(): number { - return this.persistent.length; - } - - get(index: number): ParticipationFlags | undefined { - return this.persistent.get(index) ?? undefined; - } - - set(index: number, value: ParticipationFlags): void { - this.persistent.set(index, value); - if (this.type && this.tree) this.type.tree_setProperty(this.tree, index, value); - } - - updateAllStatus(data: PersistentVector | TransientVector): void { - this.persistent.vector = data; - - if (this.type && this.tree) { - const packedData = new Uint8Array(data.length); - data.forEach((d, i) => (packedData[i] = d)); - this.tree.rootNode = unsafeUint8ArrayToTree(packedData, this.type.getChunkDepth()); - this.type.tree_setLength(this.tree, data.length); - } - } - - push(value: ParticipationFlags): number { - this.persistent.push(value); - if (this.type && this.tree) this.type.tree_push(this.tree, value); - return this.persistent.length; - } - - pop(): ParticipationFlags { - const popped = this.persistent.pop(); - if (this.type && this.tree) this.type.tree_pop(this.tree); - if (popped === undefined) return (undefined as unknown) as ParticipationFlags; - return popped; - } - - *[Symbol.iterator](): Iterator { - for (const data of this.persistent) { - yield data; - } - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - find(fn: (value: ParticipationFlags, index: number, list: this) => boolean): ParticipationFlags | undefined { - return; - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - findIndex(fn: (value: ParticipationFlags, index: number, list: this) => boolean): number { - return -1; - } - - forEach(fn: (value: ParticipationFlags, index: number, list: this) => void): void { - this.persistent.forEach((value, index) => (fn as (value: ParticipationFlags, index: number) => void)(value, index)); - } - - map(fn: (value: ParticipationFlags, index: number) => T): T[] { - return this.persistent.map((value, index) => fn(value, index)); - } - - forEachStatus(fn: (value: ParticipationFlags, index: number, list: this) => void): void { - this.persistent.forEach(fn as (t: ParticipationFlags, i: number) => void); - } - - mapStatus(fn: (value: ParticipationFlags, index: number) => T): T[] { - return this.persistent.map((value, index) => fn(value, index)); - } -} - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const CachedEpochParticipationProxyHandler: ProxyHandler = { - get(target: CachedEpochParticipation, key: PropertyKey): unknown { - if (!Number.isNaN(Number(String(key)))) { - return target.get(key as number); - } else if (target[key as keyof CachedEpochParticipation] !== undefined) { - return target[key as keyof CachedEpochParticipation]; - } else { - if (target.type && target.tree) { - const treeBacked = target.type.createTreeBacked(target.tree); - if (key in treeBacked) { - return treeBacked[key as keyof TreeBacked>]; - } - } - return undefined; - } - }, - set(target: CachedEpochParticipation, key: PropertyKey, value: ParticipationFlags): boolean { - if (!Number.isNaN(Number(key))) { - target.set(key as number, value); - return true; - } - return false; - }, -}; diff --git a/packages/beacon-state-transition/src/cache/cachedInactivityScoreList.ts b/packages/beacon-state-transition/src/cache/cachedInactivityScoreList.ts deleted file mode 100644 index dbea96d767a..00000000000 --- a/packages/beacon-state-transition/src/cache/cachedInactivityScoreList.ts +++ /dev/null @@ -1,97 +0,0 @@ -import {BasicListType, List, TreeBacked} from "@chainsafe/ssz"; -import {Number64} from "@chainsafe/lodestar-types"; -import {Tree} from "@chainsafe/persistent-merkle-tree"; -import {MutableVector} from "@chainsafe/persistent-ts"; - -/** - * Inactivity score implementation that synchronizes changes between two underlying implementations: - * an immutable-js-style backing and a merkle tree backing - */ -export class CachedInactivityScoreList implements List { - [index: number]: Number64; - tree: Tree; - type: BasicListType>; - persistent: MutableVector; - - constructor(type: BasicListType>, tree: Tree, persistent: MutableVector) { - this.type = type; - this.tree = tree; - this.persistent = persistent; - } - - get length(): number { - return this.persistent.length; - } - - get(index: number): Number64 | undefined { - return this.persistent.get(index) ?? undefined; - } - - set(index: number, value: Number64): void { - this.persistent.set(index, value); - this.type.tree_setProperty(this.tree, index, value); - } - - setMultiple(newValues: Map): void { - // TODO: based on newValues.size to determine we build the tree from scratch or not - for (const [index, value] of newValues.entries()) { - this.set(index, value); - } - } - - push(value: Number64): number { - this.persistent.push(value); - return this.type.tree_push(this.tree, value); - } - - pop(): Number64 { - this.type.tree_pop(this.tree); - return this.persistent.pop() as Number64; - } - - *[Symbol.iterator](): Iterator { - yield* this.persistent[Symbol.iterator](); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - find(fn: (value: Number64, index: number, list: this) => boolean): Number64 | undefined { - return; - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - findIndex(fn: (value: Number64, index: number, list: this) => boolean): number { - return -1; - } - - forEach(fn: (value: Number64, index: number, list: this) => void): void { - this.persistent.forEach(fn as (value: Number64, index: number) => void); - } - - map(fn: (value: Number64, index: number) => T): T[] { - return this.persistent.map(fn); - } -} - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const CachedInactivityScoreListProxyHandler: ProxyHandler = { - get(target: CachedInactivityScoreList, key: PropertyKey): unknown { - if (!Number.isNaN(Number(String(key)))) { - return target.get(key as number); - } else if (target[key as keyof CachedInactivityScoreList] !== undefined) { - return target[key as keyof CachedInactivityScoreList]; - } else { - const treeBacked = target.type.createTreeBacked(target.tree); - if (key in treeBacked) { - return treeBacked[key as keyof TreeBacked>]; - } - return undefined; - } - }, - set(target: CachedInactivityScoreList, key: PropertyKey, value: Number64): boolean { - if (!Number.isNaN(Number(key))) { - target.set(key as number, value); - return true; - } - return false; - }, -}; diff --git a/packages/beacon-state-transition/src/cache/effectiveBalanceIncrements.ts b/packages/beacon-state-transition/src/cache/effectiveBalanceIncrements.ts index b14b2d4797f..63c6924397f 100644 --- a/packages/beacon-state-transition/src/cache/effectiveBalanceIncrements.ts +++ b/packages/beacon-state-transition/src/cache/effectiveBalanceIncrements.ts @@ -1,3 +1,6 @@ +import {EFFECTIVE_BALANCE_INCREMENT} from "@chainsafe/lodestar-params"; +import {BeaconStateAllForks} from "../types"; + /** * Alias to allow easier refactoring. * TODO: Estimate the risk of future proof of MAX_EFFECTIVE_BALANCE_INCREMENT < 255 @@ -21,3 +24,17 @@ export function getEffectiveBalanceIncrementsWithLen(validatorCount: number): Ef return new Uint8Array(byteLen); } + +/** + * Shows how EffectiveBalanceIncrements is meant to be populated. + * In practice this function should not be used, since it more efficient to loop the validators array once and do + * more tasks than only populating effectiveBalanceIncrements + */ +export function getEffectiveBalanceIncrements(state: BeaconStateAllForks): EffectiveBalanceIncrements { + const validatorsArr = state.validators.getAllReadonlyValues(); + const effectiveBalanceIncrements = new Uint8Array(validatorsArr.length); + for (let i = 0; i < validatorsArr.length; i++) { + effectiveBalanceIncrements[i] = Math.floor(validatorsArr[i].effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); + } + return effectiveBalanceIncrements; +} diff --git a/packages/beacon-state-transition/src/cache/epochContext.ts b/packages/beacon-state-transition/src/cache/epochContext.ts index d0d121fa55e..d5428cd90df 100644 --- a/packages/beacon-state-transition/src/cache/epochContext.ts +++ b/packages/beacon-state-transition/src/cache/epochContext.ts @@ -1,19 +1,8 @@ -import {BitList, List, readonlyValuesListOfLeafNodeStruct} from "@chainsafe/ssz"; import bls, {CoordType} from "@chainsafe/bls"; +import {BLSSignature, CommitteeIndex, Epoch, Slot, ValidatorIndex, phase0, SyncPeriod} from "@chainsafe/lodestar-types"; +import {createIBeaconConfig, IBeaconConfig, IChainConfig} from "@chainsafe/lodestar-config"; import { - BLSSignature, - CommitteeIndex, - Epoch, - Slot, - ValidatorIndex, - phase0, - allForks, - Number64, - altair, - SyncPeriod, -} from "@chainsafe/lodestar-types"; -import {IBeaconConfig} from "@chainsafe/lodestar-config"; -import { + ATTESTATION_SUBNET_COUNT, EFFECTIVE_BALANCE_INCREMENT, FAR_FUTURE_EPOCH, GENESIS_EPOCH, @@ -22,21 +11,20 @@ import { WEIGHT_DENOMINATOR, } from "@chainsafe/lodestar-params"; import {LodestarError} from "@chainsafe/lodestar-utils"; - import { computeActivationExitEpoch, computeEpochAtSlot, - computeProposers, computeStartSlotAtEpoch, getChurnLimit, isActiveValidator, isAggregatorFromCommitteeLength, - zipIndexesCommitteeBits, + computeProposers, computeSyncPeriodAtEpoch, } from "../util"; import {computeEpochShuffling, IEpochShuffling} from "../util/epochShuffling"; import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements"; import {Index2PubkeyCache, PubkeyIndexMap, syncPubkeys} from "./pubkeyCache"; +import {BeaconStateAllForks, BeaconStateAltair} from "./types"; import { computeSyncCommitteeCache, getSyncCommitteeCache, @@ -48,11 +36,15 @@ import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../ut /** `= PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)` */ export const PROPOSER_WEIGHT_FACTOR = PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT); +export type EpochContextImmutableData = { + config: IBeaconConfig; + pubkey2index: PubkeyIndexMap; + index2pubkey: Index2PubkeyCache; +}; + export type EpochContextOpts = { - pubkey2index?: PubkeyIndexMap; - index2pubkey?: Index2PubkeyCache; - skipSyncPubkeys?: boolean; skipSyncCommitteeCache?: boolean; + skipSyncPubkeys?: boolean; }; /** @@ -149,16 +141,9 @@ export class EpochContext { */ exitQueueChurn: number; - /** - * Returns a SyncCommitteeCache. (Note: phase0 has no sync committee, and returns an empty cache) - * - validatorIndices (of the committee members) - * - validatorIndexMap: Map of ValidatorIndex -> syncCommitteeIndexes - * - * The syncCommittee is immutable and changes as a whole every ~ 27h. - * It contains fixed 512 members so it's rather small. - */ + /** TODO: Indexed SyncCommitteeCache */ currentSyncCommitteeIndexed: SyncCommitteeCache; - /** Same as currentSyncCommitteeIndexed */ + /** TODO: Indexed SyncCommitteeCache */ nextSyncCommitteeIndexed: SyncCommitteeCache; // TODO: Helper stats @@ -213,9 +198,13 @@ export class EpochContext { * * SLOW CODE - 🐢 */ - static createFromState(config: IBeaconConfig, state: allForks.BeaconState, opts?: EpochContextOpts): EpochContext { - const pubkey2index = opts?.pubkey2index || new PubkeyIndexMap(); - const index2pubkey = opts?.index2pubkey || ([] as Index2PubkeyCache); + static createFromState( + state: BeaconStateAllForks, + {config, pubkey2index, index2pubkey}: EpochContextImmutableData, + opts?: EpochContextOpts + ): EpochContext { + // syncPubkeys here to ensure EpochContextImmutableData is popualted before computing the rest of caches + // - computeSyncCommitteeCache() needs a fully populated pubkey2index cache if (!opts?.skipSyncPubkeys) { syncPubkeys(state, pubkey2index, index2pubkey); } @@ -229,7 +218,7 @@ export class EpochContext { let exitQueueEpoch = computeActivationExitEpoch(currentEpoch); let exitQueueChurn = 0; - const validators = readonlyValuesListOfLeafNodeStruct(state.validators); + const validators = state.validators.getAllReadonlyValues(); const validatorCount = validators.length; const effectiveBalanceIncrements = getEffectiveBalanceIncrementsWithLen(validatorCount); @@ -297,7 +286,7 @@ export class EpochContext { let nextSyncCommitteeIndexed: SyncCommitteeCache; // Allow to skip populating sync committee for initializeBeaconStateFromEth1() if (afterAltairFork && !opts?.skipSyncCommitteeCache) { - const altairState = state as altair.BeaconState; + const altairState = state as BeaconStateAltair; currentSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.currentSyncCommittee, pubkey2index); nextSyncCommitteeIndexed = computeSyncCommitteeCache(altairState.nextSyncCommittee, pubkey2index); } else { @@ -337,7 +326,7 @@ export class EpochContext { syncParticipantReward, syncProposerReward, baseRewardPerIncrement, - totalActiveBalanceIncrements: totalActiveBalanceIncrements, + totalActiveBalanceIncrements, churnLimit, exitQueueEpoch, exitQueueChurn, @@ -351,7 +340,7 @@ export class EpochContext { /** * Copies a given EpochContext while avoiding copying its immutable parts. */ - copy(): EpochContext { + clone(): EpochContext { // warning: pubkey cache is not copied, it is shared, as eth1 is not expected to reorder validators. // Shallow copy all data from current epoch context to the next // All data is completely replaced, or only-appended @@ -388,7 +377,7 @@ export class EpochContext { * new epoch. */ afterProcessEpoch( - state: allForks.BeaconState, + state: BeaconStateAllForks, epochProcess: { nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; nextEpochTotalActiveBalanceByIncrement: number; @@ -398,6 +387,7 @@ export class EpochContext { this.currentShuffling = this.nextShuffling; const currEpoch = this.currentShuffling.epoch; const nextEpoch = currEpoch + 1; + this.nextShuffling = computeEpochShuffling(state, epochProcess.nextEpochShufflingActiveValidatorIndices, nextEpoch); this.proposers = computeProposers(state, this.currentShuffling, this.effectiveBalanceIncrements); @@ -467,6 +457,16 @@ export class EpochContext { return this.getShufflingAtEpoch(epoch).committeesPerSlot; } + /** + * Compute the correct subnet for a slot/committee index + */ + computeSubnetForSlot(slot: number, committeeIndex: number): number { + const slotsSinceEpochStart = slot % SLOTS_PER_EPOCH; + const committeesPerSlot = this.getCommitteeCountPerSlot(computeEpochAtSlot(slot)); + const committeesSinceEpochStart = committeesPerSlot * slotsSinceEpochStart; + return (committeesSinceEpochStart + committeeIndex) % ATTESTATION_SUBNET_COUNT; + } + getBeaconProposer(slot: Slot): ValidatorIndex { const epoch = computeEpochAtSlot(slot); if (epoch !== this.currentShuffling.epoch) { @@ -483,23 +483,17 @@ export class EpochContext { getIndexedAttestation(attestation: phase0.Attestation): phase0.IndexedAttestation { const {aggregationBits, data} = attestation; const committeeIndices = this.getBeaconCommittee(data.slot, data.index); - const attestingIndices = zipIndexesCommitteeBits(committeeIndices, aggregationBits); + const attestingIndices = aggregationBits.intersectValues(committeeIndices); // sort in-place attestingIndices.sort((a, b) => a - b); return { - attestingIndices: attestingIndices as List, + attestingIndices: attestingIndices, data: data, signature: attestation.signature, }; } - getAttestingIndices(data: phase0.AttestationData, bits: BitList): ValidatorIndex[] { - const committeeIndices = this.getBeaconCommittee(data.slot, data.index); - const validatorIndices = zipIndexesCommitteeBits(committeeIndices, bits); - return validatorIndices; - } - getCommitteeAssignments( epoch: Epoch, requestedValidatorIndices: ValidatorIndex[] @@ -554,7 +548,7 @@ export class EpochContext { const committee = this.getBeaconCommittee(slot, i); if (committee.includes(validatorIndex)) { return { - validators: committee as List, + validators: committee, committeeIndex: i, slot, }; @@ -591,18 +585,6 @@ export class EpochContext { } } - effectiveBalanceIncrementsSet(index: number, effectiveBalance: number): void { - if (index >= this.effectiveBalanceIncrements.length) { - // Clone and extend effectiveBalanceIncrements - const effectiveBalanceIncrements = this.effectiveBalanceIncrements; - // Note: getEffectiveBalanceIncrementsWithLen() returns a Uint8Array larger than `index + 1` to reduce copy-ing - this.effectiveBalanceIncrements = getEffectiveBalanceIncrementsWithLen(index + 1); - this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); - } - - this.effectiveBalanceIncrements[index] = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); - } - /** * Note: The range of slots a validator has to perform duties is off by one. * The previous slot wording means that if your validator is in a sync committee for a period that runs from slot @@ -634,15 +616,37 @@ export class EpochContext { this.currentSyncCommitteeIndexed = this.nextSyncCommitteeIndexed; this.nextSyncCommitteeIndexed = getSyncCommitteeCache(nextSyncCommitteeIndices); } + + /** On phase0 -> altair fork, set both current and nextSyncCommitteeIndexed */ + setSyncCommitteesIndexed(nextSyncCommitteeIndices: number[]): void { + this.nextSyncCommitteeIndexed = getSyncCommitteeCache(nextSyncCommitteeIndices); + this.currentSyncCommitteeIndexed = this.nextSyncCommitteeIndexed; + } + + effectiveBalanceIncrementsSet(index: number, effectiveBalance: number): void { + if (index >= this.effectiveBalanceIncrements.length) { + // Clone and extend effectiveBalanceIncrements + const effectiveBalanceIncrements = this.effectiveBalanceIncrements; + this.effectiveBalanceIncrements = new Uint8Array(getEffectiveBalanceIncrementsByteLen(index + 1)); + this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); + } + + this.effectiveBalanceIncrements[index] = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); + } +} + +function getEffectiveBalanceIncrementsByteLen(validatorCount: number): number { + // TODO: Research what's the best number to minimize both memory cost and copy costs + return 1024 * Math.ceil(validatorCount / 1024); } // Copied from lodestar-api package to avoid depending on the package type AttesterDuty = { validatorIndex: ValidatorIndex; committeeIndex: CommitteeIndex; - committeeLength: Number64; - committeesAtSlot: Number64; - validatorCommitteeIndex: Number64; + committeeLength: number; + committeesAtSlot: number; + validatorCommitteeIndex: number; slot: Slot; }; @@ -657,3 +661,15 @@ type EpochContextErrorType = { }; export class EpochContextError extends LodestarError {} + +export function createEmptyEpochContextImmutableData( + chainConfig: IChainConfig, + state: Pick +): EpochContextImmutableData { + return { + config: createIBeaconConfig(chainConfig, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }; +} diff --git a/packages/beacon-state-transition/src/cache/epochProcess.ts b/packages/beacon-state-transition/src/cache/epochProcess.ts index 9fbf06c3f4a..9f33ea7fc60 100644 --- a/packages/beacon-state-transition/src/cache/epochProcess.ts +++ b/packages/beacon-state-transition/src/cache/epochProcess.ts @@ -1,15 +1,12 @@ -import {Epoch, ValidatorIndex, allForks, phase0} from "@chainsafe/lodestar-types"; +import {Epoch, ValidatorIndex} from "@chainsafe/lodestar-types"; import {intDiv} from "@chainsafe/lodestar-utils"; import { - EFFECTIVE_BALANCE_INCREMENT, EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkName, MAX_EFFECTIVE_BALANCE, } from "@chainsafe/lodestar-params"; -import {readonlyValuesListOfLeafNodeStruct} from "@chainsafe/ssz"; -import {isActiveValidator, newZeroedArray} from "../util"; import { IAttesterStatus, createIAttesterStatus, @@ -23,9 +20,11 @@ import { FLAG_CURR_TARGET_ATTESTER, FLAG_CURR_HEAD_ATTESTER, } from "../util/attesterStatus"; -import {CachedBeaconState} from "../cache/cachedBeaconState"; import {statusProcessEpoch} from "../phase0/epoch/processPendingAttestations"; -import {computeBaseRewardPerIncrement} from "../util/syncCommittee"; +import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from ".."; +import {computeBaseRewardPerIncrement} from "../util/altair"; + +/* eslint-disable @typescript-eslint/naming-convention */ /** * EpochProcess is the parent object of: @@ -39,7 +38,7 @@ import {computeBaseRewardPerIncrement} from "../util/syncCommittee"; * - Only loop state.validators once for all `process_*` fns * - Only loop status array once */ -export type EpochProcess = { +export interface EpochProcess { prevEpoch: Epoch; currentEpoch: Epoch; /** @@ -54,6 +53,16 @@ export type EpochProcess = { headStakeByIncrement: number; }; currEpochUnslashedTargetStakeByIncrement: number; + + /** + * Validator indices that are either + * - active in previous epoch + * - slashed and not yet withdrawable + * + * getRewardsAndPenalties() and processInactivityUpdates() iterate this list + */ + eligibleValidatorIndices: ValidatorIndex[]; + /** * Indices which will receive the slashing penalty * ``` @@ -76,6 +85,7 @@ export type EpochProcess = { * that happen to be in the same committee, which is very unlikely. */ indicesToSlash: ValidatorIndex[]; + /** * Indices of validators that just joinned and will be eligible for the active queue. * ``` @@ -89,6 +99,7 @@ export type EpochProcess = { * For mainnet spec = 512 */ indicesEligibleForActivationQueue: ValidatorIndex[]; + /** * Indices of validators that may become active once churn and finaly allow. * ``` @@ -98,6 +109,7 @@ export type EpochProcess = { * For less than 327680 validators, churnLimit = 4 (minimum possible), so max processed is 4. */ indicesEligibleForActivation: ValidatorIndex[]; + /** * Indices of validators that will be ejected due to low balance. * ``` @@ -108,8 +120,23 @@ export type EpochProcess = { */ indicesToEject: ValidatorIndex[]; + /** + * Pre-computes status flags for faster checking of statuses during epoch transition. + * Spec requires some reward or penalty to apply to + * - eligible validators + * - un-slashed validators + * - prev attester flag set + * With a status flag to check this conditions at once we just have to mask with an OR of the conditions. + */ statuses: IAttesterStatus[]; + + /** + * balances array will be populated by processRewardsAndPenalties() and consumed by processEffectiveBalanceUpdates(). + * processRewardsAndPenalties() already has a regular Javascript array of balances. + * Then processEffectiveBalanceUpdates() needs to iterate all balances so it can re-use the array pre-computed previously. + */ balances?: number[]; + /** * Active validator indices for currentEpoch + 2. * This is only used in `afterProcessEpoch` to compute epoch shuffling, it's not efficient to calculate it at that time @@ -120,6 +147,7 @@ export type EpochProcess = { * | afterEpochProcess | read it | */ nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; + /** * Altair specific, this is total active balances for the next epoch. * This is only used in `afterProcessEpoch` to compute base reward and sync participant reward. @@ -138,19 +166,20 @@ export type EpochProcess = { * Used in `processEffectiveBalanceUpdates` to save one loop over validators after epoch process. */ isActiveNextEpoch: boolean[]; -}; +} -export function beforeProcessEpoch(state: CachedBeaconState): EpochProcess { +export function beforeProcessEpoch(state: CachedBeaconStateAllForks): EpochProcess { const {config, epochCtx} = state; const forkName = config.getForkName(state.slot); const currentEpoch = epochCtx.currentShuffling.epoch; const prevEpoch = epochCtx.previousShuffling.epoch; - // active validator indices for nextShuffling is ready, we want to precalculate for the one after that - const nextShufflingEpoch = currentEpoch + 2; const nextEpoch = currentEpoch + 1; + // active validator indices for nextShuffling is ready, we want to precalculate for the one after that + const nextEpoch2 = currentEpoch + 2; const slashingsEpoch = currentEpoch + intDiv(EPOCHS_PER_SLASHINGS_VECTOR, 2); + const eligibleValidatorIndices: ValidatorIndex[] = []; const indicesToSlash: ValidatorIndex[] = []; const indicesEligibleForActivationQueue: ValidatorIndex[] = []; const indicesEligibleForActivation: ValidatorIndex[] = []; @@ -165,13 +194,13 @@ export function beforeProcessEpoch(state: Cached // To optimize memory each validator node in `state.validators` is represented with a special node type // `BranchNodeStruct` that represents the data as struct internally. This utility grabs the struct data directrly // from the nodes without any extra transformation. The returned `validators` array contains native JS objects. - const validators = readonlyValuesListOfLeafNodeStruct(state.validators); + const validators = state.validators.getAllReadonlyValues(); const validatorCount = validators.length; // Clone before being mutated in processEffectiveBalanceUpdates epochCtx.beforeEpochTransition(); - const effectiveBalancesByIncrements = newZeroedArray(validators.length); + const effectiveBalancesByIncrements = epochCtx.effectiveBalanceIncrements; for (let i = 0; i < validatorCount; i++) { const validator = validators[i]; @@ -185,19 +214,25 @@ export function beforeProcessEpoch(state: Cached status.flags |= FLAG_UNSLASHED; } - const activePrev = isActiveValidator(validator, prevEpoch); - isActivePrevEpoch.push(activePrev); - if (activePrev || (validator.slashed && prevEpoch + 1 < validator.withdrawableEpoch)) { + const {activationEpoch, exitEpoch} = validator; + const isActivePrev = activationEpoch <= prevEpoch && prevEpoch < exitEpoch; + const isActiveCurr = activationEpoch <= currentEpoch && currentEpoch < exitEpoch; + const isActiveNext = activationEpoch <= nextEpoch && nextEpoch < exitEpoch; + const isActiveNext2 = activationEpoch <= nextEpoch2 && nextEpoch2 < exitEpoch; + + isActivePrevEpoch.push(isActivePrev); + + // Both active validators and slashed-but-not-yet-withdrawn validators are eligible to receive penalties. + // This is done to prevent self-slashing from being a way to escape inactivity leaks. + // TODO: Consider using an array of `eligibleValidatorIndices: number[]` + if (isActivePrev || (validator.slashed && prevEpoch + 1 < validator.withdrawableEpoch)) { + eligibleValidatorIndices.push(i); status.flags |= FLAG_ELIGIBLE_ATTESTER; } - const active = isActiveValidator(validator, currentEpoch); - // We track effectiveBalanceByIncrement as ETH to fit total network balance in a JS number (53 bits) - const effectiveBalanceByIncrement = Math.floor(validator.effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); - effectiveBalancesByIncrements[i] = effectiveBalanceByIncrement; - if (active) { + if (isActiveCurr) { status.active = true; - totalActiveStakeByIncrement += effectiveBalanceByIncrement; + totalActiveStakeByIncrement += effectiveBalancesByIncrements[i]; } // To optimize process_registry_updates(): @@ -249,9 +284,9 @@ export function beforeProcessEpoch(state: Cached statuses.push(status); - isActiveNextEpoch.push(isActiveValidator(validator, nextEpoch)); + isActiveNextEpoch.push(isActiveNext); - if (isActiveValidator(validator, nextShufflingEpoch)) { + if (isActiveNext2) { nextEpochShufflingActiveValidatorIndices.push(i); } } @@ -272,44 +307,44 @@ export function beforeProcessEpoch(state: Cached ); if (forkName === ForkName.phase0) { - const statePhase0 = (state as unknown) as CachedBeaconState; statusProcessEpoch( - statePhase0, + state as CachedBeaconStatePhase0, statuses, - statePhase0.previousEpochAttestations, + (state as CachedBeaconStatePhase0).previousEpochAttestations.getAllReadonly(), prevEpoch, FLAG_PREV_SOURCE_ATTESTER, FLAG_PREV_TARGET_ATTESTER, FLAG_PREV_HEAD_ATTESTER ); statusProcessEpoch( - statePhase0, + state as CachedBeaconStatePhase0, statuses, - statePhase0.currentEpochAttestations, + (state as CachedBeaconStatePhase0).currentEpochAttestations.getAllReadonly(), currentEpoch, FLAG_CURR_SOURCE_ATTESTER, FLAG_CURR_TARGET_ATTESTER, FLAG_CURR_HEAD_ATTESTER ); } else { - state.previousEpochParticipation.forEachStatus((participationFlags, i) => { + const previousEpochParticipation = (state as CachedBeaconStateAltair).previousEpochParticipation.getAll(); + for (let i = 0; i < previousEpochParticipation.length; i++) { const status = statuses[i]; // this is required to pass random spec tests in altair if (isActivePrevEpoch[i]) { - status.flags |= - // FLAG_PREV are indexes [0,1,2] - status.flags |= participationFlags; + // FLAG_PREV are indexes [0,1,2] + status.flags |= previousEpochParticipation[i]; } - }); - state.currentEpochParticipation.forEachStatus((participationFlags, i) => { + } + + const currentEpochParticipation = (state as CachedBeaconStateAltair).currentEpochParticipation.getAll(); + for (let i = 0; i < currentEpochParticipation.length; i++) { const status = statuses[i]; // this is required to pass random spec tests in altair if (status.active) { - status.flags |= - // FLAG_CURR are indexes [3,4,5], so shift by 3 - status.flags |= participationFlags << 3; + // FLAG_PREV are indexes [3,4,5], so shift by 3 + status.flags |= currentEpochParticipation[i] << 3; } - }); + } } let prevSourceUnslStake = 0; @@ -358,6 +393,7 @@ export function beforeProcessEpoch(state: Cached headStakeByIncrement: prevHeadUnslStake, }, currEpochUnslashedTargetStakeByIncrement: currTargetUnslStake, + eligibleValidatorIndices, indicesToSlash, indicesEligibleForActivationQueue, indicesEligibleForActivation, @@ -367,5 +403,8 @@ export function beforeProcessEpoch(state: Cached nextEpochTotalActiveBalanceByIncrement: 0, isActiveNextEpoch, statuses, + + // Will be assigned in processRewardsAndPenalties() + balances: undefined, }; } diff --git a/packages/beacon-state-transition/src/cache/pubkeyCache.ts b/packages/beacon-state-transition/src/cache/pubkeyCache.ts index ce623ae0b24..0bf64066bcb 100644 --- a/packages/beacon-state-transition/src/cache/pubkeyCache.ts +++ b/packages/beacon-state-transition/src/cache/pubkeyCache.ts @@ -1,6 +1,6 @@ import bls, {CoordType, PublicKey} from "@chainsafe/bls"; -import {allForks, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {ByteVector} from "@chainsafe/ssz"; +import {ValidatorIndex} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks} from "./types"; export type Index2PubkeyCache = PublicKey[]; @@ -14,7 +14,7 @@ type PubkeyHex = string; * * See https://github.com/ChainSafe/lodestar/issues/3446 */ -function toMemoryEfficientHexStr(hex: ByteVector | Uint8Array | string): string { +function toMemoryEfficientHexStr(hex: Uint8Array | string): string { if (typeof hex === "string") { if (hex.startsWith("0x")) { hex = hex.slice(2); @@ -36,7 +36,7 @@ export class PubkeyIndexMap { /** * Must support reading with string for API support where pubkeys are already strings */ - get(key: ByteVector | Uint8Array | PubkeyHex): ValidatorIndex | undefined { + get(key: Uint8Array | PubkeyHex): ValidatorIndex | undefined { return this.map.get(toMemoryEfficientHexStr(key)); } @@ -53,7 +53,7 @@ export class PubkeyIndexMap { * If pubkey caches are empty: SLOW CODE - 🐢 */ export function syncPubkeys( - state: allForks.BeaconState, + state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap, index2pubkey: Index2PubkeyCache ): void { @@ -66,7 +66,7 @@ export function syncPubkeys( const newCount = state.validators.length; for (let i = pubkey2index.size; i < newCount; i++) { - const pubkey = validators[i].pubkey.valueOf() as Uint8Array; + const pubkey = validators.getReadonly(i).pubkey; pubkey2index.set(pubkey, i); // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed. // Afterwards any public key is the state consider validated. diff --git a/packages/beacon-state-transition/src/cache/stateCache.ts b/packages/beacon-state-transition/src/cache/stateCache.ts new file mode 100644 index 00000000000..33c7ee52c4c --- /dev/null +++ b/packages/beacon-state-transition/src/cache/stateCache.ts @@ -0,0 +1,151 @@ +import {IBeaconConfig} from "@chainsafe/lodestar-config"; +import {EpochContext, EpochContextImmutableData, EpochContextOpts} from "./epochContext"; +import {BeaconStatePhase0, BeaconStateAltair, BeaconStateBellatrix, BeaconStateAllForks} from "./types"; + +export type BeaconStateCache = { + config: IBeaconConfig; + epochCtx: EpochContext; +}; + +/** + * `BeaconState` with various caches + * + * Currently contains the following: + * - The full list of network params, ssz types, and fork schedule + * - The ssz type for the state + * - The full merkle tree representation of the state + * - A cache of shufflings, committees, proposers, expanded pubkeys + * - A flat copy of validators (for fast access/iteration) + * + * ### BeaconState data representation tradeoffs: + * + * Requirements of a BeaconState: + * - Block processing and epoch processing be performant to not block the node. This functions requires to iterate over + * very large arrays fast, while doing random mutations or big mutations. After them the state must be hashed. + * Processing times: (ideal / current / maximum) + * - block processing: 20ms / 200ms / 500ms + * - epoch processing: 200ms / 2s / 4s + * + * - BeaconState must be memory efficient. Data should only be represented once in a succint manner. Uint8Arrays are + * expensive, native types are not. + * - BeaconState must be hashed efficiently. Data must be merkelized before hashing so the conversion to merkelized + * must be fast or be done already. It must must persist a hashing cache that should be structurally shared between + * states for memory efficiency. + * - BeaconState raw data changes sparsingly, so it should be structurally shared between states for memory efficiency + * + * Summary of goals: + * - Structurally share data + hashing cache + * - Very fast read and iteration over large arrays + * - Fast bulk writes and somewhat fast single writes + * - Fast merkelization of data for hashing + * + * #### state.validators + * + * 91% of all memory merkelized is state.validators. In normal network conditions state.validators changes rarely. + * However for epoch processing the entire array must be iterated and read. So we need fast reads and slow writes. + * Tradeoffs to achieve that: + * - Represent leaf data with native JS types (deserialized form) + * - Use a single Tree for structurally sharing leaf data + hashing cache + * - Keep only the root cached on leaf nodes + * - Micro-optimizations (TODO): + * - Keep also the root of the node above pubkey and withdrawal creds. Will never change + * - Keep pubkey + withdrawal creds in the same Uint8Array + * - Have a global pubkey + withdrawal creds Uint8Array global cache, like with the index2pubkey cache + * + * ------------------ + * + * _Previous JSDocs for `BeaconStateContext`_ + * + * Cache useful data associated to a specific state. + * Optimize processing speed of block processing + gossip validation while having a low memory cost. + * + * Previously BeaconStateContext included: + * ```ts + * validators: CachedValidatorList & T["validators"]; + * balances: CachedBalanceList & T["balances"]; + * inactivityScores: CachedInactivityScoreList & Number64[]; + * ``` + * + * Those caches where removed since they are no strictly necessary to make the epoch transition faster, + * but have a high memory cost. Note that all data was duplicated between the Tree and MutableVector. + * 1. TreeView, for efficient hashing + * 2. MutableVector (persistent-ts) with StructBacked validator objects for fast accessing and iteration + * + * ### validators + * state.validators is the heaviest data structure in the state. As TreeView, the leafs account for 91% with + * 200_000 validators. It requires ~ 2_000_000 Uint8Array instances with total memory of ~ 400MB. + * However its contents don't change very often. Validators only change when; + * - they first deposit + * - they dip from 32 effective balance to 31 (pretty much only when inactive for very long, or slashed) + * - they activate (once) + * - they exit (once) + * - they get slashed (max once) + * + * ### balances + * The balances array completely changes at the epoch boundary, where almost all the validator balances + * are updated. However it may have tiny changes during block processing if: + * - On a valid deposit + * - Validator gets slashed + * - On altair, the block proposer. Optimized to only happen once per block + * + * ### epochParticipation + * epochParticipation changes continuously through the epoch for each partipation bit of each valid attestation in the state. + * The entire structure is dropped after two epochs. + * + * ### inactivityScores + * inactivityScores can be changed only: + * - At the epoch transition. It only changes when a validator is offline. So it may change a bit but not + * a lot on normal network conditions. + * - During block processing, when a validator joins a new 0 entry is pushed + * + * RESULT: Don't keep a duplicated structure around always. During block processing just push to the tree. During + * epoch processing some temporary flat structures are computed but dropped after processing the epoch. + */ +export type CachedBeaconState = T & BeaconStateCache; + +export type CachedBeaconStatePhase0 = CachedBeaconState; +export type CachedBeaconStateAltair = CachedBeaconState; +export type CachedBeaconStateBellatrix = CachedBeaconState; +export type CachedBeaconStateAllForks = CachedBeaconState; + +/** + * Create CachedBeaconState computing a new EpochContext instance + */ +export function createCachedBeaconState( + state: T, + immutableData: EpochContextImmutableData, + opts?: EpochContextOpts +): T & BeaconStateCache { + return getCachedBeaconState(state, { + config: immutableData.config, + epochCtx: EpochContext.createFromState(state, immutableData, opts), + }); +} + +/** + * Attach an already computed BeaconStateCache to a BeaconState object + */ +export function getCachedBeaconState( + state: T, + cache: BeaconStateCache +): T & BeaconStateCache { + const cachedState = state as T & BeaconStateCache; + cachedState.config = cache.config; + cachedState.epochCtx = cache.epochCtx; + + // Overwrite .clone function to preserve cache + // TreeViewDU.clone() creates a new object that does not have the attached cache + const viewDUClone = cachedState.clone.bind(cachedState); + + function clone(this: T & BeaconStateCache): T & BeaconStateCache { + const viewDUCloned = viewDUClone(); + return getCachedBeaconState(viewDUCloned, { + config: this.config, + epochCtx: this.epochCtx.clone(), + }) as T & BeaconStateCache; + } + + cachedState.clone = clone as typeof viewDUClone; + + return cachedState; +} diff --git a/packages/beacon-state-transition/src/cache/syncCommitteeCache.ts b/packages/beacon-state-transition/src/cache/syncCommitteeCache.ts index 05ab3b5f6a6..d48c4d13db5 100644 --- a/packages/beacon-state-transition/src/cache/syncCommitteeCache.ts +++ b/packages/beacon-state-transition/src/cache/syncCommitteeCache.ts @@ -1,5 +1,5 @@ -import {altair, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {readonlyValues, toHexString} from "@chainsafe/ssz"; +import {ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {CompositeViewDU, toHexString} from "@chainsafe/ssz"; import {PubkeyIndexMap} from "./pubkeyCache"; type SyncComitteeValidatorIndexMap = Map; @@ -36,7 +36,7 @@ export function getSyncCommitteeCache(validatorIndices: ValidatorIndex[]): SyncC } export function computeSyncCommitteeCache( - syncCommittee: altair.SyncCommittee, + syncCommittee: CompositeViewDU, pubkey2index: PubkeyIndexMap ): SyncCommitteeCache { const validatorIndices = computeSyncCommitteeIndices(syncCommittee, pubkey2index); @@ -74,13 +74,13 @@ export function computeSyncComitteeMap(syncCommitteeIndexes: ValidatorIndex[]): * Extract validator indices from current and next sync committee */ function computeSyncCommitteeIndices( - syncCommittee: altair.SyncCommittee, + syncCommittee: CompositeViewDU, pubkey2index: PubkeyIndexMap ): ValidatorIndex[] { const validatorIndices: ValidatorIndex[] = []; - const pubkeys = readonlyValues(syncCommittee.pubkeys); + const pubkeys = syncCommittee.pubkeys.getAllReadonly(); for (const pubkey of pubkeys) { - const validatorIndex = pubkey2index.get(pubkey.valueOf() as typeof pubkey); + const validatorIndex = pubkey2index.get(pubkey); if (validatorIndex === undefined) { throw Error(`SyncCommittee pubkey is unknown ${toHexString(pubkey)}`); } diff --git a/packages/beacon-state-transition/src/cache/types.ts b/packages/beacon-state-transition/src/cache/types.ts new file mode 100644 index 00000000000..56f91bfd9c9 --- /dev/null +++ b/packages/beacon-state-transition/src/cache/types.ts @@ -0,0 +1,13 @@ +import {ssz} from "@chainsafe/lodestar-types"; +import {CompositeViewDU} from "@chainsafe/ssz"; + +export type BeaconStatePhase0 = CompositeViewDU; +export type BeaconStateAltair = CompositeViewDU; +export type BeaconStateBellatrix = CompositeViewDU; + +// Union at the TreeViewDU level +// - Works well as function argument and as generic type for allForks functions +// +// Quasy equivalent to +// CompositeViewDU +export type BeaconStateAllForks = BeaconStatePhase0 | BeaconStateAltair | BeaconStateBellatrix; diff --git a/packages/beacon-state-transition/src/index.ts b/packages/beacon-state-transition/src/index.ts index 6ac6d1886f8..325d5298ace 100644 --- a/packages/beacon-state-transition/src/index.ts +++ b/packages/beacon-state-transition/src/index.ts @@ -10,18 +10,24 @@ export * as phase0 from "./phase0"; export * as altair from "./altair"; export * as bellatrix from "./bellatrix"; export * as allForks from "./allForks"; -export {CachedBeaconState, createCachedBeaconState} from "./cache/cachedBeaconState"; export { CachedBeaconStatePhase0, CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateAllForks, - CachedBeaconStateAnyFork, + // Non-cached states + BeaconStatePhase0, + BeaconStateAltair, + BeaconStateBellatrix, + BeaconStateAllForks, } from "./types"; -export {EpochContext} from "./cache/epochContext"; +// Main state caches +export {createCachedBeaconState, BeaconStateCache} from "./cache/stateCache"; +export {EpochContext, EpochContextImmutableData, createEmptyEpochContextImmutableData} from "./cache/epochContext"; export {EpochProcess, beforeProcessEpoch} from "./cache/epochProcess"; -export {PubkeyIndexMap, Index2PubkeyCache} from "./cache/pubkeyCache"; +// Aux data-structures +export {PubkeyIndexMap, Index2PubkeyCache} from "./cache/pubkeyCache"; export {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsZeroed} from "./cache/effectiveBalanceIncrements"; diff --git a/packages/beacon-state-transition/src/phase0/block/index.ts b/packages/beacon-state-transition/src/phase0/block/index.ts index eb322b4e28a..e80bec82d5c 100644 --- a/packages/beacon-state-transition/src/phase0/block/index.ts +++ b/packages/beacon-state-transition/src/phase0/block/index.ts @@ -1,5 +1,5 @@ import {phase0} from "@chainsafe/lodestar-types"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processBlockHeader, processEth1Data, processRandao} from "../../allForks/block"; import {processOperations} from "./processOperations"; import {processAttestation, validateAttestation} from "./processAttestation"; @@ -22,8 +22,8 @@ export { }; export function processBlock(state: CachedBeaconStatePhase0, block: phase0.BeaconBlock, verifySignatures = true): void { - processBlockHeader(state as CachedBeaconStateAllForks, block); - processRandao(state as CachedBeaconStateAllForks, block, verifySignatures); - processEth1Data(state as CachedBeaconStateAllForks, block.body); + processBlockHeader(state, block); + processRandao(state, block, verifySignatures); + processEth1Data(state, block.body.eth1Data); processOperations(state, block.body, verifySignatures); } diff --git a/packages/beacon-state-transition/src/phase0/block/processAttestation.ts b/packages/beacon-state-transition/src/phase0/block/processAttestation.ts index 044a2b28c56..1f4d98ae21d 100644 --- a/packages/beacon-state-transition/src/phase0/block/processAttestation.ts +++ b/packages/beacon-state-transition/src/phase0/block/processAttestation.ts @@ -22,9 +22,9 @@ export function processAttestation( const slot = state.slot; const data = attestation.data; - validateAttestation(state as CachedBeaconStateAllForks, attestation); + validateAttestation(state, attestation); - const pendingAttestation = ssz.phase0.PendingAttestation.createTreeBackedFromStruct({ + const pendingAttestation = ssz.phase0.PendingAttestation.toViewDU({ data: data, aggregationBits: attestation.aggregationBits, inclusionDelay: slot - data.slot, @@ -51,13 +51,7 @@ export function processAttestation( state.previousEpochAttestations.push(pendingAttestation); } - if ( - !isValidIndexedAttestation( - state as CachedBeaconStateAllForks, - epochCtx.getIndexedAttestation(attestation), - verifySignature - ) - ) { + if (!isValidIndexedAttestation(state, epochCtx.getIndexedAttestation(attestation), verifySignature)) { throw new Error("Attestation is not valid"); } } @@ -96,10 +90,10 @@ export function validateAttestation(state: CachedBeaconStateAllForks, attestatio } const committee = epochCtx.getBeaconCommittee(data.slot, data.index); - if (attestation.aggregationBits.length !== committee.length) { + if (attestation.aggregationBits.bitLen !== committee.length) { throw new Error( "Attestation aggregation bits length does not match committee length: " + - `aggregationBitsLength=${attestation.aggregationBits.length} committeeLength=${committee.length}` + `aggregationBitsLength=${attestation.aggregationBits.bitLen} committeeLength=${committee.length}` ); } } diff --git a/packages/beacon-state-transition/src/phase0/block/processAttesterSlashing.ts b/packages/beacon-state-transition/src/phase0/block/processAttesterSlashing.ts index b2a44d873f3..58d7593b9a4 100644 --- a/packages/beacon-state-transition/src/phase0/block/processAttesterSlashing.ts +++ b/packages/beacon-state-transition/src/phase0/block/processAttesterSlashing.ts @@ -1,7 +1,7 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processAttesterSlashing as processAttesterSlashingAllForks} from "../../allForks/block"; export function processAttesterSlashing( @@ -9,10 +9,5 @@ export function processAttesterSlashing( attesterSlashing: phase0.AttesterSlashing, verifySignatures = true ): void { - processAttesterSlashingAllForks( - ForkName.phase0, - state as CachedBeaconStateAllForks, - attesterSlashing, - verifySignatures - ); + processAttesterSlashingAllForks(ForkName.phase0, state, attesterSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/phase0/block/processDeposit.ts b/packages/beacon-state-transition/src/phase0/block/processDeposit.ts index 8695899c595..aec62bb0de1 100644 --- a/packages/beacon-state-transition/src/phase0/block/processDeposit.ts +++ b/packages/beacon-state-transition/src/phase0/block/processDeposit.ts @@ -1,9 +1,9 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processDeposit as processDepositAllForks} from "../../allForks/block"; export function processDeposit(state: CachedBeaconStatePhase0, deposit: phase0.Deposit): void { - processDepositAllForks(ForkName.phase0, state as CachedBeaconStateAllForks, deposit); + processDepositAllForks(ForkName.phase0, state, deposit); } diff --git a/packages/beacon-state-transition/src/phase0/block/processOperations.ts b/packages/beacon-state-transition/src/phase0/block/processOperations.ts index 659d6cfc033..17b3e7abbe3 100644 --- a/packages/beacon-state-transition/src/phase0/block/processOperations.ts +++ b/packages/beacon-state-transition/src/phase0/block/processOperations.ts @@ -1,4 +1,3 @@ -import {List, readonlyValues} from "@chainsafe/ssz"; import {phase0} from "@chainsafe/lodestar-types"; import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; @@ -9,14 +8,6 @@ import {processAttestation} from "./processAttestation"; import {processDeposit} from "./processDeposit"; import {processVoluntaryExit} from "./processVoluntaryExit"; -type Operation = - | phase0.ProposerSlashing - | phase0.AttesterSlashing - | phase0.Attestation - | phase0.Deposit - | phase0.VoluntaryExit; -type OperationFunction = (state: CachedBeaconStatePhase0, op: Operation, verify: boolean) => void; - export function processOperations( state: CachedBeaconStatePhase0, body: phase0.BeaconBlockBody, @@ -30,15 +21,19 @@ export function processOperations( ); } - for (const [operations, processOp] of [ - [body.proposerSlashings, processProposerSlashing], - [body.attesterSlashings, processAttesterSlashing], - [body.attestations, processAttestation], - [body.deposits, processDeposit], - [body.voluntaryExits, processVoluntaryExit], - ] as [List, OperationFunction][]) { - for (const op of readonlyValues(operations)) { - processOp(state, op, verifySignatures); - } + for (const proposerSlashing of body.proposerSlashings) { + processProposerSlashing(state, proposerSlashing, verifySignatures); + } + for (const attesterSlashing of body.attesterSlashings) { + processAttesterSlashing(state, attesterSlashing, verifySignatures); + } + for (const attestation of body.attestations) { + processAttestation(state, attestation, verifySignatures); + } + for (const deposit of body.deposits) { + processDeposit(state, deposit); + } + for (const voluntaryExit of body.voluntaryExits) { + processVoluntaryExit(state, voluntaryExit, verifySignatures); } } diff --git a/packages/beacon-state-transition/src/phase0/block/processProposerSlashing.ts b/packages/beacon-state-transition/src/phase0/block/processProposerSlashing.ts index 865bb09f7a6..b9275251a6e 100644 --- a/packages/beacon-state-transition/src/phase0/block/processProposerSlashing.ts +++ b/packages/beacon-state-transition/src/phase0/block/processProposerSlashing.ts @@ -1,6 +1,6 @@ import {phase0} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processProposerSlashing as processProposerSlashingAllForks} from "../../allForks/block"; export function processProposerSlashing( @@ -8,10 +8,5 @@ export function processProposerSlashing( proposerSlashing: phase0.ProposerSlashing, verifySignatures = true ): void { - processProposerSlashingAllForks( - ForkName.phase0, - state as CachedBeaconStateAllForks, - proposerSlashing, - verifySignatures - ); + processProposerSlashingAllForks(ForkName.phase0, state, proposerSlashing, verifySignatures); } diff --git a/packages/beacon-state-transition/src/phase0/block/processVoluntaryExit.ts b/packages/beacon-state-transition/src/phase0/block/processVoluntaryExit.ts index 8c8d6ac921c..97adf7173a9 100644 --- a/packages/beacon-state-transition/src/phase0/block/processVoluntaryExit.ts +++ b/packages/beacon-state-transition/src/phase0/block/processVoluntaryExit.ts @@ -1,5 +1,5 @@ import {phase0} from "@chainsafe/lodestar-types"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../../types"; +import {CachedBeaconStatePhase0} from "../../types"; import {processVoluntaryExitAllForks} from "../../allForks/block"; export function processVoluntaryExit( @@ -7,5 +7,5 @@ export function processVoluntaryExit( signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature = true ): void { - processVoluntaryExitAllForks(state as CachedBeaconStateAllForks, signedVoluntaryExit, verifySignature); + processVoluntaryExitAllForks(state, signedVoluntaryExit, verifySignature); } diff --git a/packages/beacon-state-transition/src/phase0/epoch/getAttestationDeltas.ts b/packages/beacon-state-transition/src/phase0/epoch/getAttestationDeltas.ts index 3f8f89fbb03..38a2f42a42a 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/getAttestationDeltas.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/getAttestationDeltas.ts @@ -1,4 +1,4 @@ -import {bigIntSqrt} from "@chainsafe/lodestar-utils"; +import {bigIntSqrt, bnToNum} from "@chainsafe/lodestar-utils"; import {BASE_REWARDS_PER_EPOCH as BASE_REWARDS_PER_EPOCH_CONST} from "../../constants"; import {newZeroedArray} from "../../util"; import {EpochProcess, CachedBeaconStatePhase0} from "../../types"; @@ -64,11 +64,11 @@ export function getAttestationDeltas(state: CachedBeaconStatePhase0, epochProces const prevEpochHeadStakeByIncrement = epochProcess.prevEpochUnslashedStake.headStakeByIncrement; // sqrt first, before factoring out the increment for later usage - const balanceSqRoot = Number(bigIntSqrt(totalBalanceInGwei)); + const balanceSqRoot = bnToNum(bigIntSqrt(totalBalanceInGwei)); const finalityDelay = epochProcess.prevEpoch - state.finalizedCheckpoint.epoch; const BASE_REWARDS_PER_EPOCH = BASE_REWARDS_PER_EPOCH_CONST; - const proposerRewardQuotient = Number(PROPOSER_REWARD_QUOTIENT); + const proposerRewardQuotient = PROPOSER_REWARD_QUOTIENT; const isInInactivityLeak = finalityDelay > MIN_EPOCHS_TO_INACTIVITY_PENALTY; // effectiveBalance is multiple of EFFECTIVE_BALANCE_INCREMENT and less than MAX_EFFECTIVE_BALANCE @@ -122,6 +122,7 @@ export function getAttestationDeltas(state: CachedBeaconStatePhase0, epochProces rewards[status.proposerIndex] += proposerReward; rewards[i] += Math.floor(maxAttesterReward / status.inclusionDelay); } + if (hasMarkers(status.flags, FLAG_ELIGIBLE_ATTESTER)) { // expected FFG source if (hasMarkers(status.flags, FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED)) { @@ -160,5 +161,6 @@ export function getAttestationDeltas(state: CachedBeaconStatePhase0, epochProces } } } + return [rewards, penalties]; } diff --git a/packages/beacon-state-transition/src/phase0/epoch/index.ts b/packages/beacon-state-transition/src/phase0/epoch/index.ts index 01eb3ce9d5d..1fe9a87e39e 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/index.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/index.ts @@ -1,4 +1,4 @@ -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStatePhase0, EpochProcess} from "../../types"; import { processJustificationAndFinalization, processRegistryUpdates, @@ -13,18 +13,18 @@ import {processSlashings} from "./processSlashings"; import {getAttestationDeltas} from "./getAttestationDeltas"; import {processParticipationRecordUpdates} from "./processParticipationRecordUpdates"; -export {processRewardsAndPenalties, processSlashings, getAttestationDeltas}; +export {processRewardsAndPenalties, processSlashings, getAttestationDeltas, processParticipationRecordUpdates}; export function processEpoch(state: CachedBeaconStatePhase0, epochProcess: EpochProcess): void { - processJustificationAndFinalization(state as CachedBeaconStateAllForks, epochProcess); + processJustificationAndFinalization(state, epochProcess); processRewardsAndPenalties(state, epochProcess); - processRegistryUpdates(state as CachedBeaconStateAllForks, epochProcess); + processRegistryUpdates(state, epochProcess); processSlashings(state, epochProcess); // inline processFinalUpdates() to follow altair and for clarity - processEth1DataReset(state as CachedBeaconStateAllForks, epochProcess); - processEffectiveBalanceUpdates(state as CachedBeaconStateAllForks, epochProcess); - processSlashingsReset(state as CachedBeaconStateAllForks, epochProcess); - processRandaoMixesReset(state as CachedBeaconStateAllForks, epochProcess); - processHistoricalRootsUpdate(state as CachedBeaconStateAllForks, epochProcess); + processEth1DataReset(state, epochProcess); + processEffectiveBalanceUpdates(state, epochProcess); + processSlashingsReset(state, epochProcess); + processRandaoMixesReset(state, epochProcess); + processHistoricalRootsUpdate(state, epochProcess); processParticipationRecordUpdates(state); } diff --git a/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts b/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts index 75640d1c88e..1a6d3569bb7 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates.ts @@ -1,5 +1,4 @@ -import {phase0} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; import {CachedBeaconStatePhase0} from "../../types"; /** @@ -9,5 +8,7 @@ import {CachedBeaconStatePhase0} from "../../types"; export function processParticipationRecordUpdates(state: CachedBeaconStatePhase0): void { // rotate current/previous epoch attestations state.previousEpochAttestations = state.currentEpochAttestations; - state.currentEpochAttestations = ([] as phase0.PendingAttestation[]) as List; + + // Reset list to empty + state.currentEpochAttestations = ssz.phase0.EpochAttestations.defaultViewDU(); } diff --git a/packages/beacon-state-transition/src/phase0/epoch/processPendingAttestations.ts b/packages/beacon-state-transition/src/phase0/epoch/processPendingAttestations.ts index 1a2e6e349cf..3f93235ac41 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/processPendingAttestations.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/processPendingAttestations.ts @@ -1,8 +1,7 @@ -import {Epoch, phase0, ssz} from "@chainsafe/lodestar-types"; -import {List, readonlyValues} from "@chainsafe/ssz"; +import {Epoch, phase0} from "@chainsafe/lodestar-types"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {CachedBeaconStatePhase0} from "../../types"; -import {computeStartSlotAtEpoch, getBlockRootAtSlot, zipIndexesCommitteeBits} from "../../util"; -import {IAttesterStatus} from "../../util/attesterStatus"; +import {computeStartSlotAtEpoch, getBlockRootAtSlot, IAttesterStatus} from "../../util"; /** * Mutates `statuses` from all pending attestations. @@ -18,33 +17,39 @@ import {IAttesterStatus} from "../../util/attesterStatus"; export function statusProcessEpoch( state: CachedBeaconStatePhase0, statuses: IAttesterStatus[], - attestations: List, + attestations: phase0.PendingAttestation[], epoch: Epoch, sourceFlag: number, targetFlag: number, headFlag: number ): void { const {epochCtx, slot: stateSlot} = state; - const rootType = ssz.Root; const prevEpoch = epochCtx.previousShuffling.epoch; if (attestations.length === 0) { return; } + + // Prevent frequent object get of external CommonJS dependencies + const byteArrayEqualsFn = byteArrayEquals; + const actualTargetBlockRoot = getBlockRootAtSlot(state, computeStartSlotAtEpoch(epoch)); - for (const att of readonlyValues(attestations)) { - const aggregationBits = att.aggregationBits; + + for (const att of attestations) { + // Ignore empty BitArray, from spec test minimal/phase0/epoch_processing/participation_record_updates updated_participation_record + // See https://github.com/ethereum/consensus-specs/issues/2825 + if (att.aggregationBits.bitLen === 0) { + continue; + } + const attData = att.data; const inclusionDelay = att.inclusionDelay; const proposerIndex = att.proposerIndex; const attSlot = attData.slot; - const committeeIndex = attData.index; - const attBeaconBlockRoot = attData.beaconBlockRoot; - const attTarget = attData.target; - const attVotedTargetRoot = rootType.equals(attTarget.root, actualTargetBlockRoot); + const attVotedTargetRoot = byteArrayEqualsFn(attData.target.root, actualTargetBlockRoot); const attVotedHeadRoot = - attSlot < stateSlot && rootType.equals(attBeaconBlockRoot, getBlockRootAtSlot(state, attSlot)); - const committee = epochCtx.getBeaconCommittee(attSlot, committeeIndex); - const participants = zipIndexesCommitteeBits(committee, aggregationBits); + attSlot < stateSlot && byteArrayEqualsFn(attData.beaconBlockRoot, getBlockRootAtSlot(state, attSlot)); + const committee = epochCtx.getBeaconCommittee(attSlot, attData.index); + const participants = att.aggregationBits.intersectValues(committee); if (epoch === prevEpoch) { for (const p of participants) { diff --git a/packages/beacon-state-transition/src/phase0/epoch/processRewardsAndPenalties.ts b/packages/beacon-state-transition/src/phase0/epoch/processRewardsAndPenalties.ts index 3d5a70ece82..be4e16930fe 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/processRewardsAndPenalties.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/processRewardsAndPenalties.ts @@ -1,13 +1,12 @@ import {ForkName} from "@chainsafe/lodestar-params"; import {processRewardsAndPenaltiesAllForks} from "../../allForks/epoch/processRewardsAndPenalties"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStatePhase0, EpochProcess} from "../../types"; /** * Iterate over all validator and compute rewards and penalties to apply to balances. * - * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags - * are true, worst case: FLAG_UNSLASHED + FLAG_ELIGIBLE_ATTESTER + FLAG_PREV_* + * PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags are set */ export function processRewardsAndPenalties(state: CachedBeaconStatePhase0, epochProcess: EpochProcess): void { - processRewardsAndPenaltiesAllForks(ForkName.phase0, state as CachedBeaconStateAllForks, epochProcess); + processRewardsAndPenaltiesAllForks(ForkName.phase0, state, epochProcess); } diff --git a/packages/beacon-state-transition/src/phase0/epoch/processSlashings.ts b/packages/beacon-state-transition/src/phase0/epoch/processSlashings.ts index 2882e6d9da5..96369abeb63 100644 --- a/packages/beacon-state-transition/src/phase0/epoch/processSlashings.ts +++ b/packages/beacon-state-transition/src/phase0/epoch/processSlashings.ts @@ -1,7 +1,7 @@ import {ForkName} from "@chainsafe/lodestar-params"; -import {CachedBeaconStatePhase0, CachedBeaconStateAllForks, EpochProcess} from "../../types"; +import {CachedBeaconStatePhase0, EpochProcess} from "../../types"; import {processSlashingsAllForks} from "../../allForks/epoch/processSlashings"; export function processSlashings(state: CachedBeaconStatePhase0, epochProcess: EpochProcess): void { - processSlashingsAllForks(ForkName.phase0, state as CachedBeaconStateAllForks, epochProcess); + processSlashingsAllForks(ForkName.phase0, state, epochProcess); } diff --git a/packages/beacon-state-transition/src/types.ts b/packages/beacon-state-transition/src/types.ts index 83712160712..37db2138824 100644 --- a/packages/beacon-state-transition/src/types.ts +++ b/packages/beacon-state-transition/src/types.ts @@ -1,15 +1,11 @@ -import {allForks, altair, bellatrix, phase0} from "@chainsafe/lodestar-types"; -import {CachedBeaconState} from "./cache/cachedBeaconState"; - export {EpochContext} from "./cache/epochContext"; export {EpochProcess} from "./cache/epochProcess"; -export type CachedBeaconStatePhase0 = CachedBeaconState; -export type CachedBeaconStateAltair = CachedBeaconState; -export type CachedBeaconStateBellatrix = CachedBeaconState; -export type CachedBeaconStateAllForks = CachedBeaconState; -export type CachedBeaconStateAnyFork = - | CachedBeaconStatePhase0 - | CachedBeaconStateAltair - | CachedBeaconStateBellatrix - | CachedBeaconStateAllForks; +export { + CachedBeaconStatePhase0, + CachedBeaconStateAltair, + CachedBeaconStateBellatrix, + CachedBeaconStateAllForks, +} from "./cache/stateCache"; + +export {BeaconStatePhase0, BeaconStateAltair, BeaconStateBellatrix, BeaconStateAllForks} from "./cache/types"; diff --git a/packages/beacon-state-transition/src/util/aggregationBits.ts b/packages/beacon-state-transition/src/util/aggregationBits.ts deleted file mode 100644 index 62b9b534e15..00000000000 --- a/packages/beacon-state-transition/src/util/aggregationBits.ts +++ /dev/null @@ -1,189 +0,0 @@ -import {BitList, BitListType, BitVector, isTreeBacked, TreeBacked, Type} from "@chainsafe/ssz"; -import {ssz} from "@chainsafe/lodestar-types"; -import {LodestarError} from "@chainsafe/lodestar-utils"; - -const BITS_PER_BYTE = 8; -/** Globally cache this information. @see getUint8ByteToBitBooleanArray */ -const uint8ByteToBitBooleanArrays: boolean[][] = []; - -/** - * Given a byte (0 -> 255), return a Array of boolean with length = 8, big endian. - * Ex: 1 => [true false false false false false false false] - * 5 => [true false true false false fase false false] - */ -export function getUint8ByteToBitBooleanArray(byte: number): boolean[] { - if (uint8ByteToBitBooleanArrays[byte] === undefined) { - uint8ByteToBitBooleanArrays[byte] = computeUint8ByteToBitBooleanArray(byte); - } - return uint8ByteToBitBooleanArrays[byte]; -} - -/** @see getUint8ByteToBitBooleanArray */ -function computeUint8ByteToBitBooleanArray(byte: number): boolean[] { - // this returns little endian - const binaryStr = byte.toString(2); - const binaryLength = binaryStr.length; - return Array.from({length: BITS_PER_BYTE}, (_, j) => { - if (j < binaryLength) { - return binaryStr[binaryLength - j - 1] === "1" ? true : false; - } else { - return false; - } - }); -} - -/** zipIndexes for CommitteeBits. @see zipIndexes */ -export function zipIndexesCommitteeBits(indexes: number[], bits: TreeBacked | BitVector): number[] { - return zipIndexes(indexes, bits, ssz.phase0.CommitteeBits)[0]; -} - -/** zipIndexes for SyncCommitteeBits. @see zipIndexes */ -export function zipIndexesSyncCommitteeBits(indexes: number[], bits: TreeBacked | BitVector): number[] { - return zipIndexes(indexes, bits, ssz.altair.SyncCommitteeBits)[0]; -} - -/** Similar to zipIndexesSyncCommitteeBits but we extract both participant and unparticipant indices*/ -export function zipAllIndexesSyncCommitteeBits( - indexes: number[], - bits: TreeBacked | BitVector -): [number[], number[]] { - return zipIndexes(indexes, bits, ssz.altair.SyncCommitteeBits); -} - -/** - * Performant indexing of a BitList, both as struct or TreeBacked - * Return [0] as participant indices and [1] as unparticipant indices - * @see zipIndexesInBitListTreeBacked - */ -export function zipIndexes( - indexes: number[], - bitlist: TreeBacked | BitArr, - sszType: Type -): [number[], number[]] { - if (isTreeBacked(bitlist)) { - return zipIndexesTreeBacked(indexes, bitlist, sszType); - } else { - const attestingIndices = []; - const unattestingIndices = []; - for (let i = 0, len = indexes.length; i < len; i++) { - if (bitlist[i]) { - attestingIndices.push(indexes[i]); - } else { - unattestingIndices.push(indexes[i]); - } - } - return [attestingIndices, unattestingIndices]; - } -} - -/** - * Returns [0] as indices that participated in `bitlist` and [1] as indices that did not participated in `bitlist`. - * Participation of `indexes[i]` means that the bit at position `i` in `bitlist` is true. - * - * Previously we computed this information with `readonlyValues(TreeBacked)`. - * However this approach is very inneficient since the SSZ parsing of BitList is not optimized. - * This function uses a precomputed array of booleans `Uint8 -> boolean[]` @see uint8ByteToBitBooleanArrays. - * This approach is x15 times faster. - */ -export function zipIndexesTreeBacked( - indexes: number[], - bits: TreeBacked, - sszType: Type -): [number[], number[]] { - const bytes = bitsToUint8Array(bits, sszType); - - const participantIndices: number[] = []; - const unparticipantIndices: number[] = []; - - // Iterate over each byte of bits - for (let iByte = 0, byteLen = bytes.length; iByte < byteLen; iByte++) { - // Get the precomputed boolean array for this byte - const booleansInByte = getUint8ByteToBitBooleanArray(bytes[iByte]); - // For each bit in the byte check participation and add to indexesSelected array - for (let iBit = 0; iBit < BITS_PER_BYTE; iBit++) { - const committeeIndex = indexes[iByte * BITS_PER_BYTE + iBit]; - if (committeeIndex !== undefined) { - if (booleansInByte[iBit]) { - participantIndices.push(committeeIndex); - } else { - unparticipantIndices.push(committeeIndex); - } - } - } - } - - return [participantIndices, unparticipantIndices]; -} - -/** - * Efficiently extract the Uint8Array inside a `TreeBacked` structure. - * @see zipIndexesInBitListTreeBacked for reasoning and advantatges. - */ -export function bitsToUint8Array( - bits: TreeBacked, - sszType: Type -): Uint8Array { - const tree = bits.tree; - const treeType = (sszType as unknown) as BitListType; - const chunkCount = treeType.tree_getChunkCount(tree); - const chunkDepth = treeType.getChunkDepth(); - const nodeIterator = tree.iterateNodesAtDepth(chunkDepth, 0, chunkCount); - const chunks: Uint8Array[] = []; - for (const node of nodeIterator) { - chunks.push(node.root); - } - // the last chunk has 32 bytes but we don't use all of them - return Buffer.concat(chunks).subarray(0, Math.ceil(bits.length / BITS_PER_BYTE)); -} - -/** - * Variant to extract a single bit (for un-aggregated attestations) - */ -export function getSingleBitIndex(bits: BitList | TreeBacked): number { - let index: number | null = null; - - if (isTreeBacked(bits)) { - const bytes = bitsToUint8Array(bits, ssz.phase0.CommitteeBits); - - // Iterate over each byte of bits - for (let iByte = 0, byteLen = bytes.length; iByte < byteLen; iByte++) { - // If it's exactly zero, there won't be any indexes, continue early - if (bytes[iByte] === 0) { - continue; - } - - // Get the precomputed boolean array for this byte - const booleansInByte = getUint8ByteToBitBooleanArray(bytes[iByte]); - // For each bit in the byte check participation and add to indexesSelected array - for (let iBit = 0; iBit < BITS_PER_BYTE; iBit++) { - if (booleansInByte[iBit] === true) { - if (index !== null) throw new AggregationBitsError({code: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET}); - index = iByte * BITS_PER_BYTE + iBit; - } - } - } - } else { - for (let i = 0, len = bits.length; i < len; i++) { - if (bits[i] === true) { - if (index !== null) throw new AggregationBitsError({code: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET}); - index = i; - } - } - } - - if (index === null) { - throw new AggregationBitsError({code: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET}); - } else { - return index; - } -} - -export enum AggregationBitsErrorCode { - NOT_EXACTLY_ONE_BIT_SET = "AGGREGATION_BITS_ERROR_NOT_EXACTLY_ONE_BIT_SET", -} - -type AggregationBitsErrorType = { - code: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET; -}; - -export class AggregationBitsError extends LodestarError {} diff --git a/packages/beacon-state-transition/src/util/aggregator.ts b/packages/beacon-state-transition/src/util/aggregator.ts index c39bda9926f..2595ea45678 100644 --- a/packages/beacon-state-transition/src/util/aggregator.ts +++ b/packages/beacon-state-transition/src/util/aggregator.ts @@ -1,4 +1,4 @@ -import {hash} from "@chainsafe/ssz"; +import {digest} from "@chainsafe/as-sha256"; import {BLSSignature} from "@chainsafe/lodestar-types"; import {intDiv, bytesToBigInt} from "@chainsafe/lodestar-utils"; import { @@ -29,5 +29,5 @@ export function isAggregatorFromCommitteeLength(committeeLength: number, slotSig * Using bytesToInt() may cause isSelectionProofValid() to always return false. */ export function isSelectionProofValid(sig: BLSSignature, modulo: number): boolean { - return bytesToBigInt(hash(sig.valueOf() as Uint8Array).slice(0, 8)) % BigInt(modulo) === ZERO_BIGINT; + return bytesToBigInt(digest(sig).slice(0, 8)) % BigInt(modulo) === ZERO_BIGINT; } diff --git a/packages/beacon-state-transition/src/util/altair.ts b/packages/beacon-state-transition/src/util/altair.ts new file mode 100644 index 00000000000..00e716c9d68 --- /dev/null +++ b/packages/beacon-state-transition/src/util/altair.ts @@ -0,0 +1,13 @@ +import {BASE_REWARD_FACTOR, EFFECTIVE_BALANCE_INCREMENT} from "@chainsafe/lodestar-params"; +import {bigIntSqrt, bnToNum} from "@chainsafe/lodestar-utils"; + +/** + * Before we manage bigIntSqrt(totalActiveStake) as BigInt and return BigInt. + * bigIntSqrt(totalActiveStake) should fit a number (2 ** 53 -1 max) + **/ +export function computeBaseRewardPerIncrement(totalActiveStakeByIncrement: number): number { + return Math.floor( + (EFFECTIVE_BALANCE_INCREMENT * BASE_REWARD_FACTOR) / + bnToNum(bigIntSqrt(BigInt(totalActiveStakeByIncrement) * BigInt(EFFECTIVE_BALANCE_INCREMENT))) + ); +} diff --git a/packages/beacon-state-transition/src/util/array.ts b/packages/beacon-state-transition/src/util/array.ts index 43930a0a9d1..bb763d1359b 100644 --- a/packages/beacon-state-transition/src/util/array.ts +++ b/packages/beacon-state-transition/src/util/array.ts @@ -30,3 +30,24 @@ export function newFilledArray(n: number, val: T): T[] { } return arr; } + +/** + * Returns an array with all values not in the participants array. + * - All elements in values must be unique + * - Does NOT require sorting + */ +export function getUnparticipantValues(participants: T[], values: T[]): T[] { + const unparticipants: T[] = []; + + let j = 0; + for (let i = 0; i < values.length; i++) { + if (values[i] === participants[j]) { + // Included + j++; + } else { + unparticipants.push(values[i]); + } + } + + return unparticipants; +} diff --git a/packages/beacon-state-transition/src/util/attesterStatus.ts b/packages/beacon-state-transition/src/util/attesterStatus.ts index ea09d251d06..908e3b75cc0 100644 --- a/packages/beacon-state-transition/src/util/attesterStatus.ts +++ b/packages/beacon-state-transition/src/util/attesterStatus.ts @@ -8,9 +8,9 @@ export const FLAG_CURR_HEAD_ATTESTER = 1 << 5; export const FLAG_UNSLASHED = 1 << 6; export const FLAG_ELIGIBLE_ATTESTER = 1 << 7; // Precompute OR flags -export const FLAG_PREV_SOURCE_ATTESTER_OR_UNSLASHED = FLAG_PREV_SOURCE_ATTESTER | FLAG_UNSLASHED; -export const FLAG_PREV_TARGET_ATTESTER_OR_UNSLASHED = FLAG_PREV_TARGET_ATTESTER | FLAG_UNSLASHED; -export const FLAG_PREV_HEAD_ATTESTER_OR_UNSLASHED = FLAG_PREV_HEAD_ATTESTER | FLAG_UNSLASHED; +export const FLAG_PREV_SOURCE_ATTESTER_UNSLASHED = FLAG_PREV_SOURCE_ATTESTER | FLAG_UNSLASHED; +export const FLAG_PREV_TARGET_ATTESTER_UNSLASHED = FLAG_PREV_TARGET_ATTESTER | FLAG_UNSLASHED; +export const FLAG_PREV_HEAD_ATTESTER_UNSLASHED = FLAG_PREV_HEAD_ATTESTER | FLAG_UNSLASHED; /** * During the epoch transition, additional data is precomputed to avoid traversing any state a second diff --git a/packages/beacon-state-transition/src/util/balance.ts b/packages/beacon-state-transition/src/util/balance.ts index 590bc0d23d1..e17e7233238 100644 --- a/packages/beacon-state-transition/src/util/balance.ts +++ b/packages/beacon-state-transition/src/util/balance.ts @@ -3,10 +3,11 @@ */ import {EFFECTIVE_BALANCE_INCREMENT} from "@chainsafe/lodestar-params"; -import {allForks, Gwei, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {Gwei, ValidatorIndex} from "@chainsafe/lodestar-types"; import {bigIntMax} from "@chainsafe/lodestar-utils"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements"; -import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types"; +import {BeaconStateAllForks} from ".."; +import {CachedBeaconStateAllForks} from "../types"; /** * Return the combined effective balance of the [[indices]]. @@ -14,27 +15,24 @@ import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types"; * * SLOW CODE - 🐢 */ -export function getTotalBalance(state: allForks.BeaconState, indices: ValidatorIndex[]): Gwei { - return bigIntMax( - BigInt(EFFECTIVE_BALANCE_INCREMENT), - indices.reduce( - // TODO: Use a fast cache to get the effective balance 🐢 - (total: Gwei, index: ValidatorIndex): Gwei => total + BigInt(state.validators[index].effectiveBalance), - BigInt(0) - ) - ); +export function getTotalBalance(state: BeaconStateAllForks, indices: ValidatorIndex[]): Gwei { + let total = BigInt(0); + + // TODO: Use a fast cache to get the effective balance 🐢 + const validatorsArr = state.validators.getAllReadonlyValues(); + for (let i = 0; i < indices.length; i++) { + total += BigInt(validatorsArr[indices[i]].effectiveBalance); + } + + return bigIntMax(BigInt(EFFECTIVE_BALANCE_INCREMENT), total); } /** * Increase the balance for a validator with the given ``index`` by ``delta``. */ -export function increaseBalance( - state: CachedBeaconStateAllForks | CachedBeaconStateAltair, - index: ValidatorIndex, - delta: number -): void { +export function increaseBalance(state: BeaconStateAllForks, index: ValidatorIndex, delta: number): void { // TODO: Inline this - state.balanceList.applyDelta(index, delta); + state.balances.set(index, state.balances.get(index) + delta); } /** @@ -42,12 +40,10 @@ export function increaseBalance( * * Set to ``0`` when underflow. */ -export function decreaseBalance( - state: CachedBeaconStateAllForks | CachedBeaconStateAltair, - index: ValidatorIndex, - delta: number -): void { - state.balanceList.applyDelta(index, -delta); +export function decreaseBalance(state: BeaconStateAllForks, index: ValidatorIndex, delta: number): void { + const newBalance = state.balances.get(index) - delta; + // TODO: Is it necessary to protect against underflow here? Add unit test + state.balances.set(index, Math.max(0, newBalance)); } /** @@ -58,10 +54,10 @@ export function decreaseBalance( export function getEffectiveBalanceIncrementsZeroInactive( justifiedState: CachedBeaconStateAllForks ): EffectiveBalanceIncrements { - const {activeIndices} = justifiedState.currentShuffling; - // 5x faster than using readonlyValuesListOfLeafNodeStruct + const {activeIndices} = justifiedState.epochCtx.currentShuffling; + // 5x faster than reading from state.validators, with validator Nodes as values const validatorCount = justifiedState.validators.length; - const {effectiveBalanceIncrements} = justifiedState; + const {effectiveBalanceIncrements} = justifiedState.epochCtx; // Slice up to `validatorCount` since it won't be mutated, nor accessed beyond `validatorCount` const effectiveBalanceIncrementsZeroInactive = effectiveBalanceIncrements.slice(0, validatorCount); diff --git a/packages/beacon-state-transition/src/util/blockRoot.ts b/packages/beacon-state-transition/src/util/blockRoot.ts index f32aee953a3..3863af40c5b 100644 --- a/packages/beacon-state-transition/src/util/blockRoot.ts +++ b/packages/beacon-state-transition/src/util/blockRoot.ts @@ -8,24 +8,25 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {ZERO_HASH} from "../constants"; import {computeStartSlotAtEpoch} from "./epoch"; import {SLOTS_PER_HISTORICAL_ROOT} from "@chainsafe/lodestar-params"; +import {BeaconStateAllForks} from "../types"; /** * Return the block root at a recent [[slot]]. */ -export function getBlockRootAtSlot(state: allForks.BeaconState, slot: Slot): Root { +export function getBlockRootAtSlot(state: BeaconStateAllForks, slot: Slot): Root { if (slot >= state.slot) { throw Error(`Can only get block root in the past currentSlot=${state.slot} slot=${slot}`); } if (slot < state.slot - SLOTS_PER_HISTORICAL_ROOT) { throw Error(`Cannot get block root more than ${SLOTS_PER_HISTORICAL_ROOT} in the past`); } - return state.blockRoots[slot % SLOTS_PER_HISTORICAL_ROOT]; + return state.blockRoots.get(slot % SLOTS_PER_HISTORICAL_ROOT); } /** * Return the block root at the start of a recent [[epoch]]. */ -export function getBlockRoot(state: allForks.BeaconState, epoch: Epoch): Root { +export function getBlockRoot(state: BeaconStateAllForks, epoch: Epoch): Root { return getBlockRootAtSlot(state, computeStartSlotAtEpoch(epoch)); } /** diff --git a/packages/beacon-state-transition/src/util/domain.ts b/packages/beacon-state-transition/src/util/domain.ts index 997dc32c500..d74a859ad29 100644 --- a/packages/beacon-state-transition/src/util/domain.ts +++ b/packages/beacon-state-transition/src/util/domain.ts @@ -1,7 +1,7 @@ /** * @module chain/stateTransition/util */ -import {Epoch, Version, Root, DomainType, allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {Epoch, Version, Root, DomainType, phase0, ssz} from "@chainsafe/lodestar-types"; // Only used by processDeposit + lightclient /** @@ -18,7 +18,7 @@ export function computeDomain(domainType: DomainType, forkVersion: Version, gene /** * Return the ForkVersion at an epoch from a Fork type */ -export function getForkVersion(fork: allForks.BeaconState["fork"], epoch: Epoch): Version { +export function getForkVersion(fork: phase0.Fork, epoch: Epoch): Version { return epoch < fork.epoch ? fork.previousVersion : fork.currentVersion; } diff --git a/packages/beacon-state-transition/src/util/epoch.ts b/packages/beacon-state-transition/src/util/epoch.ts index bd411677fc5..7262cd2a605 100644 --- a/packages/beacon-state-transition/src/util/epoch.ts +++ b/packages/beacon-state-transition/src/util/epoch.ts @@ -34,14 +34,14 @@ export function computeActivationExitEpoch(epoch: Epoch): Epoch { /** * Return the current epoch of the given state. */ -export function getCurrentEpoch(state: allForks.BeaconState): Epoch { +export function getCurrentEpoch(state: Pick): Epoch { return computeEpochAtSlot(state.slot); } /** * Return the previous epoch of the given state. */ -export function getPreviousEpoch(state: allForks.BeaconState): Epoch { +export function getPreviousEpoch(state: Pick): Epoch { const currentEpoch = getCurrentEpoch(state); if (currentEpoch === GENESIS_EPOCH) { return GENESIS_EPOCH; diff --git a/packages/beacon-state-transition/src/util/epochShuffling.ts b/packages/beacon-state-transition/src/util/epochShuffling.ts index 00558b673e5..91283fbdc56 100644 --- a/packages/beacon-state-transition/src/util/epochShuffling.ts +++ b/packages/beacon-state-transition/src/util/epochShuffling.ts @@ -1,4 +1,4 @@ -import {Epoch, ValidatorIndex, allForks} from "@chainsafe/lodestar-types"; +import {Epoch, ValidatorIndex} from "@chainsafe/lodestar-types"; import {intDiv} from "@chainsafe/lodestar-utils"; import { DOMAIN_BEACON_ATTESTER, @@ -6,7 +6,17 @@ import { SLOTS_PER_EPOCH, TARGET_COMMITTEE_SIZE, } from "@chainsafe/lodestar-params"; -import {getSeed, unshuffleList} from "."; +import {BeaconStateAllForks} from "../types"; +import {getSeed} from "./seed"; +import {unshuffleList} from "./shuffle"; + +/** + * Readonly interface for IEpochShuffling. + */ +export interface IReadonlyEpochShuffling { + readonly epoch: Epoch; + readonly committees: Readonly; +} export interface IEpochShuffling { /** @@ -47,7 +57,7 @@ export function computeCommitteeCount(activeValidatorCount: number): number { } export function computeEpochShuffling( - state: allForks.BeaconState, + state: BeaconStateAllForks, activeIndices: ValidatorIndex[], epoch: Epoch ): IEpochShuffling { diff --git a/packages/beacon-state-transition/src/util/finality.ts b/packages/beacon-state-transition/src/util/finality.ts index bac4dfcc7b3..93f3ca46d61 100644 --- a/packages/beacon-state-transition/src/util/finality.ts +++ b/packages/beacon-state-transition/src/util/finality.ts @@ -2,7 +2,8 @@ import {MIN_EPOCHS_TO_INACTIVITY_PENALTY} from "@chainsafe/lodestar-params"; import {CachedBeaconStateAllForks} from "../types"; export function getFinalityDelay(state: CachedBeaconStateAllForks): number { - return state.previousShuffling.epoch - state.finalizedCheckpoint.epoch; + // previousEpoch = epoch - 1 + return state.epochCtx.epoch - 1 - state.finalizedCheckpoint.epoch; } /** diff --git a/packages/beacon-state-transition/src/util/genesis.ts b/packages/beacon-state-transition/src/util/genesis.ts index 0fd78479a2c..2c538c11d31 100644 --- a/packages/beacon-state-transition/src/util/genesis.ts +++ b/packages/beacon-state-transition/src/util/genesis.ts @@ -1,4 +1,3 @@ -import {List, TreeBacked} from "@chainsafe/ssz"; import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; import { EFFECTIVE_BALANCE_INCREMENT, @@ -8,25 +7,21 @@ import { GENESIS_SLOT, MAX_EFFECTIVE_BALANCE, } from "@chainsafe/lodestar-params"; -import { - allForks, - altair, - Bytes32, - bellatrix, - Number64, - phase0, - Root, - ssz, - ValidatorIndex, -} from "@chainsafe/lodestar-types"; +import {Bytes32, phase0, Root, ssz, TimeSeconds} from "@chainsafe/lodestar-types"; +import {processDeposit} from "../allForks"; +import {CachedBeaconStateAllForks, BeaconStateAllForks} from "../types"; import {computeEpochAtSlot} from "./epoch"; import {getActiveValidatorIndices} from "./validator"; import {getTemporaryBlockHeader} from "./blockRoot"; -import {getNextSyncCommittee} from "../util/syncCommittee"; -import {processDeposit} from "../allForks"; -import {createCachedBeaconState} from "../cache/cachedBeaconState"; -import {CachedBeaconStateAllForks} from "../types"; +import {CompositeViewDU, ListCompositeType} from "@chainsafe/ssz"; +import {newFilledArray} from "./array"; +import {getNextSyncCommittee} from "./syncCommittee"; +import {createCachedBeaconState} from "../cache/stateCache"; +import {EpochContextImmutableData} from "../cache/epochContext"; + +type DepositDataRootListType = ListCompositeType; +type DepositDataRootViewDU = CompositeViewDU; // TODO: Refactor to work with non-phase0 genesis state @@ -35,7 +30,7 @@ import {CachedBeaconStateAllForks} from "../types"; * @param config * @param state */ -export function isValidGenesisState(config: IChainForkConfig, state: allForks.BeaconState): boolean { +export function isValidGenesisState(config: IChainForkConfig, state: BeaconStateAllForks): boolean { return state.genesisTime >= config.MIN_GENESIS_TIME && isValidGenesisValidators(config, state); } @@ -44,7 +39,7 @@ export function isValidGenesisState(config: IChainForkConfig, state: allForks.Be * @param config * @param state */ -export function isValidGenesisValidators(config: IChainForkConfig, state: allForks.BeaconState): boolean { +export function isValidGenesisValidators(config: IChainForkConfig, state: BeaconStateAllForks): boolean { return ( getActiveValidatorIndices(state, computeEpochAtSlot(GENESIS_SLOT)).length >= config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT @@ -57,14 +52,16 @@ export function isValidGenesisValidators(config: IChainForkConfig, state: allFor * SLOW CODE - 🐢 */ export function getGenesisBeaconState( - config: IBeaconConfig, + config: IChainForkConfig, genesisEth1Data: phase0.Eth1Data, latestBlockHeader: phase0.BeaconBlockHeader -): CachedBeaconStateAllForks { +): BeaconStateAllForks { // Seed RANDAO with Eth1 entropy - const randaoMixes = Array(EPOCHS_PER_HISTORICAL_VECTOR).fill(genesisEth1Data.blockHash); + const randaoMixes = newFilledArray(EPOCHS_PER_HISTORICAL_VECTOR, genesisEth1Data.blockHash); + + const beaconStateType = config.getForkTypes(GENESIS_SLOT).BeaconState; + const state = beaconStateType.defaultViewDU(); - const state = config.getForkTypes(GENESIS_SLOT).BeaconState.defaultTreeBacked(); // MISC state.slot = GENESIS_SLOT; const version = config.getForkVersion(GENESIS_SLOT); @@ -74,31 +71,24 @@ export function getGenesisBeaconState( const previousForkIndex = Math.max(0, forkIndex - 1); const previousForkName = allForkNames[previousForkIndex]; const previousFork = config.forks[previousForkName]; + // the altair genesis spec test requires previous version to be phase0 although ALTAIR_FORK_EPOCH=0 - state.fork = { + state.fork = ssz.phase0.Fork.toViewDU({ previousVersion: previousFork.version, currentVersion: version, epoch: computeEpochAtSlot(GENESIS_SLOT), - } as phase0.Fork; + }); // Validator registry // Randomness and committees - state.latestBlockHeader = latestBlockHeader; + state.latestBlockHeader = ssz.phase0.BeaconBlockHeader.toViewDU(latestBlockHeader); // Ethereum 1.0 chain data - state.eth1Data = genesisEth1Data; - state.randaoMixes = randaoMixes; + state.eth1Data = ssz.phase0.Eth1Data.toViewDU(genesisEth1Data); + state.randaoMixes = ssz.phase0.RandaoMixes.toViewDU(randaoMixes); - // We need a CachedBeaconState to run processDeposit() which uses various caches. - // However at this point the state's syncCommittees are not known. - // This function can be called by: - // - 1. genesis spec tests: Don't care about the committee cache - // - 2. genesis builder: Only supports starting from genesis at phase0 fork - // - 3. interop state: Only supports starting from genesis at phase0 fork - // So it's okay to skip syncing the sync committee cache here and expect it to be - // populated latter when the altair fork happens for cases 2, 3. - return createCachedBeaconState(config, state, {skipSyncCommitteeCache: true}); + return state; } /** @@ -107,9 +97,9 @@ export function getGenesisBeaconState( * @param state BeaconState * @param eth1BlockHash eth1 block hash */ -export function applyEth1BlockHash(state: allForks.BeaconState, eth1BlockHash: Bytes32): void { +export function applyEth1BlockHash(state: CachedBeaconStateAllForks, eth1BlockHash: Bytes32): void { state.eth1Data.blockHash = eth1BlockHash; - state.randaoMixes = Array(EPOCHS_PER_HISTORICAL_VECTOR).fill(eth1BlockHash); + state.randaoMixes = ssz.phase0.RandaoMixes.toViewDU(newFilledArray(EPOCHS_PER_HISTORICAL_VECTOR, eth1BlockHash)); } /** @@ -120,7 +110,7 @@ export function applyEth1BlockHash(state: allForks.BeaconState, eth1BlockHash: B */ export function applyTimestamp( config: IChainForkConfig, - state: TreeBacked, + state: CachedBeaconStateAllForks, eth1Timestamp: number ): void { state.genesisTime = eth1Timestamp + config.GENESIS_DELAY; @@ -143,12 +133,16 @@ export function applyDeposits( config: IChainForkConfig, state: CachedBeaconStateAllForks, newDeposits: phase0.Deposit[], - fullDepositDataRootList?: TreeBacked> -): ValidatorIndex[] { + fullDepositDataRootList?: DepositDataRootViewDU +): {activatedValidatorCount: number} { const depositDataRootList: Root[] = []; - if (fullDepositDataRootList) { - for (let index = 0; index < state.eth1Data.depositCount; index++) { - depositDataRootList.push(fullDepositDataRootList[index]); + + const fullDepositDataRootArr = fullDepositDataRootList ? fullDepositDataRootList.getAllReadonlyValues() : null; + + if (fullDepositDataRootArr) { + const depositCount = state.eth1Data.depositCount; + for (let index = 0; index < depositCount; index++) { + depositDataRootList.push(fullDepositDataRootArr[index]); } } @@ -157,13 +151,13 @@ export function applyDeposits( const {DepositData, DepositDataRootList} = ssz.phase0; for (const [index, deposit] of newDeposits.entries()) { - if (fullDepositDataRootList) { - depositDataRootList.push(fullDepositDataRootList[index + initDepositCount]); - state.eth1Data.depositRoot = DepositDataRootList.hashTreeRoot(depositDataRootList as List); + if (fullDepositDataRootArr) { + depositDataRootList.push(fullDepositDataRootArr[index + initDepositCount]); + state.eth1Data.depositRoot = DepositDataRootList.hashTreeRoot(depositDataRootList); } else if (depositDatas) { const depositDataList = depositDatas.slice(0, index + 1); state.eth1Data.depositRoot = DepositDataRootList.hashTreeRoot( - depositDataList.map((d) => DepositData.hashTreeRoot(d)) as List + depositDataList.map((d) => DepositData.hashTreeRoot(d)) ); } @@ -173,81 +167,104 @@ export function applyDeposits( processDeposit(forkName, state, deposit); } - const activeValidatorIndices: ValidatorIndex[] = []; // Process activations - // validators are edited, so we're not iterating (read-only) through the validators - const validatorLength = state.validators.length; - for (let index = 0; index < validatorLength; index++) { - const validator = state.validators[index]; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const balance = state.balanceList.get(index)!; + const {epochCtx} = state; + const balancesArr = state.balances.getAll(); + const validatorCount = state.validators.length; + let activatedValidatorCount = 0; + + for (let i = 0; i < validatorCount; i++) { + // For the case if effective balance has to be updated, get a mutable view + const validator = state.validators.get(i); + + // Already active, ignore + if (validator.activationEpoch === GENESIS_EPOCH) { + continue; + } + + const balance = balancesArr[i]; const effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); + validator.effectiveBalance = effectiveBalance; - state.effectiveBalanceIncrementsSet(index, effectiveBalance); + epochCtx.effectiveBalanceIncrementsSet(i, effectiveBalance); if (validator.effectiveBalance === MAX_EFFECTIVE_BALANCE) { validator.activationEligibilityEpoch = GENESIS_EPOCH; validator.activationEpoch = GENESIS_EPOCH; - activeValidatorIndices.push(index); + activatedValidatorCount++; } - // If state is a CachedBeaconState<> validator has to be re-assigned manually - state.validators[index] = validator; } // Set genesis validators root for domain separation and chain versioning - state.genesisValidatorsRoot = config - .getForkTypes(state.slot) - .BeaconState.fields.validators.hashTreeRoot(state.validators); - return activeValidatorIndices; + // .hashTreeRoot() automatically commits() + state.genesisValidatorsRoot = state.validators.hashTreeRoot(); + + return {activatedValidatorCount}; } /** * Mainly used for spec test. * * SLOW CODE - 🐢 - * - * @param config - * @param eth1BlockHash - * @param eth1Timestamp - * @param deposits */ export function initializeBeaconStateFromEth1( config: IChainForkConfig, + immutableData: EpochContextImmutableData, eth1BlockHash: Bytes32, - eth1Timestamp: Number64, + eth1Timestamp: TimeSeconds, deposits: phase0.Deposit[], - fullDepositDataRootList?: TreeBacked>, - executionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultTreeBacked() -): TreeBacked { - const state = getGenesisBeaconState( - // CachedBeaconcState is used for convinience only, we return TreeBacked anyway + fullDepositDataRootList?: DepositDataRootViewDU, + executionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultViewDU() +): CachedBeaconStateAllForks { + const stateView = getGenesisBeaconState( + // CachedBeaconcState is used for convinience only, we return BeaconStateAllForks anyway // so it's safe to do a cast here, we can't use get domain until we have genesisValidatorRoot config as IBeaconConfig, ssz.phase0.Eth1Data.defaultValue(), getTemporaryBlockHeader(config, config.getForkTypes(GENESIS_SLOT).BeaconBlock.defaultValue()) ); + // We need a CachedBeaconState to run processDeposit() which uses various caches. + // However at this point the state's syncCommittees are not known. + // This function can be called by: + // - 1. genesis spec tests: Don't care about the committee cache + // - 2. genesis builder: Only supports starting from genesis at phase0 fork + // - 3. interop state: Only supports starting from genesis at phase0 fork + // So it's okay to skip syncing the sync committee cache here and expect it to be + // populated latter when the altair fork happens for cases 2, 3. + const state = createCachedBeaconState(stateView, immutableData, {skipSyncCommitteeCache: true}); + applyTimestamp(config, state, eth1Timestamp); applyEth1BlockHash(state, eth1BlockHash); // Process deposits - const activeValidatorIndices = applyDeposits(config, state, deposits, fullDepositDataRootList); + applyDeposits(config, state, deposits, fullDepositDataRootList); + + // Commit before reading all validators in `getActiveValidatorIndices()` + state.commit(); + const activeValidatorIndices = getActiveValidatorIndices(state, computeEpochAtSlot(GENESIS_SLOT)); if (GENESIS_SLOT >= config.ALTAIR_FORK_EPOCH) { - const syncCommittees = getNextSyncCommittee(state, activeValidatorIndices, state.effectiveBalanceIncrements); - const stateAltair = state as TreeBacked; + const {syncCommittee} = getNextSyncCommittee( + state, + activeValidatorIndices, + state.epochCtx.effectiveBalanceIncrements + ); + const stateAltair = state as CompositeViewDU; stateAltair.fork.previousVersion = config.ALTAIR_FORK_VERSION; stateAltair.fork.currentVersion = config.ALTAIR_FORK_VERSION; - stateAltair.currentSyncCommittee = syncCommittees; - stateAltair.nextSyncCommittee = syncCommittees; + stateAltair.currentSyncCommittee = ssz.altair.SyncCommittee.toViewDU(syncCommittee); + stateAltair.nextSyncCommittee = ssz.altair.SyncCommittee.toViewDU(syncCommittee); } if (GENESIS_SLOT >= config.BELLATRIX_FORK_EPOCH) { - const stateBellatrix = state as TreeBacked; + const stateBellatrix = state as CompositeViewDU; stateBellatrix.fork.previousVersion = config.BELLATRIX_FORK_VERSION; stateBellatrix.fork.currentVersion = config.BELLATRIX_FORK_VERSION; stateBellatrix.latestExecutionPayloadHeader = executionPayloadHeader; } + state.commit(); + return state; } diff --git a/packages/beacon-state-transition/src/util/index.ts b/packages/beacon-state-transition/src/util/index.ts index 08075da6bfc..e3a963ff996 100644 --- a/packages/beacon-state-transition/src/util/index.ts +++ b/packages/beacon-state-transition/src/util/index.ts @@ -2,7 +2,6 @@ * @module chain/stateTransition/util */ -export * from "./aggregationBits"; export * from "./aggregator"; export * from "./array"; export * from "./attestation"; @@ -22,6 +21,5 @@ export * from "./signatureSets"; export * from "./signingRoot"; export * from "./slot"; export * from "./syncCommittee"; -export * from "./unsafeUint8ArrayToTree"; export * from "./validator"; export * from "./weakSubjectivity"; diff --git a/packages/beacon-state-transition/src/util/interop.ts b/packages/beacon-state-transition/src/util/interop.ts index 3b41a684520..7e43bf9d3c4 100644 --- a/packages/beacon-state-transition/src/util/interop.ts +++ b/packages/beacon-state-transition/src/util/interop.ts @@ -1,5 +1,5 @@ import {toBufferBE} from "bigint-buffer"; -import {hash} from "@chainsafe/ssz"; +import {digest} from "@chainsafe/as-sha256"; import bls, {SecretKey} from "@chainsafe/bls"; import {bytesToBigInt, intToBytes} from "@chainsafe/lodestar-utils"; @@ -17,6 +17,6 @@ export function interopSecretKeys(validatorCount: number): SecretKey[] { export function interopSecretKey(index: number): SecretKey { const CURVE_ORDER = getCurveOrder(); - const secretKeyBytes = toBufferBE(bytesToBigInt(hash(intToBytes(index, 32))) % CURVE_ORDER, 32); + const secretKeyBytes = toBufferBE(bytesToBigInt(digest(intToBytes(index, 32))) % CURVE_ORDER, 32); return bls.SecretKey.fromBytes(secretKeyBytes); } diff --git a/packages/beacon-state-transition/src/util/seed.ts b/packages/beacon-state-transition/src/util/seed.ts index 77adc2e7039..86f30bc293a 100644 --- a/packages/beacon-state-transition/src/util/seed.ts +++ b/packages/beacon-state-transition/src/util/seed.ts @@ -2,8 +2,8 @@ * @module chain/stateTransition/util */ -import {hash} from "@chainsafe/ssz"; -import {Epoch, Bytes32, DomainType, allForks, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {digest} from "@chainsafe/as-sha256"; +import {Epoch, Bytes32, DomainType, ValidatorIndex} from "@chainsafe/lodestar-types"; import {assert, bytesToBigInt, intToBytes} from "@chainsafe/lodestar-utils"; import { DOMAIN_BEACON_PROPOSER, @@ -16,17 +16,17 @@ import { SLOTS_PER_EPOCH, SYNC_COMMITTEE_SIZE, } from "@chainsafe/lodestar-params"; +import {BeaconStateAllForks} from "../types"; import {computeStartSlotAtEpoch} from "./epoch"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements"; -import {IEpochShuffling} from "./epochShuffling"; import {computeEpochAtSlot} from "./epoch"; /** * Compute proposer indices for an epoch */ export function computeProposers( - state: allForks.BeaconState, - shuffling: IEpochShuffling, + state: BeaconStateAllForks, + shuffling: {epoch: Epoch; activeIndices: ValidatorIndex[]}, effectiveBalanceIncrements: EffectiveBalanceIncrements ): number[] { const epochSeed = getSeed(state, shuffling.epoch, DOMAIN_BEACON_PROPOSER); @@ -37,7 +37,7 @@ export function computeProposers( computeProposerIndex( effectiveBalanceIncrements, shuffling.activeIndices, - hash(Buffer.concat([epochSeed, intToBytes(slot, 8)])) + digest(Buffer.concat([epochSeed, intToBytes(slot, 8)])) ) ); } @@ -54,7 +54,9 @@ export function computeProposerIndex( indices: ValidatorIndex[], seed: Uint8Array ): ValidatorIndex { - assert.gt(indices.length, 0, "Validator indices must not be empty"); + if (indices.length === 0) { + throw Error("Validator indices must not be empty"); + } // TODO: Inline outside this function const MAX_RANDOM_BYTE = 2 ** 8 - 1; @@ -64,7 +66,7 @@ export function computeProposerIndex( /* eslint-disable-next-line no-constant-condition */ while (true) { const candidateIndex = indices[computeShuffledIndex(i % indices.length, indices.length, seed)]; - const randByte = hash( + const randByte = digest( Buffer.concat([ seed, // @@ -72,7 +74,6 @@ export function computeProposerIndex( ]) )[i % 32]; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const effectiveBalanceIncrement = effectiveBalanceIncrements[candidateIndex]; if (effectiveBalanceIncrement * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE_INCREMENT * randByte) { return candidateIndex; @@ -95,7 +96,7 @@ export function computeProposerIndex( * SLOW CODE - 🐢 */ export function getNextSyncCommitteeIndices( - state: allForks.BeaconState, + state: BeaconStateAllForks, activeValidatorIndices: ValidatorIndex[], effectiveBalanceIncrements: EffectiveBalanceIncrements ): ValidatorIndex[] { @@ -112,7 +113,7 @@ export function getNextSyncCommitteeIndices( while (syncCommitteeIndices.length < SYNC_COMMITTEE_SIZE) { const shuffledIndex = computeShuffledIndex(i % activeValidatorCount, activeValidatorCount, seed); const candidateIndex = activeValidatorIndices[shuffledIndex]; - const randByte = hash( + const randByte = digest( Buffer.concat([ seed, // @@ -142,14 +143,14 @@ export function computeShuffledIndex(index: number, indexCount: number, seed: By let permuted = index; assert.lt(index, indexCount, "indexCount must be less than index"); assert.lte(indexCount, 2 ** 40, "indexCount too big"); - const _seed = seed.valueOf() as Uint8Array; + const _seed = seed; for (let i = 0; i < SHUFFLE_ROUND_COUNT; i++) { const pivot = Number( - bytesToBigInt(hash(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) + bytesToBigInt(digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount) ); const flip = (pivot + indexCount - permuted) % indexCount; const position = Math.max(permuted, flip); - const source = hash(Buffer.concat([_seed, intToBytes(i, 1), intToBytes(Math.floor(position / 256), 4)])); + const source = digest(Buffer.concat([_seed, intToBytes(i, 1), intToBytes(Math.floor(position / 256), 4)])); const byte = source[Math.floor((position % 256) / 8)]; const bit = (byte >> position % 8) % 2; permuted = bit ? flip : permuted; @@ -160,15 +161,15 @@ export function computeShuffledIndex(index: number, indexCount: number, seed: By /** * Return the randao mix at a recent [[epoch]]. */ -export function getRandaoMix(state: allForks.BeaconState, epoch: Epoch): Bytes32 { - return state.randaoMixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR]; +export function getRandaoMix(state: BeaconStateAllForks, epoch: Epoch): Bytes32 { + return state.randaoMixes.get(epoch % EPOCHS_PER_HISTORICAL_VECTOR); } /** * Return the seed at [[epoch]]. */ -export function getSeed(state: allForks.BeaconState, epoch: Epoch, domainType: DomainType): Uint8Array { +export function getSeed(state: BeaconStateAllForks, epoch: Epoch, domainType: DomainType): Uint8Array { const mix = getRandaoMix(state, epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1); - return hash(Buffer.concat([domainType as Buffer, intToBytes(epoch, 8), mix.valueOf() as Uint8Array])); + return digest(Buffer.concat([domainType as Buffer, intToBytes(epoch, 8), mix])); } diff --git a/packages/beacon-state-transition/src/util/shuffle.ts b/packages/beacon-state-transition/src/util/shuffle.ts index 77713b449ba..15462a899f9 100644 --- a/packages/beacon-state-transition/src/util/shuffle.ts +++ b/packages/beacon-state-transition/src/util/shuffle.ts @@ -1,7 +1,7 @@ /** * @module util/objects */ -import {hash} from "@chainsafe/ssz"; +import {digest} from "@chainsafe/as-sha256"; import {SHUFFLE_ROUND_COUNT} from "@chainsafe/lodestar-params"; import {ValidatorIndex, Bytes32} from "@chainsafe/lodestar-types"; import {assert, bytesToBigInt} from "@chainsafe/lodestar-utils"; @@ -97,7 +97,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): } // Seed is always the first 32 bytes of the hash input, we never have to change this part of the buffer. - const _seed = seed.valueOf() as Uint8Array; + const _seed = seed; Buffer.from(_seed).copy(buf, 0, 0, _SHUFFLE_H_SEED_SIZE); // initial values here are not used: overwritten first within the inner for loop. @@ -111,7 +111,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): buf[_SHUFFLE_H_SEED_SIZE] = r; // Seed is already in place, now just hash the correct part of the buffer, and take a uint64 from it, // and modulo it to get a pivot within range. - const h = hash(buf.slice(0, _SHUFFLE_H_PIVOT_VIEW_SIZE)); + const h = digest(buf.slice(0, _SHUFFLE_H_PIVOT_VIEW_SIZE)); const pivot = Number(bytesToBigInt(h.slice(0, 8)) % BigInt(listSize)) >>> 0; // Split up the for-loop in two: @@ -134,7 +134,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): // We start from the pivot position, and work back to the mirror position (of the part left to the pivot). // This makes us process each pear exactly once (instead of unnecessarily twice, like in the spec) setPositionUint32(pivot >> 8, buf); // already using first pivot byte below. - source = hash(buf); + source = digest(buf); byteV = source[(pivot & 0xff) >> 3]; for (let i = 0, j; i < mirror; i++) { @@ -145,7 +145,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): if ((j & 0xff) == 0xff) { // just overwrite the last part of the buffer, reuse the start (seed, round) setPositionUint32(j >> 8, buf); - source = hash(buf); + source = digest(buf); } // Same trick with byte retrieval. Only every 8th. @@ -171,7 +171,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): // We start at the end, and work back to the mirror point. // This makes us process each pear exactly once (instead of unnecessarily twice, like in the spec) setPositionUint32(end >> 8, buf); - source = hash(buf); + source = digest(buf); byteV = source[(end & 0xff) >> 3]; for (let i = pivot + 1, j; i < mirror; i++) { j = end - i + pivot + 1; @@ -181,7 +181,7 @@ function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): if ((j & 0xff) == 0xff) { // just overwrite the last part of the buffer, reuse the start (seed, round) setPositionUint32(j >> 8, buf); - source = hash(buf); + source = digest(buf); } // Same trick with byte retrieval. Only every 8th. diff --git a/packages/beacon-state-transition/src/util/shufflingDecisionRoot.ts b/packages/beacon-state-transition/src/util/shufflingDecisionRoot.ts index c7772fa745a..1dea08f3e54 100644 --- a/packages/beacon-state-transition/src/util/shufflingDecisionRoot.ts +++ b/packages/beacon-state-transition/src/util/shufflingDecisionRoot.ts @@ -24,7 +24,7 @@ export function proposerShufflingDecisionRoot(state: CachedBeaconStateAllForks): * can be used to key the proposer shuffling for the current epoch. */ function proposerShufflingDecisionSlot(state: CachedBeaconStateAllForks): Slot { - const startSlot = computeStartSlotAtEpoch(state.currentShuffling.epoch); + const startSlot = computeStartSlotAtEpoch(state.epochCtx.epoch); return Math.max(startSlot - 1, 0); } @@ -64,15 +64,14 @@ function attesterShufflingDecisionSlot(state: CachedBeaconStateAllForks, request * - `EpochTooHigh` when `requestedEpoch` is more than 1 after `currentEpoch`. */ function attesterShufflingDecisionEpoch(state: CachedBeaconStateAllForks, requestedEpoch: Epoch): Epoch { - const currentEpoch = state.currentShuffling.epoch; - const previouEpoch = state.previousShuffling.epoch; + const currentEpoch = state.epochCtx.epoch; // Next if (requestedEpoch === currentEpoch + 1) return currentEpoch; // Current - if (requestedEpoch === currentEpoch) return previouEpoch; + if (requestedEpoch === currentEpoch) return Math.max(currentEpoch - 1, 0); // Previous - if (requestedEpoch === currentEpoch - 1) return Math.max(previouEpoch - 1, 0); + if (requestedEpoch === currentEpoch - 1) return Math.max(currentEpoch - 2, 0); if (requestedEpoch < currentEpoch) { throw Error(`EpochTooLow: current ${currentEpoch} requested ${requestedEpoch}`); diff --git a/packages/beacon-state-transition/src/util/signatureSets.ts b/packages/beacon-state-transition/src/util/signatureSets.ts index 8a4ab3c3053..e40ac7fff21 100644 --- a/packages/beacon-state-transition/src/util/signatureSets.ts +++ b/packages/beacon-state-transition/src/util/signatureSets.ts @@ -26,10 +26,10 @@ export function verifySignatureSet(signatureSet: ISignatureSet): boolean { switch (signatureSet.type) { case SignatureSetType.single: - return signature.verify(signatureSet.pubkey, signatureSet.signingRoot.valueOf() as Uint8Array); + return signature.verify(signatureSet.pubkey, signatureSet.signingRoot); case SignatureSetType.aggregate: - return signature.verifyAggregate(signatureSet.pubkeys, signatureSet.signingRoot.valueOf() as Uint8Array); + return signature.verifyAggregate(signatureSet.pubkeys, signatureSet.signingRoot); default: throw Error("Unknown signature set type"); diff --git a/packages/beacon-state-transition/src/util/slot.ts b/packages/beacon-state-transition/src/util/slot.ts index 7647dc12c12..68b8f221d5c 100644 --- a/packages/beacon-state-transition/src/util/slot.ts +++ b/packages/beacon-state-transition/src/util/slot.ts @@ -1,27 +1,27 @@ import {IChainConfig} from "@chainsafe/lodestar-config"; import {GENESIS_SLOT, INTERVALS_PER_SLOT} from "@chainsafe/lodestar-params"; -import {Number64, Slot, Epoch} from "@chainsafe/lodestar-types"; +import {Slot, Epoch, TimeSeconds} from "@chainsafe/lodestar-types"; import {computeStartSlotAtEpoch, computeEpochAtSlot} from "."; -export function getSlotsSinceGenesis(config: IChainConfig, genesisTime: Number64): Slot { +export function getSlotsSinceGenesis(config: IChainConfig, genesisTime: TimeSeconds): Slot { const diffInSeconds = Date.now() / 1000 - genesisTime; return Math.floor(diffInSeconds / config.SECONDS_PER_SLOT); } -export function getCurrentSlot(config: IChainConfig, genesisTime: Number64): Slot { +export function getCurrentSlot(config: IChainConfig, genesisTime: TimeSeconds): Slot { return GENESIS_SLOT + getSlotsSinceGenesis(config, genesisTime); } -export function computeSlotsSinceEpochStart(slot: Slot, epoch?: Epoch): number { +export function computeSlotsSinceEpochStart(slot: Slot, epoch?: Epoch): Slot { const computeEpoch = epoch ?? computeEpochAtSlot(slot); return slot - computeStartSlotAtEpoch(computeEpoch); } -export function computeTimeAtSlot(config: IChainConfig, slot: Slot, genesisTime: Number64): Number64 { +export function computeTimeAtSlot(config: IChainConfig, slot: Slot, genesisTime: TimeSeconds): TimeSeconds { return genesisTime + slot * config.SECONDS_PER_SLOT; } -export function getCurrentInterval(config: IChainConfig, genesisTime: Number64, secondsIntoSlot: number): number { +export function getCurrentInterval(config: IChainConfig, secondsIntoSlot: number): number { const timePerInterval = Math.floor(config.SECONDS_PER_SLOT / INTERVALS_PER_SLOT); return Math.floor(secondsIntoSlot / timePerInterval); } diff --git a/packages/beacon-state-transition/src/util/syncCommittee.ts b/packages/beacon-state-transition/src/util/syncCommittee.ts index 7b0392da95d..199afdd6ba6 100644 --- a/packages/beacon-state-transition/src/util/syncCommittee.ts +++ b/packages/beacon-state-transition/src/util/syncCommittee.ts @@ -7,8 +7,9 @@ import { SYNC_REWARD_WEIGHT, WEIGHT_DENOMINATOR, } from "@chainsafe/lodestar-params"; -import {allForks, altair, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {altair, ValidatorIndex} from "@chainsafe/lodestar-types"; import {bigIntSqrt} from "@chainsafe/lodestar-utils"; +import {BeaconStateAllForks} from "../types"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements"; import {getNextSyncCommitteeIndices} from "./seed"; @@ -18,16 +19,21 @@ import {getNextSyncCommitteeIndices} from "./seed"; * SLOW CODE - 🐢 */ export function getNextSyncCommittee( - state: allForks.BeaconState, + state: BeaconStateAllForks, activeValidatorIndices: ValidatorIndex[], effectiveBalanceIncrements: EffectiveBalanceIncrements -): altair.SyncCommittee { +): {indices: ValidatorIndex[]; syncCommittee: altair.SyncCommittee} { const indices = getNextSyncCommitteeIndices(state, activeValidatorIndices, effectiveBalanceIncrements); + // Using the index2pubkey cache is slower because it needs the serialized pubkey. - const pubkeys = indices.map((index) => state.validators[index].pubkey); + const pubkeys = indices.map((index) => state.validators.get(index).pubkey); + return { - pubkeys, - aggregatePubkey: aggregatePublicKeys(pubkeys.map((pubkey) => pubkey.valueOf() as Uint8Array)), + indices, + syncCommittee: { + pubkeys, + aggregatePubkey: aggregatePublicKeys(pubkeys), + }, }; } diff --git a/packages/beacon-state-transition/src/util/unsafeUint8ArrayToTree.ts b/packages/beacon-state-transition/src/util/unsafeUint8ArrayToTree.ts deleted file mode 100644 index f2a57cfba83..00000000000 --- a/packages/beacon-state-transition/src/util/unsafeUint8ArrayToTree.ts +++ /dev/null @@ -1,26 +0,0 @@ -import {LeafNode, Node, subtreeFillToContents} from "@chainsafe/persistent-merkle-tree"; - -/** - * Convert a Uint8Array to a merkle tree, using the underlying array's underlying ArrayBuffer - * - * `data` MUST NOT be modified after this, or risk the merkle nodes being modified. - */ -export function unsafeUint8ArrayToTree(data: Uint8Array, depth: number): Node { - const leaves: LeafNode[] = []; - - // Loop 32 bytes at a time, creating leaves from the backing subarray - const maxStartIndex = data.length - 31; - for (let i = 0; i < maxStartIndex; i += 32) { - leaves.push(new LeafNode(data.subarray(i, i + 32))); - } - - // If there is any extra data at the end (less than 32 bytes), append a final leaf - const lengthMod32 = data.length % 32; - if (lengthMod32 !== 0) { - const finalChunk = new Uint8Array(32); - finalChunk.set(data.subarray(data.length - lengthMod32)); - leaves.push(new LeafNode(finalChunk)); - } - - return subtreeFillToContents(leaves, depth); -} diff --git a/packages/beacon-state-transition/src/util/validator.ts b/packages/beacon-state-transition/src/util/validator.ts index 2d146a3536f..e70c5f70631 100644 --- a/packages/beacon-state-transition/src/util/validator.ts +++ b/packages/beacon-state-transition/src/util/validator.ts @@ -2,10 +2,10 @@ * @module chain/stateTransition/util */ -import {readonlyValues} from "@chainsafe/ssz"; -import {Epoch, phase0, ValidatorIndex, allForks} from "@chainsafe/lodestar-types"; +import {Epoch, phase0, ValidatorIndex} from "@chainsafe/lodestar-types"; import {intDiv} from "@chainsafe/lodestar-utils"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {BeaconStateAllForks} from "../types"; /** * Check if [[validator]] is active @@ -26,15 +26,16 @@ export function isSlashableValidator(validator: phase0.Validator, epoch: Epoch): * * NAIVE - SLOW CODE 🐢 */ -export function getActiveValidatorIndices(state: allForks.BeaconState, epoch: Epoch): ValidatorIndex[] { +export function getActiveValidatorIndices(state: BeaconStateAllForks, epoch: Epoch): ValidatorIndex[] { const indices: ValidatorIndex[] = []; - let index = 0; - for (const validator of readonlyValues(state.validators)) { - if (isActiveValidator(validator, epoch)) { - indices.push(index); + + const validatorsArr = state.validators.getAllReadonlyValues(); + for (let i = 0; i < validatorsArr.length; i++) { + if (isActiveValidator(validatorsArr[i], epoch)) { + indices.push(i); } - index++; } + return indices; } diff --git a/packages/beacon-state-transition/src/util/weakSubjectivity.ts b/packages/beacon-state-transition/src/util/weakSubjectivity.ts index 3dfdfcf6704..169adcd109c 100644 --- a/packages/beacon-state-transition/src/util/weakSubjectivity.ts +++ b/packages/beacon-state-transition/src/util/weakSubjectivity.ts @@ -5,11 +5,11 @@ import { MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH, } from "@chainsafe/lodestar-params"; -import {allForks, Epoch, Root} from "@chainsafe/lodestar-types"; +import {Epoch, Root} from "@chainsafe/lodestar-types"; import {ssz} from "@chainsafe/lodestar-types"; import {Checkpoint} from "@chainsafe/lodestar-types/phase0"; import {toHexString} from "@chainsafe/ssz"; -import {CachedBeaconStateAllForks} from "../types"; +import {CachedBeaconStateAllForks, BeaconStateAllForks} from "../types"; import { getActiveValidatorIndices, getCurrentEpoch, @@ -20,7 +20,7 @@ import { } from ".."; export const ETH_TO_GWEI = 10 ** 9; -const SAFETY_DECAY = BigInt(10); +const SAFETY_DECAY = 10; /** * Returns the epoch of the latest weak subjectivity checkpoint for the given @@ -45,10 +45,10 @@ export function computeWeakSubjectivityPeriodCachedState( config: IChainForkConfig, state: CachedBeaconStateAllForks ): number { - const activeValidatorCount = state.currentShuffling.activeIndices.length; + const activeValidatorCount = state.epochCtx.currentShuffling.activeIndices.length; return computeWeakSubjectivityPeriodFromConstituents( activeValidatorCount, - state.totalActiveBalanceIncrements, + state.epochCtx.totalActiveBalanceIncrements, getChurnLimit(config, activeValidatorCount), config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY ); @@ -58,11 +58,12 @@ export function computeWeakSubjectivityPeriodCachedState( * Same to computeWeakSubjectivityPeriodCachedState but for normal state * This is called only 1 time at app startup so it's ok to calculate totalActiveBalanceIncrements manually */ -export function computeWeakSubjectivityPeriod(config: IChainForkConfig, state: allForks.BeaconState): number { +export function computeWeakSubjectivityPeriod(config: IChainForkConfig, state: BeaconStateAllForks): number { const activeIndices = getActiveValidatorIndices(state, getCurrentEpoch(state)); + const validators = state.validators.getAllReadonlyValues(); let totalActiveBalanceIncrements = 0; for (const index of activeIndices) { - totalActiveBalanceIncrements += Math.floor(state.validators[index].effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); + totalActiveBalanceIncrements += Math.floor(validators[index].effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); } if (totalActiveBalanceIncrements <= 1) { totalActiveBalanceIncrements = 1; @@ -86,11 +87,11 @@ export function computeWeakSubjectivityPeriodFromConstituents( // totalBalanceByIncrement = totalBalance / MAX_EFFECTIVE_BALANCE and MAX_EFFECTIVE_BALANCE = ETH_TO_GWEI atm // we need to change this calculation just in case MAX_EFFECTIVE_BALANCE != ETH_TO_GWEI const t = Math.floor(totalBalanceByIncrement / N); - const T = Number(MAX_EFFECTIVE_BALANCE / ETH_TO_GWEI); + const T = MAX_EFFECTIVE_BALANCE / ETH_TO_GWEI; const delta = churnLimit; // eslint-disable-next-line @typescript-eslint/naming-convention const Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH; - const D = Number(SAFETY_DECAY); + const D = SAFETY_DECAY; let wsPeriod = minWithdrawabilityDelay; if (T * (200 + 3 * D) < t * (200 + 12 * D)) { @@ -106,21 +107,21 @@ export function computeWeakSubjectivityPeriodFromConstituents( return wsPeriod; } -export function getLatestBlockRoot(config: IChainForkConfig, state: allForks.BeaconState): Root { +export function getLatestBlockRoot(state: BeaconStateAllForks): Root { const header = ssz.phase0.BeaconBlockHeader.clone(state.latestBlockHeader); if (ssz.Root.equals(header.stateRoot, ZERO_HASH)) { - header.stateRoot = config.getForkTypes(state.slot).BeaconState.hashTreeRoot(state); + header.stateRoot = state.hashTreeRoot(); } return ssz.phase0.BeaconBlockHeader.hashTreeRoot(header); } export function isWithinWeakSubjectivityPeriod( config: IBeaconConfig, - wsState: allForks.BeaconState, + wsState: BeaconStateAllForks, wsCheckpoint: Checkpoint ): boolean { const wsStateEpoch = computeEpochAtSlot(wsState.slot); - const blockRoot = getLatestBlockRoot(config, wsState); + const blockRoot = getLatestBlockRoot(wsState); if (!ssz.Root.equals(blockRoot, wsCheckpoint.root)) { throw new Error( `Roots do not match. expected=${toHexString(wsCheckpoint.root)}, actual=${toHexString(blockRoot)}` diff --git a/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts b/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts index c384f946a9e..d4b1d7fd4b9 100644 --- a/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/hashing.test.ts @@ -1,7 +1,6 @@ import {itBench} from "@dapplion/benchmark"; -import {phase0, ssz} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; -import {generatePerformanceStatePhase0, getPubkeys, numValidators} from "../util"; +import {ssz} from "@chainsafe/lodestar-types"; +import {generatePerfTestCachedStatePhase0, numValidators} from "../util"; import {unshuffleList} from "../../../src"; // Test cost of hashing state after some modifications @@ -9,22 +8,21 @@ import {unshuffleList} from "../../../src"; describe("BeaconState hashTreeRoot", () => { const vc = numValidators; const indicesShuffled: number[] = []; - let stateOg: TreeBacked; + let stateOg: ReturnType; before(function () { this.timeout(300_000); - const {pubkeys} = getPubkeys(); - stateOg = generatePerformanceStatePhase0(pubkeys); + stateOg = generatePerfTestCachedStatePhase0(); stateOg.hashTreeRoot(); for (let i = 0; i < vc; i++) indicesShuffled[i] = i; unshuffleList(indicesShuffled, new Uint8Array([42, 32])); }); - const validator: phase0.Validator = ssz.phase0.Validator.defaultValue(); + const validator = ssz.phase0.Validator.defaultViewDU(); const balance = 31e9; - const testCases: {id: string; noTrack?: boolean; fn: (state: TreeBacked) => void}[] = [ + const testCases: {id: string; noTrack?: boolean; fn: (state: typeof stateOg) => void}[] = [ { id: "No change", fn: () => { @@ -39,7 +37,7 @@ describe("BeaconState hashTreeRoot", () => { id: `${count} full validator`, noTrack: count < 512, fn: (state) => { - for (const i of indicesShuffled.slice(0, count)) state.validators[i] = validator; + for (const i of indicesShuffled.slice(0, count)) state.validators.set(i, validator); }, }); } @@ -50,7 +48,7 @@ describe("BeaconState hashTreeRoot", () => { noTrack: count < 512, fn: (state) => { for (const i of indicesShuffled.slice(0, count)) { - state.validators[i].effectiveBalance = balance; + state.validators.get(i).effectiveBalance = balance; } }, }); @@ -62,18 +60,19 @@ describe("BeaconState hashTreeRoot", () => { id: `${count} balances`, noTrack: count < 512, fn: (state) => { - for (const i of indicesShuffled.slice(0, count)) state.balances[i] = balance; + for (const i of indicesShuffled.slice(0, count)) state.balances.set(i, balance); }, }); } for (const {id, noTrack, fn} of testCases) { - itBench, TreeBacked>({ + itBench({ id: `BeaconState.hashTreeRoot - ${id}`, noThreshold: noTrack, beforeEach: () => { const state = stateOg.clone(); fn(state); + state.commit(); return state; }, fn: (state) => { diff --git a/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts b/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts index 602a6d82d90..2f3ba9bcefa 100644 --- a/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/util/epochContext.test.ts @@ -15,7 +15,7 @@ describe("epochCtx.getCommitteeAssignments", () => { before(function () { this.timeout(60 * 1000); - state = generatePerfTestCachedStatePhase0() as CachedBeaconStateAllForks; + state = generatePerfTestCachedStatePhase0(); epoch = computeEpochAtSlot(state.slot); // Sanity check to ensure numValidators doesn't go stale diff --git a/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts b/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts index c905793c40e..e0ba1ebdb1e 100644 --- a/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts +++ b/packages/beacon-state-transition/test/perf/allForks/util/shufflings.test.ts @@ -15,7 +15,7 @@ describe("epoch shufflings", () => { before(function () { this.timeout(60 * 1000); - state = generatePerfTestCachedStatePhase0() as CachedBeaconStateAllForks; + state = generatePerfTestCachedStatePhase0(); nextEpoch = computeEpochAtSlot(state.slot) + 1; // Sanity check to ensure numValidators doesn't go stale @@ -25,7 +25,7 @@ describe("epoch shufflings", () => { itBench({ id: `computeProposers - vc ${numValidators}`, fn: () => { - computeProposers(state, state.epochCtx.nextShuffling, state.effectiveBalanceIncrements); + computeProposers(state, state.epochCtx.nextShuffling, state.epochCtx.effectiveBalanceIncrements); }, }); @@ -39,7 +39,11 @@ describe("epoch shufflings", () => { itBench({ id: `getNextSyncCommittee - vc ${numValidators}`, fn: () => { - getNextSyncCommittee(state, state.epochCtx.nextShuffling.activeIndices, state.effectiveBalanceIncrements); + getNextSyncCommittee( + state, + state.epochCtx.nextShuffling.activeIndices, + state.epochCtx.effectiveBalanceIncrements + ); }, }); }); diff --git a/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts index ea55a1f67c2..7589569c357 100644 --- a/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/block/processAttestation.test.ts @@ -10,12 +10,16 @@ import { SLOTS_PER_EPOCH, SYNC_COMMITTEE_SIZE, } from "@chainsafe/lodestar-params"; +import {phase0} from "@chainsafe/lodestar-types"; import {altair, CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../../../src"; import {generatePerfTestCachedStateAltair, perfStateId} from "../../util"; import {BlockAltairOpts, getBlockAltair} from "../../phase0/block/util"; -import {StateAltair, StateAttestations} from "../../types"; -import {ParticipationFlags, phase0} from "@chainsafe/lodestar-types"; -import {updateEpochParticipants} from "../../../../src/altair/block/processAttestation"; +import {StateAltair} from "../../types"; + +type StateAttestations = { + state: CachedBeaconStateAllForks; + attestations: phase0.Attestation[]; +}; // Most of the cost of processAttestation in altair is for updating participation flag tree describe("altair processAttestation", () => { @@ -55,14 +59,23 @@ describe("altair processAttestation", () => { itBench({ id: `altair processAttestation - ${perfStateId} ${id}`, before: () => { - const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; - const block = getBlockAltair(state, opts); - state.hashTreeRoot(); + const state = generatePerfTestCachedStateAltair(); + const block = getBlockAltair(state as CachedBeaconStateAltair, opts); return {state, attestations: block.message.body.attestations as phase0.Attestation[]}; }, - beforeEach: ({state, attestations}) => ({state: state.clone(), attestations}), + beforeEach: ({state, attestations}) => { + const stateCloned = state.clone(); + // Populate state array cache (on the cloned instance) + (stateCloned as CachedBeaconStateAltair).previousEpochParticipation.getAll(); + (stateCloned as CachedBeaconStateAltair).currentEpochParticipation.getAll(); + (stateCloned as CachedBeaconStateAltair).balances.getAll(); + return {state: stateCloned, attestations}; + }, fn: ({state, attestations}) => { altair.processAttestations(state as CachedBeaconStateAltair, attestations, false); + state.commit(); + // After processAttestations normal case vc 250_000 it has to do 6802 hash64 ops + // state.hashTreeRoot(); }, }); } @@ -89,62 +102,27 @@ describe("altair processAttestation - CachedEpochParticipation.setStatus", () => state.hashTreeRoot(); return state; }, - beforeEach: (state) => state.clone(), - fn: (state) => { - const {currentEpochParticipation} = state; - const numAttesters = Math.floor((state.currentShuffling.activeIndices.length * ratio) / SLOTS_PER_EPOCH); - // just get committees of slot 10 - let count = 0; - for (const committees of state.currentShuffling.committees[10]) { - for (const committee of committees) { - currentEpochParticipation.set(committee, 0b111); - count++; - if (count >= numAttesters) break; - } - } - }, - }); - } - - for (const {name, ratio} of testCases) { - itBench({ - id: `altair processAttestation - updateEpochParticipants - ${name} join`, - before: () => { - const state = generatePerfTestCachedStateAltair(); - state.hashTreeRoot(); - return state; + beforeEach: (state) => { + const stateCloned = state.clone(); + // Populate state array cache (on the cloned instance) + stateCloned.currentEpochParticipation.getAll(); + return stateCloned; }, - beforeEach: (state) => state.clone(), fn: (state) => { const {currentEpochParticipation} = state; - const numAttesters = Math.floor((state.currentShuffling.activeIndices.length * ratio) / SLOTS_PER_EPOCH); + const numAttesters = Math.floor( + (state.epochCtx.currentShuffling.activeIndices.length * ratio) / SLOTS_PER_EPOCH + ); // just get committees of slot 10 let count = 0; - const epochStatuses = new Map(); - for (const committees of state.currentShuffling.committees[10]) { + for (const committees of state.epochCtx.currentShuffling.committees[10]) { for (const committee of committees) { - // currentEpochParticipation.setStatus(committee, {timelyHead: true, timelySource: true, timelyTarget: true}); - epochStatuses.set(committee, 0b111); + currentEpochParticipation.set(committee, 0b111); count++; if (count >= numAttesters) break; } } - updateEpochParticipants(epochStatuses, currentEpochParticipation, state.currentShuffling.activeIndices.length); }, }); } - - itBench({ - id: "altair processAttestation - updateAllStatus", - before: () => { - const state = generatePerfTestCachedStateAltair(); - state.hashTreeRoot(); - return state; - }, - beforeEach: (state) => state.clone(), - fn: (state) => { - const {currentEpochParticipation} = state; - currentEpochParticipation.updateAllStatus(currentEpochParticipation.persistent.vector); - }, - }); }); diff --git a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts index 40f1b0a32bc..f8f45fa46ae 100644 --- a/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/block/processBlock.test.ts @@ -1,4 +1,5 @@ import {itBench} from "@dapplion/benchmark"; +import {ssz} from "@chainsafe/lodestar-types"; import { ACTIVE_PRESET, MAX_ATTESTATIONS, @@ -9,8 +10,8 @@ import { PresetName, SYNC_COMMITTEE_SIZE, } from "@chainsafe/lodestar-params"; -import {allForks, CachedBeaconStateAllForks} from "../../../../src"; -import {generatePerfTestCachedStateAltair, perfStateId} from "../../util"; +import {allForks, CachedBeaconStateAltair} from "../../../../src"; +import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair, perfStateId} from "../../util"; import {BlockAltairOpts, getBlockAltair} from "../../phase0/block/util"; import {StateBlock} from "../../types"; @@ -31,7 +32,7 @@ import {StateBlock} from "../../types"; // // ### Verifying signatures // Signature verification is done in bulk using batch BLS verification. Performance is proportional to the amount of -// sigs to verify and the cost to construct the signature sets from TreeBacked data. +// sigs to verify and the cost to construct the signature sets from TreeView data. // // - Same as phase0 // - SyncAggregate sigs: 1 x agg (358 bits on avg) - TODO: assuming same participation as attestations @@ -99,24 +100,41 @@ describe("altair processBlock", () => { ]; for (const {id, opts} of testCases) { - itBench({ - id: `altair processBlock - ${perfStateId} ${id}`, - before: () => { - const state = generatePerfTestCachedStateAltair() as CachedBeaconStateAllForks; - const block = getBlockAltair(state, opts); - state.hashTreeRoot(); - return {state, block}; - }, - beforeEach: ({state, block}) => ({state: state.clone(), block}), - fn: ({state, block}) => { - allForks.stateTransition(state, block, { - verifyProposer: false, - verifySignatures: false, - verifyStateRoot: false, - }); - // set verifyStateRoot = false, and get the root here because the block root is wrong - state.hashTreeRoot(); - }, - }); + for (const hashState of [false, true]) { + itBench({ + id: `altair processBlock - ${perfStateId} ${id}` + (hashState ? " hashState" : ""), + before: () => { + const state = generatePerfTestCachedStateAltair(); + const block = getBlockAltair(state as CachedBeaconStateAltair, opts); + // Populate permanent root caches of the block + ssz.altair.BeaconBlock.hashTreeRoot(block.message); + // Populate tree root caches of the state + state.hashTreeRoot(); + return {state, block}; + }, + beforeEach: ({state, block}) => { + const stateCloned = state.clone(); + // Populate all state array caches (on the cloned instance) + cachedStateAltairPopulateCaches(stateCloned as CachedBeaconStateAltair); + return {state: stateCloned, block}; + }, + fn: ({state, block}) => { + const postState = allForks.stateTransition(state, block, { + verifyProposer: false, + verifySignatures: false, + verifyStateRoot: false, + }); + + // Not necessary to call commit here since it's called inside .stateTransition() + + if (hashState) { + // set verifyStateRoot = false, and get the root here because the block root is wrong + // normalcase vc 250_000: 6947 hashes - 8.683 ms + // worstcase vc 250_000: 15297 hashes - 19.121 ms + postState.hashTreeRoot(); + } + }, + }); + } } }); diff --git a/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts b/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts new file mode 100644 index 00000000000..47f19bc36e4 --- /dev/null +++ b/packages/beacon-state-transition/test/perf/altair/block/processEth1Data.test.ts @@ -0,0 +1,54 @@ +import {itBench} from "@dapplion/benchmark"; +import {ACTIVE_PRESET, PresetName, SYNC_COMMITTEE_SIZE} from "@chainsafe/lodestar-params"; +import {allForks, CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../../../../src"; +import {generatePerfTestCachedStateAltair, perfStateId} from "../../util"; +import {getBlockAltair} from "../../phase0/block/util"; +import {phase0} from "@chainsafe/lodestar-types"; + +type StateEth1Data = { + state: CachedBeaconStateAllForks; + eth1Data: phase0.Eth1Data; +}; + +// Most of the cost of processAttestation in altair is for updating participation flag tree +describe("altair processEth1Data", () => { + if (ACTIVE_PRESET !== PresetName.mainnet) { + throw Error(`ACTIVE_PRESET ${ACTIVE_PRESET} must be mainnet`); + } + + const testCases: {id: string}[] = [ + {id: "normalcase"}, + // TODO: What's a worst case for eth1Data? + // {id: "worstcase"} + ]; + + for (const {id} of testCases) { + itBench({ + id: `altair processEth1Data - ${perfStateId} ${id}`, + before: () => { + const state = generatePerfTestCachedStateAltair(); + const block = getBlockAltair(state as CachedBeaconStateAltair, { + proposerSlashingLen: 0, + attesterSlashingLen: 0, + attestationLen: 90, + depositsLen: 0, + voluntaryExitLen: 0, + bitsLen: 90, + // TODO: There's no data yet on how full syncCommittee will be. Assume same ratio of attestations + syncCommitteeBitsLen: Math.round(SYNC_COMMITTEE_SIZE * 0.7), + }); + state.hashTreeRoot(); + return {state, eth1Data: block.message.body.eth1Data}; + }, + beforeEach: ({state, eth1Data}) => { + const stateCloned = state.clone(); + // Populate nodes cache of eth1DataVotes array (on the cloned instance) + stateCloned.eth1DataVotes.getAllReadonly(); + return {state: stateCloned, eth1Data}; + }, + fn: ({state, eth1Data}) => { + allForks.processEth1Data(state, eth1Data); + }, + }); + } +}); diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts index f601fc27124..b27d9ae991f 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/epoch.test.ts @@ -28,12 +28,13 @@ describe(`altair processEpoch - ${stateId}`, () => { itBench({ id: `altair processEpoch - ${stateId}`, yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 - beforeEach: () => stateOg.value.clone() as CachedBeaconStateAllForks, + beforeEach: () => stateOg.value.clone(), fn: (state) => { const epochProcess = beforeProcessEpoch(state); altair.processEpoch(state as CachedBeaconStateAltair, epochProcess); state.epochCtx.afterProcessEpoch(state, epochProcess); // Simulate root computation through the next block to account for changes + // 74184 hash64 ops - 92.730 ms state.hashTreeRoot(); }, }); diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts index 397a7d41f27..3cf8e943180 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processRewardsAndPenalties.test.ts @@ -5,8 +5,7 @@ import {FlagFactors, generateBalanceDeltasEpochProcess} from "../../phase0/epoch import {StateAltairEpoch} from "../../types"; import {mutateInactivityScores} from "./util"; -// PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags -// are true, worst case: FLAG_UNSLASHED + FLAG_ELIGIBLE_ATTESTER + FLAG_PREV_* +// PERF: Cost = 'proportional' to $VALIDATOR_COUNT. Extra work is done per validator the more status flags are set // Worst case: // statuses: everything true diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts index 234ce3b2c94..b93ac7104fc 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/processSyncCommitteeUpdates.test.ts @@ -1,6 +1,6 @@ import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@chainsafe/lodestar-params"; import {itBench} from "@dapplion/benchmark"; -import {altair} from "../../../../src"; +import {processSyncCommitteeUpdates} from "../../../../src/altair"; import {StateAltair} from "../../types"; import {generatePerfTestCachedStateAltair, numValidators} from "../../util"; @@ -12,12 +12,19 @@ describe("altair processSyncCommitteeUpdates", () => { itBench({ id: `altair processSyncCommitteeUpdates - ${vc}`, yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 - before: () => { - const state = generatePerfTestCachedStateAltair({goBackOneSlot: true}); - state.epochCtx.epoch = EPOCHS_PER_SYNC_COMMITTEE_PERIOD - 1; - return state; + before: () => generatePerfTestCachedStateAltair({goBackOneSlot: true}), + beforeEach: (state) => { + const stateCloned = state.clone(); + // Force processSyncCommitteeUpdates to run + stateCloned.epochCtx.epoch = EPOCHS_PER_SYNC_COMMITTEE_PERIOD - 1; + return stateCloned; + }, + fn: (state) => { + const nextSyncCommitteeBefore = state.nextSyncCommittee; + processSyncCommitteeUpdates(state); + if (state.nextSyncCommittee === nextSyncCommitteeBefore) { + throw Error("nextSyncCommittee instance has not changed"); + } }, - beforeEach: (state) => state.clone(), - fn: (state) => altair.processSyncCommitteeUpdates(state), }); }); diff --git a/packages/beacon-state-transition/test/perf/altair/epoch/util.ts b/packages/beacon-state-transition/test/perf/altair/epoch/util.ts index 58f00fc47a0..802d2217511 100644 --- a/packages/beacon-state-transition/test/perf/altair/epoch/util.ts +++ b/packages/beacon-state-transition/test/perf/altair/epoch/util.ts @@ -3,6 +3,6 @@ import {CachedBeaconStateAltair} from "../../../../src"; export function mutateInactivityScores(state: CachedBeaconStateAltair, factorWithPositive: number): void { const vc = state.inactivityScores.length; for (let i = 0; i < vc; i++) { - state.inactivityScores[i] = i < vc * factorWithPositive ? 100 : 0; + state.inactivityScores.set(i, i < vc * factorWithPositive ? 100 : 0); } } diff --git a/packages/beacon-state-transition/test/perf/analyzeBlocks.ts b/packages/beacon-state-transition/test/perf/analyzeBlocks.ts index 5163d25642e..a897ab55cbe 100644 --- a/packages/beacon-state-transition/test/perf/analyzeBlocks.ts +++ b/packages/beacon-state-transition/test/perf/analyzeBlocks.ts @@ -59,8 +59,9 @@ async function run(): Promise { voluntaryExits += block.message.body.voluntaryExits.length; for (const attestation of block.message.body.attestations) { + const indexes = Array.from({length: attestation.aggregationBits.bitLen}, () => 0); + aggregationBits += attestation.aggregationBits.intersectValues(indexes).length; inclusionDistance += block.message.slot - attestation.data.slot; - aggregationBits += Array.from(attestation.aggregationBits).filter((bit) => bit === true).length; } } diff --git a/packages/beacon-state-transition/test/perf/analyzeEpochs.ts b/packages/beacon-state-transition/test/perf/analyzeEpochs.ts index c7f5172ab9d..5aaf1f8b344 100644 --- a/packages/beacon-state-transition/test/perf/analyzeEpochs.ts +++ b/packages/beacon-state-transition/test/perf/analyzeEpochs.ts @@ -8,15 +8,14 @@ import { allForks, computeEpochAtSlot, computeStartSlotAtEpoch, - CachedBeaconStateAllForks, AttesterFlags, - createCachedBeaconState, beforeProcessEpoch, parseAttesterFlags, } from "../../src"; import {Validator} from "../../lib/phase0"; import {csvAppend, readCsv} from "./csv"; import {getInfuraBeaconUrl} from "./infura"; +import {createCachedBeaconStateTest} from "../utils/state"; // Understand the real network characteristics regarding epoch transitions to accurately produce performance test data. // @@ -97,11 +96,11 @@ async function analyzeEpochs(network: NetworkName, fromEpoch?: number): Promise< const preEpoch = computeEpochAtSlot(state.slot); const nextEpochSlot = computeStartSlotAtEpoch(preEpoch + 1); - const stateTB = ssz.phase0.BeaconState.createTreeBackedFromStruct(state as phase0.BeaconState); - const postState = createCachedBeaconState(config, stateTB); + const stateTB = ssz.phase0.BeaconState.toViewDU(state as phase0.BeaconState); + const postState = createCachedBeaconStateTest(stateTB, config); const epochProcess = beforeProcessEpoch(postState); - allForks.processSlots(postState as CachedBeaconStateAllForks, nextEpochSlot, null); + allForks.processSlots(postState, nextEpochSlot, null); const validatorCount = state.validators.length; @@ -109,7 +108,7 @@ async function analyzeEpochs(network: NetworkName, fromEpoch?: number): Promise< const validatorKeys = Object.keys(validatorChangesCountZero) as (keyof typeof validatorChangesCountZero)[]; for (let i = 0; i < validatorCount; i++) { const validatorPrev = state.validators[i]; - const validatorNext = postState.validators[i]; + const validatorNext = postState.validators.get(i); for (const key of validatorKeys) { const valuePrev = validatorPrev[key]; const valueNext = validatorNext[key]; @@ -172,9 +171,7 @@ async function analyzeEpochs(network: NetworkName, fromEpoch?: number): Promise< function countAttBits(atts: phase0.PendingAttestation[]): number { let totalBits = 0; for (const att of atts) { - for (const bit of att.aggregationBits) { - if (bit) totalBits++; - } + totalBits += att.aggregationBits.getTrueBitIndexes().length; } return totalBits / atts.length; } diff --git a/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts b/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts index 5855773c5b1..f9853ed843b 100644 --- a/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts +++ b/packages/beacon-state-transition/test/perf/dataStructures/arrayish.test.ts @@ -50,7 +50,7 @@ describe("Tree (persistent-merkle-tree)", () => { const d = 40; const gih = toGindex(d, BigInt(ih)); - const n2 = new LeafNode(Buffer.alloc(32, 2)); + const n2 = LeafNode.fromRoot(Buffer.alloc(32, 2)); let tree: Tree; before(function () { @@ -89,7 +89,7 @@ describe("Tree (persistent-merkle-tree)", () => { }); function getTree(d: number, n: number): Tree { - const leaf = new LeafNode(Buffer.alloc(32, 1)); + const leaf = LeafNode.fromRoot(Buffer.alloc(32, 1)); const startIndex = BigInt(2 ** d); const tree = new Tree(zeroNode(d)); for (let i = BigInt(0), nB = BigInt(n); i < nB; i++) { diff --git a/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts b/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts index 85bdec06da5..46327e6dc76 100644 --- a/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts +++ b/packages/beacon-state-transition/test/perf/misc/aggregationBits.test.ts @@ -1,8 +1,7 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {MAX_VALIDATORS_PER_COMMITTEE} from "@chainsafe/lodestar-params"; import {ssz} from "@chainsafe/lodestar-types"; -import {BitList, List, readonlyValues, TreeBacked} from "@chainsafe/ssz"; -import {zipIndexesCommitteeBits} from "../../../src"; +import {BitArray} from "@chainsafe/ssz"; describe("aggregationBits", () => { setBenchOpts({noThreshold: true}); @@ -11,11 +10,11 @@ describe("aggregationBits", () => { const idPrefix = `aggregationBits - ${len} els`; let indexes: number[]; - let bitlistTree: TreeBacked; + let bitlistTree: BitArray; before(function () { - const aggregationBits = Array.from({length: len}, () => true); - bitlistTree = ssz.phase0.CommitteeBits.createTreeBackedFromStruct(aggregationBits as List); + const aggregationBits = BitArray.fromBoolArray(Array.from({length: len}, () => true)); + bitlistTree = ssz.phase0.CommitteeBits.toViewDU(aggregationBits); indexes = Array.from({length: len}, () => 165432); }); @@ -23,11 +22,7 @@ describe("aggregationBits", () => { // aggregationBits - 2048 els - zipIndexesInBitList 50.904 us/op 236.17 us/op 0.22 // This benchmark is very unstable in CI. We already know that zipIndexesInBitList is faster - itBench(`${idPrefix} - readonlyValues`, () => { - Array.from(readonlyValues(bitlistTree)); - }); - itBench(`${idPrefix} - zipIndexesInBitList`, () => { - zipIndexesCommitteeBits(indexes, bitlistTree); + bitlistTree.intersectValues(indexes); }); }); diff --git a/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts b/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts index 0c9ba3d069c..0a63f4fd983 100644 --- a/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts +++ b/packages/beacon-state-transition/test/perf/misc/rootEquals.test.ts @@ -13,18 +13,14 @@ describe("root equals", () => { setBenchOpts({noThreshold: true}); const stateRoot = fromHexString("0x6c86ca3c4c6688cf189421b8a68bf2dbc91521609965e6f4e207d44347061fee"); - const rootTree = ssz.Root.createTreeBackedFromStruct(stateRoot); + const rootTree = ssz.Root.toViewDU(stateRoot); // This benchmark is very unstable in CI. We already know that "ssz.Root.equals" is the fastest itBench("ssz.Root.equals", () => { ssz.Root.equals(rootTree, stateRoot); }); - itBench("ssz.Root.equals with valueOf()", () => { - ssz.Root.equals(rootTree.valueOf() as Uint8Array, stateRoot); - }); - - itBench("byteArrayEquals with valueOf()", () => { - byteArrayEquals(rootTree.valueOf() as Uint8Array, stateRoot); + itBench("byteArrayEquals", () => { + byteArrayEquals(rootTree, stateRoot); }); }); diff --git a/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts b/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts index 48adc087dca..8c11050fe88 100644 --- a/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/block/processBlock.test.ts @@ -8,7 +8,7 @@ import { MAX_VOLUNTARY_EXITS, PresetName, } from "@chainsafe/lodestar-params"; -import {allForks, CachedBeaconStateAllForks} from "../../../../src"; +import {allForks} from "../../../../src"; import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; import {BlockOpts, getBlockPhase0} from "./util"; import {StateBlock} from "../../types"; @@ -30,7 +30,7 @@ import {StateBlock} from "../../types"; // // ### Verifying signatures // Signature verification is done in bulk using batch BLS verification. Performance is proportional to the amount of -// sigs to verify and the cost to construct the signature sets from TreeBacked data. +// sigs to verify and the cost to construct the signature sets from TreeView data. // // - Proposer sig: 1 single // - RandaoReveal sig: 1 single @@ -101,7 +101,7 @@ describe("phase0 processBlock", () => { itBench({ id: `phase0 processBlock - ${perfStateId} ${id}`, before: () => { - const state = generatePerfTestCachedStatePhase0() as CachedBeaconStateAllForks; + const state = generatePerfTestCachedStatePhase0(); const block = getBlockPhase0(state, opts); state.hashTreeRoot(); return {block, state}; diff --git a/packages/beacon-state-transition/test/perf/phase0/block/util.ts b/packages/beacon-state-transition/test/perf/phase0/block/util.ts index 140940a6d52..610ebda6a2c 100644 --- a/packages/beacon-state-transition/test/perf/phase0/block/util.ts +++ b/packages/beacon-state-transition/test/perf/phase0/block/util.ts @@ -1,16 +1,17 @@ import {altair, phase0, ssz} from "@chainsafe/lodestar-types"; import {SecretKey} from "@chainsafe/blst"; +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; +import {BitArray} from "@chainsafe/ssz"; import {DOMAIN_DEPOSIT, SYNC_COMMITTEE_SIZE} from "@chainsafe/lodestar-params"; import {config} from "@chainsafe/lodestar-config/default"; -import {List} from "@chainsafe/ssz"; import { computeDomain, computeEpochAtSlot, computeSigningRoot, ZERO_HASH, CachedBeaconStateAllForks, + CachedBeaconStateAltair, } from "../../../../src"; -import {LeafNode} from "@chainsafe/persistent-merkle-tree"; import {getBlockRoot, getBlockRootAtSlot} from "../../../../src"; export type BlockOpts = { @@ -47,7 +48,7 @@ export function getBlockPhase0( const attesterSlashingStartIndex = proposerSlashingStartIndex + proposerSlashingLen * exitedIndexStep; const voluntaryExitStartIndex = attesterSlashingStartIndex + attesterSlashingLen * bitsLen * exitedIndexStep; - const proposerSlashings = ([] as phase0.ProposerSlashing[]) as List; + const proposerSlashings = [] as phase0.ProposerSlashing[]; for (let i = 0; i < proposerSlashingLen; i++) { const proposerIndex = proposerSlashingStartIndex + i * exitedIndexStep; proposerSlashings.push({ @@ -64,11 +65,11 @@ export function getBlockPhase0( const attSlot = stateSlot - 2; const attEpoch = computeEpochAtSlot(attSlot); - const attesterSlashings = ([] as phase0.AttesterSlashing[]) as List; + const attesterSlashings = [] as phase0.AttesterSlashing[]; for (let i = 0; i < attesterSlashingLen; i++) { // Double vote for 128 participants const startIndex = attesterSlashingStartIndex + i * bitsLen * exitedIndexStep; - const attestingIndices = linspace(startIndex, bitsLen, exitedIndexStep) as List; + const attestingIndices = linspace(startIndex, bitsLen, exitedIndexStep); const attData: phase0.AttestationData = { slot: attSlot, @@ -91,8 +92,8 @@ export function getBlockPhase0( }); } - const attestations = ([] as phase0.Attestation[]) as List; - const committeeCountPerSlot = preState.getCommitteeCountPerSlot(attEpoch); + const attestations = [] as phase0.Attestation[]; + const committeeCountPerSlot = preState.epochCtx.getCommitteeCountPerSlot(attEpoch); const attSource = attEpoch === stateEpoch ? preState.currentJustifiedCheckpoint : preState.previousJustifiedCheckpoint; for (let i = 0; i < attestationLen; i++) { @@ -100,7 +101,7 @@ export function getBlockPhase0( const attCommittee = preState.epochCtx.getBeaconCommittee(attSlot, attIndex); // Spread attesting indices through the whole range, offset on each attestation attestations.push({ - aggregationBits: getAggregationBits(attCommittee.length, bitsLen) as List, + aggregationBits: getAggregationBits(attCommittee.length, bitsLen), data: { slot: attSlot, index: attIndex, @@ -115,7 +116,7 @@ export function getBlockPhase0( // Moved to different function since it's a bit complex const deposits = getDeposits(preState, depositsLen); - const voluntaryExits = ([] as phase0.SignedVoluntaryExit[]) as List; + const voluntaryExits = [] as phase0.SignedVoluntaryExit[]; for (let i = 0; i < voluntaryExitLen; i++) { voluntaryExits.push({ message: { @@ -127,10 +128,10 @@ export function getBlockPhase0( } const slot = preState.slot + 1; - return ssz.phase0.SignedBeaconBlock.createTreeBackedFromStruct({ + return { message: { slot, - proposerIndex: preState.getBeaconProposer(slot), + proposerIndex: preState.epochCtx.getBeaconProposer(slot), parentRoot: ssz.phase0.BeaconBlockHeader.hashTreeRoot(preState.latestBlockHeader), // TODO: Compute the state root properly! stateRoot: rootA, @@ -152,22 +153,26 @@ export function getBlockPhase0( }, }, signature: emptySig, - }); + }; } /** * Get an altair block. * This mutates the input preState as well to mark attestations not seen by the network. */ -export function getBlockAltair(preState: CachedBeaconStateAllForks, opts: BlockAltairOpts): altair.SignedBeaconBlock { +export function getBlockAltair(preState: CachedBeaconStateAltair, opts: BlockAltairOpts): altair.SignedBeaconBlock { const emptySig = Buffer.alloc(96); - const phase0Block = getBlockPhase0(preState as CachedBeaconStateAllForks, opts); + const phase0Block = getBlockPhase0(preState, opts); const stateEpoch = computeEpochAtSlot(preState.slot); for (const attestation of phase0Block.message.body.attestations) { const attEpoch = computeEpochAtSlot(attestation.data.slot); const epochParticipation = attEpoch === stateEpoch ? preState.currentEpochParticipation : preState.previousEpochParticipation; - const attestingIndices = preState.getAttestingIndices(attestation.data, attestation.aggregationBits); + + const committeeindices = preState.epochCtx.getBeaconCommittee(attestation.data.slot, attestation.data.index); + const attestingIndices = attestation.aggregationBits.intersectValues(committeeindices); + + // TODO: Is this necessary? for (const index of attestingIndices) { epochParticipation.set(index, 0); } @@ -191,13 +196,18 @@ export function getBlockAltair(preState: CachedBeaconStateAllForks, opts: BlockA * Generate valid deposits with valid signatures and valid merkle proofs. * NOTE: Mutates `preState` to add the new `eth1Data.depositRoot` */ -function getDeposits(preState: CachedBeaconStateAllForks, count: number): List { - const depositRootTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); +function getDeposits(preState: CachedBeaconStateAllForks, count: number): phase0.Deposit[] { + const depositRootViewDU = ssz.phase0.DepositDataRootList.toViewDU([]); const depositCount = preState.eth1Data.depositCount; const withdrawalCredentials = Buffer.alloc(32, 0xee); const depositsData: phase0.DepositData[] = []; - const deposits = ([] as phase0.Deposit[]) as List; + const deposits = [] as phase0.Deposit[]; + + // Fill depositRootViewDU up to depositCount + // Instead of actually filling it, just mutate the length to allow .set() + depositRootViewDU["_length"] = depositCount + count; + depositRootViewDU["dirtyLength"] = true; for (let i = 0; i < count; i++) { const sk = SecretKey.fromBytes(Buffer.alloc(32, i + 1)); @@ -206,28 +216,30 @@ function getDeposits(preState: CachedBeaconStateAllForks, count: number): List

0) { preState.eth1Data.depositCount = depositCount + count; - preState.eth1Data.depositRoot = depositRootTree.hashTreeRoot(); + preState.eth1Data.depositRoot = depositRootViewDU.hashTreeRoot(); } return deposits; @@ -242,10 +254,10 @@ function linspace(from: number, count: number, step: number): number[] { return arr; } -function getAggregationBits(len: number, participants: number): boolean[] { - const bits: boolean[] = []; - for (let i = 0; i < len; i++) { - bits.push(i < participants); +function getAggregationBits(len: number, participants: number): BitArray { + const bits = BitArray.fromBitLen(len); + for (let i = 0; i < participants; i++) { + bits.set(i, true); } return bits; } diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts index c1094142447..dadd10022ea 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/afterProcessEpoch.test.ts @@ -1,5 +1,5 @@ import {itBench} from "@dapplion/benchmark"; -import {CachedBeaconStateAllForks, beforeProcessEpoch} from "../../../../src"; +import {beforeProcessEpoch} from "../../../../src"; import {StateEpoch} from "../../types"; import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; @@ -13,7 +13,7 @@ describe("phase0 afterProcessEpoch", () => { before: () => { const state = generatePerfTestCachedStatePhase0({goBackOneSlot: true}); const epochProcess = beforeProcessEpoch(state); - return {state: state as CachedBeaconStateAllForks, epochProcess}; + return {state: state, epochProcess}; }, beforeEach: ({state, epochProcess}) => ({state: state.clone(), epochProcess}), fn: ({state, epochProcess}) => { diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts index ef9b6905f15..b0984acc7ca 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/beforeProcessEpoch.test.ts @@ -1,5 +1,5 @@ import {itBench} from "@dapplion/benchmark"; -import {beforeProcessEpoch, CachedBeaconStateAllForks} from "../../../../src"; +import {beforeProcessEpoch} from "../../../../src"; import {State} from "../../types"; import {generatePerfTestCachedStatePhase0, perfStateId} from "../../util"; @@ -12,7 +12,7 @@ describe("phase0 beforeProcessEpoch", () => { itBench({ id: `phase0 beforeProcessEpoch - ${perfStateId}`, yieldEventLoopAfterEach: true, // So SubTree(s)'s WeakRef can be garbage collected https://github.com/nodejs/node/issues/39902 - before: () => generatePerfTestCachedStatePhase0({goBackOneSlot: true}) as CachedBeaconStateAllForks, + before: () => generatePerfTestCachedStatePhase0({goBackOneSlot: true}), beforeEach: (state) => state.clone(), fn: (state) => { beforeProcessEpoch(state); diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts index 1cdfb25e6aa..76259bea469 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/epoch.test.ts @@ -28,7 +28,7 @@ describe(`phase0 processEpoch - ${stateId}`, () => { itBench({ id: `phase0 processEpoch - ${stateId}`, - beforeEach: () => stateOg.value.clone() as CachedBeaconStateAllForks, + beforeEach: () => stateOg.value.clone(), fn: (state) => { const epochProcess = beforeProcessEpoch(state); phase0.processEpoch(state as CachedBeaconStatePhase0, epochProcess); diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts index fa36d37d47e..b0c91221a4c 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processEffectiveBalanceUpdates.test.ts @@ -1,15 +1,10 @@ import {itBench} from "@dapplion/benchmark"; import {ssz} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/default"; -import { - allForks, - beforeProcessEpoch, - CachedBeaconStateAllForks, - createCachedBeaconState, - EpochProcess, -} from "../../../../src"; +import {allForks, beforeProcessEpoch, CachedBeaconStateAllForks, EpochProcess} from "../../../../src"; import {numValidators} from "../../util"; import {StateEpoch} from "../../types"; +import {createCachedBeaconStateTest} from "../../../utils/state"; // PERF: Cost 'proportional' to $VALIDATOR_COUNT, to iterate over all balances. Then cost is proportional to the amount // of validators whose effectiveBalance changed. Worst case is a massive network leak or a big slashing event which @@ -54,16 +49,16 @@ function getEffectiveBalanceTestData( state: CachedBeaconStateAllForks; epochProcess: EpochProcess; } { - const stateTree = ssz.phase0.BeaconState.defaultTreeBacked(); + const stateTree = ssz.phase0.BeaconState.defaultViewDU(); stateTree.slot = 1; - const activeValidator = { - ...ssz.phase0.Validator.defaultTreeBacked(), + const activeValidator = ssz.phase0.Validator.toViewDU({ + ...ssz.phase0.Validator.defaultValue(), exitEpoch: Infinity, withdrawableEpoch: Infinity, // Set current effective balance to max effectiveBalance: 32e9, - }; + }); const balances: number[] = []; for (let i = 0; i < vc; i++) { @@ -76,12 +71,14 @@ function getEffectiveBalanceTestData( stateTree.validators.push(activeValidator); } - const cachedBeaconState = createCachedBeaconState(config, stateTree, {skipSyncPubkeys: true}); + stateTree.commit(); + + const cachedBeaconState = createCachedBeaconStateTest(stateTree, config, {skipSyncPubkeys: true}); const epochProcess = beforeProcessEpoch(cachedBeaconState); epochProcess.balances = balances; return { - state: cachedBeaconState as CachedBeaconStateAllForks, + state: cachedBeaconState, epochProcess: epochProcess, }; } diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts index 7d76a2ae9bc..e9943b22988 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processRegistryUpdates.test.ts @@ -90,7 +90,7 @@ function getRegistryUpdatesTestData( epochProcess.indicesEligibleForActivation = linspace(lengths.indicesEligibleForActivation); return { - state: state as CachedBeaconStateAllForks, + state, epochProcess, }; } diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts index 572bc6b5cfe..880d2e42283 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/processSlashingsAllForks.test.ts @@ -54,7 +54,7 @@ function getProcessSlashingsTestData( epochProcess.indicesToSlash = linspace(indicesToSlashLen); return { - state: state as CachedBeaconStateAllForks, + state, epochProcess, }; } diff --git a/packages/beacon-state-transition/test/perf/phase0/epoch/util.ts b/packages/beacon-state-transition/test/perf/phase0/epoch/util.ts index 7dfcfe79d62..91c863674dc 100644 --- a/packages/beacon-state-transition/test/perf/phase0/epoch/util.ts +++ b/packages/beacon-state-transition/test/perf/phase0/epoch/util.ts @@ -1,4 +1,4 @@ -import {AttesterFlags, IAttesterStatus, toAttesterFlags} from "../../../../src"; +import {AttesterFlags, FLAG_ELIGIBLE_ATTESTER, hasMarkers, IAttesterStatus, toAttesterFlags} from "../../../../src"; import {CachedBeaconStatePhase0, CachedBeaconStateAltair, EpochProcess} from "../../../../src/types"; /** @@ -13,8 +13,17 @@ export function generateBalanceDeltasEpochProcess( ): EpochProcess { const vc = state.validators.length; + const statuses = generateStatuses(state.validators.length, flagFactors); + const eligibleValidatorIndices: number[] = []; + for (let i = 0; i < statuses.length; i++) { + if (hasMarkers(statuses[i].flags, FLAG_ELIGIBLE_ATTESTER)) { + eligibleValidatorIndices.push(i); + } + } + const epochProcess: Partial = { - statuses: generateStatuses(state.validators.length, flagFactors), + statuses, + eligibleValidatorIndices, totalActiveStakeByIncrement: vc, baseRewardPerIncrement: 726, prevEpochUnslashedStake: { @@ -32,17 +41,17 @@ export type FlagFactors = Record | number; function generateStatuses(vc: number, flagFactors: FlagFactors): IAttesterStatus[] { const totalProposers = 32; - const statuses: IAttesterStatus[] = []; + const statuses = new Array(vc); for (let i = 0; i < vc; i++) { // Set to number to set all validators to the same value if (typeof flagFactors === "number") { - statuses.push({ + statuses[i] = { flags: flagFactors, proposerIndex: i % totalProposers, inclusionDelay: 1 + (i % 4), active: true, - }); + }; } else { // Use a factor to set some validators to this flag const flagsObj: AttesterFlags = { @@ -55,12 +64,12 @@ function generateStatuses(vc: number, flagFactors: FlagFactors): IAttesterStatus unslashed: i < vc * flagFactors.unslashed, // 6 eligibleAttester: i < vc * flagFactors.eligibleAttester, // 7 }; - statuses.push({ + statuses[i] = { flags: toAttesterFlags(flagsObj), proposerIndex: i % totalProposers, inclusionDelay: 1 + (i % 4), active: true, - }); + }; } } diff --git a/packages/beacon-state-transition/test/perf/types.ts b/packages/beacon-state-transition/test/perf/types.ts index 67238e68b47..7cc0dd3379f 100644 --- a/packages/beacon-state-transition/test/perf/types.ts +++ b/packages/beacon-state-transition/test/perf/types.ts @@ -1,4 +1,4 @@ -import {allForks, phase0, CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../../src"; +import {allForks, CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../../src"; import {EpochProcess} from "../../src/types"; // Type aliases to typesafe itBench() calls @@ -6,10 +6,6 @@ import {EpochProcess} from "../../src/types"; export type State = CachedBeaconStateAllForks; export type StateAltair = CachedBeaconStateAltair; export type StateBlock = {state: CachedBeaconStateAllForks; block: allForks.SignedBeaconBlock}; -export type StateAttestations = { - state: CachedBeaconStateAllForks; - attestations: phase0.Attestation[]; -}; export type StateEpoch = {state: CachedBeaconStateAllForks; epochProcess: EpochProcess}; export type StatePhase0Epoch = {state: CachedBeaconStatePhase0; epochProcess: EpochProcess}; export type StateAltairEpoch = {state: CachedBeaconStateAltair; epochProcess: EpochProcess}; diff --git a/packages/beacon-state-transition/test/perf/util.ts b/packages/beacon-state-transition/test/perf/util.ts index 8358b832bf3..1701fc0bfe0 100644 --- a/packages/beacon-state-transition/test/perf/util.ts +++ b/packages/beacon-state-transition/test/perf/util.ts @@ -1,47 +1,50 @@ import fs from "node:fs"; import path from "node:path"; import {config} from "@chainsafe/lodestar-config/default"; -import {phase0, ssz, Slot, altair, ParticipationFlags} from "@chainsafe/lodestar-types"; +import {phase0, ssz, Slot, altair} from "@chainsafe/lodestar-types"; import bls, {CoordType, PublicKey, SecretKey} from "@chainsafe/bls"; -import {fromHexString, List, TreeBacked} from "@chainsafe/ssz"; +import {BitArray, fromHexString} from "@chainsafe/ssz"; import { allForks, interopSecretKey, computeEpochAtSlot, getActiveValidatorIndices, PubkeyIndexMap, + newFilledArray, createCachedBeaconState, - getNextSyncCommittee, computeCommitteeCount, } from "../../src"; -import {createIChainForkConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; -import {CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair} from "../../src/types"; +import {createIBeaconConfig, createIChainForkConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; +import { + CachedBeaconStateAllForks, + CachedBeaconStatePhase0, + CachedBeaconStateAltair, + BeaconStatePhase0, + BeaconStateAltair, +} from "../../src/types"; import {profilerLogger} from "../utils/logger"; import {interopPubkeysCached} from "../utils/interop"; -import {PendingAttestation} from "@chainsafe/lodestar-types/phase0"; -import {intDiv} from "@chainsafe/lodestar-utils"; import { - EFFECTIVE_BALANCE_INCREMENT, EPOCHS_PER_ETH1_VOTING_PERIOD, EPOCHS_PER_HISTORICAL_VECTOR, MAX_ATTESTATIONS, - MAX_VALIDATORS_PER_COMMITTEE, + MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT, - TIMELY_HEAD_FLAG_INDEX, - TIMELY_SOURCE_FLAG_INDEX, - TIMELY_TARGET_FLAG_INDEX, } from "@chainsafe/lodestar-params"; import {NetworkName, networksChainConfig} from "@chainsafe/lodestar-config/networks"; import {getClient} from "@chainsafe/lodestar-api"; import {getInfuraBeaconUrl} from "./infura"; import {testCachePath} from "../cache"; +import {getNextSyncCommittee} from "../../src/util/syncCommittee"; +import {createCachedBeaconStateTest} from "../utils/state"; +import {getEffectiveBalanceIncrements} from "../../src/cache/effectiveBalanceIncrements"; -let phase0State: TreeBacked | null = null; +let phase0State: BeaconStatePhase0 | null = null; let phase0CachedState23637: CachedBeaconStatePhase0 | null = null; let phase0CachedState23638: CachedBeaconStatePhase0 | null = null; -let phase0SignedBlock: TreeBacked | null = null; -let altairState: TreeBacked | null = null; +let phase0SignedBlock: phase0.SignedBeaconBlock | null = null; +let altairState: BeaconStateAltair | null = null; let altairCachedState23637: CachedBeaconStateAltair | null = null; let altairCachedState23638: CachedBeaconStateAltair | null = null; const logger = profilerLogger(); @@ -88,10 +91,8 @@ export function getSecretKeyFromIndexCached(validatorIndex: number): SecretKey { return sk; } -export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean}): CachedBeaconStatePhase0 { - // Generate only some publicKeys - const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(); - +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +function getPubkeyCaches({pubkeysMod, pubkeysModObj}: ReturnType) { // Manually sync pubkeys to prevent doing BLS opts 110_000 times const pubkey2index = new PubkeyIndexMap(); const index2pubkey = [] as PublicKey[]; @@ -102,19 +103,105 @@ export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean index2pubkey.push(pubkeyObj); } - const origState = generatePerformanceStatePhase0(pubkeys); + // Since most pubkeys are equal the size of pubkey2index is not numValidators. + // Fill with junk up to numValidators + for (let i = pubkey2index.size; i < numValidators; i++) { + const buf = Buffer.alloc(48, 0); + buf.writeInt32LE(i); + pubkey2index.set(buf, i); + } + + return {pubkey2index, index2pubkey}; +} + +export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean}): CachedBeaconStatePhase0 { + // Generate only some publicKeys + const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(); + const {pubkey2index, index2pubkey} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); + + if (!phase0State) { + const state = buildPerformanceStatePhase0(); + + // no justificationBits + phase0State = ssz.phase0.BeaconState.toViewDU(state); + logger.verbose("Loaded phase0 state", { + slot: state.slot, + numValidators: state.validators.length, + }); + + // cache roots + phase0State.hashTreeRoot(); + } + if (!phase0CachedState23637) { - const state = origState.clone(); + const state = phase0State.clone(); state.slot -= 1; - phase0CachedState23637 = createCachedBeaconState(config, state, { + phase0CachedState23637 = createCachedBeaconState(state, { + config: createIBeaconConfig(config, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - skipSyncPubkeys: true, }); + + const currentEpoch = computeEpochAtSlot(state.slot - 1); + const previousEpoch = currentEpoch - 1; + + // previous epoch attestations + const numPrevAttestations = SLOTS_PER_EPOCH * MAX_ATTESTATIONS; + const activeValidatorCount = pubkeys.length; + const committeesPerSlot = computeCommitteeCount(activeValidatorCount); + for (let i = 0; i < numPrevAttestations; i++) { + const slotInEpoch = i % SLOTS_PER_EPOCH; + const slot = previousEpoch * SLOTS_PER_EPOCH + slotInEpoch; + const index = i % committeesPerSlot; + const shuffling = phase0CachedState23637.epochCtx.getShufflingAtEpoch(previousEpoch); + const committee = shuffling.committees[slotInEpoch][index]; + phase0CachedState23637.previousEpochAttestations.push( + ssz.phase0.PendingAttestation.toViewDU({ + aggregationBits: BitArray.fromBoolArray(Array.from({length: committee.length}, () => true)), + data: { + beaconBlockRoot: phase0CachedState23637.blockRoots.get(slotInEpoch % SLOTS_PER_HISTORICAL_ROOT), + index, + slot, + source: state.previousJustifiedCheckpoint, + target: state.currentJustifiedCheckpoint, + }, + inclusionDelay: 1, + proposerIndex: i, + }) + ); + } + + // current epoch attestations + const numCurAttestations = (SLOTS_PER_EPOCH - 1) * MAX_ATTESTATIONS; + for (let i = 0; i < numCurAttestations; i++) { + const slotInEpoch = i % SLOTS_PER_EPOCH; + const slot = currentEpoch * SLOTS_PER_EPOCH + slotInEpoch; + const index = i % committeesPerSlot; + const shuffling = phase0CachedState23637.epochCtx.getShufflingAtEpoch(previousEpoch); + const committee = shuffling.committees[slotInEpoch][index]; + + phase0CachedState23637.currentEpochAttestations.push( + ssz.phase0.PendingAttestation.toViewDU({ + aggregationBits: BitArray.fromBoolArray(Array.from({length: committee.length}, () => true)), + data: { + beaconBlockRoot: phase0CachedState23637.blockRoots.get(slotInEpoch % SLOTS_PER_HISTORICAL_ROOT), + index, + slot, + source: state.currentJustifiedCheckpoint, + target: { + epoch: currentEpoch, + root: phase0CachedState23637.blockRoots.get((currentEpoch * SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT), + }, + }, + inclusionDelay: 1, + proposerIndex: i, + }) + ); + } } if (!phase0CachedState23638) { phase0CachedState23638 = allForks.processSlots( - phase0CachedState23637 as CachedBeaconStateAllForks, + phase0CachedState23637, phase0CachedState23637.slot + 1 ) as CachedBeaconStatePhase0; phase0CachedState23638.slot += 1; @@ -124,34 +211,38 @@ export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean return resultingState.clone(); } +export function cachedStateAltairPopulateCaches(state: CachedBeaconStateAltair): void { + // Populate caches + state.blockRoots.getAllReadonly(); + state.eth1DataVotes.getAllReadonly(); + state.validators.getAllReadonly(); + state.balances.getAll(); + state.previousEpochParticipation.getAll(); + state.currentEpochParticipation.getAll(); + state.inactivityScores.getAll(); +} + export function generatePerfTestCachedStateAltair(opts?: {goBackOneSlot: boolean}): CachedBeaconStateAltair { const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(); + const {pubkey2index, index2pubkey} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}); + // eslint-disable-next-line @typescript-eslint/naming-convention const altairConfig = createIChainForkConfig({ALTAIR_FORK_EPOCH: 0}); - // Manually sync pubkeys to prevent doing BLS opts 110_000 times - const pubkey2index = new PubkeyIndexMap(); - const index2pubkey = [] as PublicKey[]; - for (let i = 0; i < numValidators; i++) { - const pubkey = pubkeysMod[i % keypairsMod]; - const pubkeyObj = pubkeysModObj[i % keypairsMod]; - pubkey2index.set(pubkey, i); - index2pubkey.push(pubkeyObj); - } - const origState = generatePerformanceStateAltair(pubkeys); + if (!altairCachedState23637) { const state = origState.clone(); state.slot -= 1; - altairCachedState23637 = createCachedBeaconState(altairConfig, state, { + altairCachedState23637 = createCachedBeaconState(state, { + config: createIBeaconConfig(altairConfig, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - skipSyncPubkeys: true, }); } if (!altairCachedState23638) { altairCachedState23638 = allForks.processSlots( - altairCachedState23637 as CachedBeaconStateAllForks, + altairCachedState23637, altairCachedState23637.slot + 1 ) as CachedBeaconStateAltair; altairCachedState23638.slot += 1; @@ -164,36 +255,33 @@ export function generatePerfTestCachedStateAltair(opts?: {goBackOneSlot: boolean /** * This is generated from Medalla state 756416 */ -export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): TreeBacked { +export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): BeaconStateAltair { if (!altairState) { const pubkeys = pubkeysArg || getPubkeys().pubkeys; - const defaultState = ssz.altair.BeaconState.defaultValue(); - const state = buildPerformanceStateAllForks(defaultState) as altair.BeaconState; - const TIMELY_SOURCE = 1 << TIMELY_SOURCE_FLAG_INDEX; - const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX; - const TIMELY_HEAD = 1 << TIMELY_HEAD_FLAG_INDEX; - const fullParticpation = TIMELY_SOURCE | TIMELY_TARGET | TIMELY_HEAD; - if (fullParticpation !== 7) { - throw new Error("Not correct fullParticipation"); - } - state.previousEpochParticipation = Array.from( - {length: pubkeys.length}, - () => fullParticpation - ) as List; - state.currentEpochParticipation = Array.from( - {length: pubkeys.length}, - () => fullParticpation - ) as List; - state.inactivityScores = Array.from({length: pubkeys.length}, (_, i) => i % 2) as List; + const statePhase0 = buildPerformanceStatePhase0(); + const state = (statePhase0 as allForks.BeaconState) as altair.BeaconState; + + state.previousEpochParticipation = newFilledArray(pubkeys.length, 0b111); + state.currentEpochParticipation = state.previousEpochParticipation; + state.inactivityScores = Array.from({length: pubkeys.length}, (_, i) => i % 2); + + // Placeholder syncCommittees + state.currentSyncCommittee = ssz.altair.SyncCommittee.defaultValue(); + state.nextSyncCommittee = state.currentSyncCommittee; + + // Now the state is fully populated to convert to ViewDU + altairState = ssz.altair.BeaconState.toViewDU(state); + + // Now set correct syncCommittees const epoch = computeEpochAtSlot(state.slot); - const activeValidatorIndices = getActiveValidatorIndices(state, epoch); - const effectiveBalanceIncrements = new Uint8Array( - Array.from(state.validators).map((v) => v.effectiveBalance / EFFECTIVE_BALANCE_INCREMENT) - ); - const syncCommittee = getNextSyncCommittee(state, activeValidatorIndices, effectiveBalanceIncrements); + const activeValidatorIndices = getActiveValidatorIndices(altairState, epoch); + + const effectiveBalanceIncrements = getEffectiveBalanceIncrements(altairState); + const {syncCommittee} = getNextSyncCommittee(altairState, activeValidatorIndices, effectiveBalanceIncrements); state.currentSyncCommittee = syncCommittee; state.nextSyncCommittee = syncCommittee; - altairState = ssz.altair.BeaconState.createTreeBackedFromStruct(state); + + altairState = ssz.altair.BeaconState.toViewDU(state); logger.verbose("Loaded phase0 state", { slot: altairState.slot, numValidators: altairState.validators.length, @@ -204,152 +292,99 @@ export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): TreeB return altairState.clone(); } -/** - * This is generated from Medalla state 756416 - */ -export function generatePerformanceStatePhase0(pubkeysArg?: Uint8Array[]): TreeBacked { - if (!phase0State) { - const pubkeys = pubkeysArg || getPubkeys().pubkeys; - const defaultState = ssz.phase0.BeaconState.defaultValue(); - const state = buildPerformanceStateAllForks(defaultState) as phase0.BeaconState; - const currentEpoch = computeEpochAtSlot(state.slot - 1); - const previousEpoch = currentEpoch - 1; - // previous epoch attestations - const numPrevAttestations = SLOTS_PER_EPOCH * MAX_ATTESTATIONS; - const activeValidatorCount = pubkeys.length; - const committeesPerSlot = computeCommitteeCount(activeValidatorCount); - state.previousEpochAttestations = Array.from({length: numPrevAttestations}, (_, i) => { - const slotInEpoch = intDiv(i, MAX_ATTESTATIONS); - return { - aggregationBits: Array.from({length: MAX_VALIDATORS_PER_COMMITTEE}, () => true) as List, - data: { - beaconBlockRoot: state.blockRoots[slotInEpoch % SLOTS_PER_HISTORICAL_ROOT], - index: i % committeesPerSlot, - slot: previousEpoch * SLOTS_PER_EPOCH + slotInEpoch, - source: state.previousJustifiedCheckpoint, - target: state.currentJustifiedCheckpoint, - }, - inclusionDelay: 1, - proposerIndex: i, - }; - }) as List; - // current epoch attestations - const numCurAttestations = (SLOTS_PER_EPOCH - 1) * MAX_ATTESTATIONS; - state.currentEpochAttestations = Array.from({length: numCurAttestations}, (_, i) => { - const slotInEpoch = intDiv(i, MAX_ATTESTATIONS); - return { - aggregationBits: Array.from({length: MAX_VALIDATORS_PER_COMMITTEE}, () => true) as List, - data: { - beaconBlockRoot: state.blockRoots[slotInEpoch % SLOTS_PER_HISTORICAL_ROOT], - index: i % committeesPerSlot, - slot: currentEpoch * SLOTS_PER_EPOCH + slotInEpoch, - source: state.currentJustifiedCheckpoint, - target: { - epoch: currentEpoch, - root: state.blockRoots[(currentEpoch * SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT], - }, - }, - inclusionDelay: 1, - proposerIndex: i, - }; - }) as List; - // no justificationBits - phase0State = ssz.phase0.BeaconState.createTreeBackedFromStruct(state); - logger.verbose("Loaded phase0 state", { - slot: phase0State.slot, - numValidators: phase0State.validators.length, - }); - // cache roots - phase0State.hashTreeRoot(); - } - return phase0State.clone(); -} - /** * This is generated from Medalla block 756417 */ -export function generatePerformanceBlockPhase0(): TreeBacked { +export function generatePerformanceBlockPhase0(): phase0.SignedBeaconBlock { if (!phase0SignedBlock) { const block = ssz.phase0.SignedBeaconBlock.defaultValue(); const parentState = generatePerfTestCachedStatePhase0(); block.message.slot = parentState.slot; - block.message.proposerIndex = parentState.getBeaconProposer(parentState.slot); + block.message.proposerIndex = parentState.epochCtx.getBeaconProposer(parentState.slot); block.message.parentRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(parentState.latestBlockHeader); block.message.stateRoot = fromHexString("0x6c86ca3c4c6688cf189421b8a68bf2dbc91521609965e6f4e207d44347061fee"); block.message.body.randaoReveal = fromHexString( "0x8a5d2673c48f22f6ed19462efec35645db490df29eed2f56321dbe4a89b2463b0c902095a7ab74a2dc5b7f67edb1a19507ea3d4361d5af9cb0a524945c91638dfd6568841486813a2c45142659d6d9403f5081febb123a7931edbc248b9d0025" ); // eth1Data, graffiti, attestations - phase0SignedBlock = ssz.phase0.SignedBeaconBlock.createTreeBackedFromStruct(block); + phase0SignedBlock = block; logger.verbose("Loaded block", {slot: phase0SignedBlock.message.slot}); } - return phase0SignedBlock.clone(); + + return phase0SignedBlock; } -function buildPerformanceStateAllForks(state: allForks.BeaconState, pubkeysArg?: Uint8Array[]): allForks.BeaconState { +function buildPerformanceStatePhase0(pubkeysArg?: Uint8Array[]): phase0.BeaconState { + const slot = epoch * SLOTS_PER_EPOCH; const pubkeys = pubkeysArg || getPubkeys().pubkeys; - state.genesisTime = 1596546008; - state.genesisValidatorsRoot = fromHexString("0x04700007fabc8282644aed6d1c7c9e21d38a03a0c4ba193f3afe428824b3a673"); - state.slot = epoch * SLOTS_PER_EPOCH; - state.fork = { - currentVersion: fromHexString("0x00000001"), - previousVersion: fromHexString("0x00000001"), - epoch: 0, - }; - state.latestBlockHeader = { - slot: state.slot - 1, - proposerIndex: 80882, - parentRoot: fromHexString("0x5b83c3078e474b86af60043eda82a34c3c2e5ebf83146b14d9d909aea4163ef2"), - stateRoot: fromHexString("0x2761ae355e8a53c11e0e37d5e417f8984db0c53fa83f1bc65f89c6af35a196a7"), - bodyRoot: fromHexString("0x249a1962eef90e122fa2447040bfac102798b1dba9c73e5593bc5aa32eb92bfd"), - }; - state.blockRoots = Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)); - state.stateRoots = Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)); - // historicalRoots - state.eth1Data = { - depositCount: pubkeys.length, - depositRoot: fromHexString("0xcb1f89a924cfd31224823db5a41b1643f10faa7aedf231f1e28887f6ee98c047"), - blockHash: fromHexString("0x701fb2869ce16d0f1d14f6705725adb0dec6799da29006dfc6fff83960298f21"), - }; - state.eth1DataVotes = (Array.from( + const currentEpoch = computeEpochAtSlot(slot - 1); + + return { + // Misc + genesisTime: 1596546008, + genesisValidatorsRoot: fromHexString("0x04700007fabc8282644aed6d1c7c9e21d38a03a0c4ba193f3afe428824b3a673"), + slot: epoch * SLOTS_PER_EPOCH, + fork: { + currentVersion: fromHexString("0x00000001"), + previousVersion: fromHexString("0x00000001"), + epoch: 0, + }, + // History + latestBlockHeader: { + slot: slot - 1, + proposerIndex: 80882, + parentRoot: fromHexString("0x5b83c3078e474b86af60043eda82a34c3c2e5ebf83146b14d9d909aea4163ef2"), + stateRoot: fromHexString("0x2761ae355e8a53c11e0e37d5e417f8984db0c53fa83f1bc65f89c6af35a196a7"), + bodyRoot: fromHexString("0x249a1962eef90e122fa2447040bfac102798b1dba9c73e5593bc5aa32eb92bfd"), + }, + blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)), + stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)), + historicalRoots: [], + // Eth1 + eth1Data: { + depositCount: pubkeys.length, + depositRoot: fromHexString("0xcb1f89a924cfd31224823db5a41b1643f10faa7aedf231f1e28887f6ee98c047"), + blockHash: fromHexString("0x701fb2869ce16d0f1d14f6705725adb0dec6799da29006dfc6fff83960298f21"), + }, // minus one so that inserting 1 from block works - {length: EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH - 1}, - (_, i) => { - return { - depositCount: i, - depositRoot: Buffer.alloc(32, i), - blockHash: Buffer.alloc(32, i), - }; - } - ) as unknown) as List; - state.eth1DepositIndex = pubkeys.length; - state.validators = pubkeys.map((_, i) => ({ - pubkey: pubkeys[i], - withdrawalCredentials: Buffer.alloc(32, i), - effectiveBalance: 31000000000, - slashed: false, - activationEligibilityEpoch: 0, - activationEpoch: 0, - exitEpoch: Infinity, - withdrawableEpoch: Infinity, - })) as List; - state.balances = Array.from({length: pubkeys.length}, () => 31217089836) as List; - state.randaoMixes = Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, (_, i) => Buffer.alloc(32, i)); - // no slashings - const currentEpoch = computeEpochAtSlot(state.slot - 1); - state.previousJustifiedCheckpoint = { - epoch: currentEpoch - 2, - root: fromHexString("0x3fe60bf06a57b0956cd1f8181d26649cf8bf79e48bf82f55562e04b33d4785d4"), + eth1DataVotes: newFilledArray(EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH - 1, { + depositCount: 1, + depositRoot: Buffer.alloc(32, 1), + blockHash: Buffer.alloc(32, 1), + }), + eth1DepositIndex: pubkeys.length, + // Registry + validators: pubkeys.map((_, i) => ({ + pubkey: pubkeys[i], + withdrawalCredentials: Buffer.alloc(32, i), + effectiveBalance: 31000000000, + slashed: false, + activationEligibilityEpoch: 0, + activationEpoch: 0, + exitEpoch: Infinity, + withdrawableEpoch: Infinity, + })), + balances: Array.from({length: pubkeys.length}, () => 31217089836), + randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, (_, i) => Buffer.alloc(32, i)), + // Slashings + slashings: ssz.phase0.Slashings.defaultValue(), + previousEpochAttestations: [], + currentEpochAttestations: [], + // Finality + justificationBits: BitArray.fromBitLen(4), + previousJustifiedCheckpoint: { + epoch: currentEpoch - 2, + root: fromHexString("0x3fe60bf06a57b0956cd1f8181d26649cf8bf79e48bf82f55562e04b33d4785d4"), + }, + currentJustifiedCheckpoint: { + epoch: currentEpoch - 1, + root: fromHexString("0x3ba0913d2fb5e4cbcfb0d39eb15803157c1e769d63b8619285d8fdabbd8181c7"), + }, + finalizedCheckpoint: { + epoch: currentEpoch - 3, + root: fromHexString("0x122b8ff579d0c8f8a8b66326bdfec3f685007d2842f01615a0768870961ccc17"), + }, }; - state.currentJustifiedCheckpoint = { - epoch: currentEpoch - 1, - root: fromHexString("0x3ba0913d2fb5e4cbcfb0d39eb15803157c1e769d63b8619285d8fdabbd8181c7"), - }; - state.finalizedCheckpoint = { - epoch: currentEpoch - 3, - root: fromHexString("0x122b8ff579d0c8f8a8b66326bdfec3f685007d2842f01615a0768870961ccc17"), - }; - return state; } export function generateTestCachedBeaconStateOnlyValidators({ @@ -372,13 +407,13 @@ export function generateTestCachedBeaconStateOnlyValidators({ index2pubkey.push(pubkeyObj); } - const state = ssz.phase0.BeaconState.defaultTreeBacked(); + const state = ssz.phase0.BeaconState.defaultViewDU(); state.slot = slot; - const activeValidator = ssz.phase0.Validator.createTreeBackedFromStruct({ + const activeValidator = ssz.phase0.Validator.toViewDU({ pubkey: Buffer.alloc(48, 0), withdrawalCredentials: Buffer.alloc(32, 0), - effectiveBalance: 31000000000, + effectiveBalance: MAX_EFFECTIVE_BALANCE, slashed: false, activationEligibilityEpoch: 0, activationEpoch: 0, @@ -389,16 +424,26 @@ export function generateTestCachedBeaconStateOnlyValidators({ for (let i = 0; i < vc; i++) { const validator = activeValidator.clone(); validator.pubkey = pubkeys[i]; - state.validators[i] = validator; + state.validators.push(validator); } - state.balances = Array.from({length: pubkeys.length}, () => 31217089836) as List; - state.randaoMixes = Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, (_, i) => Buffer.alloc(32, i)); + state.balances = ssz.phase0.Balances.toViewDU(newFilledArray(pubkeys.length, MAX_EFFECTIVE_BALANCE)); + state.randaoMixes = ssz.phase0.RandaoMixes.toViewDU( + newFilledArray(EPOCHS_PER_HISTORICAL_VECTOR, Buffer.alloc(32, 0xdd)) + ); + + // Commit ViewDU changes + state.commit(); + + // Sanity check for .commit() above + if (state.validators.length !== vc) { + throw Error(`Wrong number of validators in the state: ${state.validators.length} !== ${vc}`); + } - return createCachedBeaconState(config, state as TreeBacked, { + return createCachedBeaconState(state, { + config: createIBeaconConfig(config, state.genesisValidatorsRoot), pubkey2index, index2pubkey, - skipSyncPubkeys: true, }); } @@ -471,6 +516,6 @@ export async function getNetworkCachedState( fs.writeFileSync(filepath, stateSsz); } - const stateTB = config.getForkTypes(slot).BeaconState.createTreeBackedFromBytes(stateSsz); - return createCachedBeaconState(config, stateTB); + const stateView = config.getForkTypes(slot).BeaconState.deserializeToViewDU(stateSsz); + return createCachedBeaconStateTest(stateView, config); } diff --git a/packages/beacon-state-transition/test/unit/block/isValidIndexedAttestation.test.ts b/packages/beacon-state-transition/test/unit/block/isValidIndexedAttestation.test.ts index 9cba6c12fc5..bbd78b4a961 100644 --- a/packages/beacon-state-transition/test/unit/block/isValidIndexedAttestation.test.ts +++ b/packages/beacon-state-transition/test/unit/block/isValidIndexedAttestation.test.ts @@ -1,25 +1,21 @@ -import {List, TreeBacked} from "@chainsafe/ssz"; import {config} from "@chainsafe/lodestar-config/default"; import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; -import {allForks, ssz} from "@chainsafe/lodestar-types"; import {generateAttestationData} from "../../utils/attestation"; import {expect} from "chai"; import {EMPTY_SIGNATURE} from "../../../src"; -import {phase0, createCachedBeaconState} from "../../../src"; -import {generateState} from "../../utils/state"; +import {phase0} from "../../../src"; +import {generateCachedState} from "../../utils/state"; import {generateValidators} from "../../utils/validator"; describe("validate indexed attestation", () => { - const treeState = ssz.phase0.BeaconState.createTreeBackedFromStruct( - generateState({ - validators: generateValidators(100, { - balance: MAX_EFFECTIVE_BALANCE, - activation: 0, - withdrawableEpoch: FAR_FUTURE_EPOCH, - exit: FAR_FUTURE_EPOCH, - }), - }) - ); + const state = generateCachedState(config, { + validators: generateValidators(100, { + balance: MAX_EFFECTIVE_BALANCE, + activation: 0, + withdrawableEpoch: FAR_FUTURE_EPOCH, + exit: FAR_FUTURE_EPOCH, + }), + }); const testValues = [ { @@ -42,10 +38,9 @@ describe("validate indexed attestation", () => { for (const testValue of testValues) { it(testValue.name, function () { const attestationData = generateAttestationData(0, 1); - const state = createCachedBeaconState(config, treeState.clone() as TreeBacked); const indexedAttestation: phase0.IndexedAttestation = { - attestingIndices: testValue.indices as List, + attestingIndices: testValue.indices, data: attestationData, signature: EMPTY_SIGNATURE, }; diff --git a/packages/beacon-state-transition/test/unit/cachedBeaconState.test.ts b/packages/beacon-state-transition/test/unit/cachedBeaconState.test.ts new file mode 100644 index 00000000000..08fd8f44c5a --- /dev/null +++ b/packages/beacon-state-transition/test/unit/cachedBeaconState.test.ts @@ -0,0 +1,57 @@ +import {ssz} from "@chainsafe/lodestar-types"; +import {toHexString} from "@chainsafe/lodestar-utils"; +import {expect} from "chai"; +import {createCachedBeaconStateTest} from "../utils/state"; + +describe("CachedBeaconState", () => { + it("Clone and mutate", () => { + const stateView = ssz.altair.BeaconState.defaultViewDU(); + const state1 = createCachedBeaconStateTest(stateView); + const state2 = state1.clone(); + + state1.slot = 1; + expect(state2.slot).to.equal(0, "state2.slot was mutated"); + + const prevRoot = state2.currentJustifiedCheckpoint.root; + const newRoot = Buffer.alloc(32, 1); + state1.currentJustifiedCheckpoint.root = newRoot; + expect(toHexString(state2.currentJustifiedCheckpoint.root)).to.equal( + toHexString(prevRoot), + "state2.currentJustifiedCheckpoint.root was mutated" + ); + + state1.epochCtx.epoch = 1; + expect(state2.epochCtx.epoch).to.equal(0, "state2.epochCtx.epoch was mutated"); + }); + + it("Auto-commit on hashTreeRoot", () => { + // Use Checkpoint instead of BeaconState to speed up the test + const cp1 = ssz.phase0.Checkpoint.defaultViewDU(); + const cp2 = ssz.phase0.Checkpoint.defaultViewDU(); + + cp1.epoch = 1; + cp2.epoch = 1; + + // Only commit state1 beforehand + cp1.commit(); + expect(toHexString(cp1.hashTreeRoot())).to.equal( + toHexString(cp2.hashTreeRoot()), + ".hashTreeRoot() does not automatically commit" + ); + }); + + it("Auto-commit on serialize", () => { + const cp1 = ssz.phase0.Checkpoint.defaultViewDU(); + const cp2 = ssz.phase0.Checkpoint.defaultViewDU(); + + cp1.epoch = 1; + cp2.epoch = 1; + + // Only commit state1 beforehand + cp1.commit(); + expect(toHexString(cp1.serialize())).to.equal( + toHexString(cp2.serialize()), + ".serialize() does not automatically commit" + ); + }); +}); diff --git a/packages/beacon-state-transition/test/unit/signatureSets/signatureSets.test.ts b/packages/beacon-state-transition/test/unit/signatureSets/signatureSets.test.ts index 44bc9961b63..de4f5e13624 100644 --- a/packages/beacon-state-transition/test/unit/signatureSets/signatureSets.test.ts +++ b/packages/beacon-state-transition/test/unit/signatureSets/signatureSets.test.ts @@ -1,19 +1,19 @@ import crypto from "node:crypto"; import bls from "@chainsafe/bls"; -import {BitList, List, TreeBacked} from "@chainsafe/ssz"; import {config} from "@chainsafe/lodestar-config/default"; -import {ValidatorIndex, BLSSignature, ssz} from "@chainsafe/lodestar-types"; +import {ValidatorIndex, BLSSignature} from "@chainsafe/lodestar-types"; import {ZERO_HASH} from "../../../src/constants"; -import {generateState} from "../../utils/state"; +import {generateCachedState} from "../../utils/state"; import {generateValidators} from "../../utils/validator"; import {expect} from "chai"; -import {phase0, createCachedBeaconState, allForks} from "../../../src"; +import {phase0, allForks} from "../../../src"; import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; +import {BitArray} from "@chainsafe/ssz"; + +const EMPTY_SIGNATURE = Buffer.alloc(96); describe("signatureSets", () => { it("should aggregate all signatures from a block", () => { - const EMPTY_SIGNATURE = Buffer.alloc(96); - const block: phase0.BeaconBlock = { slot: 0, proposerIndex: 0, @@ -32,20 +32,17 @@ describe("signatureSets", () => { {proposerIndex: 0, signature: EMPTY_SIGNATURE}, {proposerIndex: 0, signature: EMPTY_SIGNATURE} ), - ] as List, + ], attesterSlashings: [ getMockAttesterSlashings( - {attestingIndices: [0] as List, signature: EMPTY_SIGNATURE}, - {attestingIndices: [0] as List, signature: EMPTY_SIGNATURE} + {attestingIndices: [0], signature: EMPTY_SIGNATURE}, + {attestingIndices: [0], signature: EMPTY_SIGNATURE} ), - ] as List, - attestations: [ - getMockAttestations({attestingIndices: [0] as List, signature: EMPTY_SIGNATURE}), - ] as List, - deposits: ([] as phase0.Deposit[]) as List, - voluntaryExits: [ - getMockSignedVoluntaryExit({validatorIndex: 0, signature: EMPTY_SIGNATURE}), - ] as List, + ], + // Set to 1 since there's only one validator per committee + attestations: [getMockAttestations(1)], + deposits: [] as phase0.Deposit[], + voluntaryExits: [getMockSignedVoluntaryExit({validatorIndex: 0, signature: EMPTY_SIGNATURE})], }, }; @@ -64,10 +61,7 @@ describe("signatureSets", () => { validator.pubkey = bls.SecretKey.fromKeygen().toPublicKey().toBytes(); } - const state = createCachedBeaconState( - config, - ssz.phase0.BeaconState.createTreeBackedFromStruct(generateState({validators})) as TreeBacked - ); + const state = generateCachedState(config, {validators}); const signatureSets = allForks.getAllBlockSignatureSets(state, signedBlock); expect(signatureSets.length).to.equal( @@ -113,7 +107,7 @@ function getMockSignedBeaconBlockHeader(data: IBlockProposerData): phase0.Signed } interface IIndexAttestationData { - attestingIndices: List; + attestingIndices: ValidatorIndex[]; signature: BLSSignature; } @@ -148,11 +142,11 @@ function getAttestationData(): phase0.AttestationData { }; } -function getMockAttestations(data: IIndexAttestationData): phase0.Attestation { +function getMockAttestations(bitLen: number): phase0.Attestation { return { - aggregationBits: [true] as BitList, + aggregationBits: BitArray.fromSingleBit(bitLen, 0), data: getAttestationData(), - signature: data.signature, + signature: EMPTY_SIGNATURE, }; } diff --git a/packages/beacon-state-transition/test/unit/util/aggregationBits.test.ts b/packages/beacon-state-transition/test/unit/util/aggregationBits.test.ts deleted file mode 100644 index d3ea9d57d29..00000000000 --- a/packages/beacon-state-transition/test/unit/util/aggregationBits.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -import {MAX_VALIDATORS_PER_COMMITTEE, SYNC_COMMITTEE_SIZE} from "@chainsafe/lodestar-params"; -import {ssz} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; -import {expect} from "chai"; -import { - getUint8ByteToBitBooleanArray, - bitsToUint8Array, - zipIndexesSyncCommitteeBits, - zipAllIndexesSyncCommitteeBits, - getSingleBitIndex, - AggregationBitsErrorCode, -} from "../../../src"; - -const BITS_PER_BYTE = 8; - -describe("aggregationBits", function () { - const testCases: {name: string; data: boolean[]; numBytes: number}[] = [ - {name: "8 bits all true", data: [true, true, true, true, true, true, true, true], numBytes: 1}, - {name: "8 bits with true and false", data: [false, false, false, false, false, true, false, true], numBytes: 1}, - { - name: "10 bits with true and false", - data: [false, false, false, false, false, true, false, true, true, true], - numBytes: 2, - }, - { - name: "" + MAX_VALIDATORS_PER_COMMITTEE + " bits all true", - data: Array.from({length: MAX_VALIDATORS_PER_COMMITTEE}, () => true), - numBytes: Math.ceil(MAX_VALIDATORS_PER_COMMITTEE / 8), - }, - ]; - - it("getUint8ByteToBitBooleanArray", () => { - expect(getUint8ByteToBitBooleanArray(1)).to.be.deep.equal([true, false, false, false, false, false, false, false]); - expect(getUint8ByteToBitBooleanArray(5)).to.be.deep.equal([true, false, true, false, false, false, false, false]); - }); - - for (const {name, data, numBytes} of testCases) { - it(name, () => { - const tree = ssz.phase0.CommitteeBits.createTreeBackedFromStruct(data as List); - const aggregationBytes = bitsToUint8Array(tree, ssz.phase0.CommitteeBits); - expect(aggregationBytes.length).to.be.equal(numBytes, "number of bytes is incorrect"); - const aggregationBits: boolean[] = []; - for (let i = 0; i < tree.length; i++) { - aggregationBits.push(getAggregationBit(aggregationBytes, i)); - } - expect(aggregationBits).to.be.deep.equal(data, "incorrect extracted aggregationBits"); - }); - } - - it("getUint8ByteToBitBooleanArray - all values in 8 bytes", () => { - for (let i = 0; i <= 0xff; i++) { - const boolArr = getUint8ByteToBitBooleanArray(i); - const tree = ssz.phase0.CommitteeBits.createTreeBackedFromStruct(boolArr as List); - const bytes = tree.serialize(); - expect(bytes[0]).to.equal(i, `Wrong serialization of ${i}: ${JSON.stringify(boolArr)}`); - } - }); -}); - -describe("zipIndexesSyncCommitteeBits and zipAllIndexesSyncCommitteeBits", function () { - const committeeIndices = Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => i * 2); - const pivot = 3; - // 3 first bits are true - const syncCommitteeBits = Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => { - return i < pivot ? true : false; - }); - - // remaining bits are false - const expectedUnparticipantIndices: number[] = []; - for (let i = pivot; i < SYNC_COMMITTEE_SIZE; i++) { - expectedUnparticipantIndices.push(committeeIndices[i]); - } - - it("should extract from TreeBacked SyncAggregate", function () { - const syncAggregate = ssz.altair.SyncAggregate.defaultTreeBacked(); - syncAggregate.syncCommitteeBits = syncCommitteeBits; - expect(zipIndexesSyncCommitteeBits(committeeIndices, syncAggregate.syncCommitteeBits)).to.be.deep.equal( - [0, 2, 4], - "Incorrect participant indices from TreeBacked SyncAggregate" - ); - const [participantIndices, unparticipantIndices] = zipAllIndexesSyncCommitteeBits( - committeeIndices, - syncAggregate.syncCommitteeBits - ); - expect(participantIndices).to.be.deep.equal( - [0, 2, 4], - "Incorrect participant indices from TreeBacked SyncAggregate" - ); - expect(unparticipantIndices).to.be.deep.equal( - expectedUnparticipantIndices, - "Incorrect unparticipant indices from TreeBacked SyncAggregate" - ); - }); - - it("should extract from struct SyncAggregate", function () { - const syncAggregate = ssz.altair.SyncAggregate.defaultValue(); - syncAggregate.syncCommitteeBits = syncCommitteeBits; - expect(zipIndexesSyncCommitteeBits(committeeIndices, syncAggregate.syncCommitteeBits)).to.be.deep.equal( - [0, 2, 4], - "Incorrect participant indices from struct SyncAggregate" - ); - const [participantIndices, unparticipantIndices] = zipAllIndexesSyncCommitteeBits( - committeeIndices, - syncAggregate.syncCommitteeBits - ); - expect(participantIndices).to.be.deep.equal([0, 2, 4], "Incorrect participant indices from struct SyncAggregate"); - expect(unparticipantIndices).to.be.deep.equal( - expectedUnparticipantIndices, - "Incorrect unparticipant indices from struct SyncAggregate" - ); - }); -}); - -describe("getSingleBitIndex", () => { - const len = MAX_VALIDATORS_PER_COMMITTEE; - - const testCases: {id: string; bitList: boolean[]; res: number | Error | string}[] = [ - {id: "bit 0 true", bitList: [true, false, false, false, false, false, false, false, false, false], res: 0}, - {id: "bit 4 true", bitList: [false, false, false, false, true, false, false, false, false, false], res: 4}, - {id: "bit 9 true", bitList: [false, false, false, false, false, false, false, false, false, true], res: 9}, - { - id: "2 bits true", - bitList: [true, false, false, false, true, false, false, false, false, false], - res: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET, - }, - { - id: `${len} all true`, - bitList: Array.from({length: len}, () => true), - res: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET, - }, - { - id: `${len} all false`, - bitList: Array.from({length: len}, () => false), - res: AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET, - }, - ]; - - for (const {id, res, bitList} of testCases) { - const struct = bitList as List; - const treeBacked = ssz.phase0.CommitteeBits.createTreeBackedFromStruct(struct); - - it(`${id} - struct`, () => { - if (typeof res === "number") { - expect(getSingleBitIndex(struct)).to.equal(res); - } else { - expect(() => getSingleBitIndex(struct)).to.throw(res as Error); - } - }); - - it(`${id} - treeBacked`, () => { - if (typeof res === "number") { - expect(getSingleBitIndex(treeBacked)).to.equal(res); - } else { - expect(() => getSingleBitIndex(treeBacked)).to.throw(res as Error); - } - }); - } -}); - -/** - * Get aggregation bit (true/false) from an aggregation bytes array and validator index in committee. - * Notice: If we want to access the bit in batch, using this method is not efficient, check the performance - * test for an example of how to do that. - */ -export function getAggregationBit(attBytes: number[] | Uint8Array, indexInCommittee: number): boolean { - const byteIndex = Math.floor(indexInCommittee / BITS_PER_BYTE); - const indexInByte = indexInCommittee % BITS_PER_BYTE; - return getUint8ByteToBitBooleanArray(attBytes[byteIndex])[indexInByte]; -} diff --git a/packages/beacon-state-transition/test/unit/util/balance.test.ts b/packages/beacon-state-transition/test/unit/util/balance.test.ts index a09b9c374e8..e6ebe2723ad 100644 --- a/packages/beacon-state-transition/test/unit/util/balance.test.ts +++ b/packages/beacon-state-transition/test/unit/util/balance.test.ts @@ -1,15 +1,14 @@ import {assert, expect} from "chai"; import {config as minimalConfig} from "@chainsafe/lodestar-config/default"; -import {List, readonlyValuesListOfLeafNodeStruct} from "@chainsafe/ssz"; import {EFFECTIVE_BALANCE_INCREMENT} from "@chainsafe/lodestar-params"; -import {phase0, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {ValidatorIndex} from "@chainsafe/lodestar-types"; import {increaseBalance, decreaseBalance, getTotalBalance, isActiveValidator} from "../../../src/util"; +import {getEffectiveBalanceIncrementsZeroed, getEffectiveBalanceIncrementsZeroInactive} from "../../../src"; import {generateValidators} from "../../utils/validator"; import {generateCachedState, generateState} from "../../utils/state"; -import {getEffectiveBalanceIncrementsZeroInactive, getEffectiveBalanceIncrementsZeroed} from "../../../src"; describe("getTotalBalance", () => { it("should return correct balances", () => { @@ -19,7 +18,7 @@ describe("getTotalBalance", () => { for (const v of validators) { v.effectiveBalance = validatorBalance; } - const state: phase0.BeaconState = generateState({validators: validators}); + const state = generateState({validators: validators}); const validatorIndices: ValidatorIndex[] = Array.from({length: num}, (_, i) => i); const result = getTotalBalance(state, validatorIndices); @@ -30,8 +29,8 @@ describe("getTotalBalance", () => { it("should return correct balances", () => { const num = 5; const validators = generateValidators(num); - const balances = Array.from({length: num}, () => 0) as List; - const state: phase0.BeaconState = generateState({validators: validators, balances}); + const balances = Array.from({length: num}, () => 0); + const state = generateState({validators: validators, balances}); const validatorIndices: ValidatorIndex[] = Array.from({length: num}, (_, i) => i); const result = getTotalBalance(state, validatorIndices); @@ -43,16 +42,13 @@ describe("getTotalBalance", () => { describe("increaseBalance", () => { it("should add to a validators balance", () => { const state = generateCachedState(); + state.balances.push(0); + expect(state.balances.get(0)).to.be.equal(0); - state.validators.push(generateValidators(1)[0]); - state.balanceList.push(0); - expect(state.balanceList.get(0)).to.be.equal(0); - expect(state.balances[0]).to.be.equal(0); const delta = 5; for (let i = 1; i < 10; i++) { increaseBalance(state, 0, delta); - expect(state.balanceList.get(0)).to.be.equal(delta * i); - expect(state.balances[0]).to.be.equal(delta * i); + expect(state.balances.get(0)).to.be.equal(delta * i); } }); }); @@ -60,29 +56,27 @@ describe("increaseBalance", () => { describe("decreaseBalance", () => { it("should subtract from a validators balance", () => { const state = generateCachedState(); - state.validators.push(generateValidators(1)[0]); const initial = 100; - state.balanceList.push(initial); + state.balances.push(initial); + const delta = 5; for (let i = 1; i < 10; i++) { decreaseBalance(state, 0, delta); - expect(state.balanceList.get(0)).to.be.equal(initial - delta * i); - expect(state.balances[0]).to.be.equal(initial - delta * i); + expect(state.balances.get(0)).to.be.equal(initial - delta * i); } }); + it("should not make a validators balance < 0", () => { const state = generateCachedState(); - state.validators.push(generateValidators(1)[0]); const initial = 10; - state.balanceList.push(initial); + state.balances.push(initial); const delta = 11; decreaseBalance(state, 0, delta); - expect(state.balanceList.get(0)).to.be.equal(0); - expect(state.balances[0]).to.be.equal(0); + expect(state.balances.get(0)).to.be.equal(0); }); }); -describe("getEffectiveBalances", () => { +describe("getEffectiveBalanceIncrementsZeroInactive", () => { it("should get correct effective balances", () => { const justifiedState = generateCachedState(minimalConfig, { validators: [ @@ -92,10 +86,10 @@ describe("getEffectiveBalances", () => { ...generateValidators(4, {activation: 0, exit: Infinity, balance: 32e9}), // not active ...generateValidators(5, {activation: Infinity, exit: Infinity, balance: 32e9}), - ] as List, + ], }); - const justifiedEpoch = justifiedState.currentShuffling.epoch; - const validators = readonlyValuesListOfLeafNodeStruct(justifiedState.validators); + const justifiedEpoch = justifiedState.epochCtx.currentShuffling.epoch; + const validators = justifiedState.validators.getAllReadonlyValues(); const effectiveBalances = getEffectiveBalanceIncrementsZeroed(validators.length); for (let i = 0, len = validators.length; i < len; i++) { diff --git a/packages/beacon-state-transition/test/unit/util/cachedBeaconState.test.ts b/packages/beacon-state-transition/test/unit/util/cachedBeaconState.test.ts index 2d0e5280ff2..d1e411f6474 100644 --- a/packages/beacon-state-transition/test/unit/util/cachedBeaconState.test.ts +++ b/packages/beacon-state-transition/test/unit/util/cachedBeaconState.test.ts @@ -1,10 +1,16 @@ +import {createIBeaconConfig} from "@chainsafe/lodestar-config"; import {config} from "@chainsafe/lodestar-config/default"; import {ssz} from "@chainsafe/lodestar-types"; -import {createCachedBeaconState} from "../../../src"; +import {createCachedBeaconState, PubkeyIndexMap} from "../../../src"; describe("CachedBeaconState", () => { it("Create empty CachedBeaconState", () => { - const emptyState = ssz.phase0.BeaconState.defaultTreeBacked(); - createCachedBeaconState(config, emptyState); + const emptyState = ssz.phase0.BeaconState.defaultViewDU(); + + createCachedBeaconState(emptyState, { + config: createIBeaconConfig(config, emptyState.genesisValidatorsRoot), + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }); }); }); diff --git a/packages/beacon-state-transition/test/unit/util/epoch.test.ts b/packages/beacon-state-transition/test/unit/util/epoch.test.ts index 8afdb55cf1d..ececf95c038 100644 --- a/packages/beacon-state-transition/test/unit/util/epoch.test.ts +++ b/packages/beacon-state-transition/test/unit/util/epoch.test.ts @@ -1,7 +1,7 @@ import {assert} from "chai"; import {GENESIS_SLOT, MAX_SEED_LOOKAHEAD} from "@chainsafe/lodestar-params"; -import {phase0, Epoch, Slot} from "@chainsafe/lodestar-types"; +import {Epoch, Slot} from "@chainsafe/lodestar-types"; import { computeStartSlotAtEpoch, getPreviousEpoch, @@ -61,7 +61,7 @@ describe("getPreviousEpoch", () => { for (const testValue of testValues) { it("epoch should return previous epoch", () => { - const state: phase0.BeaconState = generateState({slot: testValue.slot}); + const state = generateState({slot: testValue.slot}); const result = getPreviousEpoch(state); assert.equal(result, testValue.expectedEpoch); }); diff --git a/packages/beacon-state-transition/test/unit/util/flags.test.ts b/packages/beacon-state-transition/test/unit/util/flags.test.ts new file mode 100644 index 00000000000..6566ef8205b --- /dev/null +++ b/packages/beacon-state-transition/test/unit/util/flags.test.ts @@ -0,0 +1,40 @@ +import {expect} from "chai"; + +describe("Altair status flags", () => { + for (let prev = 0b000; prev <= 0b111; prev++) { + for (let att = 0b000; att <= 0b111; att++) { + it(`prevFlag ${toStr(prev)} attFlag ${toStr(att)}`, () => { + expect( + // Actual function + toStr(getResFlags(prev, att)) + ).to.equal( + // Naive but correct implementation + toStr(getResFlagsNaive(prev, att)) + ); + }); + } + } +}); + +function toStr(flags: number): string { + return flags.toString(2).padStart(3, "0"); +} + +function getResFlags(prev: number, att: number): number { + return ~prev & att; +} + +function getResFlagsNaive(prev: number, att: number): number { + let out = 0; + + for (let i = 0; i < 3; i++) { + const mask = 1 << i; + const hasPrev = (prev & mask) === mask; + const hasAtt = (att & mask) === mask; + if (!hasPrev && hasAtt) { + out |= mask; + } + } + + return out; +} diff --git a/packages/beacon-state-transition/test/unit/util/interface.test.ts b/packages/beacon-state-transition/test/unit/util/interface.test.ts deleted file mode 100644 index c48f9782aa1..00000000000 --- a/packages/beacon-state-transition/test/unit/util/interface.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import {config} from "@chainsafe/lodestar-config/default"; -import {ssz} from "@chainsafe/lodestar-types"; -import {fromHexString, List, TreeBacked} from "@chainsafe/ssz"; -import {expect} from "chai"; -import {phase0, CachedBeaconStatePhase0, createCachedBeaconState} from "../../../src"; -import {generateState} from "../../utils/state"; - -const NUM_VALIDATORS = 1001; - -describe("CachedBeaconState", function () { - let state: TreeBacked; - let wrappedState: CachedBeaconStatePhase0; - - before(function () { - this.timeout(0); - const validators: phase0.Validator[] = []; - for (let i = 0; i < NUM_VALIDATORS; i++) { - validators.push({ - pubkey: fromHexString( - // randomly pregenerated pubkey - "0x84105a985058fc8740a48bf1ede9d223ef09e8c6b1735ba0a55cf4a9ff2ff92376b778798365e488dab07a652eb04576" - ), - withdrawalCredentials: Buffer.alloc(32), - effectiveBalance: 1000000, - slashed: false, - activationEligibilityEpoch: i + 10, - activationEpoch: i, - exitEpoch: i + 20, - withdrawableEpoch: i + 30, - }); - } - const defaultState = generateState({validators: validators as List}); - state = ssz.phase0.BeaconState.createTreeBackedFromStruct(defaultState); - }); - - beforeEach(() => { - state = state.clone(); - wrappedState = createCachedBeaconState(config, state); - }); - - it("should read the same value of TreeBacked", () => { - expect(state.validators[1000].activationEpoch).to.be.equal(1000); - expect(wrappedState.validators[1000].activationEpoch).to.be.equal(1000); - }); - - it("should modify both state and wrappedState", () => { - const oldFlatValidator = wrappedState.validators[1000]; - const validator = wrappedState.validators[1000]; - validator.activationEpoch = 2020; - validator.exitEpoch = 2030; - - expect(wrappedState.validators[1000].activationEpoch).to.be.equal(2020); - expect(wrappedState.validators[1000].exitEpoch).to.be.equal(2030); - // other property is the same - expect(wrappedState.validators[1000].effectiveBalance).to.be.equal(oldFlatValidator.effectiveBalance); - expect(wrappedState.validators[1000].slashed).to.be.equal(oldFlatValidator.slashed); - expect(wrappedState.validators[1000].activationEligibilityEpoch).to.be.equal( - oldFlatValidator.activationEligibilityEpoch - ); - expect(wrappedState.validators[1000].withdrawableEpoch).to.be.equal(oldFlatValidator.withdrawableEpoch); - - expect(state.validators[1000].activationEpoch).to.be.equal(2020); - expect(state.validators[1000].exitEpoch).to.be.equal(2030); - // other property is the same - expect(state.validators[1000].effectiveBalance).to.be.equal(oldFlatValidator.effectiveBalance); - expect(state.validators[1000].slashed).to.be.equal(oldFlatValidator.slashed); - expect(state.validators[1000].activationEligibilityEpoch).to.be.equal(oldFlatValidator.activationEligibilityEpoch); - expect(state.validators[1000].withdrawableEpoch).to.be.equal(oldFlatValidator.withdrawableEpoch); - }); - - it("should add validator to both state and wrappedState", () => { - wrappedState.validators.push({ - pubkey: Buffer.alloc(48), - withdrawalCredentials: Buffer.alloc(32), - effectiveBalance: 1000000, - slashed: false, - activationEligibilityEpoch: NUM_VALIDATORS + 10, - activationEpoch: NUM_VALIDATORS, - exitEpoch: NUM_VALIDATORS + 20, - withdrawableEpoch: NUM_VALIDATORS + 30, - }); - - expect(wrappedState.validators.length).to.be.equal(NUM_VALIDATORS + 1); - expect(state.validators.length).to.be.equal(NUM_VALIDATORS + 1); - expect(wrappedState.validators[NUM_VALIDATORS].activationEpoch).to.be.equal(NUM_VALIDATORS); - expect(state.validators[NUM_VALIDATORS].activationEpoch).to.be.equal(NUM_VALIDATORS); - }); -}); diff --git a/packages/beacon-state-transition/test/unit/util/seed.test.ts b/packages/beacon-state-transition/test/unit/util/seed.test.ts index e5623ce2b7b..071a6afd9f5 100644 --- a/packages/beacon-state-transition/test/unit/util/seed.test.ts +++ b/packages/beacon-state-transition/test/unit/util/seed.test.ts @@ -1,27 +1,30 @@ -import {assert} from "chai"; +import {expect} from "chai"; import {GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {getRandaoMix} from "../../../src/util"; import {generateState} from "../../utils/state"; +import {toHexString} from "@chainsafe/ssz"; describe("getRandaoMix", () => { + const randaoMix1 = Buffer.alloc(32, 1); + const randaoMix2 = Buffer.alloc(32, 2); + it("should return first randao mix for GENESIS_EPOCH", () => { // Empty state in 2nd epoch - const state = generateState({ - slot: GENESIS_SLOT + SLOTS_PER_EPOCH, - randaoMixes: [Buffer.from([0xab]), Buffer.from([0xcd])], - }); + const state = generateState({slot: GENESIS_SLOT + SLOTS_PER_EPOCH}); + state.randaoMixes.set(0, randaoMix1); + const res = getRandaoMix(state, GENESIS_EPOCH); - assert(Buffer.from(res as Uint8Array).equals(Uint8Array.from([0xab]))); + expect(toHexString(res)).to.equal(toHexString(randaoMix1)); }); it("should return second randao mix for GENESIS_EPOCH + 1", () => { // Empty state in 2nd epoch - const state = generateState({ - slot: GENESIS_SLOT + SLOTS_PER_EPOCH * 2, - randaoMixes: [Buffer.from([0xab]), Buffer.from([0xcd]), Buffer.from([0xef])], - }); + const state = generateState({slot: GENESIS_SLOT + SLOTS_PER_EPOCH * 2}); + state.randaoMixes.set(0, randaoMix1); + state.randaoMixes.set(1, randaoMix2); + const res = getRandaoMix(state, GENESIS_EPOCH + 1); - assert(Buffer.from(res as Uint8Array).equals(Uint8Array.from([0xcd]))); + expect(toHexString(res)).to.equal(toHexString(randaoMix2)); }); }); diff --git a/packages/beacon-state-transition/test/unit/util/validator.test.ts b/packages/beacon-state-transition/test/unit/util/validator.test.ts index dd1e8186242..bceaa996a30 100644 --- a/packages/beacon-state-transition/test/unit/util/validator.test.ts +++ b/packages/beacon-state-transition/test/unit/util/validator.test.ts @@ -1,7 +1,6 @@ import {assert, expect} from "chai"; -import {List} from "@chainsafe/ssz"; -import {phase0} from "@chainsafe/lodestar-types"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {getActiveValidatorIndices, isActiveValidator, isSlashableValidator} from "../../../src/util"; @@ -17,10 +16,11 @@ describe("getActiveValidatorIndices", () => { const state = generateState(); const activationEpoch = 1; const exitEpoch = 10; - state.validators = Array.from({length: 10}, () => - generateValidator({activation: activationEpoch, exit: exitEpoch}) - ) as List; - const allActiveIndices = Array.from(state.validators).map((_, i) => i); + state.validators = ssz.phase0.Validators.toViewDU( + Array.from({length: 10}, () => generateValidator({activation: activationEpoch, exit: exitEpoch})) + ); + + const allActiveIndices = state.validators.getAllReadonlyValues().map((_, i) => i); const allInactiveIndices: any = []; assert.deepEqual(getActiveValidatorIndices(state, activationEpoch), allActiveIndices); assert.deepEqual(getActiveValidatorIndices(state, exitEpoch), allInactiveIndices); diff --git a/packages/beacon-state-transition/test/utils/attestation.ts b/packages/beacon-state-transition/test/utils/attestation.ts index c30c2dddc1f..b3dc827e001 100644 --- a/packages/beacon-state-transition/test/utils/attestation.ts +++ b/packages/beacon-state-transition/test/utils/attestation.ts @@ -1,5 +1,5 @@ -import {List} from "@chainsafe/ssz"; import {phase0, Epoch} from "@chainsafe/lodestar-types"; +import {BitArray} from "@chainsafe/ssz"; /** * Generates a fake attestation data for test purposes. @@ -26,7 +26,7 @@ export function generateAttestationData(sourceEpoch: Epoch, targetEpoch: Epoch): export function generateEmptyAttestation(): phase0.Attestation { return { - aggregationBits: Array.from({length: 64}, () => false) as List, + aggregationBits: BitArray.fromBitLen(64), data: { slot: 1, index: 0, diff --git a/packages/beacon-state-transition/test/utils/block.ts b/packages/beacon-state-transition/test/utils/block.ts index 7c165890c6c..867d7d63906 100644 --- a/packages/beacon-state-transition/test/utils/block.ts +++ b/packages/beacon-state-transition/test/utils/block.ts @@ -1,5 +1,4 @@ import crypto from "node:crypto"; -import {List} from "@chainsafe/ssz"; import {phase0} from "@chainsafe/lodestar-types"; import {ZERO_HASH} from "../../src/constants"; @@ -17,11 +16,11 @@ export function generateEmptyBlock(): phase0.BeaconBlock { depositCount: 0, }, graffiti: crypto.randomBytes(32), - proposerSlashings: ([] as phase0.ProposerSlashing[]) as List, - attesterSlashings: ([] as phase0.AttesterSlashing[]) as List, - attestations: ([] as phase0.Attestation[]) as List, - deposits: ([] as phase0.Deposit[]) as List, - voluntaryExits: ([] as phase0.SignedVoluntaryExit[]) as List, + proposerSlashings: [] as phase0.ProposerSlashing[], + attesterSlashings: [] as phase0.AttesterSlashing[], + attestations: [] as phase0.Attestation[], + deposits: [] as phase0.Deposit[], + voluntaryExits: [] as phase0.SignedVoluntaryExit[], }, }; } diff --git a/packages/beacon-state-transition/test/utils/specTestTypes/stateTestCase.ts b/packages/beacon-state-transition/test/utils/specTestTypes/stateTestCase.ts deleted file mode 100644 index eeb1e14829d..00000000000 --- a/packages/beacon-state-transition/test/utils/specTestTypes/stateTestCase.ts +++ /dev/null @@ -1,6 +0,0 @@ -import {phase0} from "@chainsafe/lodestar-types"; - -export interface IStateTestCase { - pre: phase0.BeaconState; - post: phase0.BeaconState; -} diff --git a/packages/beacon-state-transition/test/utils/state.ts b/packages/beacon-state-transition/test/utils/state.ts index 2564ab61168..73763236d66 100644 --- a/packages/beacon-state-transition/test/utils/state.ts +++ b/packages/beacon-state-transition/test/utils/state.ts @@ -1,4 +1,3 @@ -import {List, Vector} from "@chainsafe/ssz"; import {config as minimalConfig} from "@chainsafe/lodestar-config/default"; import { EPOCHS_PER_HISTORICAL_VECTOR, @@ -14,8 +13,16 @@ import {ZERO_HASH} from "../../src/constants"; import {newZeroedBigIntArray} from "../../src/util"; import {generateEmptyBlock} from "./block"; -import {CachedBeaconStateAllForks, createCachedBeaconState} from "../../src"; -import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import { + BeaconStatePhase0, + CachedBeaconStateAllForks, + BeaconStateAllForks, + createCachedBeaconState, + PubkeyIndexMap, +} from "../../src"; +import {createIBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; +import {BeaconStateCache} from "../../src/cache/stateCache"; +import {EpochContextOpts} from "../../src/cache/epochContext"; /** * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed. @@ -28,8 +35,8 @@ type TestBeaconState = Partial; * @param {TestBeaconState} opts * @returns {BeaconState} */ -export function generateState(opts?: TestBeaconState): phase0.BeaconState { - return { +export function generateState(opts?: TestBeaconState): BeaconStatePhase0 { + return ssz.phase0.BeaconState.toViewDU({ genesisTime: Math.floor(Date.now() / 1000), genesisValidatorsRoot: ZERO_HASH, slot: GENESIS_SLOT, @@ -47,21 +54,21 @@ export function generateState(opts?: TestBeaconState): phase0.BeaconState { }, blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - historicalRoots: ([] as Vector[]) as List>, + historicalRoots: [], eth1Data: { depositRoot: Buffer.alloc(32), blockHash: Buffer.alloc(32), depositCount: 0, }, - eth1DataVotes: ([] as phase0.Eth1Data[]) as List, + eth1DataVotes: [], eth1DepositIndex: 0, - validators: ([] as phase0.Validator[]) as List, - balances: ([] as number[]) as List, + validators: [], + balances: [], randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), slashings: newZeroedBigIntArray(EPOCHS_PER_SLASHINGS_VECTOR), - previousEpochAttestations: ([] as phase0.PendingAttestation[]) as List, - currentEpochAttestations: ([] as phase0.PendingAttestation[]) as List, - justificationBits: [false, false, false, false], + previousEpochAttestations: [], + currentEpochAttestations: [], + justificationBits: ssz.phase0.JustificationBits.defaultValue(), previousJustifiedCheckpoint: { epoch: GENESIS_EPOCH, root: ZERO_HASH, @@ -75,7 +82,7 @@ export function generateState(opts?: TestBeaconState): phase0.BeaconState { root: ZERO_HASH, }, ...opts, - }; + }); } export function generateCachedState( @@ -83,5 +90,27 @@ export function generateCachedState( opts: TestBeaconState = {} ): CachedBeaconStateAllForks { const state = generateState(opts); - return createCachedBeaconState(config, config.getForkTypes(state.slot).BeaconState.createTreeBackedFromStruct(state)); + return createCachedBeaconState(state, { + config: createIBeaconConfig(config, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }); +} + +export function createCachedBeaconStateTest( + state: T, + configCustom: IChainForkConfig = config, + opts?: EpochContextOpts +): T & BeaconStateCache { + return createCachedBeaconState( + state, + { + config: createIBeaconConfig(configCustom, state.genesisValidatorsRoot), + // This is a test state, there's no need to have a global shared cache of keys + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }, + opts + ); } diff --git a/packages/beacon-state-transition/test/utils/validator.ts b/packages/beacon-state-transition/test/utils/validator.ts index 5927eff067b..ff63e94ba7d 100644 --- a/packages/beacon-state-transition/test/utils/validator.ts +++ b/packages/beacon-state-transition/test/utils/validator.ts @@ -1,4 +1,4 @@ -import {fromHexString, List} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {FAR_FUTURE_EPOCH} from "@chainsafe/lodestar-params"; import {phase0} from "@chainsafe/lodestar-types"; @@ -40,6 +40,6 @@ export function generateValidator(opts: IValidatorGeneratorOpts = {}): phase0.Va * @param {number} n * @returns {Validator[]} */ -export function generateValidators(n: number, opts?: IValidatorGeneratorOpts): List { - return Array.from({length: n}, () => generateValidator(opts)) as List; +export function generateValidators(n: number, opts?: IValidatorGeneratorOpts): phase0.Validator[] { + return Array.from({length: n}, () => generateValidator(opts)); } diff --git a/packages/cli/package.json b/packages/cli/package.json index db0a70279ed..496a34aff48 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -49,6 +49,7 @@ ], "dependencies": { "@chainsafe/abort-controller": "^3.0.1", + "@chainsafe/as-sha256": "^0.3.0", "@chainsafe/bls": "6.0.3", "@chainsafe/bls-keygen": "^0.3.0", "@chainsafe/bls-keystore": "^2.0.0", @@ -64,7 +65,7 @@ "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", "@chainsafe/lodestar-validator": "^0.36.0", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "^0.9.0", "@types/lockfile": "^1.0.1", "bip39": "^3.0.2", "deepmerge": "^4.2.2", diff --git a/packages/cli/src/cmds/account/cmds/validator/slashingProtection/export.ts b/packages/cli/src/cmds/account/cmds/validator/slashingProtection/export.ts index 44d66332165..bb7d3aaa697 100644 --- a/packages/cli/src/cmds/account/cmds/validator/slashingProtection/export.ts +++ b/packages/cli/src/cmds/account/cmds/validator/slashingProtection/export.ts @@ -1,5 +1,4 @@ import {InterchangeFormatVersion} from "@chainsafe/lodestar-validator"; -import {Json} from "@chainsafe/ssz"; import {ICliCommand, writeFile} from "../../../../../util"; import {IGlobalArgs} from "../../../../../options"; import {IAccountValidatorArgs} from "../options"; @@ -41,7 +40,7 @@ export const exportCmd: ICliCommand, - wsState: TreeBacked, + store: BeaconStateAllForks, + wsState: BeaconStateAllForks, wsCheckpoint: Checkpoint -): Promise<{anchorState: TreeBacked; wsCheckpoint: Checkpoint}> { +): Promise<{anchorState: BeaconStateAllForks; wsCheckpoint: Checkpoint}> { // Check if the store's state and wsState are compatible if ( store.genesisTime !== wsState.genesisTime || @@ -50,7 +49,7 @@ async function initAndVerifyWeakSubjectivityState( let anchorCheckpoint = wsCheckpoint; if (store.slot > wsState.slot) { anchorState = store; - anchorCheckpoint = getCheckpointFromState(config, store); + anchorCheckpoint = getCheckpointFromState(store); logger.verbose( "Db state is ahead of the provided checkpoint state, using the db state to initialize the beacon chain" ); @@ -82,7 +81,7 @@ export async function initBeaconState( db: IBeaconDb, logger: ILogger, signal: AbortSignal -): Promise<{anchorState: TreeBacked; wsCheckpoint?: Checkpoint}> { +): Promise<{anchorState: BeaconStateAllForks; wsCheckpoint?: Checkpoint}> { // fetch the latest state stored in the db // this will be used in all cases, if it exists, either used during verification of a weak subjectivity state, or used directly as the anchor state const lastDbState = await db.stateArchive.lastValue(); @@ -99,7 +98,7 @@ export async function initBeaconState( const genesisStateFile = args.genesisStateFile || getGenesisFileUrl(args.network || defaultNetwork); if (genesisStateFile && !args.forceGenesis) { const stateBytes = await downloadOrLoadFile(genesisStateFile); - let anchorState = getStateTypeFromBytes(chainForkConfig, stateBytes).createTreeBackedFromBytes(stateBytes); + let anchorState = getStateTypeFromBytes(chainForkConfig, stateBytes).deserializeToViewDU(stateBytes); const config = createIBeaconConfig(chainForkConfig, anchorState.genesisValidatorsRoot); anchorState = await initStateFromAnchorState(config, db, logger, anchorState); return {anchorState}; @@ -111,12 +110,12 @@ export async function initBeaconState( } async function initFromWSState( - lastDbState: TreeBacked | null, + lastDbState: BeaconStateAllForks | null, wssOpts: WSSOptions, chainForkConfig: IChainForkConfig, db: IBeaconDb, logger: ILogger -): Promise<{anchorState: TreeBacked; wsCheckpoint?: Checkpoint}> { +): Promise<{anchorState: BeaconStateAllForks; wsCheckpoint?: Checkpoint}> { if (wssOpts.weakSubjectivityStateFile) { // weak subjectivity sync from a provided state file: // if a weak subjectivity checkpoint has been provided, it is used for additional verification @@ -124,12 +123,12 @@ async function initFromWSState( const {weakSubjectivityStateFile, weakSubjectivityCheckpoint} = wssOpts; const stateBytes = await downloadOrLoadFile(weakSubjectivityStateFile); - const wsState = getStateTypeFromBytes(chainForkConfig, stateBytes).createTreeBackedFromBytes(stateBytes); + const wsState = getStateTypeFromBytes(chainForkConfig, stateBytes).deserializeToViewDU(stateBytes); const config = createIBeaconConfig(chainForkConfig, wsState.genesisValidatorsRoot); const store = lastDbState ?? wsState; const checkpoint = weakSubjectivityCheckpoint ? getCheckpointFromArg(weakSubjectivityCheckpoint) - : getCheckpointFromState(config, wsState); + : getCheckpointFromState(wsState); return initAndVerifyWeakSubjectivityState(config, db, logger, store, wsState, checkpoint); } else if (wssOpts.weakSubjectivitySyncLatest) { // weak subjectivity sync from a state that needs to be fetched: diff --git a/packages/cli/src/cmds/dev/handler.ts b/packages/cli/src/cmds/dev/handler.ts index bbac52f94df..a3ca1cdb493 100644 --- a/packages/cli/src/cmds/dev/handler.ts +++ b/packages/cli/src/cmds/dev/handler.ts @@ -78,7 +78,7 @@ export async function devHandler(args: IDevArgs & IGlobalArgs): Promise { if (args.genesisStateFile) { const state = config .getForkTypes(GENESIS_SLOT) - .BeaconState.createTreeBackedFromBytes(await fs.promises.readFile(args.genesisStateFile)); + .BeaconState.deserializeToViewDU(await fs.promises.readFile(args.genesisStateFile)); anchorState = await initStateFromAnchorState(config, db, logger, state); } else { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion diff --git a/packages/cli/src/config/beaconNodeOptions.ts b/packages/cli/src/config/beaconNodeOptions.ts index 559d8e3688d..8df66b6ccba 100644 --- a/packages/cli/src/config/beaconNodeOptions.ts +++ b/packages/cli/src/config/beaconNodeOptions.ts @@ -1,5 +1,4 @@ import deepmerge from "deepmerge"; -import {Json} from "@chainsafe/ssz"; import {defaultOptions, IBeaconNodeOptions} from "@chainsafe/lodestar"; import {isPlainObject, RecursivePartial} from "@chainsafe/lodestar-utils"; import {writeFile, readFile} from "../util"; @@ -53,7 +52,7 @@ export class BeaconNodeOptions { } export function writeBeaconNodeOptions(filename: string, config: Partial): void { - writeFile(filename, config as Json); + writeFile(filename, config); } /** diff --git a/packages/cli/src/config/peerId.ts b/packages/cli/src/config/peerId.ts index cbaf9ba5d96..5726a77e6de 100644 --- a/packages/cli/src/config/peerId.ts +++ b/packages/cli/src/config/peerId.ts @@ -1,5 +1,4 @@ import PeerId from "peer-id"; -import {Json} from "@chainsafe/ssz"; import {writeFile, readFile} from "../util"; export async function createPeerId(): Promise { @@ -7,7 +6,7 @@ export async function createPeerId(): Promise { } export function writePeerId(filepath: string, peerId: PeerId): void { - writeFile(filepath, (peerId.toJSON() as unknown) as Json); + writeFile(filepath, peerId.toJSON()); } export async function readPeerId(filepath: string): Promise { diff --git a/packages/cli/src/depositContract/depositData.ts b/packages/cli/src/depositContract/depositData.ts index b103aee54ec..bc7a541ae8b 100644 --- a/packages/cli/src/depositContract/depositData.ts +++ b/packages/cli/src/depositContract/depositData.ts @@ -1,5 +1,6 @@ import {ethers} from "ethers"; -import {hash, Json, toHexString} from "@chainsafe/ssz"; +import {digest} from "@chainsafe/as-sha256"; +import {toHexString} from "@chainsafe/ssz"; import {phase0, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import bls, {SecretKey, PublicKey} from "@chainsafe/bls"; @@ -15,13 +16,12 @@ function getDepositInterface(): ethers.utils.Interface { export function decodeEth1TxData(bytes: string, amount: string): {depositData: phase0.DepositData; root: string} { const depositContract = getDepositInterface(); - const inputs: Json = depositContract.decodeFunctionData("deposit", bytes); + const inputs = depositContract.decodeFunctionData("deposit", bytes); const {deposit_data_root: root} = inputs; const depositData: phase0.DepositData = ssz.phase0.DepositData.fromJson( // attach `amount` to decoded deposit inputs so it can be parsed to a DepositData - {...inputs, amount}, - {case: "snake"} + {...inputs, amount} ); // Sanity check @@ -38,7 +38,7 @@ export function encodeDepositData( config: IChainForkConfig ): string { const pubkey = signingKey.toPublicKey().toBytes(); - const withdrawalCredentials = Buffer.concat([BLS_WITHDRAWAL_PREFIX, hash(withdrawalPublicKey.toBytes()).slice(1)]); + const withdrawalCredentials = Buffer.concat([BLS_WITHDRAWAL_PREFIX, digest(withdrawalPublicKey.toBytes()).slice(1)]); // deposit data with empty signature to sign const depositData: phase0.DepositData = { diff --git a/packages/cli/src/networks/index.ts b/packages/cli/src/networks/index.ts index 71ed21675ef..dfddd02cd90 100644 --- a/packages/cli/src/networks/index.ts +++ b/packages/cli/src/networks/index.ts @@ -2,12 +2,11 @@ import {SLOTS_PER_EPOCH, ForkName} from "@chainsafe/lodestar-params"; import {getClient} from "@chainsafe/lodestar-api"; import {IBeaconNodeOptions} from "@chainsafe/lodestar"; import {IChainConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; -import {allForks} from "@chainsafe/lodestar-types"; import {Checkpoint} from "@chainsafe/lodestar-types/phase0"; import {RecursivePartial, fromHex} from "@chainsafe/lodestar-utils"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; // eslint-disable-next-line no-restricted-imports import {getStateTypeFromBytes} from "@chainsafe/lodestar/lib/util/multifork"; -import {TreeBacked} from "@chainsafe/ssz"; import fs from "node:fs"; import got from "got"; import * as mainnet from "./mainnet"; @@ -145,7 +144,7 @@ export function enrsToNetworkConfig(enrs: string[]): RecursivePartial; wsCheckpoint: Checkpoint}> { +): Promise<{wsState: BeaconStateAllForks; wsCheckpoint: Checkpoint}> { try { let wsCheckpoint; const api = getClient(config, {baseUrl: weakSubjectivityServerUrl}); @@ -162,7 +161,7 @@ export async function fetchWeakSubjectivityState( ? api.debug.getState(`${stateSlot}`, "ssz") : api.debug.getStateV2(`${stateSlot}`, "ssz")); - return {wsState: getStateTypeFromBytes(config, stateBytes).createTreeBackedFromBytes(stateBytes), wsCheckpoint}; + return {wsState: getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes), wsCheckpoint}; } catch (e) { throw new Error("Unable to fetch weak subjectivity state: " + (e as Error).message); } diff --git a/packages/cli/src/util/file.ts b/packages/cli/src/util/file.ts index 75b5840a128..7245581747a 100644 --- a/packages/cli/src/util/file.ts +++ b/packages/cli/src/util/file.ts @@ -4,7 +4,6 @@ import stream from "node:stream"; import {promisify} from "node:util"; import got from "got"; import {load, dump, FAILSAFE_SCHEMA, Schema, Type} from "js-yaml"; -import {Json} from "@chainsafe/ssz"; export const yamlSchema = new Schema({ include: [FAILSAFE_SCHEMA], @@ -36,7 +35,7 @@ export enum FileFormat { /** * Parse file contents as Json. */ -export function parse(contents: string, fileFormat: FileFormat): T { +export function parse(contents: string, fileFormat: FileFormat): T { switch (fileFormat) { case FileFormat.json: return JSON.parse(contents) as T; @@ -51,7 +50,7 @@ export function parse(contents: string, fileFormat: FileFormat): T { /** * Stringify file contents. */ -export function stringify(obj: T, fileFormat: FileFormat): string { +export function stringify(obj: unknown, fileFormat: FileFormat): string { let contents: string; switch (fileFormat) { case FileFormat.json: @@ -72,7 +71,7 @@ export function stringify(obj: T, fileFormat: FileFormat): string { * * Serialize either to json, yaml, or toml */ -export function writeFile(filepath: string, obj: Json): void { +export function writeFile(filepath: string, obj: unknown): void { mkdir(path.dirname(filepath)); const fileFormat = path.extname(filepath).substr(1); fs.writeFileSync(filepath, stringify(obj, fileFormat as FileFormat), "utf-8"); @@ -84,7 +83,7 @@ export function writeFile(filepath: string, obj: Json): void { * Parse either from json, yaml, or toml * Optional acceptedFormats object can be passed which can be an array of accepted formats, in future can be extended to include parseFn for the accepted formats */ -export function readFile(filepath: string, acceptedFormats?: Json[]): T { +export function readFile(filepath: string, acceptedFormats?: string[]): T { const fileFormat = path.extname(filepath).substr(1); if (acceptedFormats && !acceptedFormats.includes(fileFormat)) throw new Error(`UnsupportedFileFormat: ${filepath}`); const contents = fs.readFileSync(filepath, "utf-8"); @@ -95,7 +94,7 @@ export function readFile(filepath: string, acceptedFormats?: Json[]): * @see readFile * If `filepath` does not exist returns null */ -export function readFileIfExists(filepath: string, acceptedFormats?: Json[]): T | null { +export function readFileIfExists(filepath: string, acceptedFormats?: string[]): T | null { try { return readFile(filepath, acceptedFormats); } catch (e) { diff --git a/packages/config/package.json b/packages/config/package.json index b9aae98fddf..34fdf657591 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -44,6 +44,6 @@ "dependencies": { "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", - "@chainsafe/ssz": "^0.8.20" + "@chainsafe/ssz": "^0.9.0" } } diff --git a/packages/config/src/chainConfig/sszTypes.ts b/packages/config/src/chainConfig/sszTypes.ts deleted file mode 100644 index d8e8a851e18..00000000000 --- a/packages/config/src/chainConfig/sszTypes.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {ByteVectorType, ContainerType} from "@chainsafe/ssz"; -import {ssz, StringType} from "@chainsafe/lodestar-types"; -import {IChainConfig} from "./types"; - -const ByteVector20 = new ByteVectorType({ - length: 20, -}); - -export const ChainConfig = new ContainerType({ - fields: { - PRESET_BASE: new StringType(), - - // Transition - TERMINAL_TOTAL_DIFFICULTY: ssz.Uint256, - TERMINAL_BLOCK_HASH: ssz.Root, - TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: ssz.Number64, - - // Genesis - MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: ssz.Number64, - MIN_GENESIS_TIME: ssz.Number64, - GENESIS_FORK_VERSION: ssz.Version, - GENESIS_DELAY: ssz.Number64, - - // Forking - // Altair - ALTAIR_FORK_VERSION: ssz.Version, - ALTAIR_FORK_EPOCH: ssz.Number64, - // Bellatrix - BELLATRIX_FORK_VERSION: ssz.Version, - BELLATRIX_FORK_EPOCH: ssz.Number64, - // Sharding - SHARDING_FORK_VERSION: ssz.Version, - SHARDING_FORK_EPOCH: ssz.Number64, - - // Time parameters - SECONDS_PER_SLOT: ssz.Number64, - SECONDS_PER_ETH1_BLOCK: ssz.Number64, - MIN_VALIDATOR_WITHDRAWABILITY_DELAY: ssz.Number64, - SHARD_COMMITTEE_PERIOD: ssz.Number64, - ETH1_FOLLOW_DISTANCE: ssz.Number64, - - // Validator cycle - INACTIVITY_SCORE_BIAS: ssz.Number64, - INACTIVITY_SCORE_RECOVERY_RATE: ssz.Number64, - EJECTION_BALANCE: ssz.Number64, - MIN_PER_EPOCH_CHURN_LIMIT: ssz.Number64, - CHURN_LIMIT_QUOTIENT: ssz.Number64, - PROPOSER_SCORE_BOOST: ssz.Number64, - - // Deposit contract - DEPOSIT_CHAIN_ID: ssz.Number64, - DEPOSIT_NETWORK_ID: ssz.Number64, - DEPOSIT_CONTRACT_ADDRESS: ByteVector20, - }, - // Expected and container fields are the same here - expectedCase: "notransform", -}); diff --git a/packages/config/src/forkConfig/index.ts b/packages/config/src/forkConfig/index.ts index 3b5b6258a45..28bfcc77a00 100644 --- a/packages/config/src/forkConfig/index.ts +++ b/packages/config/src/forkConfig/index.ts @@ -43,7 +43,7 @@ export function createIForkConfig(config: IChainConfig): IForkConfig { return this.getForkInfo(slot).version; }, getForkTypes(slot: Slot): allForks.AllForksSSZTypes { - return ssz.allForks[this.getForkName(slot)]; + return ssz.allForks[this.getForkName(slot)] as allForks.AllForksSSZTypes; }, }; } diff --git a/packages/config/src/genesisConfig/index.ts b/packages/config/src/genesisConfig/index.ts index bec34c1b5da..9b53fc631ef 100644 --- a/packages/config/src/genesisConfig/index.ts +++ b/packages/config/src/genesisConfig/index.ts @@ -1,6 +1,6 @@ import {ForkName} from "@chainsafe/lodestar-params"; import {DomainType, ForkDigest, phase0, Root, Slot, ssz, Version} from "@chainsafe/lodestar-types"; -import {ByteVector, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {IChainForkConfig} from "../beaconConfig"; import {ForkDigestHex, ICachedGenesis} from "./types"; export {IForkDigestContext} from "./types"; @@ -89,7 +89,7 @@ function computeForkDataRoot(currentVersion: Version, genesisValidatorsRoot: Roo return ssz.phase0.ForkData.hashTreeRoot(forkData); } -function toHexStringNoPrefix(hex: string | ByteVector): string { +function toHexStringNoPrefix(hex: string | Uint8Array): string { return strip0xPrefix(typeof hex === "string" ? hex : toHexString(hex)); } diff --git a/packages/db/package.json b/packages/db/package.json index 84c79abee81..434067cddda 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -38,7 +38,7 @@ "dependencies": { "@chainsafe/lodestar-config": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "^0.9.0", "@types/levelup": "^4.3.3", "it-all": "^1.0.2", "level": "^7.0.0", diff --git a/packages/db/src/abstractRepository.ts b/packages/db/src/abstractRepository.ts index 956130795ac..ea2cf58f0d8 100644 --- a/packages/db/src/abstractRepository.ts +++ b/packages/db/src/abstractRepository.ts @@ -1,5 +1,5 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {ArrayLike, Type} from "@chainsafe/ssz"; +import {Type} from "@chainsafe/ssz"; import {BUCKET_LENGTH} from "."; import {IFilterOptions, IKeyValue} from "./controller"; import {Db} from "./controller/interface"; @@ -11,8 +11,8 @@ export type Id = Uint8Array | string | number | bigint; /** * Repository is a high level kv storage - * managing a Uint8rray to Uint8rray kv database - * It translates typed keys and values to Uint8rrays required by the underlying database + * managing a Uint8Array to Uint8Array kv database + * It translates typed keys and values to Uint8Arrays required by the underlying database * * By default, SSZ-encoded values, * indexed by root @@ -100,7 +100,7 @@ export abstract class Repository { await this.delete(this.getId(value)); } - async batchPut(items: ArrayLike>): Promise { + async batchPut(items: IKeyValue[]): Promise { this.dbWriteMetrics?.inc(); await this.db.batchPut( Array.from({length: items.length}, (_, i) => ({ @@ -111,7 +111,7 @@ export abstract class Repository { } // Similar to batchPut but we support value as Uint8Array - async batchPutBinary(items: ArrayLike>): Promise { + async batchPutBinary(items: IKeyValue[]): Promise { this.dbWriteMetrics?.inc(); await this.db.batchPut( Array.from({length: items.length}, (_, i) => ({ @@ -121,12 +121,12 @@ export abstract class Repository { ); } - async batchDelete(ids: ArrayLike): Promise { + async batchDelete(ids: I[]): Promise { this.dbWriteMetrics?.inc(); await this.db.batchDelete(Array.from({length: ids.length}, (_, i) => this.encodeKey(ids[i]))); } - async batchAdd(values: ArrayLike): Promise { + async batchAdd(values: T[]): Promise { await this.batchPut( Array.from({length: values.length}, (_, i) => ({ key: this.getId(values[i]), @@ -135,7 +135,7 @@ export abstract class Repository { ); } - async batchRemove(values: ArrayLike): Promise { + async batchRemove(values: T[]): Promise { await this.batchDelete(Array.from({length: values.length}, (ignored, i) => this.getId(values[i]))); } diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 59ec657e0c7..0eb4925a040 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -42,7 +42,7 @@ "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/ssz": "^0.8.20" + "@chainsafe/ssz": "^0.9.0" }, "keywords": [ "ethereum", diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index c6f168d120a..94fdcb94c5c 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -1,4 +1,4 @@ -import {isTreeBacked, readonlyValues, toHexString, TreeBacked} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {SAFE_SLOTS_TO_UPDATE_JUSTIFIED, SLOTS_PER_HISTORICAL_ROOT, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {Slot, ValidatorIndex, phase0, allForks, ssz, RootHex, Epoch, Root} from "@chainsafe/lodestar-types"; import { @@ -9,6 +9,7 @@ import { ZERO_HASH, bellatrix, EffectiveBalanceIncrements, + BeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {IChainConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; @@ -270,7 +271,7 @@ export class ForkChoice implements IForkChoice { * `justifiedBalances` balances of justified state which is updated synchronously. * This ensures that the forkchoice is never out of sync. */ - onBlock(block: allForks.BeaconBlock, state: allForks.BeaconState, preCachedData?: OnBlockPrecachedData): void { + onBlock(block: allForks.BeaconBlock, state: BeaconStateAllForks, preCachedData?: OnBlockPrecachedData): void { const {parentRoot, slot} = block; const parentRootHex = toHexString(parentRoot); // Parent block must be known @@ -388,7 +389,7 @@ export class ForkChoice implements IForkChoice { throw Error("Missing blockDelaySec info for proposerBoost"); } - const proposerInterval = getCurrentInterval(this.config, state.genesisTime, blockDelaySec); + const proposerInterval = getCurrentInterval(this.config, blockDelaySec); if (proposerInterval < 1) { this.proposerBoostRoot = blockRootHex; this.synced = false; @@ -396,7 +397,7 @@ export class ForkChoice implements IForkChoice { } const targetSlot = computeStartSlotAtEpoch(computeEpochAtSlot(slot)); - const targetRoot = slot === targetSlot ? blockRoot : state.blockRoots[targetSlot % SLOTS_PER_HISTORICAL_ROOT]; + const targetRoot = slot === targetSlot ? blockRoot : state.blockRoots.get(targetSlot % SLOTS_PER_HISTORICAL_ROOT); // This does not apply a vote to the block, it just makes fork choice aware of the block so // it can still be identified as the head even if it doesn't have any votes. @@ -466,7 +467,7 @@ export class ForkChoice implements IForkChoice { this.validateOnAttestation(attestation, slot, blockRootHex, targetEpoch); if (slot < this.fcStore.currentSlot) { - for (const validatorIndex of readonlyValues(attestation.attestingIndices)) { + for (const validatorIndex of attestation.attestingIndices) { this.addLatestMessage(validatorIndex, targetEpoch, blockRootHex); } } else { @@ -478,7 +479,7 @@ export class ForkChoice implements IForkChoice { // ``` this.queuedAttestations.add({ slot: slot, - attestingIndices: Array.from(readonlyValues(attestation.attestingIndices)), + attestingIndices: attestation.attestingIndices, blockRoot: blockRootHex, targetEpoch, }); @@ -733,7 +734,7 @@ export class ForkChoice implements IForkChoice { * * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/fork-choice.md#should_update_justified_checkpoint */ - private shouldUpdateJustifiedCheckpoint(state: allForks.BeaconState): boolean { + private shouldUpdateJustifiedCheckpoint(state: BeaconStateAllForks): boolean { const {slot, currentJustifiedCheckpoint} = state; const newJustifiedCheckpoint = currentJustifiedCheckpoint; @@ -803,24 +804,20 @@ export class ForkChoice implements IForkChoice { } const attestationData = indexedAttestation.data; - // Only cache attestation data root hex if it's tree backed since it's available. - if ( - isTreeBacked(attestationData) && - this.validatedAttestationDatas.has( - toHexString(((attestationData as unknown) as TreeBacked).tree.root) - ) - ) { - return; - } + // AttestationData is expected to internally cache its root to make this hashTreeRoot() call free + const attestationCacheKey = toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestationData)); - this.validateAttestationData(indexedAttestation.data, slot, blockRootHex, targetEpoch); + if (!this.validatedAttestationDatas.has(attestationCacheKey)) { + this.validateAttestationData(indexedAttestation.data, slot, blockRootHex, targetEpoch, attestationCacheKey); + } } private validateAttestationData( attestationData: phase0.AttestationData, slot: Slot, beaconBlockRootHex: string, - targetEpoch: Epoch + targetEpoch: Epoch, + attestationCacheKey: string ): void { const epochNow = computeEpochAtSlot(this.fcStore.currentSlot); const targetRootHex = toHexString(attestationData.target.root); @@ -920,12 +917,7 @@ export class ForkChoice implements IForkChoice { }); } - // Only cache attestation data root hex if it's tree backed since it's available. - if (isTreeBacked(attestationData)) { - this.validatedAttestationDatas.add( - toHexString(((attestationData as unknown) as TreeBacked).tree.root) - ); - } + this.validatedAttestationDatas.add(attestationCacheKey); } /** diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index 3cf6c6e15ad..030b116731f 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -1,4 +1,5 @@ import {EffectiveBalanceIncrements} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {Epoch, Slot, ValidatorIndex, phase0, allForks, Root, RootHex} from "@chainsafe/lodestar-types"; import {IProtoBlock, ExecutionStatus} from "../protoArray/interface"; import {CheckpointWithHex} from "./store"; @@ -57,7 +58,7 @@ export interface IForkChoice { * `preCachedData` includes data necessary for validation included in the spec but some data is * pre-fetched in advance to keep the fork-choice fully syncronous */ - onBlock(block: allForks.BeaconBlock, state: allForks.BeaconState, preCachedData?: OnBlockPrecachedData): void; + onBlock(block: allForks.BeaconBlock, state: BeaconStateAllForks, preCachedData?: OnBlockPrecachedData): void; /** * Register `attestation` with the fork choice DAG so that it may influence future calls to `getHead`. * diff --git a/packages/fork-choice/src/forkChoice/store.ts b/packages/fork-choice/src/forkChoice/store.ts index 899b9a6fc24..b63c96beb84 100644 --- a/packages/fork-choice/src/forkChoice/store.ts +++ b/packages/fork-choice/src/forkChoice/store.ts @@ -75,7 +75,7 @@ export class ForkChoiceStore implements IForkChoiceStore { export function toCheckpointWithHex(checkpoint: phase0.Checkpoint): CheckpointWithHex { // `valueOf` coerses the checkpoint, which may be tree-backed, into a javascript object // See https://github.com/ChainSafe/lodestar/issues/2258 - const root = checkpoint.root.valueOf() as Uint8Array; + const root = checkpoint.root; return { epoch: checkpoint.epoch, root, diff --git a/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts index fcc6e6c1584..9d5370f0d0c 100644 --- a/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/perf/forkChoice/forkChoice.test.ts @@ -122,12 +122,14 @@ describe("ForkChoice", () => { const aggregatedAttestations: IndexedAttestation[] = []; const averageAggregatorsPerSlot = 11; for (let committeeIndex = 0; committeeIndex < ATTESTATION_SUBNET_COUNT; committeeIndex++) { - const tbAttestationData = ssz.phase0.AttestationData.createTreeBackedFromStruct({ + const tbAttestationData = { ...attestationDataOmitIndex, index: committeeIndex, - }); + }; + // cache the root - tbAttestationData.hashTreeRoot(); + ssz.phase0.AttestationData.hashTreeRoot(tbAttestationData); + for (let aggregator = 0; aggregator < averageAggregatorsPerSlot; aggregator++) { // same data, different signatures aggregatedAttestations.push({ diff --git a/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts b/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts index fe61839037e..d9666f1fb72 100644 --- a/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts +++ b/packages/fork-choice/test/perf/protoArray/computeDeltas.test.ts @@ -1,7 +1,7 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {expect} from "chai"; import { - CachedBeaconStateAllForks, + CachedBeaconStateAltair, computeStartSlotAtEpoch, EffectiveBalanceIncrements, getEffectiveBalanceIncrementsZeroed, @@ -19,7 +19,7 @@ function flagIsTimelySource(flag: number): boolean { } describe("computeDeltas", () => { - let originalState: CachedBeaconStateAllForks; + let originalState: CachedBeaconStateAltair; const indices: Map = new Map(); let oldBalances: EffectiveBalanceIncrements; let newBalances: EffectiveBalanceIncrements; @@ -30,15 +30,13 @@ describe("computeDeltas", () => { before(function () { this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow - originalState = (generatePerfTestCachedStateAltair({ - goBackOneSlot: true, - }) as unknown) as CachedBeaconStateAllForks; - const numPreviousEpochParticipation = originalState.previousEpochParticipation.persistent - .toArray() - .filter((flags) => flagIsTimelySource(flags)).length; - const numCurrentEpochParticipation = originalState.currentEpochParticipation.persistent - .toArray() - .filter((flags) => flagIsTimelySource(flags)).length; + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}); + + const previousEpochParticipationArr = originalState.previousEpochParticipation.getAll(); + const currentEpochParticipationArr = originalState.currentEpochParticipation.getAll(); + + const numPreviousEpochParticipation = previousEpochParticipationArr.filter(flagIsTimelySource).length; + const numCurrentEpochParticipation = currentEpochParticipationArr.filter(flagIsTimelySource).length; expect(numPreviousEpochParticipation).to.equal(250000, "Wrong numPreviousEpochParticipation"); expect(numCurrentEpochParticipation).to.equal(250000, "Wrong numCurrentEpochParticipation"); @@ -66,8 +64,8 @@ describe("computeDeltas", () => { id: "computeDeltas", beforeEach: () => { const votes: IVoteTracker[] = []; - const epoch = originalState.currentShuffling.epoch; - const committee = originalState.getBeaconCommittee(computeStartSlotAtEpoch(epoch), 0); + const epoch = originalState.epochCtx.currentShuffling.epoch; + const committee = originalState.epochCtx.getBeaconCommittee(computeStartSlotAtEpoch(epoch), 0); for (let i = 0; i < 250000; i++) { if (committee.includes(i)) { votes.push({ diff --git a/packages/keymanager-server/package.json b/packages/keymanager-server/package.json index 741a30e50f1..e63cd8842aa 100644 --- a/packages/keymanager-server/package.json +++ b/packages/keymanager-server/package.json @@ -52,7 +52,7 @@ "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", "@chainsafe/lodestar-validator": "^0.36.0", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "^0.9.0", "lockfile": "^1.0.4", "fastify": "3.15.1", "fastify-cors": "^6.0.1", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index a3905c754e9..7f1f2c9fa5c 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -42,8 +42,8 @@ "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/persistent-merkle-tree": "^0.3.7", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/persistent-merkle-tree": "^0.4.0", + "@chainsafe/ssz": "^0.9.0", "cross-fetch": "^3.1.4", "mitt": "^3.0.0" }, diff --git a/packages/light-client/src/index.ts b/packages/light-client/src/index.ts index 08b7c7cad23..503edd3f9ca 100644 --- a/packages/light-client/src/index.ts +++ b/packages/light-client/src/index.ts @@ -6,7 +6,7 @@ import {altair, phase0, RootHex, ssz, SyncPeriod} from "@chainsafe/lodestar-type import {createIBeaconConfig, IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; import {TreeOffsetProof} from "@chainsafe/persistent-merkle-tree"; import {isErrorAborted, sleep} from "@chainsafe/lodestar-utils"; -import {fromHexString, Path, toHexString} from "@chainsafe/ssz"; +import {fromHexString, JsonPath, toHexString} from "@chainsafe/ssz"; import {getCurrentSlot, slotWithFutureTolerance, timeUntilNextEpoch} from "./utils/clock"; import {isBetterUpdate, LightclientUpdateStats} from "./utils/update"; import {deserializeSyncCommittee, isEmptyHeader, sumBits} from "./utils/utils"; @@ -229,7 +229,7 @@ export class Lightclient { } /** Returns header since head may change during request */ - async getHeadStateProof(paths: Path[]): Promise<{proof: TreeOffsetProof; header: phase0.BeaconBlockHeader}> { + async getHeadStateProof(paths: JsonPath[]): Promise<{proof: TreeOffsetProof; header: phase0.BeaconBlockHeader}> { const header = this.head.header; const stateId = toHexString(header.stateRoot); const res = await this.api.lightclient.getStateProof(stateId, paths); diff --git a/packages/light-client/src/utils/domain.ts b/packages/light-client/src/utils/domain.ts index 0d7ea0fd757..2450637691b 100644 --- a/packages/light-client/src/utils/domain.ts +++ b/packages/light-client/src/utils/domain.ts @@ -1,6 +1,6 @@ // Only used by processDeposit + lightclient -import {Epoch, Version, Root, DomainType, allForks, phase0, ssz, Domain} from "@chainsafe/lodestar-types"; +import {Epoch, Version, Root, DomainType, phase0, ssz, Domain} from "@chainsafe/lodestar-types"; import {Type} from "@chainsafe/ssz"; /** @@ -17,7 +17,7 @@ export function computeDomain(domainType: DomainType, forkVersion: Version, gene /** * Return the ForkVersion at an epoch from a Fork type */ -export function getForkVersion(fork: allForks.BeaconState["fork"], epoch: Epoch): Version { +export function getForkVersion(fork: phase0.Fork, epoch: Epoch): Version { return epoch < fork.epoch ? fork.previousVersion : fork.currentVersion; } diff --git a/packages/light-client/src/utils/utils.ts b/packages/light-client/src/utils/utils.ts index f74172409b3..6f9798f5979 100644 --- a/packages/light-client/src/utils/utils.ts +++ b/packages/light-client/src/utils/utils.ts @@ -1,17 +1,11 @@ import {PublicKey} from "@chainsafe/bls"; import {altair, Root, ssz} from "@chainsafe/lodestar-types"; import {BeaconBlockHeader} from "@chainsafe/lodestar-types/phase0"; -import {ArrayLike, BitVector} from "@chainsafe/ssz"; +import {BitArray} from "@chainsafe/ssz"; import {SyncCommitteeFast} from "../types"; -export function sumBits(bits: ArrayLike): number { - let sum = 0; - for (const bit of bits) { - if (bit) { - sum++; - } - } - return sum; +export function sumBits(bits: BitArray): number { + return bits.getTrueBitIndexes().length; } export function isZeroHash(root: Root): boolean { @@ -23,7 +17,7 @@ export function isZeroHash(root: Root): boolean { return true; } -export function assertZeroHashes(rootArray: ArrayLike, expectedLength: number, errorMessage: string): void { +export function assertZeroHashes(rootArray: Root[], expectedLength: number, errorMessage: string): void { if (rootArray.length !== expectedLength) { throw Error(`Wrong length ${errorMessage}`); } @@ -38,16 +32,9 @@ export function assertZeroHashes(rootArray: ArrayLike, expectedLength: num /** * Util to guarantee that all bits have a corresponding pubkey */ -export function getParticipantPubkeys(pubkeys: ArrayLike, bits: BitVector): T[] { - const participantPubkeys: T[] = []; - for (let i = 0; i < bits.length; i++) { - if (bits[i]) { - if (pubkeys[i] === undefined) throw Error(`No pubkey ${i} in syncCommittee`); - participantPubkeys.push(pubkeys[i]); - } - } - - return participantPubkeys; +export function getParticipantPubkeys(pubkeys: T[], bits: BitArray): T[] { + // BitArray.intersectValues() checks the length is correct + return bits.intersectValues(pubkeys); } export function toBlockHeader(block: altair.BeaconBlock): BeaconBlockHeader { @@ -61,7 +48,7 @@ export function toBlockHeader(block: altair.BeaconBlock): BeaconBlockHeader { } function deserializePubkeys(pubkeys: altair.LightClientUpdate["nextSyncCommittee"]["pubkeys"]): PublicKey[] { - return Array.from(pubkeys).map((pk) => PublicKey.fromBytes(pk.valueOf() as Uint8Array)); + return Array.from(pubkeys).map((pk) => PublicKey.fromBytes(pk)); } function serializePubkeys(pubkeys: PublicKey[]): altair.LightClientUpdate["nextSyncCommittee"]["pubkeys"] { @@ -71,7 +58,7 @@ function serializePubkeys(pubkeys: PublicKey[]): altair.LightClientUpdate["nextS export function deserializeSyncCommittee(syncCommittee: altair.SyncCommittee): SyncCommitteeFast { return { pubkeys: deserializePubkeys(syncCommittee.pubkeys), - aggregatePubkey: PublicKey.fromBytes(syncCommittee.aggregatePubkey.valueOf() as Uint8Array), + aggregatePubkey: PublicKey.fromBytes(syncCommittee.aggregatePubkey), }; } diff --git a/packages/light-client/src/validation.ts b/packages/light-client/src/validation.ts index 0af2e36267e..ecd3c15cd76 100644 --- a/packages/light-client/src/validation.ts +++ b/packages/light-client/src/validation.ts @@ -66,10 +66,10 @@ export function assertValidFinalityProof(update: altair.LightClientUpdate): void if ( !isValidMerkleBranch( ssz.phase0.BeaconBlockHeader.hashTreeRoot(update.finalizedHeader), - Array.from(update.finalityBranch).map((i) => i.valueOf() as Uint8Array), + update.finalityBranch, FINALIZED_ROOT_DEPTH, FINALIZED_ROOT_INDEX, - update.attestedHeader.stateRoot.valueOf() as Uint8Array + update.attestedHeader.stateRoot ) ) { throw Error("Invalid finality header merkle branch"); @@ -96,10 +96,10 @@ export function assertValidSyncCommitteeProof(update: altair.LightClientUpdate): if ( !isValidMerkleBranch( ssz.altair.SyncCommittee.hashTreeRoot(update.nextSyncCommittee), - Array.from(update.nextSyncCommitteeBranch).map((i) => i.valueOf() as Uint8Array), + update.nextSyncCommitteeBranch, NEXT_SYNC_COMMITTEE_DEPTH, NEXT_SYNC_COMMITTEE_INDEX, - activeHeader(update).stateRoot.valueOf() as Uint8Array + activeHeader(update).stateRoot ) ) { throw Error("Invalid next sync committee merkle branch"); @@ -155,9 +155,7 @@ export function assertValidSignedHeader( domain: config.getDomain(DOMAIN_SYNC_COMMITTEE, signedHeaderSlot), }); - if ( - !isValidBlsAggregate(participantPubkeys, signingRoot, syncAggregate.syncCommitteeSignature.valueOf() as Uint8Array) - ) { + if (!isValidBlsAggregate(participantPubkeys, signingRoot, syncAggregate.syncCommitteeSignature)) { throw Error("Invalid aggregate signature"); } } diff --git a/packages/light-client/test/lightclientApiServer.ts b/packages/light-client/test/lightclientApiServer.ts index aa4cde97561..1696512c977 100644 --- a/packages/light-client/test/lightclientApiServer.ts +++ b/packages/light-client/test/lightclientApiServer.ts @@ -4,14 +4,15 @@ import {registerRoutes} from "@chainsafe/lodestar-api/server"; import fastifyCors from "fastify-cors"; import querystring from "querystring"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {Path, TreeBacked} from "@chainsafe/ssz"; -import {allForks, altair, RootHex, SyncPeriod} from "@chainsafe/lodestar-types"; +import {JsonPath} from "@chainsafe/ssz"; +import {altair, RootHex, SyncPeriod} from "@chainsafe/lodestar-types"; import {Proof} from "@chainsafe/persistent-merkle-tree"; +import {BeaconStateAltair} from "./types"; /* eslint-disable @typescript-eslint/no-unsafe-member-access */ export type IStateRegen = { - getStateByRoot(stateRoot: string): Promise>; + getStateByRoot(stateRoot: string): Promise; }; export type IBlockCache = { @@ -39,12 +40,12 @@ export async function startServer(opts: ServerOpts, config: IChainForkConfig, ap } export class LightclientServerApi implements routes.lightclient.Api { - readonly states = new Map>(); + readonly states = new Map(); readonly updates = new Map(); readonly snapshots = new Map(); latestHeadUpdate: routes.lightclient.LightclientHeaderUpdate | null = null; - async getStateProof(stateId: string, paths: Path[]): Promise<{data: Proof}> { + async getStateProof(stateId: string, paths: JsonPath[]): Promise<{data: Proof}> { const state = this.states.get(stateId); if (!state) throw Error(`stateId ${stateId} not available`); return {data: state.createProof(paths)}; diff --git a/packages/light-client/test/naive/update.ts b/packages/light-client/test/naive/update.ts index 39f342a2e8f..5402fc01f77 100644 --- a/packages/light-client/test/naive/update.ts +++ b/packages/light-client/test/naive/update.ts @@ -70,7 +70,7 @@ export function processLightClientUpdate( // Note that (2) means that the current light client design needs finality. // It may be changed to re-organizable light client design. See the on-going issue https://github.com/ethereum/consensus-specs/issues/2315. if ( - sumBits(update.syncAggregate.syncCommitteeBits) * 3 >= update.syncAggregate.syncCommitteeBits.length * 2 && + sumBits(update.syncAggregate.syncCommitteeBits) * 3 >= update.syncAggregate.syncCommitteeBits.bitLen * 2 && !isEmptyHeader(update.finalizedHeader) ) { applyLightClientUpdate(store.snapshot, update); diff --git a/packages/light-client/test/prepareUpdateNaive.ts b/packages/light-client/test/prepareUpdateNaive.ts index 7ce559f733c..753eef11678 100644 --- a/packages/light-client/test/prepareUpdateNaive.ts +++ b/packages/light-client/test/prepareUpdateNaive.ts @@ -1,16 +1,17 @@ -import {altair, phase0, Root} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; +import {altair, phase0, Root, ssz} from "@chainsafe/lodestar-types"; +import {CompositeViewDU} from "@chainsafe/ssz"; import {FINALIZED_ROOT_GINDEX, NEXT_SYNC_COMMITTEE_GINDEX, SLOTS_PER_HISTORICAL_ROOT} from "@chainsafe/lodestar-params"; import {computeEpochAtSlot} from "../src/utils/clock"; import {getForkVersion} from "../src/utils/domain"; +import {Tree} from "@chainsafe/persistent-merkle-tree"; export interface IBeaconChainLc { getBlockHeaderByRoot(blockRoot: Root): Promise; - getStateByRoot(stateRoot: Root): Promise>; + getStateByRoot(stateRoot: Root): Promise>; } /** - * From a TreeBacked state, return an update to be consumed by a light client + * From a TreeView state, return an update to be consumed by a light client * Spec v1.0.1 */ export async function prepareUpdateNaive( @@ -83,7 +84,7 @@ export async function prepareUpdateNaive( // Get the finality block root that sync committees have signed in blockA const syncAttestedSlot = stateWithSyncAggregate.slot - 1; // Inlined `getBlockRootAtSlot()` - const syncAttestedBlockRoot = stateWithSyncAggregate.blockRoots[syncAttestedSlot % SLOTS_PER_HISTORICAL_ROOT]; + const syncAttestedBlockRoot = stateWithSyncAggregate.blockRoots.get(syncAttestedSlot % SLOTS_PER_HISTORICAL_ROOT); const syncAttestedBlockHeader = await chain.getBlockHeaderByRoot(syncAttestedBlockRoot); // Get the ForkVersion used in the syncAggregate, as verified in the state transition fn @@ -93,17 +94,22 @@ export async function prepareUpdateNaive( // Get the finalized state defined in the block "attested" by the current sync committee const syncAttestedState = await chain.getStateByRoot(syncAttestedBlockHeader.stateRoot); const finalizedCheckpointBlockHeader = await chain.getBlockHeaderByRoot(syncAttestedState.finalizedCheckpoint.root); + // Prove that the `finalizedCheckpointRoot` belongs in that block - const finalityBranch = syncAttestedState.tree.getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); + syncAttestedState.commit(); + const syncAttestedStateTree = new Tree(syncAttestedState.node); + const finalityBranch = syncAttestedStateTree.getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); // Get `nextSyncCommittee` from a finalized state so the lightclient can safely transition to the next committee const finalizedCheckpointState = await chain.getStateByRoot(finalizedCheckpointBlockHeader.stateRoot); // Prove that the `nextSyncCommittee` is included in a finalized state "attested" by the current sync committee - const nextSyncCommitteeBranch = finalizedCheckpointState.tree.getSingleProof(BigInt(NEXT_SYNC_COMMITTEE_GINDEX)); + finalizedCheckpointState.commit(); + const finalizedCheckpointStateTree = new Tree(finalizedCheckpointState.node); + const nextSyncCommitteeBranch = finalizedCheckpointStateTree.getSingleProof(BigInt(NEXT_SYNC_COMMITTEE_GINDEX)); return { attestedHeader: syncAttestedBlockHeader, - nextSyncCommittee: finalizedCheckpointState.nextSyncCommittee, + nextSyncCommittee: finalizedCheckpointState.nextSyncCommittee.toValue(), nextSyncCommitteeBranch: nextSyncCommitteeBranch, finalizedHeader: finalizedCheckpointBlockHeader, finalityBranch: finalityBranch, diff --git a/packages/light-client/test/types.ts b/packages/light-client/test/types.ts new file mode 100644 index 00000000000..3f10cd32b1c --- /dev/null +++ b/packages/light-client/test/types.ts @@ -0,0 +1,4 @@ +import {CompositeViewDU} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; + +export type BeaconStateAltair = CompositeViewDU; diff --git a/packages/light-client/test/unit/sync.test.ts b/packages/light-client/test/unit/sync.test.ts index 88dca4e4671..400b4d934f8 100644 --- a/packages/light-client/test/unit/sync.test.ts +++ b/packages/light-client/test/unit/sync.test.ts @@ -1,5 +1,6 @@ import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks, BeaconStateAltair} from "@chainsafe/lodestar-beacon-state-transition"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {routes, Api} from "@chainsafe/lodestar-api"; import {chainConfig as chainConfigDef} from "@chainsafe/lodestar-config/default"; import {createIBeaconConfig, IChainConfig} from "@chainsafe/lodestar-config"; @@ -13,7 +14,7 @@ import { committeeUpdateToHeadUpdate, lastInMap, } from "../utils"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {expect} from "chai"; const SOME_HASH = Buffer.alloc(32, 0xff); @@ -94,7 +95,7 @@ describe("Lightclient sync", () => { // Test fetching a proof // First create a state with some known data const executionStateRoot = Buffer.alloc(32, 0xee); - const state = ssz.bellatrix.BeaconState.defaultTreeBacked(); + const state = ssz.bellatrix.BeaconState.defaultViewDU(); state.latestExecutionPayloadHeader.stateRoot = executionStateRoot; // Track head + reference states with some known data @@ -113,7 +114,7 @@ describe("Lightclient sync", () => { // Provide the state to the lightclient server impl. Only the last one to test proof fetching if (slot === targetSlot) { - lightclientServerApi.states.set(toHexString(stateRoot), state as TreeBacked); + lightclientServerApi.states.set(toHexString(stateRoot), (state as BeaconStateAllForks) as BeaconStateAltair); } // Emit a new head update with the custom state root @@ -140,7 +141,7 @@ describe("Lightclient sync", () => { // Fetch proof of "latestExecutionPayloadHeader.stateRoot" const {proof, header} = await lightclient.getHeadStateProof([["latestExecutionPayloadHeader", "stateRoot"]]); - const recoveredState = ssz.bellatrix.BeaconState.createTreeBackedFromProof(header.stateRoot as Uint8Array, proof); + const recoveredState = ssz.bellatrix.BeaconState.createFromProof(proof, header.stateRoot); expect(toHexString(recoveredState.latestExecutionPayloadHeader.stateRoot)).to.equal( toHexString(executionStateRoot), "Recovered executionStateRoot from getHeadStateProof() not correct" diff --git a/packages/light-client/test/unit/syncNaive.test.ts b/packages/light-client/test/unit/syncNaive.test.ts index a99f1a9f117..054978dc1b8 100644 --- a/packages/light-client/test/unit/syncNaive.test.ts +++ b/packages/light-client/test/unit/syncNaive.test.ts @@ -1,7 +1,8 @@ import {expect} from "chai"; import {SecretKey} from "@chainsafe/bls"; import {altair, phase0, Root, ssz, SyncPeriod} from "@chainsafe/lodestar-types"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; +import {BeaconStateAltair} from "@chainsafe/lodestar-beacon-state-transition"; +import {toHexString} from "@chainsafe/ssz"; import {chainConfig} from "@chainsafe/lodestar-config/default"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; import {processLightClientUpdate} from "../naive/update"; @@ -121,10 +122,10 @@ class MockBeaconChainLc implements IBeaconChainLc { return blockHeader; } - async getStateByRoot(stateRoot: Root): Promise> { + async getStateByRoot(stateRoot: Root): Promise { const rootHex = toHexString(stateRoot); const state = this.states.get(rootHex); if (!state) throw Error(`No state for ${rootHex}`); - return ssz.altair.BeaconState.createTreeBackedFromStruct(state); + return ssz.altair.BeaconState.toViewDU(state); } } diff --git a/packages/light-client/test/unit/validation.test.ts b/packages/light-client/test/unit/validation.test.ts index 7edd9db5fed..1ebbed576b8 100644 --- a/packages/light-client/test/unit/validation.test.ts +++ b/packages/light-client/test/unit/validation.test.ts @@ -1,4 +1,5 @@ import {aggregatePublicKeys, PublicKey, SecretKey} from "@chainsafe/bls"; +import {Tree} from "@chainsafe/persistent-merkle-tree"; import {altair, ssz} from "@chainsafe/lodestar-types"; import {chainConfig} from "@chainsafe/lodestar-config/default"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; @@ -44,27 +45,29 @@ describe("validateLightClientUpdate", () => { }; // finalizedCheckpointState must have `nextSyncCommittee` - const finalizedCheckpointState = ssz.altair.BeaconState.defaultTreeBacked(); - finalizedCheckpointState.nextSyncCommittee = nextSyncCommittee; + const finalizedCheckpointState = ssz.altair.BeaconState.defaultViewDU(); + finalizedCheckpointState.nextSyncCommittee = ssz.altair.SyncCommittee.toViewDU(nextSyncCommittee); // Prove it - const nextSyncCommitteeBranch = finalizedCheckpointState.tree.getSingleProof(BigInt(NEXT_SYNC_COMMITTEE_GINDEX)); + const nextSyncCommitteeBranch = new Tree(finalizedCheckpointState.node).getSingleProof( + BigInt(NEXT_SYNC_COMMITTEE_GINDEX) + ); // update.header must have stateRoot to finalizedCheckpointState const finalizedHeader = defaultBeaconBlockHeader(updateHeaderSlot); - finalizedHeader.stateRoot = ssz.altair.BeaconState.hashTreeRoot(finalizedCheckpointState); + finalizedHeader.stateRoot = finalizedCheckpointState.hashTreeRoot(); // syncAttestedState must have `header` as finalizedCheckpoint - const syncAttestedState = ssz.altair.BeaconState.defaultTreeBacked(); - syncAttestedState.finalizedCheckpoint = { + const syncAttestedState = ssz.altair.BeaconState.defaultViewDU(); + syncAttestedState.finalizedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ epoch: 0, root: ssz.phase0.BeaconBlockHeader.hashTreeRoot(finalizedHeader), - }; + }); // Prove it - const finalityBranch = syncAttestedState.tree.getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); + const finalityBranch = new Tree(syncAttestedState.node).getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); // finalityHeader must have stateRoot to syncAttestedState const syncAttestedBlockHeader = defaultBeaconBlockHeader(attestedHeaderSlot); - syncAttestedBlockHeader.stateRoot = ssz.altair.BeaconState.hashTreeRoot(syncAttestedState); + syncAttestedBlockHeader.stateRoot = syncAttestedState.hashTreeRoot(); const forkVersion = ssz.Bytes4.defaultValue(); const signingRoot = getSyncAggregateSigningRoot(config, syncAttestedBlockHeader); diff --git a/packages/light-client/test/utils.ts b/packages/light-client/test/utils.ts index c7f73af2981..ce282498f19 100644 --- a/packages/light-client/test/utils.ts +++ b/packages/light-client/test/utils.ts @@ -13,7 +13,7 @@ import { } from "@chainsafe/lodestar-params"; import {altair, phase0, Slot, ssz, SyncPeriod} from "@chainsafe/lodestar-types"; import {hash} from "@chainsafe/persistent-merkle-tree"; -import {fromHexString, List} from "@chainsafe/ssz"; +import {BitArray, fromHexString} from "@chainsafe/ssz"; import {SyncCommitteeFast} from "../src/types"; import {computeSigningRoot} from "../src/utils/domain"; import {getLcLoggerConsole} from "../src/utils/logger"; @@ -38,7 +38,7 @@ export function signAndAggregate(message: Uint8Array, sks: SecretKey[]): altair. const sigs = sks.map((sk) => sk.sign(message)); const aggSig = Signature.aggregate(sigs).toBytes(); return { - syncCommitteeBits: sks.map(() => true), + syncCommitteeBits: BitArray.fromBoolArray(sks.map(() => true)), syncCommitteeSignature: aggSig, }; } @@ -85,7 +85,7 @@ export function getInteropSyncCommittee(period: SyncPeriod): SyncCommitteeKeys { const sigs = Array.from({length: SYNC_COMMITTEE_SIZE}, () => sig); const aggSig = Signature.aggregate(sigs).toBytes(); return { - syncCommitteeBits: Array.from({length: SYNC_COMMITTEE_SIZE}, () => true), + syncCommitteeBits: BitArray.fromBoolArray(sigs.map(() => true)), syncCommitteeSignature: aggSig, }; } @@ -219,12 +219,12 @@ export function generateValidator(opts: Partial = {}): phase0. /** * Generates n number of validators, for tests purposes only. */ -export function generateValidators(n: number, opts?: Partial): List { - return Array.from({length: n}, () => generateValidator(opts)) as List; +export function generateValidators(n: number, opts?: Partial): phase0.Validator[] { + return Array.from({length: n}, () => generateValidator(opts)); } -export function generateBalances(n: number): List { - return Array.from({length: n}, () => 32e9) as List; +export function generateBalances(n: number): number[] { + return Array.from({length: n}, () => 32e9); } /** diff --git a/packages/lodestar/package.json b/packages/lodestar/package.json index 40810ce7a8c..d549430bde7 100644 --- a/packages/lodestar/package.json +++ b/packages/lodestar/package.json @@ -61,6 +61,7 @@ }, "dependencies": { "@chainsafe/abort-controller": "^3.0.1", + "@chainsafe/as-sha256": "^0.3.0", "@chainsafe/bls": "6.0.3", "@chainsafe/discv5": "^0.6.7", "@chainsafe/libp2p-noise": "5.0.3", @@ -74,10 +75,9 @@ "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", "@chainsafe/lodestar-validator": "^0.36.0", - "@chainsafe/persistent-merkle-tree": "^0.3.7", + "@chainsafe/persistent-merkle-tree": "^0.4.0", "@chainsafe/snappy-stream": "5.0.0", - "@chainsafe/ssz": "^0.8.20", - "@chainsafe/as-sha256": "^0.2.4", + "@chainsafe/ssz": "^0.9.0", "@ethersproject/abi": "^5.0.0", "@types/datastore-level": "^3.0.0", "datastore-core": "^7.0.1", diff --git a/packages/lodestar/src/api/impl/beacon/blocks/index.ts b/packages/lodestar/src/api/impl/beacon/blocks/index.ts index fdd541ac9ad..90ca8ae2a28 100644 --- a/packages/lodestar/src/api/impl/beacon/blocks/index.ts +++ b/packages/lodestar/src/api/impl/beacon/blocks/index.ts @@ -125,7 +125,7 @@ export function getBeaconBlockApi({ if (slot < head.slot && head.slot <= slot + SLOTS_PER_HISTORICAL_ROOT) { const state = chain.getHeadState(); - return {data: state.blockRoots[slot % SLOTS_PER_HISTORICAL_ROOT]}; + return {data: state.blockRoots.get(slot % SLOTS_PER_HISTORICAL_ROOT)}; } } else if (blockId === "head") { const head = chain.forkChoice.getHead(); diff --git a/packages/lodestar/src/api/impl/beacon/index.ts b/packages/lodestar/src/api/impl/beacon/index.ts index 445f6748bac..d3fed6541a5 100644 --- a/packages/lodestar/src/api/impl/beacon/index.ts +++ b/packages/lodestar/src/api/impl/beacon/index.ts @@ -24,7 +24,7 @@ export function getBeaconApi( return { data: { genesisForkVersion, - genesisTime: BigInt(chain.genesisTime), + genesisTime: chain.genesisTime, genesisValidatorsRoot: chain.genesisValidatorsRoot, }, }; diff --git a/packages/lodestar/src/api/impl/beacon/state/index.ts b/packages/lodestar/src/api/impl/beacon/state/index.ts index eb67cf74827..d356f72ef34 100644 --- a/packages/lodestar/src/api/impl/beacon/state/index.ts +++ b/packages/lodestar/src/api/impl/beacon/state/index.ts @@ -1,9 +1,8 @@ import {routes} from "@chainsafe/lodestar-api"; // eslint-disable-next-line no-restricted-imports import {Api as IBeaconStateApi} from "@chainsafe/lodestar-api/lib/routes/beacon/state"; -import {allForks, altair} from "@chainsafe/lodestar-types"; -import {readonlyValues} from "@chainsafe/ssz"; import { + BeaconStateAllForks, CachedBeaconStateAltair, computeEpochAtSlot, getCurrentEpoch, @@ -19,14 +18,14 @@ import { } from "./utils"; export function getBeaconStateApi({chain, config, db}: Pick): IBeaconStateApi { - async function getState(stateId: routes.beacon.StateId): Promise { + async function getState(stateId: routes.beacon.StateId): Promise { return await resolveStateId(config, chain, db, stateId); } return { async getStateRoot(stateId) { const state = await getState(stateId); - return {data: config.getForkTypes(state.slot).BeaconState.hashTreeRoot(state)}; + return {data: state.hashTreeRoot()}; }, async getStateFork(stateId) { @@ -49,21 +48,21 @@ export function getBeaconStateApi({chain, config, db}: Pick { - return { - index, - balance, - }; - }); - return {data: balances}; + // TODO: This loops over the entire state, it's a DOS vector + const balancesArr = state.balances.getAll(); + const resp: routes.beacon.ValidatorBalance[] = []; + for (let i = 0; i < balancesArr.length; i++) { + resp.push({index: i, balance: balancesArr[i]}); + } + return {data: resp}; }, async getEpochCommittees(stateId, filters) { @@ -171,7 +172,7 @@ export function getBeaconStateApi({chain, config, db}: Pick { +): Promise { const state = await resolveStateIdOrNull(config, chain, db, stateId, opts); if (!state) { throw new ApiError(404, `No state found for id '${stateId}'`); @@ -48,7 +50,7 @@ async function resolveStateIdOrNull( db: IBeaconDb, stateId: routes.beacon.StateId, opts?: ResolveStateIdOpts -): Promise { +): Promise { stateId = stateId.toLowerCase(); if (stateId === "head" || stateId === "genesis" || stateId === "finalized" || stateId === "justified") { return await stateByName(db, chain.stateCache, chain.forkChoice, stateId); @@ -117,7 +119,7 @@ async function stateByName( stateCache: StateContextCache, forkChoice: IForkChoice, stateId: routes.beacon.StateId -): Promise { +): Promise { switch (stateId) { case "head": return stateCache.get(forkChoice.getHead().stateRoot) ?? null; @@ -136,7 +138,7 @@ async function stateByRoot( db: IBeaconDb, stateCache: StateContextCache, stateId: routes.beacon.StateId -): Promise { +): Promise { if (stateId.startsWith("0x")) { const stateRoot = stateId; const cachedStateCtx = stateCache.get(stateRoot); @@ -154,7 +156,7 @@ async function stateBySlot( forkChoice: IForkChoice, slot: Slot, opts?: ResolveStateIdOpts -): Promise { +): Promise { const blockSummary = forkChoice.getCanonicalBlockAtSlot(slot); if (blockSummary) { const state = stateCache.get(blockSummary.stateRoot); @@ -172,17 +174,20 @@ async function stateBySlot( export function filterStateValidatorsByStatuses( statuses: string[], - state: allForks.BeaconState, + state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap, currentEpoch: Epoch ): routes.beacon.ValidatorResponse[] { const responses: routes.beacon.ValidatorResponse[] = []; - const validators = Array.from(state.validators); - const filteredValidators = validators.filter((v) => statuses.includes(getValidatorStatus(v, currentEpoch))); - for (const validator of readonlyValues(filteredValidators)) { + const validatorsArr = state.validators.getAllReadonlyValues(); + const statusSet = new Set(statuses); + + for (const validator of validatorsArr) { + const validatorStatus = getValidatorStatus(validator, currentEpoch); + const validatorIndex = getStateValidatorIndex(validator.pubkey, state, pubkey2index); - if (validatorIndex !== undefined && statuses?.includes(getValidatorStatus(validator, currentEpoch))) { - responses.push(toValidatorResponse(validatorIndex, validator, state.balances[validatorIndex], currentEpoch)); + if (validatorIndex !== undefined && statusSet.has(validatorStatus)) { + responses.push(toValidatorResponse(validatorIndex, validator, state.balances.get(validatorIndex), currentEpoch)); } } return responses; @@ -197,8 +202,10 @@ async function getNearestArchivedState( slot: Slot ): Promise { const states = db.stateArchive.valuesStream({lte: slot, reverse: true}); - const state = (await states[Symbol.asyncIterator]().next()).value as TreeBacked; - return createCachedBeaconState(config, state); + const state = (await states[Symbol.asyncIterator]().next()).value as BeaconStateAllForks; + // TODO - PENDING: Don't create new immutable caches here + // see https://github.com/ChainSafe/lodestar/issues/3683 + return createCachedBeaconState(state, createEmptyEpochContextImmutableData(config, state)); } async function getFinalizedState( @@ -228,8 +235,8 @@ async function getFinalizedState( } export function getStateValidatorIndex( - id: routes.beacon.ValidatorId | ByteVector, - state: allForks.BeaconState, + id: routes.beacon.ValidatorId | BLSPubkey, + state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap ): number | undefined { let validatorIndex: ValidatorIndex | undefined; diff --git a/packages/lodestar/src/api/impl/debug/index.ts b/packages/lodestar/src/api/impl/debug/index.ts index a8f24b7f635..4596f709a8d 100644 --- a/packages/lodestar/src/api/impl/debug/index.ts +++ b/packages/lodestar/src/api/impl/debug/index.ts @@ -23,7 +23,7 @@ export function getDebugApi({ if (format === "ssz") { // Casting to any otherwise Typescript doesn't like the multi-type return // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-explicit-any - return config.getForkTypes(state.slot).BeaconState.serialize(state) as any; + return state.serialize() as any; } else { return {data: state}; } @@ -34,7 +34,7 @@ export function getDebugApi({ if (format === "ssz") { // Casting to any otherwise Typescript doesn't like the multi-type return // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-explicit-any - return config.getForkTypes(state.slot).BeaconState.serialize(state) as any; + return state.serialize() as any; } else { return {data: state, version: config.getForkName(state.slot)}; } diff --git a/packages/lodestar/src/api/impl/events/index.ts b/packages/lodestar/src/api/impl/events/index.ts index 2a58c98c0b4..69f4b0d5a5e 100644 --- a/packages/lodestar/src/api/impl/events/index.ts +++ b/packages/lodestar/src/api/impl/events/index.ts @@ -37,7 +37,7 @@ export function getEventsApi({chain, config}: Pick toHexString(getBlockRootAtSlot(state, Math.max(computeStartSlotAtEpoch(epoch) - 1, 0))) ); diff --git a/packages/lodestar/src/api/impl/lightclient/index.ts b/packages/lodestar/src/api/impl/lightclient/index.ts index 6cb1192e8e2..392eab3882e 100644 --- a/packages/lodestar/src/api/impl/lightclient/index.ts +++ b/packages/lodestar/src/api/impl/lightclient/index.ts @@ -2,8 +2,8 @@ import {ApiModules} from "../types"; import {resolveStateId} from "../beacon/state/utils"; import {routes} from "@chainsafe/lodestar-api"; import {linspace} from "../../../util/numpy"; -import {fromHexString, isCompositeType} from "@chainsafe/ssz"; -import {ProofType} from "@chainsafe/persistent-merkle-tree"; +import {fromHexString} from "@chainsafe/ssz"; +import {ProofType, Tree} from "@chainsafe/persistent-merkle-tree"; import {IApiOptions} from "../../options"; // TODO: Import from lightclient/server package @@ -17,31 +17,18 @@ export function getLightclientApi( const maxGindicesInProof = opts.maxGindicesInProof ?? 512; return { - async getStateProof(stateId, paths) { + async getStateProof(stateId, jsonPaths) { const state = await resolveStateId(config, chain, db, stateId); - // eslint-disable-next-line @typescript-eslint/naming-convention - const BeaconState = config.getForkTypes(state.slot).BeaconState; - const stateTreeBacked = BeaconState.createTreeBackedFromStruct(state); - const tree = stateTreeBacked.tree; - const gindicesSet = new Set(); + // Commit any changes before computing the state root. In normal cases the state should have no changes here + state.commit(); + const stateNode = state.node; + const tree = new Tree(stateNode); - for (const path of paths) { - // Logic from TreeBacked#createProof is (mostly) copied here to expose the # of gindices in the proof - const {type, gindex} = BeaconState.getPathInfo(path); - if (!isCompositeType(type)) { - gindicesSet.add(gindex); - } else { - // if the path subtype is composite, include the gindices of all the leaves - const gindexes = type.tree_getLeafGindices( - type.hasVariableSerializedLength() ? tree.getSubtree(gindex) : undefined, - gindex - ); - for (const gindex of gindexes) { - gindicesSet.add(gindex); - } - } - } + const gindexes = state.type.tree_createProofGindexes(stateNode, jsonPaths); + // TODO: Is it necessary to de-duplicate? + // It's not a problem if we overcount gindexes + const gindicesSet = new Set(gindexes); if (gindicesSet.size > maxGindicesInProof) { throw new Error("Requested proof is too large."); diff --git a/packages/lodestar/src/api/impl/lodestar/index.ts b/packages/lodestar/src/api/impl/lodestar/index.ts index 5313dba23a5..1b699a18730 100644 --- a/packages/lodestar/src/api/impl/lodestar/index.ts +++ b/packages/lodestar/src/api/impl/lodestar/index.ts @@ -2,7 +2,7 @@ import PeerId from "peer-id"; import {Multiaddr} from "multiaddr"; import {routes} from "@chainsafe/lodestar-api"; import {getLatestWeakSubjectivityCheckpointEpoch} from "@chainsafe/lodestar-beacon-state-transition"; -import {Json, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {ssz} from "@chainsafe/lodestar-types"; import {BeaconChain} from "../../../chain"; @@ -179,7 +179,7 @@ export function getLodestarApi({ }; } -function regenRequestToJson(config: IChainForkConfig, regenRequest: RegenRequest): Json { +function regenRequestToJson(config: IChainForkConfig, regenRequest: RegenRequest): unknown { switch (regenRequest.key) { case "getBlockSlotState": return { diff --git a/packages/lodestar/src/api/impl/validator/index.ts b/packages/lodestar/src/api/impl/validator/index.ts index 4e8de918b42..0d9dff34ea8 100644 --- a/packages/lodestar/src/api/impl/validator/index.ts +++ b/packages/lodestar/src/api/impl/validator/index.ts @@ -64,7 +64,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: if (!genesisBlockRoot) { // Close to genesis the genesis block may not be available in the DB if (state.slot < SLOTS_PER_HISTORICAL_ROOT) { - genesisBlockRoot = state.blockRoots[0]; + genesisBlockRoot = state.blockRoots.get(0); } const genesisBlock = await chain.getCanonicalBlockAtSlot(GENESIS_SLOT); @@ -285,7 +285,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: // Gather indexes to get pubkeys in batch (performance optimization) for (let i = 0; i < SLOTS_PER_EPOCH; i++) { // getBeaconProposer ensures the requested epoch is correct - const validatorIndex = state.getBeaconProposer(startSlot + i); + const validatorIndex = state.epochCtx.getBeaconProposer(startSlot + i); indexes.push(validatorIndex); } @@ -446,7 +446,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: await Promise.all([ chain.aggregatedAttestationPool.add( signedAggregateAndProof.message.aggregate, - indexedAttestation.attestingIndices.valueOf() as ValidatorIndex[], + indexedAttestation.attestingIndices, committeeIndices ), network.gossip.publishBeaconAggregateAndProof(signedAggregateAndProof), diff --git a/packages/lodestar/src/api/impl/validator/utils.ts b/packages/lodestar/src/api/impl/validator/utils.ts index f24d936d64c..4d66c4471b8 100644 --- a/packages/lodestar/src/api/impl/validator/utils.ts +++ b/packages/lodestar/src/api/impl/validator/utils.ts @@ -1,7 +1,6 @@ -import {computeSlotsSinceEpochStart} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, computeSlotsSinceEpochStart} from "@chainsafe/lodestar-beacon-state-transition"; import {ATTESTATION_SUBNET_COUNT} from "@chainsafe/lodestar-params"; -import {allForks, BLSPubkey, CommitteeIndex, phase0, Slot, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {BranchNodeStruct, TreeValue, List} from "@chainsafe/ssz"; +import {BLSPubkey, CommitteeIndex, Slot, ValidatorIndex} from "@chainsafe/lodestar-types"; export function computeSubnetForCommitteesAtSlot( slot: Slot, @@ -23,11 +22,10 @@ export function computeSubnetForCommitteesAtSlot( * See benchmark -> packages/lodestar/test/perf/api/impl/validator/attester.test.ts */ export function getPubkeysForIndices( - validators: allForks.BeaconState["validators"], + validators: BeaconStateAllForks["validators"], indexes: ValidatorIndex[] ): BLSPubkey[] { const validatorsLen = validators.length; // Get once, it's expensive - const validatorsTree = ((validators as unknown) as TreeValue>).tree; const pubkeys: BLSPubkey[] = []; for (let i = 0, len = indexes.length; i < len; i++) { @@ -37,26 +35,9 @@ export function getPubkeysForIndices( } // NOTE: This could be optimized further by traversing the tree optimally with .getNodes() - const gindex = ssz.phase0.Validators.getGindexBitStringAtChunkIndex(index); - const node = validatorsTree.getNode(gindex) as BranchNodeStruct; - pubkeys.push(node.value.pubkey); + const validator = validators.getReadonly(index); + pubkeys.push(validator.pubkey); } return pubkeys; } - -/** - * Uses special BranchNodeStruct state.validators data structure to optimize getting pubkeys. - * Type-unsafe: assumes state.validators[i] is of BranchNodeStruct type. - */ -export function getPubkeysForIndex(validators: allForks.BeaconState["validators"], index: ValidatorIndex): BLSPubkey { - const validatorsLen = validators.length; - if (index >= validatorsLen) { - throw Error(`validatorIndex ${index} too high. Current validator count ${validatorsLen}`); - } - - const validatorsTree = ((validators as unknown) as TreeValue>).tree; - const gindex = ssz.phase0.Validators.getGindexBitStringAtChunkIndex(index); - const node = validatorsTree.getNode(gindex) as BranchNodeStruct; - return node.value.pubkey; -} diff --git a/packages/lodestar/src/chain/blocks/importBlock.ts b/packages/lodestar/src/chain/blocks/importBlock.ts index 13d6c0063fc..0384e930ddf 100644 --- a/packages/lodestar/src/chain/blocks/importBlock.ts +++ b/packages/lodestar/src/chain/blocks/importBlock.ts @@ -1,5 +1,5 @@ import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {readonlyValues, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {allForks} from "@chainsafe/lodestar-types"; import { CachedBeaconStateAllForks, @@ -138,7 +138,7 @@ export async function importBlock(chain: ImportBlockModules, fullyVerifiedBlock: // If current epoch is N, and block is epoch X, block may include attestations for epoch X or X - 1. // The latest block that is useful is at epoch N - 1 which may include attestations for epoch N - 1 or N - 2. if (!skipImportingAttestations && blockEpoch >= currentEpoch - FORK_CHOICE_ATT_EPOCH_LIMIT) { - const attestations = Array.from(readonlyValues(block.message.body.attestations)); + const attestations = block.message.body.attestations; const rootCache = new altair.RootCache(postState); const parentSlot = chain.forkChoice.getBlock(block.message.parentRoot)?.slot; const invalidAttestationErrorsByCode = new Map(); diff --git a/packages/lodestar/src/chain/blocks/utils/checkpoint.ts b/packages/lodestar/src/chain/blocks/utils/checkpoint.ts index 8b3d30f8220..732dc480f36 100644 --- a/packages/lodestar/src/chain/blocks/utils/checkpoint.ts +++ b/packages/lodestar/src/chain/blocks/utils/checkpoint.ts @@ -7,7 +7,6 @@ import {ZERO_HASH} from "../../../constants"; * Compute a Checkpoint type from `state.latestBlockHeader` */ export function getCheckpointFromState(checkpointState: CachedBeaconStateAllForks): phase0.Checkpoint { - const config = checkpointState.config; const slot = checkpointState.slot; if (slot % SLOTS_PER_EPOCH !== 0) { @@ -16,7 +15,7 @@ export function getCheckpointFromState(checkpointState: CachedBeaconStateAllFork const blockHeader = ssz.phase0.BeaconBlockHeader.clone(checkpointState.latestBlockHeader); if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { - blockHeader.stateRoot = config.getForkTypes(slot).BeaconState.hashTreeRoot(checkpointState); + blockHeader.stateRoot = checkpointState.hashTreeRoot(); } return { diff --git a/packages/lodestar/src/chain/blocks/verifyBlock.ts b/packages/lodestar/src/chain/blocks/verifyBlock.ts index fe90c900f0e..829c21cf037 100644 --- a/packages/lodestar/src/chain/blocks/verifyBlock.ts +++ b/packages/lodestar/src/chain/blocks/verifyBlock.ts @@ -1,4 +1,3 @@ -import {ssz} from "@chainsafe/lodestar-types"; import { CachedBeaconStateAllForks, computeStartSlotAtEpoch, @@ -19,6 +18,7 @@ import {IStateRegenerator, RegenCaller} from "../regen"; import {IBlsVerifier} from "../bls"; import {FullyVerifiedBlock, PartiallyVerifiedBlock} from "./types"; import {ExecutePayloadStatus} from "../../executionEngine/interface"; +import {byteArrayEquals} from "../../util/bytes"; export type VerifyBlockModules = { bls: IBlsVerifier; @@ -161,7 +161,7 @@ export async function verifyBlockStateTransition( if (useBlsBatchVerify && !validSignatures) { const signatureSets = validProposerSignature ? allForks.getAllBlockSignatureSetsExceptProposer(postState, block) - : allForks.getAllBlockSignatureSets(postState as CachedBeaconStateAllForks, block); + : allForks.getAllBlockSignatureSets(postState, block); if ( signatureSets.length > 0 && @@ -176,13 +176,7 @@ export async function verifyBlockStateTransition( let executionStatus: ExecutionStatus; if (executionPayloadEnabled) { // TODO: Handle better notifyNewPayload() returning error is syncing - const execResult = await chain.executionEngine.notifyNewPayload( - // executionPayload must be serialized as JSON and the TreeBacked structure breaks the baseFeePerGas serializer - // For clarity and since it's needed anyway, just send the struct representation at this level such that - // notifyNewPayload() can expect a regular JS object. - // TODO: If blocks are no longer TreeBacked, remove. - executionPayloadEnabled.valueOf() as typeof executionPayloadEnabled - ); + const execResult = await chain.executionEngine.notifyNewPayload(executionPayloadEnabled); switch (execResult.status) { case ExecutePayloadStatus.VALID: @@ -286,11 +280,11 @@ export async function verifyBlockStateTransition( } // Check state root matches - if (!ssz.Root.equals(block.message.stateRoot, postState.tree.root)) { + if (!byteArrayEquals(block.message.stateRoot, postState.hashTreeRoot())) { throw new BlockError(block, { code: BlockErrorCode.INVALID_STATE_ROOT, - root: postState.tree.root, - expectedRoot: block.message.stateRoot.valueOf() as Uint8Array, + root: postState.hashTreeRoot(), + expectedRoot: block.message.stateRoot, preState, postState, }); diff --git a/packages/lodestar/src/chain/bls/multithread/index.ts b/packages/lodestar/src/chain/bls/multithread/index.ts index 48c45bb6ac4..64fe6662434 100644 --- a/packages/lodestar/src/chain/bls/multithread/index.ts +++ b/packages/lodestar/src/chain/bls/multithread/index.ts @@ -166,7 +166,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { opts, sets: setsWorker.map((s) => ({ publicKey: getAggregatedPubkey(s).toBytes(this.format), - message: s.signingRoot.valueOf() as Uint8Array, + message: s.signingRoot, signature: s.signature, })), }) diff --git a/packages/lodestar/src/chain/bls/singleThread.ts b/packages/lodestar/src/chain/bls/singleThread.ts index 4d97268a6ab..42d74d89cea 100644 --- a/packages/lodestar/src/chain/bls/singleThread.ts +++ b/packages/lodestar/src/chain/bls/singleThread.ts @@ -17,7 +17,7 @@ export class BlsSingleThreadVerifier implements IBlsVerifier { return verifySignatureSetsMaybeBatch( sets.map((set) => ({ publicKey: getAggregatedPubkey(set), - message: set.signingRoot.valueOf() as Uint8Array, + message: set.signingRoot, signature: set.signature, })) ); diff --git a/packages/lodestar/src/chain/chain.ts b/packages/lodestar/src/chain/chain.ts index 557c1aa2fce..f65c48bd916 100644 --- a/packages/lodestar/src/chain/chain.ts +++ b/packages/lodestar/src/chain/chain.ts @@ -3,12 +3,19 @@ */ import fs from "node:fs"; -import {CachedBeaconStateAllForks, computeStartSlotAtEpoch} from "@chainsafe/lodestar-beacon-state-transition"; +import { + BeaconStateAllForks, + CachedBeaconStateAllForks, + computeStartSlotAtEpoch, + createCachedBeaconState, + Index2PubkeyCache, + PubkeyIndexMap, +} from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {IForkChoice} from "@chainsafe/lodestar-fork-choice"; -import {allForks, Number64, Root, phase0, Slot, RootHex} from "@chainsafe/lodestar-types"; +import {allForks, UintNum64, Root, phase0, Slot, RootHex} from "@chainsafe/lodestar-types"; import {ILogger} from "@chainsafe/lodestar-utils"; -import {fromHexString, TreeBacked} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {AbortController} from "@chainsafe/abort-controller"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants"; import {IBeaconDb} from "../db"; @@ -22,7 +29,7 @@ import {IBeaconChain, SSZObjectType} from "./interface"; import {IChainOptions} from "./options"; import {IStateRegenerator, QueuedStateRegenerator, RegenCaller} from "./regen"; import {initializeForkChoice} from "./forkChoice"; -import {restoreStateCaches} from "./initState"; +import {computeAnchorCheckpoint} from "./initState"; import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool} from "./bls"; import { SeenAttesters, @@ -46,7 +53,7 @@ import {PrecomputeNextEpochTransitionScheduler} from "./precomputeNextEpochTrans import {ReprocessController} from "./reprocess"; export class BeaconChain implements IBeaconChain { - readonly genesisTime: Number64; + readonly genesisTime: UintNum64; readonly genesisValidatorsRoot: Root; readonly eth1: IEth1ForBlockProduction; readonly executionEngine: IExecutionEngine; @@ -78,6 +85,10 @@ export class BeaconChain implements IBeaconChain { readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); readonly seenContributionAndProof = new SeenContributionAndProof(); + // Global state caches + readonly pubkey2index: PubkeyIndexMap; + readonly index2pubkey: Index2PubkeyCache; + protected readonly blockProcessor: BlockProcessor; protected readonly db: IBeaconDb; protected readonly logger: ILogger; @@ -101,7 +112,7 @@ export class BeaconChain implements IBeaconChain { db: IBeaconDb; logger: ILogger; metrics: IMetrics | null; - anchorState: TreeBacked; + anchorState: BeaconStateAllForks; eth1: IEth1ForBlockProduction; executionEngine: IExecutionEngine; } @@ -113,7 +124,7 @@ export class BeaconChain implements IBeaconChain { this.metrics = metrics; this.genesisTime = anchorState.genesisTime; this.anchorStateLatestBlockSlot = anchorState.latestBlockHeader.slot; - this.genesisValidatorsRoot = anchorState.genesisValidatorsRoot.valueOf() as Uint8Array; + this.genesisValidatorsRoot = anchorState.genesisValidatorsRoot; this.eth1 = eth1; this.executionEngine = executionEngine; @@ -127,7 +138,21 @@ export class BeaconChain implements IBeaconChain { const clock = new LocalClock({config, emitter, genesisTime: this.genesisTime, signal}); const stateCache = new StateContextCache({metrics}); const checkpointStateCache = new CheckpointStateCache({metrics}); - const cachedState = restoreStateCaches(config, stateCache, checkpointStateCache, anchorState); + + // Initialize single global instance of state caches + this.pubkey2index = new PubkeyIndexMap(); + this.index2pubkey = []; + + // Restore state caches + const cachedState = createCachedBeaconState(anchorState, { + config, + pubkey2index: this.pubkey2index, + index2pubkey: this.index2pubkey, + }); + const {checkpoint} = computeAnchorCheckpoint(config, anchorState); + stateCache.add(cachedState); + checkpointStateCache.add(checkpoint, cachedState); + const forkChoice = initializeForkChoice( config, emitter, @@ -204,10 +229,6 @@ export class BeaconChain implements IBeaconChain { await this.opPool.toPersisted(this.db); } - getGenesisTime(): Number64 { - return this.genesisTime; - } - getHeadState(): CachedBeaconStateAllForks { // head state should always exist const head = this.forkChoice.getHead(); diff --git a/packages/lodestar/src/chain/eventHandlers.ts b/packages/lodestar/src/chain/eventHandlers.ts index 4b69484d344..9fae7140a26 100644 --- a/packages/lodestar/src/chain/eventHandlers.ts +++ b/packages/lodestar/src/chain/eventHandlers.ts @@ -103,7 +103,7 @@ export function onForkVersion(this: BeaconChain, version: Version): void { export function onCheckpoint(this: BeaconChain, cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void { this.logger.verbose("Checkpoint processed", toCheckpointHex(cp)); - this.metrics?.currentValidators.set({status: "active"}, state.currentShuffling.activeIndices.length); + this.metrics?.currentValidators.set({status: "active"}, state.epochCtx.currentShuffling.activeIndices.length); const parentBlockSummary = this.forkChoice.getBlock(state.latestBlockHeader.parentRoot); if (parentBlockSummary) { diff --git a/packages/lodestar/src/chain/factory/block/body.ts b/packages/lodestar/src/chain/factory/block/body.ts index 39b4a42bd15..bc4cb83d4ca 100644 --- a/packages/lodestar/src/chain/factory/block/body.ts +++ b/packages/lodestar/src/chain/factory/block/body.ts @@ -2,7 +2,6 @@ * @module chain/blockAssembly */ -import {List} from "@chainsafe/ssz"; import { Bytes96, Bytes32, @@ -62,17 +61,17 @@ export async function assembleBody( const [attesterSlashings, proposerSlashings, voluntaryExits] = chain.opPool.getSlashingsAndExits(currentState); const attestations = chain.aggregatedAttestationPool.getAttestationsForBlock(currentState); - const {eth1Data, deposits} = await chain.eth1.getEth1DataAndDeposits(currentState as CachedBeaconStateAllForks); + const {eth1Data, deposits} = await chain.eth1.getEth1DataAndDeposits(currentState); const blockBody: phase0.BeaconBlockBody = { randaoReveal, graffiti, eth1Data, - proposerSlashings: proposerSlashings as List, - attesterSlashings: attesterSlashings as List, - attestations: attestations as List, - deposits: deposits as List, - voluntaryExits: voluntaryExits as List, + proposerSlashings, + attesterSlashings, + attestations, + deposits, + voluntaryExits, }; const blockEpoch = computeEpochAtSlot(blockSlot); @@ -151,7 +150,7 @@ async function prepareExecutionPayload( } const timestamp = computeTimeAtSlot(chain.config, state.slot, state.genesisTime); - const prevRandao = getRandaoMix(state, state.currentShuffling.epoch); + const prevRandao = getRandaoMix(state, state.epochCtx.epoch); const payloadId = await chain.executionEngine.notifyForkchoiceUpdate(parentHash, finalizedBlockHash, { timestamp, prevRandao, diff --git a/packages/lodestar/src/chain/factory/block/index.ts b/packages/lodestar/src/chain/factory/block/index.ts index 9d125293534..210b81b894a 100644 --- a/packages/lodestar/src/chain/factory/block/index.ts +++ b/packages/lodestar/src/chain/factory/block/index.ts @@ -3,7 +3,6 @@ */ import {CachedBeaconStateAllForks, allForks} from "@chainsafe/lodestar-beacon-state-transition"; -import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {Bytes32, Bytes96, ExecutionAddress, Root, Slot} from "@chainsafe/lodestar-types"; import {fromHexString} from "@chainsafe/ssz"; @@ -38,7 +37,7 @@ export async function assembleBlock( const block: allForks.BeaconBlock = { slot, - proposerIndex: state.getBeaconProposer(slot), + proposerIndex: state.epochCtx.getBeaconProposer(slot), parentRoot: parentBlockRoot, stateRoot: ZERO_HASH, body: await assembleBody(chain, state, { @@ -51,7 +50,7 @@ export async function assembleBlock( }), }; - block.stateRoot = computeNewStateRoot({config: chain.config, metrics}, state, block); + block.stateRoot = computeNewStateRoot({metrics}, state, block); return block; } @@ -62,7 +61,7 @@ export async function assembleBlock( * epoch transition which happen at slot % 32 === 0) */ function computeNewStateRoot( - {config, metrics}: {config: IChainForkConfig; metrics: IMetrics | null}, + {metrics}: {metrics: IMetrics | null}, state: CachedBeaconStateAllForks, block: allForks.BeaconBlock ): Root { @@ -70,5 +69,5 @@ function computeNewStateRoot( // verifySignatures = false since the data to assemble the block is trusted allForks.processBlock(postState, block, {verifySignatures: false}, metrics); - return config.getForkTypes(state.slot).BeaconState.hashTreeRoot(postState); + return postState.hashTreeRoot(); } diff --git a/packages/lodestar/src/chain/genesis/genesis.ts b/packages/lodestar/src/chain/genesis/genesis.ts index 79938eab3bf..a495f4f519f 100644 --- a/packages/lodestar/src/chain/genesis/genesis.ts +++ b/packages/lodestar/src/chain/genesis/genesis.ts @@ -2,10 +2,10 @@ * @module chain/genesis */ -import {TreeBacked, List} from "@chainsafe/ssz"; -import {GENESIS_SLOT} from "@chainsafe/lodestar-params"; -import {Root, phase0, allForks, ssz} from "@chainsafe/lodestar-types"; +import {GENESIS_EPOCH, GENESIS_SLOT} from "@chainsafe/lodestar-params"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; import {AbortSignal} from "@chainsafe/abort-controller"; import { getTemporaryBlockHeader, @@ -13,16 +13,18 @@ import { applyDeposits, applyTimestamp, applyEth1BlockHash, - isValidGenesisState, - isValidGenesisValidators, CachedBeaconStateAllForks, createCachedBeaconState, + BeaconStateAllForks, + createEmptyEpochContextImmutableData, + getActiveValidatorIndices, } from "@chainsafe/lodestar-beacon-state-transition"; import {ILogger} from "@chainsafe/lodestar-utils"; import {IEth1Provider} from "../../eth1"; import {IEth1StreamParams} from "../../eth1/interface"; import {getDepositsAndBlockStreamForGenesis, getDepositsStream} from "../../eth1/stream"; import {IGenesisBuilder, IGenesisResult} from "./interface"; +import {DepositTree} from "../../db/repositories/depositDataRoot"; export interface IGenesisBuilderKwargs { config: IChainForkConfig; @@ -31,8 +33,8 @@ export interface IGenesisBuilderKwargs { /** Use to restore pending progress */ pendingStatus?: { - state: TreeBacked; - depositTree: TreeBacked>; + state: BeaconStateAllForks; + depositTree: DepositTree; lastProcessedBlockNumber: number; }; @@ -42,8 +44,8 @@ export interface IGenesisBuilderKwargs { export class GenesisBuilder implements IGenesisBuilder { // Expose state to persist on error - state: CachedBeaconStateAllForks; - depositTree: TreeBacked>; + readonly state: CachedBeaconStateAllForks; + readonly depositTree: DepositTree; /** Is null if no block has been processed yet */ lastProcessedBlockNumber: number | null = null; @@ -56,12 +58,13 @@ export class GenesisBuilder implements IGenesisBuilder { private readonly fromBlock: number; private readonly logEvery = 30 * 1000; private lastLog = 0; + /** Current count of active validators in the state */ + private activatedValidatorCount: number; constructor({config, eth1Provider, logger, signal, pendingStatus, maxBlocksPerPoll}: IGenesisBuilderKwargs) { // at genesis builder, there is no genesis validator so we don't have a real IBeaconConfig // but we need IBeaconConfig to temporarily create CachedBeaconState, the cast here is safe since we don't use any getDomain here - // the use of state as CachedBeaconState is just for convenient, IGenesisResult returns TreeBacked anyway - this.config = config as IBeaconConfig; + // the use of state as CachedBeaconState is just for convenient, IGenesisResult returns TreeView anyway this.eth1Provider = eth1Provider; this.logger = logger; this.signal = signal; @@ -70,20 +73,27 @@ export class GenesisBuilder implements IGenesisBuilder { maxBlocksPerPoll: maxBlocksPerPoll ?? 10000, }; + let stateView: BeaconStateAllForks; + if (pendingStatus) { this.logger.info("Restoring pending genesis state", {block: pendingStatus.lastProcessedBlockNumber}); - this.state = createCachedBeaconState(this.config, pendingStatus.state); + stateView = pendingStatus.state; this.depositTree = pendingStatus.depositTree; this.fromBlock = Math.max(pendingStatus.lastProcessedBlockNumber + 1, this.eth1Provider.deployBlock); } else { - this.state = getGenesisBeaconState( - this.config, + stateView = getGenesisBeaconState( + config, ssz.phase0.Eth1Data.defaultValue(), - getTemporaryBlockHeader(this.config, config.getForkTypes(GENESIS_SLOT).BeaconBlock.defaultValue()) + getTemporaryBlockHeader(config, config.getForkTypes(GENESIS_SLOT).BeaconBlock.defaultValue()) ); - this.depositTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); + this.depositTree = ssz.phase0.DepositDataRootList.defaultViewDU(); this.fromBlock = this.eth1Provider.deployBlock; } + + // TODO - PENDING: Ensure EpochContextImmutableData is created only once + this.state = createCachedBeaconState(stateView, createEmptyEpochContextImmutableData(config, stateView)); + this.config = this.state.config; + this.activatedValidatorCount = getActiveValidatorIndices(stateView, GENESIS_EPOCH).length; } /** @@ -109,7 +119,10 @@ export class GenesisBuilder implements IGenesisBuilder { applyEth1BlockHash(this.state, block.blockHash); this.lastProcessedBlockNumber = block.blockNumber; - if (isValidGenesisState(this.config, this.state)) { + if ( + this.state.genesisTime >= this.config.MIN_GENESIS_TIME && + this.activatedValidatorCount >= this.config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + ) { this.logger.info("Found genesis state", {blockNumber: block.blockNumber}); return { state: this.state, @@ -136,7 +149,7 @@ export class GenesisBuilder implements IGenesisBuilder { this.applyDeposits(depositEvents); this.lastProcessedBlockNumber = blockNumber; - if (isValidGenesisValidators(this.config, this.state)) { + if (this.activatedValidatorCount >= this.config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT) { this.logger.info("Found enough genesis validators", {blockNumber}); return blockNumber; } else { @@ -155,13 +168,19 @@ export class GenesisBuilder implements IGenesisBuilder { .map((depositEvent) => { this.depositCache.add(depositEvent.index); this.depositTree.push(ssz.phase0.DepositData.hashTreeRoot(depositEvent.depositData)); + const gindex = toGindex(this.depositTree.type.depth, BigInt(depositEvent.index)); + + // Apply changes from the push above + this.depositTree.commit(); + const depositTreeNode = this.depositTree.node; return { - proof: this.depositTree.tree.getSingleProof(this.depositTree.type.getPropertyGindex(depositEvent.index)), + proof: new Tree(depositTreeNode).getSingleProof(gindex), data: depositEvent.depositData, }; }); - applyDeposits(this.config, this.state, newDeposits, this.depositTree); + const {activatedValidatorCount} = applyDeposits(this.config, this.state, newDeposits, this.depositTree); + this.activatedValidatorCount += activatedValidatorCount; // TODO: If necessary persist deposits here to this.db.depositData, this.db.depositDataRoot } diff --git a/packages/lodestar/src/chain/genesis/interface.ts b/packages/lodestar/src/chain/genesis/interface.ts index 5f3cf17bc15..35fb8f1d1cb 100644 --- a/packages/lodestar/src/chain/genesis/interface.ts +++ b/packages/lodestar/src/chain/genesis/interface.ts @@ -1,10 +1,11 @@ -import {TreeBacked, List} from "@chainsafe/ssz"; -import {allForks, Root} from "@chainsafe/lodestar-types"; +import {ssz} from "@chainsafe/lodestar-types"; +import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {CompositeViewDU, VectorCompositeType} from "@chainsafe/ssz"; import {Eth1Block} from "../../eth1/interface"; export interface IGenesisResult { - state: TreeBacked; - depositTree: TreeBacked>; + state: CachedBeaconStateAllForks; + depositTree: CompositeViewDU>; block: Eth1Block; } diff --git a/packages/lodestar/src/chain/initState.ts b/packages/lodestar/src/chain/initState.ts index 6a373614fab..2192429167d 100644 --- a/packages/lodestar/src/chain/initState.ts +++ b/packages/lodestar/src/chain/initState.ts @@ -6,14 +6,14 @@ import {AbortSignal} from "@chainsafe/abort-controller"; import { blockToHeader, computeEpochAtSlot, - createCachedBeaconState, phase0, + BeaconStateAllForks, CachedBeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {allForks, ssz} from "@chainsafe/lodestar-types"; -import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; +import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {ILogger} from "@chainsafe/lodestar-utils"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {GENESIS_SLOT, ZERO_HASH} from "../constants"; import {IBeaconDb} from "../db"; @@ -21,7 +21,6 @@ import {Eth1Provider} from "../eth1"; import {IMetrics} from "../metrics"; import {GenesisBuilder} from "./genesis/genesis"; import {IGenesisResult} from "./genesis/interface"; -import {CheckpointStateCache, StateContextCache} from "./stateCache"; import {Eth1Options} from "../eth1/options"; export async function persistGenesisResult( @@ -32,7 +31,7 @@ export async function persistGenesisResult( await Promise.all([ db.stateArchive.add(genesisResult.state), db.blockArchive.add(genesisBlock), - db.depositDataRoot.putList(genesisResult.depositTree), + db.depositDataRoot.putList(genesisResult.depositTree.getAllReadonlyValues()), db.eth1Data.put(genesisResult.block.timestamp, { ...genesisResult.block, depositCount: genesisResult.depositTree.length, @@ -44,7 +43,7 @@ export async function persistGenesisResult( export async function persistAnchorState( config: IChainForkConfig, db: IBeaconDb, - anchorState: TreeBacked + anchorState: BeaconStateAllForks ): Promise { if (anchorState.slot === GENESIS_SLOT) { const genesisBlock = createGenesisBlock(config, anchorState); @@ -60,11 +59,11 @@ export async function persistAnchorState( export function createGenesisBlock( config: IChainForkConfig, - genesisState: allForks.BeaconState + genesisState: BeaconStateAllForks ): allForks.SignedBeaconBlock { const types = config.getForkTypes(GENESIS_SLOT); const genesisBlock = types.SignedBeaconBlock.defaultValue(); - const stateRoot = types.BeaconState.hashTreeRoot(genesisState); + const stateRoot = genesisState.hashTreeRoot(); genesisBlock.message.stateRoot = stateRoot; return genesisBlock; } @@ -84,7 +83,7 @@ export async function initStateFromEth1({ logger: ILogger; opts: Eth1Options; signal: AbortSignal; -}): Promise> { +}): Promise { logger.info("Listening to eth1 for genesis state"); const statePreGenesis = await db.preGenesisState.get(); @@ -104,9 +103,11 @@ export async function initStateFromEth1({ try { const genesisResult = await builder.waitForGenesis(); + + // Note: .hashTreeRoot() automatically commits() const genesisBlock = createGenesisBlock(config, genesisResult.state); const types = config.getForkTypes(GENESIS_SLOT); - const stateRoot = types.BeaconState.hashTreeRoot(genesisResult.state); + const stateRoot = genesisResult.state.hashTreeRoot(); const blockRoot = types.BeaconBlock.hashTreeRoot(genesisBlock.message); logger.info("Initializing genesis state", { @@ -125,8 +126,12 @@ export async function initStateFromEth1({ } catch (e) { if (builder.lastProcessedBlockNumber != null) { logger.info("Persisting genesis state", {block: builder.lastProcessedBlockNumber}); + + // Commit changed before serializing + builder.state.commit(); + await db.preGenesisState.put(builder.state); - await db.depositDataRoot.putList(builder.depositTree); + await db.depositDataRoot.putList(builder.depositTree.getAllReadonlyValues()); await db.preGenesisStateLastProcessedBlock.put(builder.lastProcessedBlockNumber); } throw e; @@ -140,7 +145,7 @@ export async function initStateFromDb( config: IChainForkConfig, db: IBeaconDb, logger: ILogger -): Promise> { +): Promise { const state = await db.stateArchive.lastValue(); if (!state) { throw new Error("No state exists in database"); @@ -149,10 +154,10 @@ export async function initStateFromDb( logger.info("Initializing beacon state from db", { slot: state.slot, epoch: computeEpochAtSlot(state.slot), - stateRoot: toHexString(config.getForkTypes(state.slot).BeaconState.hashTreeRoot(state)), + stateRoot: toHexString(state.hashTreeRoot()), }); - return state as TreeBacked; + return state; } /** @@ -162,12 +167,12 @@ export async function initStateFromAnchorState( config: IChainForkConfig, db: IBeaconDb, logger: ILogger, - anchorState: TreeBacked -): Promise> { + anchorState: BeaconStateAllForks +): Promise { logger.info("Initializing beacon state from anchor state", { slot: anchorState.slot, epoch: computeEpochAtSlot(anchorState.slot), - stateRoot: toHexString(config.getForkTypes(anchorState.slot).BeaconState.hashTreeRoot(anchorState)), + stateRoot: toHexString(anchorState.hashTreeRoot()), }); await persistAnchorState(config, db, anchorState); @@ -175,26 +180,7 @@ export async function initStateFromAnchorState( return anchorState; } -/** - * Restore a beacon state to the state cache. - */ -export function restoreStateCaches( - config: IBeaconConfig, - stateCache: StateContextCache, - checkpointStateCache: CheckpointStateCache, - state: TreeBacked -): CachedBeaconStateAllForks { - const {checkpoint} = computeAnchorCheckpoint(config, state); - - const cachedBeaconState = createCachedBeaconState(config, state); - - // store state in state caches - void stateCache.add(cachedBeaconState); - checkpointStateCache.add(checkpoint, cachedBeaconState); - return cachedBeaconState; -} - -export function initBeaconMetrics(metrics: IMetrics, state: TreeBacked): void { +export function initBeaconMetrics(metrics: IMetrics, state: BeaconStateAllForks): void { metrics.headSlot.set(state.slot); metrics.previousJustifiedEpoch.set(state.previousJustifiedCheckpoint.epoch); metrics.currentJustifiedEpoch.set(state.currentJustifiedCheckpoint.epoch); @@ -203,21 +189,21 @@ export function initBeaconMetrics(metrics: IMetrics, state: TreeBacked; /** Persist in-memory data to the DB. Call at least once before stopping the process */ persistToDisk(): Promise; - getGenesisTime(): Number64; getHeadState(): CachedBeaconStateAllForks; getHeadStateAtCurrentEpoch(): Promise; diff --git a/packages/lodestar/src/chain/lightClient/index.ts b/packages/lodestar/src/chain/lightClient/index.ts index 6e9043ffebf..4ac657f7d19 100644 --- a/packages/lodestar/src/chain/lightClient/index.ts +++ b/packages/lodestar/src/chain/lightClient/index.ts @@ -7,7 +7,7 @@ import { } from "@chainsafe/lodestar-beacon-state-transition"; import {ILogger} from "@chainsafe/lodestar-utils"; import {routes} from "@chainsafe/lodestar-api"; -import {BitVector, toHexString} from "@chainsafe/ssz"; +import {BitArray, CompositeViewDU, toHexString} from "@chainsafe/ssz"; import {IBeaconDb} from "../../db"; import {IMetrics} from "../../metrics"; import {MapDef, pruneSetToMax} from "../../util/map"; @@ -180,9 +180,7 @@ export class LightClientServer { this.zero = { finalizedHeader: ssz.phase0.BeaconBlockHeader.defaultValue(), - finalityBranch: ssz.altair.LightClientUpdate.getPropertyType( - "finalityBranch" - ).defaultValue() as altair.LightClientUpdate["finalityBranch"], + finalityBranch: ssz.altair.LightClientUpdate.fields["finalityBranch"].defaultValue(), }; } @@ -500,10 +498,13 @@ export class LightClientServer { }); } - private async storeSyncCommittee(syncCommittee: altair.SyncCommittee, syncCommitteeRoot: Uint8Array): Promise { + private async storeSyncCommittee( + syncCommittee: CompositeViewDU, + syncCommitteeRoot: Uint8Array + ): Promise { const isKnown = await this.db.syncCommittee.has(syncCommitteeRoot); if (!isKnown) { - await this.db.syncCommittee.put(syncCommitteeRoot, syncCommittee); + await this.db.syncCommittee.putBinary(syncCommitteeRoot, syncCommittee.serialize()); } } @@ -563,12 +564,6 @@ export function isBetterUpdate( return prevUpdate.attestedHeader.slot > nextSyncAttestedData.attestedHeader.slot; } -export function sumBits(bits: BitVector): number { - let sum = 0; - for (const bit of bits) { - if (bit) { - sum++; - } - } - return sum; +export function sumBits(bits: BitArray): number { + return bits.getTrueBitIndexes().length; } diff --git a/packages/lodestar/src/chain/lightClient/proofs.ts b/packages/lodestar/src/chain/lightClient/proofs.ts index 544f8414eeb..5d41242c9f7 100644 --- a/packages/lodestar/src/chain/lightClient/proofs.ts +++ b/packages/lodestar/src/chain/lightClient/proofs.ts @@ -1,10 +1,11 @@ -import {altair} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {FINALIZED_ROOT_GINDEX} from "@chainsafe/lodestar-params"; +import {Tree} from "@chainsafe/persistent-merkle-tree"; import {SyncCommitteeWitness} from "./types"; -export function getSyncCommitteesWitness(state: TreeBacked): SyncCommitteeWitness { - const n1 = state.tree.rootNode; +export function getSyncCommitteesWitness(state: BeaconStateAllForks): SyncCommitteeWitness { + state.commit(); + const n1 = state.node; const n3 = n1.right; // [1]0110 const n6 = n3.left; // 1[0]110 const n13 = n6.right; // 10[1]10 @@ -37,6 +38,7 @@ export function getCurrentSyncCommitteeBranch(syncCommitteesWitness: SyncCommitt return [syncCommitteesWitness.nextSyncCommitteeRoot, ...syncCommitteesWitness.witness]; } -export function getFinalizedRootProof(state: TreeBacked): Uint8Array[] { - return state.tree.getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); +export function getFinalizedRootProof(state: BeaconStateAllForks): Uint8Array[] { + state.commit(); + return new Tree(state.node).getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); } diff --git a/packages/lodestar/src/chain/opPools/aggregatedAttestationPool.ts b/packages/lodestar/src/chain/opPools/aggregatedAttestationPool.ts index edf79c496a7..8ecd286c036 100644 --- a/packages/lodestar/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/lodestar/src/chain/opPools/aggregatedAttestationPool.ts @@ -6,16 +6,15 @@ import { SLOTS_PER_EPOCH, TIMELY_SOURCE_FLAG_INDEX, } from "@chainsafe/lodestar-params"; -import {Epoch, ParticipationFlags, Slot, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {Epoch, Slot, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; import { CachedBeaconStateAllForks, CachedBeaconStatePhase0, CachedBeaconStateAltair, computeEpochAtSlot, phase0, - zipIndexesCommitteeBits, } from "@chainsafe/lodestar-beacon-state-transition"; -import {BitList, List, readonlyValues, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {MapDef} from "../../util/map"; import {pruneBySlot} from "./utils"; import {InsertOutcome} from "./types"; @@ -89,7 +88,7 @@ export class AggregatedAttestationPool { */ getAttestationsForBlock(state: CachedBeaconStateAllForks): phase0.Attestation[] { const stateSlot = state.slot; - const stateEpoch = state.currentShuffling.epoch; + const stateEpoch = state.epochCtx.epoch; const statePrevEpoch = stateEpoch - 1; const forkName = state.config.getForkName(stateSlot); @@ -195,8 +194,11 @@ export class AggregatedAttestationPool { const phase0State = state as CachedBeaconStatePhase0; const stateEpoch = computeEpochAtSlot(state.slot); - const previousEpochParticipants = extractParticipation(phase0State.previousEpochAttestations, state); - const currentEpochParticipants = extractParticipation(phase0State.currentEpochAttestations, state); + const previousEpochParticipants = extractParticipation( + phase0State.previousEpochAttestations.getAllReadonly(), + state + ); + const currentEpochParticipants = extractParticipation(phase0State.currentEpochAttestations.getAllReadonly(), state); return (epoch: Epoch) => { return epoch === stateEpoch @@ -216,8 +218,8 @@ export class AggregatedAttestationPool { // check for altair block already const altairState = state as CachedBeaconStateAltair; const stateEpoch = computeEpochAtSlot(state.slot); - const previousParticipation = altairState.previousEpochParticipation.persistent.toArray(); - const currentParticipation = altairState.currentEpochParticipation.persistent.toArray(); + const previousParticipation = altairState.previousEpochParticipation.getAll(); + const currentParticipation = altairState.currentEpochParticipation.getAll(); return (epoch: Epoch, committee: number[]) => { const participationStatus = @@ -270,6 +272,7 @@ export class MatchingDataAttestationGroup { for (const [i, existingAttestation] of this.attestations.entries()) { const existingAttestingIndices = existingAttestation.attestingIndices; const numIntersection = + // TODO: Intersect the uint8arrays from BitArray directly, it's probably much faster existingAttestingIndices.size >= attestingIndices.size ? intersection(existingAttestingIndices, attestingIndices) : intersection(attestingIndices, existingAttestingIndices); @@ -335,34 +338,26 @@ export function aggregateInto(attestation1: AttestationWithIndex, attestation2: } // Merge bits of attestation2 into attestation1 - bitArrayMergeOrWith(attestation1.attestation.aggregationBits, attestation2.attestation.aggregationBits); + attestation1.attestation.aggregationBits.mergeOrWith(attestation2.attestation.aggregationBits); - const signature1 = bls.Signature.fromBytes( - attestation1.attestation.signature.valueOf() as Uint8Array, - undefined, - true - ); - const signature2 = bls.Signature.fromBytes( - attestation2.attestation.signature.valueOf() as Uint8Array, - undefined, - true - ); + const signature1 = bls.Signature.fromBytes(attestation1.attestation.signature, undefined, true); + const signature2 = bls.Signature.fromBytes(attestation2.attestation.signature, undefined, true); attestation1.attestation.signature = bls.Signature.aggregate([signature1, signature2]).toBytes(); } export function extractParticipation( - attestations: List, + attestations: phase0.PendingAttestation[], state: CachedBeaconStateAllForks ): Set { const {epochCtx} = state; const allParticipants = new Set(); - for (const att of readonlyValues(attestations)) { + for (const att of attestations) { const aggregationBits = att.aggregationBits; const attData = att.data; const attSlot = attData.slot; const committeeIndex = attData.index; const committee = epochCtx.getBeaconCommittee(attSlot, committeeIndex); - const participants = zipIndexesCommitteeBits(committee, aggregationBits); + const participants = aggregationBits.intersectValues(committee); for (const participant of participants) { allParticipants.add(participant); } @@ -398,17 +393,6 @@ export function isValidAttestationData( return ssz.phase0.Checkpoint.equals(data.source, justifiedCheckpoint); } -/** - * Returns true if the `TIMELY_SOURCE` bit in a `ParticipationFlags` is set - */ -export function flagIsTimelySource(flag: ParticipationFlags): boolean { +function flagIsTimelySource(flag: number): boolean { return (flag & TIMELY_SOURCE) === TIMELY_SOURCE; } - -function bitArrayMergeOrWith(bits1: BitList, bits2: BitList): void { - for (let i = 0; i < bits2.length; i++) { - if (bits2[i]) { - bits1[i] = true; - } - } -} diff --git a/packages/lodestar/src/chain/opPools/attestationPool.ts b/packages/lodestar/src/chain/opPools/attestationPool.ts index ef687175554..b46dba69874 100644 --- a/packages/lodestar/src/chain/opPools/attestationPool.ts +++ b/packages/lodestar/src/chain/opPools/attestationPool.ts @@ -1,6 +1,6 @@ import {phase0, Slot, Root, ssz} from "@chainsafe/lodestar-types"; import bls, {PointFormat, Signature} from "@chainsafe/bls"; -import {BitList, readonlyValues, toHexString} from "@chainsafe/ssz"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types"; import {pruneBySlot} from "./utils"; import {MapDef} from "../../util/map"; @@ -23,7 +23,7 @@ const MAX_ATTESTATIONS_PER_SLOT = 16_384; type AggregateFast = { data: phase0.Attestation["data"]; - aggregationBits: boolean[]; + aggregationBits: BitArray; signature: Signature; }; @@ -161,21 +161,21 @@ export class AttestationPool { * Aggregate a new contribution into `aggregate` mutating it */ function aggregateAttestationInto(aggregate: AggregateFast, attestation: phase0.Attestation): InsertOutcome { - const bitIndex = bitArrayGetSingleTrueBit(attestation.aggregationBits); + const bitIndex = attestation.aggregationBits.getSingleTrueBit(); // Should never happen, attestations are verified against this exact condition before if (bitIndex === null) { throw Error("Invalid attestation not exactly one bit set"); } - if (aggregate.aggregationBits[bitIndex] === true) { + if (aggregate.aggregationBits.get(bitIndex) === true) { return InsertOutcome.AlreadyKnown; } - aggregate.aggregationBits[bitIndex] = true; + aggregate.aggregationBits.set(bitIndex, true); aggregate.signature = Signature.aggregate([ aggregate.signature, - bls.Signature.fromBytes(attestation.signature.valueOf() as Uint8Array, undefined, true), + bls.Signature.fromBytes(attestation.signature, undefined, true), ]); return InsertOutcome.Aggregated; } @@ -186,8 +186,9 @@ function aggregateAttestationInto(aggregate: AggregateFast, attestation: phase0. function attestationToAggregate(attestation: phase0.Attestation): AggregateFast { return { data: attestation.data, - aggregationBits: Array.from(readonlyValues(attestation.aggregationBits)), - signature: bls.Signature.fromBytes(attestation.signature.valueOf() as Uint8Array, undefined, true), + // clone because it will be mutated + aggregationBits: attestation.aggregationBits.clone(), + signature: bls.Signature.fromBytes(attestation.signature, undefined, true), }; } @@ -197,16 +198,7 @@ function attestationToAggregate(attestation: phase0.Attestation): AggregateFast function fastToAttestation(aggFast: AggregateFast): phase0.Attestation { return { data: aggFast.data, - aggregationBits: aggFast.aggregationBits as BitList, + aggregationBits: aggFast.aggregationBits, signature: aggFast.signature.toBytes(PointFormat.compressed), }; } - -function bitArrayGetSingleTrueBit(bits: BitList): number | null { - for (const [index, participated] of Array.from(readonlyValues(bits)).entries()) { - if (participated) { - return index; - } - } - return null; -} diff --git a/packages/lodestar/src/chain/opPools/opPool.ts b/packages/lodestar/src/chain/opPools/opPool.ts index 460f9797993..e427d955f88 100644 --- a/packages/lodestar/src/chain/opPools/opPool.ts +++ b/packages/lodestar/src/chain/opPools/opPool.ts @@ -3,6 +3,7 @@ import { computeEpochAtSlot, allForks, getAttesterSlashableIndices, + BeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {Repository, Id} from "@chainsafe/lodestar-db"; import {MAX_PROPOSER_SLASHINGS, MAX_VOLUNTARY_EXITS} from "@chainsafe/lodestar-params"; @@ -126,7 +127,7 @@ export class OpPool { for (const proposerSlashing of this.proposerSlashings.values()) { const index = proposerSlashing.signedHeader1.message.proposerIndex; - const validator = state.validators[index]; + const validator = state.validators.get(index); if (!validator.slashed && validator.activationEpoch <= stateEpoch && stateEpoch < validator.withdrawableEpoch) { proposerSlashings.push(proposerSlashing); // Set of validators to be slashed, so we don't attempt to construct invalid attester slashings. @@ -143,7 +144,7 @@ export class OpPool { const slashableIndices = new Set(); for (let i = 0; i < attesterSlashing.intersectingIndices.length; i++) { const index = attesterSlashing.intersectingIndices[i]; - const validator = state.validators[index]; + const validator = state.validators.get(index); // If we already have a slashing for this index, we can continue on to the next slashing if (toBeSlashedIndices.has(index)) { @@ -198,7 +199,7 @@ export class OpPool { /** * Prune all types of transactions given the latest head state */ - pruneAll(headState: allForks.BeaconState): void { + pruneAll(headState: BeaconStateAllForks): void { this.pruneAttesterSlashings(headState); this.pruneProposerSlashings(headState); this.pruneVoluntaryExits(headState); @@ -207,7 +208,7 @@ export class OpPool { /** * Prune attester slashings for all slashed or withdrawn validators. */ - private pruneAttesterSlashings(headState: allForks.BeaconState): void { + private pruneAttesterSlashings(headState: BeaconStateAllForks): void { const finalizedEpoch = headState.finalizedCheckpoint.epoch; attesterSlashing: for (const [key, attesterSlashing] of this.attesterSlashings.entries()) { // Slashings that don't slash any validators can be dropped @@ -219,7 +220,7 @@ export class OpPool { // // We cannot check the `slashed` field since the `head` is not finalized and // a fork could un-slash someone. - if (headState.validators[index].exitEpoch > finalizedEpoch) { + if (headState.validators.get(index).exitEpoch > finalizedEpoch) { continue attesterSlashing; } } @@ -232,11 +233,11 @@ export class OpPool { /** * Prune proposer slashings for validators which are exited in the finalized epoch. */ - private pruneProposerSlashings(headState: allForks.BeaconState): void { + private pruneProposerSlashings(headState: BeaconStateAllForks): void { const finalizedEpoch = headState.finalizedCheckpoint.epoch; for (const [key, proposerSlashing] of this.proposerSlashings.entries()) { const index = proposerSlashing.signedHeader1.message.proposerIndex; - if (headState.validators[index].exitEpoch <= finalizedEpoch) { + if (headState.validators.get(index).exitEpoch <= finalizedEpoch) { this.proposerSlashings.delete(key); } } @@ -246,7 +247,7 @@ export class OpPool { * Call after finalizing * Prune if validator has already exited at or before the finalized checkpoint of the head. */ - private pruneVoluntaryExits(headState: allForks.BeaconState): void { + private pruneVoluntaryExits(headState: BeaconStateAllForks): void { const finalizedEpoch = headState.finalizedCheckpoint.epoch; for (const [key, voluntaryExit] of this.voluntaryExits.entries()) { // TODO: Improve this simplistic condition diff --git a/packages/lodestar/src/chain/opPools/syncCommitteeMessagePool.ts b/packages/lodestar/src/chain/opPools/syncCommitteeMessagePool.ts index d0fdf8f4440..a95953b000a 100644 --- a/packages/lodestar/src/chain/opPools/syncCommitteeMessagePool.ts +++ b/packages/lodestar/src/chain/opPools/syncCommitteeMessagePool.ts @@ -1,8 +1,7 @@ import bls, {PointFormat, Signature} from "@chainsafe/bls"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; -import {newFilledArray} from "@chainsafe/lodestar-beacon-state-transition"; import {altair, Root, Slot, SubcommitteeIndex} from "@chainsafe/lodestar-types"; -import {BitList, toHexString} from "@chainsafe/ssz"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {MapDef} from "../../util/map"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types"; import {pruneBySlot} from "./utils"; @@ -20,7 +19,7 @@ const SLOTS_RETAINED = 3; const MAX_ITEMS_PER_SLOT = 512; type ContributionFast = Omit & { - aggregationBits: boolean[]; + aggregationBits: BitArray; signature: Signature; }; @@ -84,7 +83,7 @@ export class SyncCommitteeMessagePool { return { ...contribution, - aggregationBits: contribution.aggregationBits as BitList, + aggregationBits: contribution.aggregationBits, signature: contribution.signature.toBytes(PointFormat.compressed), }; } @@ -107,15 +106,14 @@ function aggregateSignatureInto( signature: altair.SyncCommitteeMessage, indexInSubcommittee: number ): InsertOutcome { - if (contribution.aggregationBits[indexInSubcommittee] === true) { + if (contribution.aggregationBits.get(indexInSubcommittee) === true) { return InsertOutcome.AlreadyKnown; } - contribution.aggregationBits[indexInSubcommittee] = true; - + contribution.aggregationBits.set(indexInSubcommittee, true); contribution.signature = Signature.aggregate([ contribution.signature, - bls.Signature.fromBytes(signature.signature.valueOf() as Uint8Array, undefined, true), + bls.Signature.fromBytes(signature.signature, undefined, true), ]); return InsertOutcome.Aggregated; } @@ -129,14 +127,13 @@ function signatureToAggregate( indexInSubcommittee: number ): ContributionFast { const indexesPerSubnet = Math.floor(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT); - const aggregationBits = newFilledArray(indexesPerSubnet, false); - aggregationBits[indexInSubcommittee] = true; + const aggregationBits = BitArray.fromSingleBit(indexesPerSubnet, indexInSubcommittee); return { slot: signature.slot, beaconBlockRoot: signature.beaconBlockRoot, subcommitteeIndex: subnet, aggregationBits, - signature: bls.Signature.fromBytes(signature.signature.valueOf() as Uint8Array, undefined, true), + signature: bls.Signature.fromBytes(signature.signature, undefined, true), }; } diff --git a/packages/lodestar/src/chain/opPools/syncContributionAndProofPool.ts b/packages/lodestar/src/chain/opPools/syncContributionAndProofPool.ts index 5213821729a..f93219510f4 100644 --- a/packages/lodestar/src/chain/opPools/syncContributionAndProofPool.ts +++ b/packages/lodestar/src/chain/opPools/syncContributionAndProofPool.ts @@ -1,8 +1,8 @@ import bls, {Signature} from "@chainsafe/bls"; -import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; +import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params"; import {altair, Slot, Root, ssz} from "@chainsafe/lodestar-types"; -import {newFilledArray, G2_POINT_AT_INFINITY} from "@chainsafe/lodestar-beacon-state-transition"; -import {readonlyValues, toHexString} from "@chainsafe/ssz"; +import {G2_POINT_AT_INFINITY} from "@chainsafe/lodestar-beacon-state-transition"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {MapDef} from "../../util/map"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types"; import {pruneBySlot} from "./utils"; @@ -19,11 +19,14 @@ const SLOTS_RETAINED = 8; */ const MAX_ITEMS_PER_SLOT = 512; +// SyncContributionAndProofPool constructor ensures SYNC_COMMITTEE_SUBNET_SIZE is multiple of 8 +const SYNC_COMMITTEE_SUBNET_BYTES = SYNC_COMMITTEE_SUBNET_SIZE / 8; + /** * A one-one mapping to SyncContribution with fast data structure to help speed up the aggregation. */ export type SyncContributionFast = { - syncSubcommitteeBits: boolean[]; + syncSubcommitteeBits: BitArray; numParticipants: number; syncSubcommitteeSignature: Uint8Array; }; @@ -44,6 +47,13 @@ export class SyncContributionAndProofPool { private lowestPermissibleSlot = 0; + constructor() { + // Param guarantee for optimizations below that merge syncSubcommitteeBits as bytes + if (SYNC_COMMITTEE_SUBNET_SIZE % 8 !== 0) { + throw Error("SYNC_COMMITTEE_SUBNET_SIZE must be multiple of 8"); + } + } + /** * Only call this once we pass all validation. */ @@ -55,7 +65,7 @@ export class SyncContributionAndProofPool { // Reject if too old. if (slot < lowestPermissibleSlot) { - throw new OpPoolError({code: OpPoolErrorCode.SLOT_TOO_LOW, slot, lowestPermissibleSlot}); + return InsertOutcome.Old; } // Limit object per slot @@ -118,9 +128,9 @@ export function replaceIfBetter( return InsertOutcome.NotBetterThan; } - bestContribution.syncSubcommitteeBits = Array.from(readonlyValues(newContribution.aggregationBits)); + bestContribution.syncSubcommitteeBits = newContribution.aggregationBits; bestContribution.numParticipants = newNumParticipants; - bestContribution.syncSubcommitteeSignature = newContribution.signature as Uint8Array; + bestContribution.syncSubcommitteeSignature = newContribution.signature; return InsertOutcome.NewData; } @@ -133,10 +143,10 @@ export function contributionToFast( ): SyncContributionFast { return { // No need to clone, aggregationBits are not mutated, only replaced - syncSubcommitteeBits: Array.from(readonlyValues(contribution.aggregationBits)), + syncSubcommitteeBits: contribution.aggregationBits, numParticipants, // No need to deserialize, signatures are not aggregated until when calling .getAggregate() - syncSubcommitteeSignature: contribution.signature as Uint8Array, + syncSubcommitteeSignature: contribution.signature, }; } @@ -146,14 +156,14 @@ export function contributionToFast( */ export function aggregate(bestContributionBySubnet: Map): altair.SyncAggregate { // check for empty/undefined bestContributionBySubnet earlier - const syncCommitteeBits = newFilledArray(SYNC_COMMITTEE_SIZE, false); - const subnetSize = Math.floor(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT); + const syncCommitteeBits = BitArray.fromBitLen(SYNC_COMMITTEE_SIZE); + const signatures: Signature[] = []; for (const [subnet, bestContribution] of bestContributionBySubnet.entries()) { - const indexOffset = subnet * subnetSize; + const byteOffset = subnet * SYNC_COMMITTEE_SUBNET_BYTES; - for (const [index, participated] of bestContribution.syncSubcommitteeBits.entries()) { - if (participated) syncCommitteeBits[indexOffset + index] = true; + for (let i = 0; i < SYNC_COMMITTEE_SUBNET_BYTES; i++) { + syncCommitteeBits.uint8Array[byteOffset + i] = bestContribution.syncSubcommitteeBits.uint8Array[i]; } signatures.push(bls.Signature.fromBytes(bestContribution.syncSubcommitteeSignature, undefined, true)); diff --git a/packages/lodestar/src/chain/regen/regen.ts b/packages/lodestar/src/chain/regen/regen.ts index 5bcc31873f3..0543607abf9 100644 --- a/packages/lodestar/src/chain/regen/regen.ts +++ b/packages/lodestar/src/chain/regen/regen.ts @@ -157,12 +157,8 @@ export class StateRegenerator implements IStateRegenerator { } for (const b of blocksToReplay.reverse()) { - const structBlock = await this.modules.db.block.get(fromHexString(b.blockRoot)); - if (!structBlock) { - throw Error(`No block found for ${b.blockRoot}`); - } - const block = this.modules.config.getForkTypes(b.slot).SignedBeaconBlock.createTreeBackedFromStruct(structBlock); - if (block === undefined) { + const block = await this.modules.db.block.get(fromHexString(b.blockRoot)); + if (!block) { throw new RegenError({ code: RegenErrorCode.BLOCK_NOT_IN_DB, blockRoot: b.blockRoot, @@ -196,7 +192,7 @@ export class StateRegenerator implements IStateRegenerator { } } - return state as CachedBeaconStateAllForks; + return state; } private findFirstStateBlock(stateRoot: RootHex): IProtoBlock { diff --git a/packages/lodestar/src/chain/stateCache/stateContextCache.ts b/packages/lodestar/src/chain/stateCache/stateContextCache.ts index 9f45d044c36..0e2b576a6db 100644 --- a/packages/lodestar/src/chain/stateCache/stateContextCache.ts +++ b/packages/lodestar/src/chain/stateCache/stateContextCache.ts @@ -1,5 +1,5 @@ -import {ByteVector, toHexString} from "@chainsafe/ssz"; -import {Epoch, RootHex} from "@chainsafe/lodestar-types"; +import {toHexString} from "@chainsafe/ssz"; +import {Epoch, Root, RootHex} from "@chainsafe/lodestar-types"; import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {routes} from "@chainsafe/lodestar-api"; import {IMetrics} from "../../metrics"; @@ -58,7 +58,7 @@ export class StateContextCache { } } - delete(root: ByteVector): void { + delete(root: Root): void { const key = toHexString(root); const item = this.cache.get(key); if (!item) return; @@ -66,7 +66,7 @@ export class StateContextCache { this.cache.delete(key); } - batchDelete(roots: ByteVector[]): void { + batchDelete(roots: Root[]): void { roots.map((root) => this.delete(root)); } diff --git a/packages/lodestar/src/chain/validation/aggregateAndProof.ts b/packages/lodestar/src/chain/validation/aggregateAndProof.ts index de3bd166b28..264eecc2a06 100644 --- a/packages/lodestar/src/chain/validation/aggregateAndProof.ts +++ b/packages/lodestar/src/chain/validation/aggregateAndProof.ts @@ -1,11 +1,9 @@ import {ValidatorIndex} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; import { phase0, allForks, computeEpochAtSlot, isAggregatorFromCommitteeLength, - zipIndexesCommitteeBits, } from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconChain} from ".."; import {getSelectionProofSignatureSet, getAggregateAndProofSignatureSet} from "./signatureSets"; @@ -71,9 +69,9 @@ export async function validateGossipAggregateAndProof( }); const committeeIndices = getCommitteeIndices(attHeadState, attSlot, attData.index); - const attestingIndices = zipIndexesCommitteeBits(committeeIndices, aggregate.aggregationBits); + const attestingIndices = aggregate.aggregationBits.intersectValues(committeeIndices); const indexedAttestation: phase0.IndexedAttestation = { - attestingIndices: attestingIndices as List, + attestingIndices, data: attData, signature: aggregate.signature, }; @@ -102,7 +100,7 @@ export async function validateGossipAggregateAndProof( // by the validator with index aggregate_and_proof.aggregator_index. // [REJECT] The aggregator signature, signed_aggregate_and_proof.signature, is valid. // [REJECT] The signature of aggregate is valid. - const aggregator = attHeadState.index2pubkey[aggregateAndProof.aggregatorIndex]; + const aggregator = attHeadState.epochCtx.index2pubkey[aggregateAndProof.aggregatorIndex]; const signatureSets = [ getSelectionProofSignatureSet(attHeadState, attSlot, aggregator, signedAggregateAndProof), getAggregateAndProofSignatureSet(attHeadState, attEpoch, aggregator, signedAggregateAndProof), diff --git a/packages/lodestar/src/chain/validation/attestation.ts b/packages/lodestar/src/chain/validation/attestation.ts index cf9b42d8b06..71d91eed0ac 100644 --- a/packages/lodestar/src/chain/validation/attestation.ts +++ b/packages/lodestar/src/chain/validation/attestation.ts @@ -1,14 +1,11 @@ import {Epoch, Root, Slot} from "@chainsafe/lodestar-types"; import {IProtoBlock} from "@chainsafe/lodestar-fork-choice"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {List, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import { allForks, phase0, computeEpochAtSlot, - getSingleBitIndex, - AggregationBitsError, - AggregationBitsErrorCode, CachedBeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconChain} from ".."; @@ -55,15 +52,11 @@ export async function validateGossipAttestation( // (len([bit for bit in attestation.aggregation_bits if bit]) == 1, i.e. exactly 1 bit is set). // > TODO: Do this check **before** getting the target state but don't recompute zipIndexes const aggregationBits = attestation.aggregationBits; - let bitIndex: number; - try { - bitIndex = getSingleBitIndex(aggregationBits); - } catch (e) { - if (e instanceof AggregationBitsError && e.type.code === AggregationBitsErrorCode.NOT_EXACTLY_ONE_BIT_SET) { - throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET}); - } else { - throw e; - } + const bitIndex = aggregationBits.getSingleTrueBit(); + if (bitIndex === null) { + throw new AttestationError(GossipAction.REJECT, { + code: AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET, + }); } // Attestations must be for a known block. If the block is unknown, we simply drop the @@ -104,8 +97,10 @@ export async function validateGossipAttestation( // [REJECT] The number of aggregation bits matches the committee size // -- i.e. len(attestation.aggregation_bits) == len(get_beacon_committee(state, data.slot, data.index)). // > TODO: Is this necessary? Lighthouse does not do this check - if (aggregationBits.length !== committeeIndices.length) { - throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS}); + if (aggregationBits.bitLen !== committeeIndices.length) { + throw new AttestationError(GossipAction.REJECT, { + code: AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS, + }); } // LH > verify_middle_checks @@ -117,7 +112,7 @@ export async function validateGossipAttestation( // -- i.e. compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index) == subnet_id, // where committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch), // which may be pre-computed along with the committee information for the signature check. - const expectedSubnet = computeSubnetForSlot(attHeadState, attSlot, attIndex); + const expectedSubnet = attHeadState.epochCtx.computeSubnetForSlot(attSlot, attIndex); if (subnet !== null && subnet !== expectedSubnet) { throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_SUBNET_ID, @@ -138,7 +133,7 @@ export async function validateGossipAttestation( // [REJECT] The signature of attestation is valid. const indexedAttestation: phase0.IndexedAttestation = { - attestingIndices: [validatorIndex] as List, + attestingIndices: [validatorIndex], data: attData, signature: attestation.signature, }; @@ -232,7 +227,7 @@ function verifyHeadBlockIsKnown(chain: IBeaconChain, beaconBlockRoot: Root): IPr if (headBlock === null) { throw new AttestationError(GossipAction.IGNORE, { code: AttestationErrorCode.UNKNOWN_BEACON_BLOCK_ROOT, - root: toHexString(beaconBlockRoot.valueOf() as typeof beaconBlockRoot), + root: toHexString(beaconBlockRoot as typeof beaconBlockRoot), }); } @@ -292,7 +287,7 @@ export function getCommitteeIndices( attestationSlot: Slot, attestationIndex: number ): number[] { - const {committees} = attestationTargetState.getShufflingAtSlot(attestationSlot); + const {committees} = attestationTargetState.epochCtx.getShufflingAtSlot(attestationSlot); const slotCommittees = committees[attestationSlot % SLOTS_PER_EPOCH]; if (attestationIndex >= slotCommittees.length) { @@ -309,7 +304,7 @@ export function getCommitteeIndices( */ export function computeSubnetForSlot(state: CachedBeaconStateAllForks, slot: number, committeeIndex: number): number { const slotsSinceEpochStart = slot % SLOTS_PER_EPOCH; - const committeesPerSlot = state.getCommitteeCountPerSlot(computeEpochAtSlot(slot)); + const committeesPerSlot = state.epochCtx.getCommitteeCountPerSlot(computeEpochAtSlot(slot)); const committeesSinceEpochStart = committeesPerSlot * slotsSinceEpochStart; return (committeesSinceEpochStart + committeeIndex) % ATTESTATION_SUBNET_COUNT; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/aggregateAndProof.ts b/packages/lodestar/src/chain/validation/signatureSets/aggregateAndProof.ts index 16f6e1f2968..416bed8420a 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/aggregateAndProof.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/aggregateAndProof.ts @@ -24,6 +24,6 @@ export function getAggregateAndProofSignatureSet( type: SignatureSetType.single, pubkey: aggregator, signingRoot, - signature: aggregateAndProof.signature.valueOf() as Uint8Array, + signature: aggregateAndProof.signature, }; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/contributionAndProof.ts b/packages/lodestar/src/chain/validation/signatureSets/contributionAndProof.ts index 1740490bc5d..a8fd97e5463 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/contributionAndProof.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/contributionAndProof.ts @@ -21,6 +21,6 @@ export function getContributionAndProofSignatureSet( type: SignatureSetType.single, pubkey: epochCtx.index2pubkey[signedContributionAndProof.message.aggregatorIndex], signingRoot: computeSigningRoot(ssz.altair.ContributionAndProof, signingData, domain), - signature: signedContributionAndProof.signature.valueOf() as Uint8Array, + signature: signedContributionAndProof.signature, }; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/selectionProof.ts b/packages/lodestar/src/chain/validation/signatureSets/selectionProof.ts index fde52798481..dbcf2a90bc1 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/selectionProof.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/selectionProof.ts @@ -20,6 +20,6 @@ export function getSelectionProofSignatureSet( type: SignatureSetType.single, pubkey: aggregator, signingRoot: computeSigningRoot(ssz.Slot, slot, selectionProofDomain), - signature: aggregateAndProof.message.selectionProof.valueOf() as Uint8Array, + signature: aggregateAndProof.message.selectionProof, }; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/syncCommittee.ts b/packages/lodestar/src/chain/validation/signatureSets/syncCommittee.ts index c12dd4c8de8..1c5d02732c3 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/syncCommittee.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/syncCommittee.ts @@ -17,6 +17,6 @@ export function getSyncCommitteeSignatureSet( type: SignatureSetType.single, pubkey: state.epochCtx.index2pubkey[syncCommittee.validatorIndex], signingRoot: computeSigningRoot(ssz.Root, syncCommittee.beaconBlockRoot, domain), - signature: syncCommittee.signature.valueOf() as Uint8Array, + signature: syncCommittee.signature, }; } diff --git a/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeContribution.ts b/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeContribution.ts index 2184bb01906..ac323996b10 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeContribution.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeContribution.ts @@ -1,7 +1,6 @@ import {PublicKey} from "@chainsafe/bls"; import {altair, ssz} from "@chainsafe/lodestar-types"; -import {DOMAIN_SYNC_COMMITTEE, SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; -import {readonlyValues} from "@chainsafe/ssz"; +import {DOMAIN_SYNC_COMMITTEE} from "@chainsafe/lodestar-params"; import { CachedBeaconStateAltair, computeSigningRoot, @@ -19,33 +18,6 @@ export function getSyncCommitteeContributionSignatureSet( type: SignatureSetType.aggregate, pubkeys, signingRoot: computeSigningRoot(ssz.Root, contribution.beaconBlockRoot, domain), - signature: contribution.signature.valueOf() as Uint8Array, + signature: contribution.signature, }; } - -/** - * Retrieve pubkeys in contribution aggregate using epochCtx: - * - currSyncCommitteeIndexes cache - * - index2pubkey cache - */ -export function getContributionPubkeys( - state: CachedBeaconStateAltair, - contribution: altair.SyncCommitteeContribution -): PublicKey[] { - const pubkeys: PublicKey[] = []; - - const subcommitteeSize = Math.floor(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT); - const startIndex = contribution.subcommitteeIndex * subcommitteeSize; - const aggBits = Array.from(readonlyValues(contribution.aggregationBits)); - const syncCommittee = state.epochCtx.getIndexedSyncCommittee(contribution.slot); - for (const [i, bit] of aggBits.entries()) { - if (bit) { - const indexInCommittee = startIndex + i; - const validatorIndex = syncCommittee.validatorIndices[indexInCommittee]; - const pubkey = state.index2pubkey[validatorIndex]; - pubkeys.push(pubkey); - } - } - - return pubkeys; -} diff --git a/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeSelectionProof.ts b/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeSelectionProof.ts index 5a5bffc3eda..ca7428b6ea4 100644 --- a/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeSelectionProof.ts +++ b/packages/lodestar/src/chain/validation/signatureSets/syncCommitteeSelectionProof.ts @@ -22,6 +22,6 @@ export function getSyncCommitteeSelectionProofSignatureSet( type: SignatureSetType.single, pubkey: epochCtx.index2pubkey[contributionAndProof.aggregatorIndex], signingRoot: computeSigningRoot(ssz.altair.SyncAggregatorSelectionData, signingData, domain), - signature: contributionAndProof.selectionProof.valueOf() as Uint8Array, + signature: contributionAndProof.selectionProof, }; } diff --git a/packages/lodestar/src/chain/validation/syncCommitteeContributionAndProof.ts b/packages/lodestar/src/chain/validation/syncCommitteeContributionAndProof.ts index 3aa141347b1..9a160a01761 100644 --- a/packages/lodestar/src/chain/validation/syncCommitteeContributionAndProof.ts +++ b/packages/lodestar/src/chain/validation/syncCommitteeContributionAndProof.ts @@ -1,5 +1,6 @@ import {CachedBeaconStateAltair, isSyncCommitteeAggregator} from "@chainsafe/lodestar-beacon-state-transition"; -import {altair} from "@chainsafe/lodestar-types"; +import {altair, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params"; import {GossipAction, SyncCommitteeError, SyncCommitteeErrorCode} from "../errors"; import {IBeaconChain} from "../interface"; import {validateGossipSyncCommitteeExceptSig} from "./syncCommittee"; @@ -7,7 +8,6 @@ import { getSyncCommitteeSelectionProofSignatureSet, getContributionAndProofSignatureSet, getSyncCommitteeContributionSignatureSet, - getContributionPubkeys, } from "./signatureSets"; /** @@ -44,8 +44,8 @@ export async function validateSyncCommitteeGossipContributionAndProof( } // [REJECT] The contribution has participants -- that is, any(contribution.aggregation_bits) - const pubkeys = getContributionPubkeys(headState as CachedBeaconStateAltair, contribution); - if (!pubkeys.length) { + const syncCommitteeIndices = getContributionIndices(headState as CachedBeaconStateAltair, contribution); + if (!syncCommitteeIndices.length) { throw new SyncCommitteeError(GossipAction.REJECT, { code: SyncCommitteeErrorCode.NO_PARTICIPANT, }); @@ -64,6 +64,7 @@ export async function validateSyncCommitteeGossipContributionAndProof( // i.e. state.validators[contribution_and_proof.aggregator_index].pubkey in get_sync_subcommittee_pubkeys(state, contribution.subcommittee_index). // > Checked in validateGossipSyncCommitteeExceptSig() + const pubkeys = syncCommitteeIndices.map((validatorIndex) => headState.epochCtx.index2pubkey[validatorIndex]); const signatureSets = [ // [REJECT] The contribution_and_proof.selection_proof is a valid signature of the SyncAggregatorSelectionData // derived from the contribution by the validator with index contribution_and_proof.aggregator_index. @@ -86,5 +87,22 @@ export async function validateSyncCommitteeGossipContributionAndProof( // no need to add to seenSyncCommittteeContributionCache here, gossip handler will do that chain.seenContributionAndProof.add(slot, subcommitteeIndex, aggregatorIndex); - return {syncCommitteeParticipants: pubkeys.length}; + return {syncCommitteeParticipants: syncCommitteeIndices.length}; +} + +/** + * Retrieve pubkeys in contribution aggregate using epochCtx: + * - currSyncCommitteeIndexes cache + * - index2pubkey cache + */ +function getContributionIndices( + state: CachedBeaconStateAltair, + contribution: altair.SyncCommitteeContribution +): ValidatorIndex[] { + const startIndex = contribution.subcommitteeIndex * SYNC_COMMITTEE_SUBNET_SIZE; + + const syncCommittee = state.epochCtx.getIndexedSyncCommittee(contribution.slot); + // The bits in contribution.aggregationBits select validatorIndexes in the subcommittee starting at startIndex + const subcommitteeIndices = syncCommittee.validatorIndices.slice(startIndex, startIndex + SYNC_COMMITTEE_SUBNET_SIZE); + return contribution.aggregationBits.intersectValues(subcommitteeIndices); } diff --git a/packages/lodestar/src/db/repositories/blockArchive.ts b/packages/lodestar/src/db/repositories/blockArchive.ts index 163ddba855d..5aa9d3fbb69 100644 --- a/packages/lodestar/src/db/repositories/blockArchive.ts +++ b/packages/lodestar/src/db/repositories/blockArchive.ts @@ -1,5 +1,4 @@ import all from "it-all"; -import {ArrayLike} from "@chainsafe/ssz"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {Db, Repository, IKeyValue, IFilterOptions, Bucket, IDbMetrics} from "@chainsafe/lodestar-db"; import {Slot, Root, allForks, ssz} from "@chainsafe/lodestar-types"; @@ -59,7 +58,7 @@ export class BlockArchiveRepository extends Repository>): Promise { + async batchPut(items: IKeyValue[]): Promise { await Promise.all([ super.batchPut(items), Array.from(items).map((item) => { @@ -75,7 +74,7 @@ export class BlockArchiveRepository extends Repository): Promise { + async batchPutBinary(items: BlockArchiveBatchPutBinaryItem[]): Promise { await Promise.all([ super.batchPutBinary(items), Array.from(items).map((item) => storeRootIndex(this.db, item.slot, item.blockRoot)), @@ -91,7 +90,7 @@ export class BlockArchiveRepository extends Repository): Promise { + async batchRemove(values: allForks.SignedBeaconBlock[]): Promise { await Promise.all([ super.batchRemove(values), Array.from(values).map((value) => diff --git a/packages/lodestar/src/db/repositories/blockArchiveIndex.ts b/packages/lodestar/src/db/repositories/blockArchiveIndex.ts index a8dc4aeda35..3587eea954f 100644 --- a/packages/lodestar/src/db/repositories/blockArchiveIndex.ts +++ b/packages/lodestar/src/db/repositories/blockArchiveIndex.ts @@ -1,7 +1,6 @@ import {Db, encodeKey, Bucket} from "@chainsafe/lodestar-db"; -import {Slot, Root, allForks} from "@chainsafe/lodestar-types"; +import {Slot, Root, allForks, ssz} from "@chainsafe/lodestar-types"; import {intToBytes} from "@chainsafe/lodestar-utils"; -import {ContainerType} from "@chainsafe/ssz"; export async function storeRootIndex(db: Db, slot: Slot, blockRoot: Root): Promise { return db.put(getRootIndexKey(blockRoot), intToBytes(slot, 8, "be")); @@ -13,10 +12,11 @@ export async function storeParentRootIndex(db: Db, slot: Slot, parentRoot: Root) export async function deleteRootIndex( db: Db, - blockType: ContainerType, + signedBeaconBlockType: allForks.AllForksSSZTypes["SignedBeaconBlock"], block: allForks.SignedBeaconBlock ): Promise { - return db.delete(getRootIndexKey(blockType.fields["message"].hashTreeRoot(block.message))); + const beaconBlockType = (signedBeaconBlockType as typeof ssz.phase0.SignedBeaconBlock).fields["message"]; + return db.delete(getRootIndexKey(beaconBlockType.hashTreeRoot(block.message))); } export async function deleteParentRootIndex(db: Db, block: allForks.SignedBeaconBlock): Promise { @@ -24,9 +24,9 @@ export async function deleteParentRootIndex(db: Db, block: allForks.SignedBeacon } export function getParentRootIndexKey(parentRoot: Root): Uint8Array { - return encodeKey(Bucket.index_blockArchiveParentRootIndex, parentRoot.valueOf() as Uint8Array); + return encodeKey(Bucket.index_blockArchiveParentRootIndex, parentRoot); } export function getRootIndexKey(root: Root): Uint8Array { - return encodeKey(Bucket.index_blockArchiveRootIndex, root.valueOf() as Uint8Array); + return encodeKey(Bucket.index_blockArchiveRootIndex, root); } diff --git a/packages/lodestar/src/db/repositories/depositDataRoot.ts b/packages/lodestar/src/db/repositories/depositDataRoot.ts index a75252aa4c3..70700d100ee 100644 --- a/packages/lodestar/src/db/repositories/depositDataRoot.ts +++ b/packages/lodestar/src/db/repositories/depositDataRoot.ts @@ -1,11 +1,14 @@ -import {List, readonlyValues, TreeBacked, Vector} from "@chainsafe/ssz"; +import {ByteVectorType, CompositeViewDU, ListCompositeType} from "@chainsafe/ssz"; import {Root, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {bytesToInt} from "@chainsafe/lodestar-utils"; import {Db, Bucket, Repository, IKeyValue, IDbMetrics} from "@chainsafe/lodestar-db"; +// TODO: Review where is best to put this type +export type DepositTree = CompositeViewDU>; + export class DepositDataRootRepository extends Repository { - private depositRootTree?: TreeBacked>; + private depositRootTree?: DepositTree; constructor(config: IChainForkConfig, db: Db, metrics?: IDbMetrics) { super(config, db, Bucket.index_depositDataRoot, ssz.Root, metrics); @@ -21,22 +24,20 @@ export class DepositDataRootRepository extends Repository { throw new Error("Unable to create depositIndex from root"); } - async put(id: number, value: Root): Promise { - const depositRootTree = await this.getDepositRootTree(); - await super.put(id, value); - depositRootTree[id] = value as TreeBacked; + async put(index: number, value: Root): Promise { + await super.put(index, value); + await this.depositRootTreeSet(index, value); } async batchPut(items: IKeyValue[]): Promise { - const depositRootTree = await this.getDepositRootTree(); await super.batchPut(items); for (const {key, value} of items) { - depositRootTree[key] = value as TreeBacked; + await this.depositRootTreeSet(key, value); } } - async putList(list: List): Promise { - await this.batchPut(Array.from(readonlyValues(list), (value, key) => ({key, value}))); + async putList(roots: Root[]): Promise { + await this.batchPut(roots.map((root, index) => ({key: index, value: root}))); } async batchPutValues(values: {index: number; root: Root}[]): Promise { @@ -48,25 +49,29 @@ export class DepositDataRootRepository extends Repository { ); } - async getTreeBacked(depositIndex: number): Promise>> { - const depositRootTree = await this.getDepositRootTree(); - const tree = depositRootTree.clone(); - let maxIndex = tree.length - 1; - if (depositIndex > maxIndex) { - throw new Error(`Cannot get tree for unseen deposits: requested ${depositIndex}, last seen ${maxIndex}`); - } - while (maxIndex > depositIndex) { - tree.pop(); - maxIndex = tree.length - 1; + async getDepositRootTree(): Promise { + if (!this.depositRootTree) { + const values = await this.values(); + this.depositRootTree = ssz.phase0.DepositDataRootList.toViewDU(values); } - return tree; + return this.depositRootTree; } - async getDepositRootTree(): Promise>> { - if (!this.depositRootTree) { - const values = (await this.values()) as List>; - this.depositRootTree = ssz.phase0.DepositDataRootList.createTreeBackedFromStruct(values); + async getDepositRootTreeAtIndex(depositIndex: number): Promise { + const depositRootTree = await this.getDepositRootTree(); + return depositRootTree.sliceTo(depositIndex); + } + + private async depositRootTreeSet(index: number, value: Uint8Array): Promise { + const depositRootTree = await this.getDepositRootTree(); + + // TODO: Review and fix properly + if (index > depositRootTree.length) { + throw Error(`Error setting depositRootTree index ${index} > length ${depositRootTree.length}`); + } else if (index === depositRootTree.length) { + depositRootTree.push(value); + } else { + depositRootTree.set(index, value); } - return this.depositRootTree; } } diff --git a/packages/lodestar/src/db/repositories/lightclientBestPartialUpdate.ts b/packages/lodestar/src/db/repositories/lightclientBestPartialUpdate.ts index ae72d83535e..e7f41ba908c 100644 --- a/packages/lodestar/src/db/repositories/lightclientBestPartialUpdate.ts +++ b/packages/lodestar/src/db/repositories/lightclientBestPartialUpdate.ts @@ -2,12 +2,8 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {Bucket, IDatabaseController, IDbMetrics, Repository} from "@chainsafe/lodestar-db"; import {FINALIZED_ROOT_DEPTH} from "@chainsafe/lodestar-params"; import {ssz, SyncPeriod} from "@chainsafe/lodestar-types"; -import {booleanType, ContainerType, VectorType} from "@chainsafe/ssz"; -import { - PartialLightClientUpdate, - PartialLightClientUpdateFinalized, - PartialLightClientUpdateNonFinalized, -} from "../../chain/lightClient/types"; +import {BooleanType, ContainerType, VectorCompositeType} from "@chainsafe/ssz"; +import {PartialLightClientUpdate} from "../../chain/lightClient/types"; /** * Best PartialLightClientUpdate in each SyncPeriod @@ -15,42 +11,23 @@ import { * Used to prepare light client updates */ export class BestPartialLightClientUpdateRepository extends Repository { - typeFinalized = new ContainerType({ - fields: { - // isFinalized: true - isFinalized: booleanType, - attestedHeader: ssz.phase0.BeaconBlockHeader, - blockRoot: ssz.Root, - finalityBranch: new VectorType({length: FINALIZED_ROOT_DEPTH, elementType: ssz.Root}), - finalizedCheckpoint: ssz.phase0.Checkpoint, - finalizedHeader: ssz.phase0.BeaconBlockHeader, - syncAggregate: ssz.altair.SyncAggregate, - }, - casingMap: { - isFinalized: "is_finalized", - attestedHeader: "attested_header", - blockRoot: "block_root", - finalityBranch: "finality_branch", - finalizedCheckpoint: "finalized_checkpoint", - finalizedHeader: "finalized_header", - syncAggregate: "sync_aggregate", - }, + typeFinalized = new ContainerType({ + // isFinalized: true + isFinalized: new BooleanType(), + attestedHeader: ssz.phase0.BeaconBlockHeader, + blockRoot: ssz.Root, + finalityBranch: new VectorCompositeType(ssz.Root, FINALIZED_ROOT_DEPTH), + finalizedCheckpoint: ssz.phase0.Checkpoint, + finalizedHeader: ssz.phase0.BeaconBlockHeader, + syncAggregate: ssz.altair.SyncAggregate, }); - typeNonFinalized = new ContainerType({ - fields: { - // isFinalized: false - isFinalized: booleanType, - attestedHeader: ssz.phase0.BeaconBlockHeader, - blockRoot: ssz.Root, - syncAggregate: ssz.altair.SyncAggregate, - }, - casingMap: { - isFinalized: "is_finalized", - attestedHeader: "attested_header", - blockRoot: "block_root", - syncAggregate: "sync_aggregate", - }, + typeNonFinalized = new ContainerType({ + // isFinalized: false + isFinalized: new BooleanType(), + attestedHeader: ssz.phase0.BeaconBlockHeader, + blockRoot: ssz.Root, + syncAggregate: ssz.altair.SyncAggregate, }); constructor(config: IChainForkConfig, db: IDatabaseController, metrics?: IDbMetrics) { @@ -72,7 +49,7 @@ export class BestPartialLightClientUpdateRepository extends Repository { constructor(config: IChainForkConfig, db: IDatabaseController, metrics?: IDbMetrics) { - const type = new ContainerType({ - fields: { - witness: new VectorType({length: 4, elementType: ssz.Root}), - currentSyncCommitteeRoot: ssz.Root, - nextSyncCommitteeRoot: ssz.Root, - }, + const type = new ContainerType({ + witness: new VectorCompositeType(ssz.Root, 4), + currentSyncCommitteeRoot: ssz.Root, + nextSyncCommitteeRoot: ssz.Root, }); super(config, db, Bucket.lightClient_syncCommitteeWitness, type, metrics); diff --git a/packages/lodestar/src/db/repositories/stateArchive.ts b/packages/lodestar/src/db/repositories/stateArchive.ts index 001ec7acb99..c624e0e7bce 100644 --- a/packages/lodestar/src/db/repositories/stateArchive.ts +++ b/packages/lodestar/src/db/repositories/stateArchive.ts @@ -1,5 +1,5 @@ -import {ContainerType, TreeBacked} from "@chainsafe/ssz"; -import {Epoch, Root, Slot, allForks, ssz} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {Epoch, Root, Slot, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {bytesToInt} from "@chainsafe/lodestar-utils"; import {Db, Bucket, Repository, IDbMetrics} from "@chainsafe/lodestar-db"; @@ -8,40 +8,41 @@ import {getRootIndexKey, storeRootIndex} from "./stateArchiveIndex"; /* eslint-disable @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-call */ -export class StateArchiveRepository extends Repository> { +export class StateArchiveRepository extends Repository { constructor(config: IChainForkConfig, db: Db, metrics?: IDbMetrics) { - // Pick some type but won't be used - const type = (ssz.phase0.BeaconState as unknown) as ContainerType>; + // Pick some type but won't be used. Casted to any because no type can match `BeaconStateAllForks` + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + const type = ssz.phase0.BeaconState as any; super(config, db, Bucket.allForks_stateArchive, type, metrics); } // Overrides for multi-fork - encodeValue(value: allForks.BeaconState): Buffer { - return this.config.getForkTypes(value.slot).BeaconState.serialize(value) as Buffer; + encodeValue(value: BeaconStateAllForks): Uint8Array { + return value.serialize(); } - decodeValue(data: Buffer): TreeBacked { - return getStateTypeFromBytes(this.config, data).createTreeBackedFromBytes(data); + decodeValue(data: Uint8Array): BeaconStateAllForks { + return getStateTypeFromBytes(this.config, data).deserializeToViewDU(data); } // Handle key as slot - async put(key: Slot, value: TreeBacked): Promise { + async put(key: Slot, value: BeaconStateAllForks): Promise { await Promise.all([super.put(key, value), storeRootIndex(this.db, key, value.hashTreeRoot())]); } - getId(state: TreeBacked): Epoch { + getId(state: BeaconStateAllForks): Epoch { return state.slot; } - decodeKey(data: Buffer): number { + decodeKey(data: Uint8Array): number { return bytesToInt((super.decodeKey(data) as unknown) as Uint8Array, "be"); } // Index Root -> Slot - async getByRoot(stateRoot: Root): Promise | null> { + async getByRoot(stateRoot: Root): Promise { const slot = await this.getSlotByRoot(stateRoot); if (slot !== null && Number.isInteger(slot)) { return this.get(slot); diff --git a/packages/lodestar/src/db/repositories/stateArchiveIndex.ts b/packages/lodestar/src/db/repositories/stateArchiveIndex.ts index adf0838c9b8..62e585be79c 100644 --- a/packages/lodestar/src/db/repositories/stateArchiveIndex.ts +++ b/packages/lodestar/src/db/repositories/stateArchiveIndex.ts @@ -7,5 +7,5 @@ export function storeRootIndex(db: Db, slot: Slot, stateRoot: Root): Promise): Promise { + async put(value: BeaconStateAllForks): Promise { this.metrics?.dbWrites.labels({bucket: "phase0_preGenesisState"}).inc(); - await this.db.put(this.key, this.type().serialize(value) as Buffer); + await this.db.put(this.key, value.serialize()); } - async get(): Promise | null> { + async get(): Promise { this.metrics?.dbReads.labels({bucket: "phase0_preGenesisState"}).inc(); const value = await this.db.get(this.key); - return value ? this.type().createTreeBackedFromBytes(value) : null; + return value ? this.type.deserializeToViewDU(value) : null; } async delete(): Promise { await this.db.delete(this.key); } - - private type(): ContainerType { - return this.config.getForkTypes(GENESIS_SLOT).BeaconState; - } } diff --git a/packages/lodestar/src/db/single/preGenesisStateLastProcessedBlock.ts b/packages/lodestar/src/db/single/preGenesisStateLastProcessedBlock.ts index 2f2af759eee..1cd25a861dd 100644 --- a/packages/lodestar/src/db/single/preGenesisStateLastProcessedBlock.ts +++ b/packages/lodestar/src/db/single/preGenesisStateLastProcessedBlock.ts @@ -1,26 +1,26 @@ -import {NumberUintType} from "@chainsafe/ssz"; +import {UintNumberType} from "@chainsafe/ssz"; import {ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {Db, Bucket, IDbMetrics} from "@chainsafe/lodestar-db"; export class PreGenesisStateLastProcessedBlock { private readonly bucket: Bucket; - private readonly type: NumberUintType; + private readonly type: UintNumberType; private readonly db: Db; - private readonly key: Buffer; + private readonly key: Uint8Array; private readonly metrics?: IDbMetrics; constructor(config: IChainForkConfig, db: Db, metrics?: IDbMetrics) { this.db = db; - this.type = ssz.Number64; + this.type = ssz.UintNum64; this.bucket = Bucket.phase0_preGenesisStateLastProcessedBlock; - this.key = Buffer.from(new Uint8Array([this.bucket])); + this.key = new Uint8Array([this.bucket]); this.metrics = metrics; } async put(value: number): Promise { this.metrics?.dbWrites.labels({bucket: "phase0_preGenesisStateLastProcessedBlock"}).inc(); - await this.db.put(this.key, this.type.serialize(value) as Buffer); + await this.db.put(this.key, this.type.serialize(value)); } async get(): Promise { diff --git a/packages/lodestar/src/eth1/eth1DepositDataTracker.ts b/packages/lodestar/src/eth1/eth1DepositDataTracker.ts index d35be427f43..6e8da33c3c4 100644 --- a/packages/lodestar/src/eth1/eth1DepositDataTracker.ts +++ b/packages/lodestar/src/eth1/eth1DepositDataTracker.ts @@ -1,6 +1,6 @@ import {phase0, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {CachedBeaconStateAllForks, allForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {allForks, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {ErrorAborted, ILogger, isErrorAborted, sleep} from "@chainsafe/lodestar-utils"; import {AbortSignal} from "@chainsafe/abort-controller"; import {IBeaconDb} from "../db"; @@ -72,7 +72,7 @@ export class Eth1DepositDataTracker { /** * Return eth1Data and deposits ready for block production for a given state */ - async getEth1DataAndDeposits(state: CachedBeaconStateAllForks): Promise { + async getEth1DataAndDeposits(state: BeaconStateAllForks): Promise { const eth1Data = await this.getEth1Data(state); const deposits = await this.getDeposits(state, eth1Data); return {eth1Data, deposits}; @@ -82,7 +82,7 @@ export class Eth1DepositDataTracker { * Returns an eth1Data vote for a given state. * Requires internal caches to be updated regularly to return good results */ - private async getEth1Data(state: allForks.BeaconState): Promise { + private async getEth1Data(state: BeaconStateAllForks): Promise { try { const eth1VotesToConsider = await getEth1VotesToConsider( this.config, @@ -101,10 +101,7 @@ export class Eth1DepositDataTracker { * Returns deposits to be included for a given state and eth1Data vote. * Requires internal caches to be updated regularly to return good results */ - private async getDeposits( - state: CachedBeaconStateAllForks, - eth1DataVote: phase0.Eth1Data - ): Promise { + private async getDeposits(state: BeaconStateAllForks, eth1DataVote: phase0.Eth1Data): Promise { // No new deposits have to be included, continue if (eth1DataVote.depositCount === state.eth1DepositIndex) { return []; @@ -112,7 +109,7 @@ export class Eth1DepositDataTracker { // TODO: Review if this is optimal // Convert to view first to hash once and compare hashes - const eth1DataVoteView = ssz.phase0.Eth1Data.createTreeBackedFromStruct(eth1DataVote); + const eth1DataVoteView = ssz.phase0.Eth1Data.toViewDU(eth1DataVote); // Eth1 data may change due to the vote included in this block const newEth1Data = allForks.becomesNewEth1Data(state, eth1DataVoteView) ? eth1DataVoteView : state.eth1Data; diff --git a/packages/lodestar/src/eth1/index.ts b/packages/lodestar/src/eth1/index.ts index d297894becc..96f0f4504e7 100644 --- a/packages/lodestar/src/eth1/index.ts +++ b/packages/lodestar/src/eth1/index.ts @@ -1,9 +1,10 @@ import { + BeaconStateAllForks, CachedBeaconStateAllForks, computeEpochAtSlot, getCurrentSlot, } from "@chainsafe/lodestar-beacon-state-transition"; -import {allForks, Root} from "@chainsafe/lodestar-types"; +import {Root} from "@chainsafe/lodestar-types"; import {bellatrix} from "@chainsafe/lodestar-beacon-state-transition"; import {fromHexString} from "@chainsafe/ssz"; import {IEth1ForBlockProduction, Eth1DataAndDeposits, IEth1Provider, PowMergeBlock} from "./interface"; @@ -48,7 +49,7 @@ export {IEth1ForBlockProduction, IEth1Provider, Eth1Provider}; export function initializeEth1ForBlockProduction( opts: Eth1Options, modules: Pick, - anchorState: allForks.BeaconState + anchorState: BeaconStateAllForks ): IEth1ForBlockProduction { if (opts.enabled) { return new Eth1ForBlockProduction(opts, { diff --git a/packages/lodestar/src/eth1/provider/utils.ts b/packages/lodestar/src/eth1/provider/utils.ts index d18c057122f..82976611443 100644 --- a/packages/lodestar/src/eth1/provider/utils.ts +++ b/packages/lodestar/src/eth1/provider/utils.ts @@ -1,6 +1,6 @@ import {RootHex} from "@chainsafe/lodestar-types"; import {bytesToBigInt, bigIntToBytes} from "@chainsafe/lodestar-utils"; -import {ByteVector, fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ErrorParseJson} from "./jsonRpcHttpClient"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -25,7 +25,7 @@ export function isJsonRpcTruncatedError(error: Error): boolean { ); } -export function bytesToHex(bytes: Uint8Array | ByteVector): string { +export function bytesToHex(bytes: Uint8Array): string { // Handle special case in Ethereum hex formating where hex values may include a single letter // 0x0, 0x1 are valid values if (bytes.length === 1 && bytes[0] <= 0xf) { @@ -82,7 +82,7 @@ export function quantityToBytes(hex: QUANTITY): Uint8Array { * QUANTITY as defined in ethereum execution layer JSON RPC https://eth.wiki/json-rpc/API. * Compress a 32 ByteVector into a QUANTITY */ -export function bytesToQuantity(bytes: Uint8Array | ByteVector): QUANTITY { +export function bytesToQuantity(bytes: Uint8Array): QUANTITY { const bn = bytesToBigInt(bytes as Uint8Array, "le"); return numToQuantity(bn); } @@ -99,7 +99,7 @@ export function bytesToQuantity(bytes: Uint8Array | ByteVector): QUANTITY { * - WRONG: 0xf0f0f (must be even number of digits) * - WRONG: 004200 (must be prefixed 0x) */ -export function bytesToData(bytes: Uint8Array | ByteVector): DATA { +export function bytesToData(bytes: Uint8Array): DATA { return toHexString(bytes); } diff --git a/packages/lodestar/src/eth1/utils/depositContract.ts b/packages/lodestar/src/eth1/utils/depositContract.ts index 140e2c18e57..09072733831 100644 --- a/packages/lodestar/src/eth1/utils/depositContract.ts +++ b/packages/lodestar/src/eth1/utils/depositContract.ts @@ -25,12 +25,17 @@ export function parseDepositLog(log: {blockNumber: number; data: string; topics: if (values === undefined) throw Error(`DepositEvent at ${log.blockNumber} has no values`); return { blockNumber: log.blockNumber, - index: ssz.Number64.deserialize(fromHexString(values.index)), + index: parseHexNumLittleEndian(values.index), depositData: { pubkey: fromHexString(values.pubkey), withdrawalCredentials: fromHexString(values.withdrawal_credentials), - amount: ssz.Number64.deserialize(fromHexString(values.amount)), + amount: parseHexNumLittleEndian(values.amount), signature: fromHexString(values.signature), }, }; } + +function parseHexNumLittleEndian(hex: string): number { + // Can't use parseInt() because amount is a hex string in little endian + return ssz.UintNum64.deserialize(fromHexString(hex)); +} diff --git a/packages/lodestar/src/eth1/utils/deposits.ts b/packages/lodestar/src/eth1/utils/deposits.ts index 9cae58f883b..2fb3821b3a0 100644 --- a/packages/lodestar/src/eth1/utils/deposits.ts +++ b/packages/lodestar/src/eth1/utils/deposits.ts @@ -1,15 +1,17 @@ import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; -import {Root, phase0, allForks, ssz} from "@chainsafe/lodestar-types"; -import {TreeBacked, List, toHexString} from "@chainsafe/ssz"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; +import {toHexString} from "@chainsafe/ssz"; import {IFilterOptions} from "@chainsafe/lodestar-db"; -import {getTreeAtIndex} from "../../util/tree"; import {Eth1Error, Eth1ErrorCode} from "../errors"; +import {DepositTree} from "../../db/repositories/depositDataRoot"; export type DepositGetter = (indexRange: IFilterOptions, eth1Data: phase0.Eth1Data) => Promise; export async function getDeposits( // eth1_deposit_index represents the next deposit index to be added - state: allForks.BeaconState, + state: BeaconStateAllForks, eth1Data: phase0.Eth1Data, depositsGetter: DepositGetter ): Promise { @@ -38,13 +40,13 @@ export async function getDeposits( export function getDepositsWithProofs( depositEvents: phase0.DepositEvent[], - depositRootTree: TreeBacked>, + depositRootTree: DepositTree, eth1Data: phase0.Eth1Data ): phase0.Deposit[] { // Get tree at this particular depositCount to compute correct proofs - const treeAtDepositCount = getTreeAtIndex(depositRootTree, eth1Data.depositCount - 1); + const viewAtDepositCount = depositRootTree.sliceTo(eth1Data.depositCount - 1); - const depositRoot = treeAtDepositCount.hashTreeRoot(); + const depositRoot = viewAtDepositCount.hashTreeRoot(); if (!ssz.Root.equals(depositRoot, eth1Data.depositRoot)) { throw new Eth1Error({ @@ -54,8 +56,12 @@ export function getDepositsWithProofs( }); } + // Already commited for .hashTreeRoot() + const treeAtDepositCount = new Tree(viewAtDepositCount.node); + const depositTreeDepth = viewAtDepositCount.type.depth; + return depositEvents.map((log) => ({ - proof: treeAtDepositCount.tree.getSingleProof(treeAtDepositCount.type.getPropertyGindex(log.index)), + proof: treeAtDepositCount.getSingleProof(toGindex(depositTreeDepth, BigInt(log.index))), data: log.depositData, })); } diff --git a/packages/lodestar/src/eth1/utils/eth1Data.ts b/packages/lodestar/src/eth1/utils/eth1Data.ts index 5eb1088cf37..76b021745cb 100644 --- a/packages/lodestar/src/eth1/utils/eth1Data.ts +++ b/packages/lodestar/src/eth1/utils/eth1Data.ts @@ -1,8 +1,7 @@ import {Root, phase0} from "@chainsafe/lodestar-types"; -import {List, TreeBacked} from "@chainsafe/ssz"; -import {getTreeAtIndex} from "../../util/tree"; import {binarySearchLte} from "../../util/binarySearch"; import {Eth1Error, Eth1ErrorCode} from "../errors"; +import {DepositTree} from "../../db/repositories/depositDataRoot"; import {Eth1Block} from "../interface"; type BlockNumber = number; @@ -14,7 +13,7 @@ type BlockNumber = number; export async function getEth1DataForBlocks( blocks: Eth1Block[], depositDescendingStream: AsyncIterable, - depositRootTree: TreeBacked>, + depositRootTree: DepositTree, lastProcessedDepositBlockNumber: BlockNumber | null ): Promise<(phase0.Eth1Data & Eth1Block)[]> { // Exclude blocks for which there is no valid eth1 data deposit @@ -80,10 +79,7 @@ export async function getDepositsByBlockNumber( /** * Precompute a map of depositCount => depositRoot from a depositRootTree filled beforehand */ -export function getDepositRootByDepositCount( - depositCounts: number[], - depositRootTree: TreeBacked> -): Map { +export function getDepositRootByDepositCount(depositCounts: number[], depositRootTree: DepositTree): Map { // Unique + sort numerically in descending order depositCounts = [...new Set(depositCounts)].sort((a, b) => b - a); @@ -97,7 +93,7 @@ export function getDepositRootByDepositCount( const depositRootByDepositCount = new Map(); for (const depositCount of depositCounts) { - depositRootTree = getTreeAtIndex(depositRootTree, depositCount - 1); + depositRootTree = depositRootTree.sliceTo(depositCount - 1); depositRootByDepositCount.set(depositCount, depositRootTree.hashTreeRoot()); } return depositRootByDepositCount; diff --git a/packages/lodestar/src/eth1/utils/eth1Vote.ts b/packages/lodestar/src/eth1/utils/eth1Vote.ts index fd4ed82f661..38d663feb28 100644 --- a/packages/lodestar/src/eth1/utils/eth1Vote.ts +++ b/packages/lodestar/src/eth1/utils/eth1Vote.ts @@ -1,8 +1,8 @@ import {EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {allForks, phase0, RootHex} from "@chainsafe/lodestar-types"; -import {computeTimeAtSlot} from "@chainsafe/lodestar-beacon-state-transition"; -import {readonlyValues, toHexString} from "@chainsafe/ssz"; +import {phase0, RootHex} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks, computeTimeAtSlot} from "@chainsafe/lodestar-beacon-state-transition"; +import {toHex} from "@chainsafe/lodestar-utils"; export type Eth1DataGetter = ({ timestampRange, @@ -12,7 +12,7 @@ export type Eth1DataGetter = ({ export async function getEth1VotesToConsider( config: IChainForkConfig, - state: allForks.BeaconState, + state: BeaconStateAllForks, eth1DataGetter: Eth1DataGetter ): Promise { const periodStart = votingPeriodStartTime(config, state); @@ -33,7 +33,7 @@ export async function getEth1VotesToConsider( ).filter((eth1Data) => eth1Data.depositCount >= state.eth1Data.depositCount); } -export function pickEth1Vote(state: allForks.BeaconState, votesToConsider: phase0.Eth1Data[]): phase0.Eth1Data { +export function pickEth1Vote(state: BeaconStateAllForks, votesToConsider: phase0.Eth1Data[]): phase0.Eth1Data { const votesToConsiderKeys = new Set(); for (const eth1Data of votesToConsider) { votesToConsiderKeys.add(getEth1DataKey(eth1Data)); @@ -48,7 +48,7 @@ export function pickEth1Vote(state: allForks.BeaconState, votesToConsider: phase // However `votesToConsider` is an array of values since those are read from DB. // TODO: Optimize cache of known votes, to prevent re-hashing stored values. // Note: for low validator counts it's not very important, since this runs once per proposal - const eth1DataVotes = Array.from(readonlyValues(state.eth1DataVotes)); + const eth1DataVotes = state.eth1DataVotes.getAllReadonly(); for (const eth1DataVote of eth1DataVotes) { const rootHex = getEth1DataKey(eth1DataVote); @@ -129,10 +129,10 @@ function getEth1DataKey(eth1Data: phase0.Eth1Data): string { * Serialize eth1Data types to a unique string ID. It is only used for comparison. */ export function fastSerializeEth1Data(eth1Data: phase0.Eth1Data): string { - return toHexString(eth1Data.blockHash) + eth1Data.depositCount.toString(16) + toHexString(eth1Data.depositRoot); + return toHex(eth1Data.blockHash) + eth1Data.depositCount.toString(16) + toHex(eth1Data.depositRoot); } -export function votingPeriodStartTime(config: IChainForkConfig, state: allForks.BeaconState): number { +export function votingPeriodStartTime(config: IChainForkConfig, state: BeaconStateAllForks): number { const eth1VotingPeriodStartSlot = state.slot - (state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)); return computeTimeAtSlot(config, eth1VotingPeriodStartSlot, state.genesisTime); } diff --git a/packages/lodestar/src/executionEngine/interface.ts b/packages/lodestar/src/executionEngine/interface.ts index fe39cbb59cd..cf884988ab2 100644 --- a/packages/lodestar/src/executionEngine/interface.ts +++ b/packages/lodestar/src/executionEngine/interface.ts @@ -1,5 +1,4 @@ import {bellatrix, Root, RootHex} from "@chainsafe/lodestar-types"; -import {ByteVector} from "@chainsafe/ssz"; import {DATA, QUANTITY} from "../eth1/provider/utils"; // An execution engine can produce a payload id anywhere the the uint64 range @@ -52,8 +51,8 @@ export type ForkChoiceUpdateStatus = export type PayloadAttributes = { timestamp: number; - prevRandao: Uint8Array | ByteVector; - suggestedFeeRecipient: Uint8Array | ByteVector; + prevRandao: Uint8Array; + suggestedFeeRecipient: Uint8Array; }; export type ApiPayloadAttributes = { diff --git a/packages/lodestar/src/metrics/metrics.ts b/packages/lodestar/src/metrics/metrics.ts index 7a5a1c7b60e..6fbef215de8 100644 --- a/packages/lodestar/src/metrics/metrics.ts +++ b/packages/lodestar/src/metrics/metrics.ts @@ -1,9 +1,8 @@ /** * @module metrics */ -import {getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {allForks} from "@chainsafe/lodestar-types"; import {collectDefaultMetrics, Counter, Registry} from "prom-client"; import gcStats from "prometheus-gc-stats"; import {DbMetricLabels, IDbMetrics} from "@chainsafe/lodestar-db"; @@ -18,7 +17,7 @@ export type IMetrics = IBeaconMetrics & ILodestarMetrics & IValidatorMonitor & { export function createMetrics( opts: IMetricsOptions, config: IChainForkConfig, - anchorState: allForks.BeaconState, + anchorState: BeaconStateAllForks, registries: Registry[] = [] ): IMetrics { const register = new RegistryMetricCreator(); diff --git a/packages/lodestar/src/metrics/metrics/lodestar.ts b/packages/lodestar/src/metrics/metrics/lodestar.ts index 07f3486db8c..dbe4d253a0f 100644 --- a/packages/lodestar/src/metrics/metrics/lodestar.ts +++ b/packages/lodestar/src/metrics/metrics/lodestar.ts @@ -11,7 +11,7 @@ export type ILodestarMetrics = ReturnType; export function createLodestarMetrics( register: RegistryMetricCreator, metadata: IMetricsOptions["metadata"], - anchorState?: allForks.BeaconState + anchorState?: Pick ) { if (metadata) { register.static<"semver" | "branch" | "commit" | "version" | "network">({ diff --git a/packages/lodestar/src/network/gossip/encoding.ts b/packages/lodestar/src/network/gossip/encoding.ts index a1e20361639..45d249b5a81 100644 --- a/packages/lodestar/src/network/gossip/encoding.ts +++ b/packages/lodestar/src/network/gossip/encoding.ts @@ -1,6 +1,6 @@ import {compress, uncompress} from "snappyjs"; import {intToBytes} from "@chainsafe/lodestar-utils"; -import {hash} from "@chainsafe/ssz"; +import {digest} from "@chainsafe/as-sha256"; import {ForkName} from "@chainsafe/lodestar-params"; import { DEFAULT_ENCODING, @@ -102,5 +102,5 @@ export function computeMsgIdAltair(topic: GossipTopic, topicStr: string, msg: Et } function hashGossipMsgData(...dataArrToHash: Uint8Array[]): Uint8Array { - return hash(Buffer.concat(dataArrToHash)).slice(0, GOSSIP_MSGID_LENGTH); + return digest(Buffer.concat(dataArrToHash)).slice(0, GOSSIP_MSGID_LENGTH); } diff --git a/packages/lodestar/src/network/gossip/gossipsub.ts b/packages/lodestar/src/network/gossip/gossipsub.ts index 340165cef99..1e57573a796 100644 --- a/packages/lodestar/src/network/gossip/gossipsub.ts +++ b/packages/lodestar/src/network/gossip/gossipsub.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import Gossipsub from "libp2p-gossipsub"; import {messageIdToString} from "libp2p-gossipsub/src/utils/messageIdToString"; -import SHA256 from "@chainsafe/as-sha256"; +import {digest} from "@chainsafe/as-sha256"; import {ERR_TOPIC_VALIDATOR_IGNORE, ERR_TOPIC_VALIDATOR_REJECT} from "libp2p-gossipsub/src/constants"; import {InMessage, utils} from "libp2p-interfaces/src/pubsub"; import Libp2p from "libp2p"; @@ -89,7 +89,7 @@ export class Eth2Gossipsub extends Gossipsub { Dlazy: 6, scoreParams: computeGossipPeerScoreParams(modules), scoreThresholds: gossipScoreThresholds, - fastMsgIdFn: (msg: InMessage) => Buffer.from(SHA256.digest(msg.data)).toString("hex"), + fastMsgIdFn: (msg: InMessage) => Buffer.from(digest(msg.data)).toString("hex"), }); const {config, logger, metrics, signal, gossipHandlers} = modules; this.config = config; diff --git a/packages/lodestar/src/network/gossip/validation/index.ts b/packages/lodestar/src/network/gossip/validation/index.ts index eabcae6b1fa..ce917ae0f32 100644 --- a/packages/lodestar/src/network/gossip/validation/index.ts +++ b/packages/lodestar/src/network/gossip/validation/index.ts @@ -81,11 +81,7 @@ function getGossipValidatorFn( try { const sszType = getGossipSSZType(topic); const messageData = getUncompressedData(gossipMsg); - gossipObject = - // TODO: Review if it's really necessary to deserialize this as TreeBacked - topic.type === GossipType.beacon_block || topic.type === GossipType.beacon_aggregate_and_proof - ? sszType.createTreeBackedFromBytes(messageData) - : sszType.deserialize(messageData); + gossipObject = sszType.deserialize(messageData); } catch (e) { // TODO: Log the error or do something better with it throw new GossipActionError(GossipAction.REJECT, {code: (e as Error).message}); diff --git a/packages/lodestar/src/network/metadata.ts b/packages/lodestar/src/network/metadata.ts index 67bc06e74a3..b2fcf9d3d7b 100644 --- a/packages/lodestar/src/network/metadata.ts +++ b/packages/lodestar/src/network/metadata.ts @@ -1,5 +1,5 @@ import {ENR} from "@chainsafe/discv5"; -import {BitVector, toHexString} from "@chainsafe/ssz"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {ForkName} from "@chainsafe/lodestar-params"; import {altair, Epoch, phase0, ssz} from "@chainsafe/lodestar-types"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; @@ -69,22 +69,22 @@ export class MetadataController { return this._metadata.seqNumber; } - get syncnets(): BitVector { + get syncnets(): BitArray { return this._metadata.syncnets; } - set syncnets(syncnets: BitVector) { + set syncnets(syncnets: BitArray) { if (this.enr) { this.enr.set(ENRKey.syncnets, ssz.altair.SyncSubnets.serialize(syncnets)); } this._metadata.syncnets = syncnets; } - get attnets(): BitVector { + get attnets(): BitArray { return this._metadata.attnets; } - set attnets(attnets: BitVector) { + set attnets(attnets: BitArray) { if (this.enr) { this.enr.set(ENRKey.attnets, ssz.phase0.AttestationSubnets.serialize(attnets)); } diff --git a/packages/lodestar/src/network/network.ts b/packages/lodestar/src/network/network.ts index 15c3096272c..880fcd61cd5 100644 --- a/packages/lodestar/src/network/network.ts +++ b/packages/lodestar/src/network/network.ts @@ -94,7 +94,7 @@ export class Network implements INetwork { signal, gossipHandlers: gossipHandlers ?? getGossipHandlers({chain, config, logger, network: this, metrics}, opts), eth2Context: { - activeValidatorCount: chain.getHeadState().currentShuffling.activeIndices.length, + activeValidatorCount: chain.getHeadState().epochCtx.currentShuffling.activeIndices.length, currentSlot: this.clock.currentSlot, currentEpoch: this.clock.currentEpoch, }, diff --git a/packages/lodestar/src/network/peers/peerManager.ts b/packages/lodestar/src/network/peers/peerManager.ts index 4e1a48085bf..cbe80c6f29f 100644 --- a/packages/lodestar/src/network/peers/peerManager.ts +++ b/packages/lodestar/src/network/peers/peerManager.ts @@ -1,6 +1,8 @@ import LibP2p, {Connection} from "libp2p"; import PeerId from "peer-id"; import {IDiscv5DiscoveryInputOptions} from "@chainsafe/discv5"; +import {BitArray} from "@chainsafe/ssz"; +import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {allForks, altair, phase0} from "@chainsafe/lodestar-types"; import {ILogger} from "@chainsafe/lodestar-utils"; @@ -23,7 +25,6 @@ import { renderIrrelevantPeerType, } from "./utils"; import {SubnetType} from "../metadata"; -import {ATTESTATION_SUBNET_COUNT} from "@chainsafe/lodestar-params"; /** heartbeat performs regular updates such as updating reputations and performing discovery requests */ const HEARTBEAT_INTERVAL_MS = 30 * 1000; @@ -257,7 +258,7 @@ export class PeerManager { peerData.metadata = { seqNumber: metadata.seqNumber, attnets: metadata.attnets, - syncnets: (metadata as Partial).syncnets || [], + syncnets: (metadata as Partial).syncnets ?? BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_COUNT), }; } } @@ -388,8 +389,8 @@ export class PeerManager { const peerData = this.connectedPeers.get(peer.toB58String()); return { id: peer, - attnets: peerData?.metadata?.attnets ?? [], - syncnets: peerData?.metadata?.syncnets ?? [], + attnets: peerData?.metadata?.attnets ?? BitArray.fromBitLen(ATTESTATION_SUBNET_COUNT), + syncnets: peerData?.metadata?.syncnets ?? BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_COUNT), score: this.peerRpcScores.getScore(peer), }; }), @@ -635,7 +636,7 @@ function countAttnets(peerData?: PeerData): number { let count = 0; for (let i = 0; i < ATTESTATION_SUBNET_COUNT; i++) { - if (attNets[i]) count++; + if (attNets.get(i)) count++; } return count; diff --git a/packages/lodestar/src/network/peers/utils/enrSubnetsDeserialize.ts b/packages/lodestar/src/network/peers/utils/enrSubnetsDeserialize.ts index 75ed687af29..fc244f4e1a0 100644 --- a/packages/lodestar/src/network/peers/utils/enrSubnetsDeserialize.ts +++ b/packages/lodestar/src/network/peers/utils/enrSubnetsDeserialize.ts @@ -1,4 +1,5 @@ -import {getUint8ByteToBitBooleanArray, newFilledArray} from "@chainsafe/lodestar-beacon-state-transition"; +import {getUint8ByteToBitBooleanArray} from "@chainsafe/ssz"; +import {newFilledArray} from "@chainsafe/lodestar-beacon-state-transition"; import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; export const zeroAttnets = newFilledArray(ATTESTATION_SUBNET_COUNT, false); diff --git a/packages/lodestar/src/network/peers/utils/prioritizePeers.ts b/packages/lodestar/src/network/peers/utils/prioritizePeers.ts index c3bbf403e87..5cd43f5ff79 100644 --- a/packages/lodestar/src/network/peers/utils/prioritizePeers.ts +++ b/packages/lodestar/src/network/peers/utils/prioritizePeers.ts @@ -24,7 +24,12 @@ type SubnetDiscvQuery = {subnet: number; toSlot: number; maxPeersToDiscover: num * - Prioritize peers with good score */ export function prioritizePeers( - connectedPeers: {id: PeerId; attnets: phase0.AttestationSubnets; syncnets: altair.SyncSubnets; score: number}[], + connectedPeers: { + id: PeerId; + attnets: phase0.AttestationSubnets | null; + syncnets: altair.SyncSubnets | null; + score: number; + }[], activeAttnets: RequestedSubnet[], activeSyncnets: RequestedSubnet[], {targetPeers, maxPeers}: {targetPeers: number; maxPeers: number} @@ -56,7 +61,9 @@ export function prioritizePeers( for (const peer of connectedPeers) { let hasDuty = false; for (const {subnet} of subnets) { - if (peer[subnetKey][subnet]) { + const subnetBitArray = peer[subnetKey]; + // TODO: Review performance + if (subnetBitArray && subnetBitArray.get(subnet) === true) { hasDuty = true; peersPerSubnet.set(subnet, 1 + (peersPerSubnet.get(subnet) ?? 0)); } diff --git a/packages/lodestar/src/network/reqresp/encoders/responseDecode.ts b/packages/lodestar/src/network/reqresp/encoders/responseDecode.ts index 2fe758be1b6..51fe9a2e6b2 100644 --- a/packages/lodestar/src/network/reqresp/encoders/responseDecode.ts +++ b/packages/lodestar/src/network/reqresp/encoders/responseDecode.ts @@ -8,7 +8,6 @@ import { Protocol, IncomingResponseBody, ContextBytesType, - deserializeToTreeByMethod, contextBytesTypeByProtocol, getResponseSzzTypeByMethod, CONTEXT_BYTES_FORK_DIGEST_LENGTH, @@ -34,7 +33,6 @@ export function responseDecode( protocol: Protocol ): (source: AsyncIterable) => AsyncGenerator { return async function* responseDecodeSink(source) { - const deserializeToTree = deserializeToTreeByMethod[protocol.method]; const contextBytesType = contextBytesTypeByProtocol(protocol); const bufferedSource = new BufferedSource(source as AsyncGenerator); @@ -58,7 +56,7 @@ export function responseDecode( const forkName = await readForkName(forkDigestContext, bufferedSource, contextBytesType); const type = getResponseSzzTypeByMethod(protocol, forkName); - yield await readEncodedPayload(bufferedSource, protocol.encoding, type, {deserializeToTree}); + yield await readEncodedPayload(bufferedSource, protocol.encoding, type); } }; } diff --git a/packages/lodestar/src/network/reqresp/encodingStrategies/index.ts b/packages/lodestar/src/network/reqresp/encodingStrategies/index.ts index ade2c79b66d..d3356080f9a 100644 --- a/packages/lodestar/src/network/reqresp/encodingStrategies/index.ts +++ b/packages/lodestar/src/network/reqresp/encodingStrategies/index.ts @@ -6,7 +6,7 @@ import { OutgoingSerializer, } from "../types"; import {BufferedSource} from "../utils"; -import {readSszSnappyPayload, ISszSnappyOptions} from "./sszSnappy/decode"; +import {readSszSnappyPayload} from "./sszSnappy/decode"; import {writeSszSnappyPayload} from "./sszSnappy/encode"; // For more info about Ethereum Consensus request/response encoding strategies, see: @@ -23,12 +23,11 @@ import {writeSszSnappyPayload} from "./sszSnappy/encode"; export async function readEncodedPayload( bufferedSource: BufferedSource, encoding: Encoding, - type: RequestOrResponseType, - options?: ISszSnappyOptions + type: RequestOrResponseType ): Promise { switch (encoding) { case Encoding.SSZ_SNAPPY: - return await readSszSnappyPayload(bufferedSource, type, options); + return await readSszSnappyPayload(bufferedSource, type); default: throw Error("Unsupported encoding"); diff --git a/packages/lodestar/src/network/reqresp/encodingStrategies/sszSnappy/decode.ts b/packages/lodestar/src/network/reqresp/encodingStrategies/sszSnappy/decode.ts index cb42bc28d58..7540a32fe08 100644 --- a/packages/lodestar/src/network/reqresp/encodingStrategies/sszSnappy/decode.ts +++ b/packages/lodestar/src/network/reqresp/encodingStrategies/sszSnappy/decode.ts @@ -1,6 +1,5 @@ import BufferList from "bl"; import varint from "varint"; -import {CompositeType} from "@chainsafe/ssz"; import {MAX_VARINT_BYTES} from "../../../../constants"; import {BufferedSource} from "../../utils"; import {RequestOrResponseType, RequestOrIncomingResponseBody} from "../../types"; @@ -8,10 +7,6 @@ import {SnappyFramesUncompress} from "./snappyFrames/uncompress"; import {maxEncodedLen} from "./utils"; import {SszSnappyError, SszSnappyErrorCode} from "./errors"; -export interface ISszSnappyOptions { - deserializeToTree?: boolean; -} - /** * ssz_snappy encoding strategy reader. * Consumes a stream source to read encoded header and payload as defined in the spec: @@ -21,13 +16,12 @@ export interface ISszSnappyOptions { */ export async function readSszSnappyPayload( bufferedSource: BufferedSource, - type: RequestOrResponseType, - options?: ISszSnappyOptions + type: RequestOrResponseType ): Promise { const sszDataLength = await readSszSnappyHeader(bufferedSource, type); const bytes = await readSszSnappyBody(bufferedSource, sszDataLength); - return deserializeSszBody(bytes, type, options); + return deserializeSszBody(bytes, type); } /** @@ -61,8 +55,8 @@ async function readSszSnappyHeader(bufferedSource: BufferedSource, type: Request buffer.consume(varintBytes); // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type. - const minSize = type.getMinSerializedLength(); - const maxSize = type.getMaxSerializedLength(); + const minSize = type.minSize; + const maxSize = type.maxSize; if (sszDataLength < minSize) { throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength}); } @@ -130,18 +124,9 @@ async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataLength: * Deseralizes SSZ body. * `isSszTree` option allows the SignedBeaconBlock type to be deserialized as a tree */ -function deserializeSszBody( - bytes: Buffer, - type: RequestOrResponseType, - options?: ISszSnappyOptions -): T { +function deserializeSszBody(bytes: Buffer, type: RequestOrResponseType): T { try { - if (options?.deserializeToTree) { - const typeTree = (type as unknown) as CompositeType>; - return (typeTree.createTreeBackedFromBytes(bytes) as unknown) as T; - } else { - return type.deserialize(bytes) as T; - } + return type.deserialize(bytes) as T; } catch (e) { throw new SszSnappyError({code: SszSnappyErrorCode.DESERIALIZE_ERROR, deserializeError: e as Error}); } diff --git a/packages/lodestar/src/network/reqresp/handlers/beaconBlocksByRoot.ts b/packages/lodestar/src/network/reqresp/handlers/beaconBlocksByRoot.ts index 315954cbe3e..4091780464f 100644 --- a/packages/lodestar/src/network/reqresp/handlers/beaconBlocksByRoot.ts +++ b/packages/lodestar/src/network/reqresp/handlers/beaconBlocksByRoot.ts @@ -10,7 +10,7 @@ export async function* onBeaconBlocksByRoot( db: IBeaconDb ): AsyncIterable { for (const blockRoot of requestBody) { - const root = blockRoot.valueOf() as Uint8Array; + const root = blockRoot; const summary = chain.forkChoice.getBlock(root); let blockBytes: Uint8Array | null = null; diff --git a/packages/lodestar/src/network/reqresp/types.ts b/packages/lodestar/src/network/reqresp/types.ts index fe5edaf7438..f100cc73ba5 100644 --- a/packages/lodestar/src/network/reqresp/types.ts +++ b/packages/lodestar/src/network/reqresp/types.ts @@ -54,16 +54,6 @@ export const isSingleResponseChunkByMethod: {[K in Method]: boolean} = { [Method.BeaconBlocksByRoot]: false, }; -/** Deserialize some types to TreeBacked directly for more efficient hashing */ -export const deserializeToTreeByMethod: {[K in Method]: boolean} = { - [Method.Status]: false, - [Method.Goodbye]: false, - [Method.Ping]: false, - [Method.Metadata]: false, - [Method.BeaconBlocksByRange]: true, - [Method.BeaconBlocksByRoot]: true, -}; - export const CONTEXT_BYTES_FORK_DIGEST_LENGTH = 4; export enum ContextBytesType { /** 0 bytes chunk, can be ignored */ diff --git a/packages/lodestar/src/network/subnets/attnetsService.ts b/packages/lodestar/src/network/subnets/attnetsService.ts index 07add024c88..7f85d8e5c34 100644 --- a/packages/lodestar/src/network/subnets/attnetsService.ts +++ b/packages/lodestar/src/network/subnets/attnetsService.ts @@ -250,7 +250,7 @@ export class AttnetsService implements IAttnetsService { private updateMetadata(): void { const subnets = ssz.phase0.AttestationSubnets.defaultValue(); for (const subnet of this.subscriptionsRandom.getAll()) { - subnets[subnet] = true; + subnets.set(subnet, true); } // Only update metadata if necessary, setting `metadata.[key]` triggers a write to disk diff --git a/packages/lodestar/src/network/subnets/syncnetsService.ts b/packages/lodestar/src/network/subnets/syncnetsService.ts index 83cfcb1b831..453672dae4a 100644 --- a/packages/lodestar/src/network/subnets/syncnetsService.ts +++ b/packages/lodestar/src/network/subnets/syncnetsService.ts @@ -103,7 +103,7 @@ export class SyncnetsService implements ISubnetsService { private updateMetadata(): void { const subnets = ssz.altair.SyncSubnets.defaultValue(); for (const subnet of this.subscriptionsCommittee.getAll()) { - subnets[subnet] = true; + subnets.set(subnet, true); } // Only update metadata if necessary, setting `metadata.[key]` triggers a write to disk diff --git a/packages/lodestar/src/node/nodejs.ts b/packages/lodestar/src/node/nodejs.ts index 7eed45dbd9e..66e1f524ead 100644 --- a/packages/lodestar/src/node/nodejs.ts +++ b/packages/lodestar/src/node/nodejs.ts @@ -6,11 +6,11 @@ import {AbortController} from "@chainsafe/abort-controller"; import LibP2p from "libp2p"; import {Registry} from "prom-client"; -import {TreeBacked} from "@chainsafe/ssz"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; -import {allForks, phase0} from "@chainsafe/lodestar-types"; +import {phase0} from "@chainsafe/lodestar-types"; import {ILogger} from "@chainsafe/lodestar-utils"; import {Api} from "@chainsafe/lodestar-api"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconDb} from "../db"; import {INetwork, Network, getReqRespHandlers} from "../network"; @@ -47,7 +47,7 @@ export interface IBeaconNodeInitModules { db: IBeaconDb; logger: ILogger; libp2p: LibP2p; - anchorState: TreeBacked; + anchorState: BeaconStateAllForks; wsCheckpoint?: phase0.Checkpoint; metricsRegistries?: Registry[]; } diff --git a/packages/lodestar/src/node/notifier.ts b/packages/lodestar/src/node/notifier.ts index a4416d7e2cd..dec46148c12 100644 --- a/packages/lodestar/src/node/notifier.ts +++ b/packages/lodestar/src/node/notifier.ts @@ -53,7 +53,7 @@ export async function runNodeNotifier({ const headInfo = chain.forkChoice.getHead(); const headState = chain.getHeadState(); const finalizedEpoch = headState.finalizedCheckpoint.epoch; - const finalizedRoot = headState.finalizedCheckpoint.root.valueOf() as Uint8Array; + const finalizedRoot = headState.finalizedCheckpoint.root; const headSlot = headInfo.slot; timeSeries.addPoint(headSlot, Date.now()); @@ -61,7 +61,7 @@ export async function runNodeNotifier({ const finalizedCheckpointRow = `finalized: ${prettyBytes(finalizedRoot)}:${finalizedEpoch}`; const headRow = `head: ${headInfo.slot} ${prettyBytes(headInfo.blockRoot)}`; const isMergeTransitionComplete = - bellatrix.isBellatrixStateType(headState) && bellatrix.isMergeTransitionComplete(headState); + bellatrix.isBellatrixCachedStateType(headState) && bellatrix.isMergeTransitionComplete(headState); const executionInfo = isMergeTransitionComplete ? [ `execution: ${headInfo.executionStatus.toLowerCase()}(${prettyBytes( @@ -132,7 +132,7 @@ export async function runNodeNotifier({ function timeToNextHalfSlot(config: IBeaconConfig, chain: IBeaconChain): number { const msPerSlot = config.SECONDS_PER_SLOT * 1000; - const msFromGenesis = Date.now() - chain.getGenesisTime() * 1000; + const msFromGenesis = Date.now() - chain.genesisTime * 1000; const msToNextSlot = msPerSlot - (msFromGenesis % msPerSlot); return msToNextSlot > msPerSlot / 2 ? msToNextSlot - msPerSlot / 2 : msToNextSlot + msPerSlot / 2; } diff --git a/packages/lodestar/src/node/utils/interop/deposits.ts b/packages/lodestar/src/node/utils/interop/deposits.ts index 8303e20e074..8e1cf6e657f 100644 --- a/packages/lodestar/src/node/utils/interop/deposits.ts +++ b/packages/lodestar/src/node/utils/interop/deposits.ts @@ -1,5 +1,6 @@ -import {hash, TreeBacked, List} from "@chainsafe/ssz"; -import {phase0, Root, ssz} from "@chainsafe/lodestar-types"; +import {digest} from "@chainsafe/as-sha256"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; +import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import { computeDomain, @@ -8,22 +9,25 @@ import { ZERO_HASH, } from "@chainsafe/lodestar-beacon-state-transition"; import {BLS_WITHDRAWAL_PREFIX, DOMAIN_DEPOSIT, MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; +import {DepositTree} from "../../../db/repositories/depositDataRoot"; /** * Compute and return deposit data from other validators. */ export function interopDeposits( config: IChainForkConfig, - depositDataRootList: TreeBacked>, + depositDataRootList: DepositTree, validatorCount: number ): phase0.Deposit[] { - const tree = depositDataRootList.tree; + depositDataRootList.commit(); + const depositTreeDepth = depositDataRootList.type.depth; + return interopSecretKeys(validatorCount).map((secretKey, i) => { const pubkey = secretKey.toPublicKey().toBytes(); // create DepositData const data: phase0.DepositData = { pubkey, - withdrawalCredentials: Buffer.concat([BLS_WITHDRAWAL_PREFIX, hash(pubkey).slice(1)]), + withdrawalCredentials: Buffer.concat([BLS_WITHDRAWAL_PREFIX, digest(pubkey).slice(1)]), amount: MAX_EFFECTIVE_BALANCE, signature: Buffer.alloc(0), }; @@ -32,8 +36,9 @@ export function interopDeposits( data.signature = secretKey.sign(signingRoot).toBytes(); // Add to merkle tree depositDataRootList.push(ssz.phase0.DepositData.hashTreeRoot(data)); + depositDataRootList.commit(); return { - proof: tree.getSingleProof(depositDataRootList.type.getPropertyGindex(i)), + proof: new Tree(depositDataRootList.node).getSingleProof(toGindex(depositTreeDepth, BigInt(i))), data, }; }); diff --git a/packages/lodestar/src/node/utils/interop/state.ts b/packages/lodestar/src/node/utils/interop/state.ts index f540095f1f6..96eea011dba 100644 --- a/packages/lodestar/src/node/utils/interop/state.ts +++ b/packages/lodestar/src/node/utils/interop/state.ts @@ -1,8 +1,9 @@ -import {List, TreeBacked} from "@chainsafe/ssz"; -import {allForks, Bytes32, Number64, phase0, Root, ssz} from "@chainsafe/lodestar-types"; +import {Bytes32, phase0, ssz, TimeSeconds} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {initializeBeaconStateFromEth1} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, initializeBeaconStateFromEth1} from "@chainsafe/lodestar-beacon-state-transition"; +import {createEmptyEpochContextImmutableData} from "@chainsafe/lodestar-beacon-state-transition"; import {GENESIS_BASE_FEE_PER_GAS, GENESIS_GAS_LIMIT} from "@chainsafe/lodestar-params"; +import {DepositTree} from "../../../db/repositories/depositDataRoot"; export const INTEROP_BLOCK_HASH = Buffer.alloc(32, "B"); export const INTEROP_TIMESTAMP = Math.pow(2, 40); @@ -10,7 +11,7 @@ export const INTEROP_TIMESTAMP = Math.pow(2, 40); export type InteropStateOpts = { genesisTime?: number; eth1BlockHash?: Bytes32; - eth1Timestamp?: Number64; + eth1Timestamp?: TimeSeconds; }; export function getInteropState( @@ -21,9 +22,9 @@ export function getInteropState( eth1Timestamp = INTEROP_TIMESTAMP, }: InteropStateOpts, deposits: phase0.Deposit[], - fullDepositDataRootList?: TreeBacked> -): TreeBacked { - const latestPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultTreeBacked(); + fullDepositDataRootList?: DepositTree +): BeaconStateAllForks { + const latestPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.defaultViewDU(); // TODO: when having different test options, consider modifying these values latestPayloadHeader.blockHash = eth1BlockHash; latestPayloadHeader.timestamp = eth1Timestamp; @@ -32,6 +33,7 @@ export function getInteropState( latestPayloadHeader.baseFeePerGas = GENESIS_BASE_FEE_PER_GAS; const state = initializeBeaconStateFromEth1( config, + createEmptyEpochContextImmutableData(config, {genesisValidatorsRoot: Buffer.alloc(32, 0)}), eth1BlockHash, eth1Timestamp, deposits, diff --git a/packages/lodestar/src/node/utils/state.ts b/packages/lodestar/src/node/utils/state.ts index 1badbcbf413..fc376bb7296 100644 --- a/packages/lodestar/src/node/utils/state.ts +++ b/packages/lodestar/src/node/utils/state.ts @@ -1,11 +1,9 @@ -import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; -import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {interopDeposits} from "./interop/deposits"; import {getInteropState, InteropStateOpts} from "./interop/state"; -import {mkdirSync, writeFileSync} from "node:fs"; -import {dirname} from "node:path"; import {IBeaconDb} from "../../db"; -import {TreeBacked} from "@chainsafe/ssz"; import {GENESIS_SLOT} from "../../constants"; export async function initDevState( @@ -13,27 +11,22 @@ export async function initDevState( db: IBeaconDb, validatorCount: number, interopStateOpts: InteropStateOpts -): Promise> { - const deposits = interopDeposits(config, ssz.phase0.DepositDataRootList.defaultTreeBacked(), validatorCount); - await storeDeposits(config, db, deposits); +): Promise { + const deposits = interopDeposits(config, ssz.phase0.DepositDataRootList.defaultViewDU(), validatorCount); + await storeDeposits(db, deposits); const state = getInteropState( config, interopStateOpts, deposits, - await db.depositDataRoot.getTreeBacked(validatorCount - 1) + await db.depositDataRoot.getDepositRootTreeAtIndex(validatorCount - 1) ); const block = config.getForkTypes(GENESIS_SLOT).SignedBeaconBlock.defaultValue(); - block.message.stateRoot = config.getForkTypes(state.slot).BeaconState.hashTreeRoot(state); + block.message.stateRoot = state.hashTreeRoot(); await db.blockArchive.add(block); return state; } -export function storeSSZState(config: IBeaconConfig, state: TreeBacked, path: string): void { - mkdirSync(dirname(path), {recursive: true}); - writeFileSync(path, config.getForkTypes(state.slot).BeaconState.serialize(state)); -} - -async function storeDeposits(config: IChainForkConfig, db: IBeaconDb, deposits: phase0.Deposit[]): Promise { +async function storeDeposits(db: IBeaconDb, deposits: phase0.Deposit[]): Promise { for (let i = 0; i < deposits.length; i++) { await Promise.all([ db.depositEvent.put(i, { diff --git a/packages/lodestar/src/sync/backfill/backfill.ts b/packages/lodestar/src/sync/backfill/backfill.ts index 3f28d5c31b6..a6dbd2225e9 100644 --- a/packages/lodestar/src/sync/backfill/backfill.ts +++ b/packages/lodestar/src/sync/backfill/backfill.ts @@ -2,11 +2,11 @@ import {IMetrics} from "../../metrics/metrics"; import {EventEmitter} from "events"; import PeerId from "peer-id"; import {StrictEventEmitter} from "strict-event-emitter-types"; -import {blockToHeader} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, blockToHeader} from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconConfig, IChainForkConfig} from "@chainsafe/lodestar-config"; import {phase0, Root, Slot, allForks, ssz} from "@chainsafe/lodestar-types"; import {ErrorAborted, ILogger, sleep} from "@chainsafe/lodestar-utils"; -import {List, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {AbortSignal} from "@chainsafe/abort-controller"; import {IBeaconChain} from "../../chain"; @@ -20,7 +20,6 @@ import {BackfillSyncError, BackfillSyncErrorCode} from "./errors"; import {verifyBlockProposerSignature, verifyBlockSequence, BackfillBlockHeader, BackfillBlock} from "./verify"; import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {byteArrayEquals} from "../../util/bytes"; -import {TreeBacked} from "@chainsafe/ssz"; import {computeAnchorCheckpoint} from "../../chain/initState"; /** * Timeout in ms to take a break from reading a backfillBatchSize from db, as just yielding @@ -35,7 +34,7 @@ export type BackfillSyncModules = { config: IBeaconConfig; logger: ILogger; metrics: IMetrics | null; - anchorState: TreeBacked; + anchorState: BeaconStateAllForks; wsCheckpoint?: phase0.Checkpoint; signal: AbortSignal; }; @@ -745,7 +744,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} } private async syncBlockByRoot(peer: PeerId, anchorBlockRoot: Root): Promise { - const [anchorBlock] = await this.network.reqResp.beaconBlocksByRoot(peer, [anchorBlockRoot] as List); + const [anchorBlock] = await this.network.reqResp.beaconBlocksByRoot(peer, [anchorBlockRoot]); if (anchorBlock == null) throw new Error("InvalidBlockSyncedFromPeer"); // GENESIS_SLOT doesn't has valid signature diff --git a/packages/lodestar/src/sync/unknownBlock.ts b/packages/lodestar/src/sync/unknownBlock.ts index 2ae9f03e64b..77f2642c4e3 100644 --- a/packages/lodestar/src/sync/unknownBlock.ts +++ b/packages/lodestar/src/sync/unknownBlock.ts @@ -1,7 +1,7 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {ILogger} from "@chainsafe/lodestar-utils"; import {allForks, Root, RootHex} from "@chainsafe/lodestar-types"; -import {fromHexString, List, toHexString} from "@chainsafe/ssz"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; import {INetwork, NetworkEvent, PeerAction} from "../network"; import {IBeaconChain} from "../chain"; import {IMetrics} from "../metrics"; @@ -239,7 +239,7 @@ export class UnknownBlockSync { for (let i = 0; i < MAX_ATTEMPTS_PER_BLOCK; i++) { const peer = shuffledPeers[i % shuffledPeers.length]; try { - const [signedBlock] = await this.network.reqResp.beaconBlocksByRoot(peer, [blockRoot] as List); + const [signedBlock] = await this.network.reqResp.beaconBlocksByRoot(peer, [blockRoot]); // Peer does not have the block, try with next peer if (signedBlock === undefined) { diff --git a/packages/lodestar/src/util/multifork.ts b/packages/lodestar/src/util/multifork.ts index 986a5ba5fc4..167f152137f 100644 --- a/packages/lodestar/src/util/multifork.ts +++ b/packages/lodestar/src/util/multifork.ts @@ -1,7 +1,6 @@ import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {allForks, Slot} from "@chainsafe/lodestar-types"; import {bytesToInt} from "@chainsafe/lodestar-utils"; -import {ContainerType} from "@chainsafe/ssz"; /** * Slot uint64 @@ -38,7 +37,7 @@ const SLOT_BYTES_POSITION_IN_STATE = 40; export function getSignedBlockTypeFromBytes( config: IChainForkConfig, bytes: Buffer | Uint8Array -): ContainerType { +): allForks.AllForksSSZTypes["SignedBeaconBlock"] { const slot = getSlotFromBytes(bytes); return config.getForkTypes(slot).SignedBeaconBlock; } @@ -50,7 +49,7 @@ export function getSlotFromBytes(bytes: Buffer | Uint8Array): Slot { export function getStateTypeFromBytes( config: IChainForkConfig, bytes: Buffer | Uint8Array -): ContainerType { +): allForks.AllForksSSZTypes["BeaconState"] { const slot = bytesToInt(bytes.slice(SLOT_BYTES_POSITION_IN_STATE, SLOT_BYTES_POSITION_IN_STATE + SLOT_BYTE_COUNT)); return config.getForkTypes(slot).BeaconState; } diff --git a/packages/lodestar/src/util/tree.ts b/packages/lodestar/src/util/tree.ts deleted file mode 100644 index 27d2b42b47b..00000000000 --- a/packages/lodestar/src/util/tree.ts +++ /dev/null @@ -1,14 +0,0 @@ -import {TreeBacked, List} from "@chainsafe/ssz"; - -export function getTreeAtIndex(tree: TreeBacked>, index: number): TreeBacked> { - const newTree = tree.clone(); - let maxIndex = newTree.length - 1; - if (index > maxIndex) { - throw new Error(`Cannot get tree for index: ${index}, maxIndex: ${maxIndex}`); - } - while (maxIndex > index) { - newTree.pop(); - maxIndex = newTree.length - 1; - } - return newTree; -} diff --git a/packages/lodestar/test/e2e/chain/lightclient.test.ts b/packages/lodestar/test/e2e/chain/lightclient.test.ts index dc18fd6e32e..cbc82c8844a 100644 --- a/packages/lodestar/test/e2e/chain/lightclient.test.ts +++ b/packages/lodestar/test/e2e/chain/lightclient.test.ts @@ -142,10 +142,7 @@ describe("chain / lightclient", function () { throw Error(`LC head state not in cache ${stateRootHex}`); } - const stateLcFromProof = ssz.altair.BeaconState.createTreeBackedFromProof( - header.stateRoot as Uint8Array, - proof - ); + const stateLcFromProof = ssz.altair.BeaconState.createFromProof(proof, header.stateRoot as Uint8Array); expect(toHexString(stateLcFromProof.latestBlockHeader.bodyRoot)).to.equal( toHexString(lcHeadState.latestBlockHeader.bodyRoot), `Recovered 'latestBlockHeader.bodyRoot' from state ${stateRootHex} not correct` diff --git a/packages/lodestar/test/e2e/eth1/eth1ForBlockProduction.test.ts b/packages/lodestar/test/e2e/eth1/eth1ForBlockProduction.test.ts index 22a2dea9cf2..bc5fe5661ae 100644 --- a/packages/lodestar/test/e2e/eth1/eth1ForBlockProduction.test.ts +++ b/packages/lodestar/test/e2e/eth1/eth1ForBlockProduction.test.ts @@ -13,11 +13,11 @@ import {getTestnetConfig, medallaTestnetConfig} from "../../utils/testnet"; import {testLogger} from "../../utils/logger"; import {BeaconDb} from "../../../src/db"; import {generateState} from "../../utils/state"; -import {fromHexString, List, toHexString} from "@chainsafe/ssz"; -import {Root, ssz} from "@chainsafe/lodestar-types"; -import {createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {ssz} from "@chainsafe/lodestar-types"; import {Eth1Provider} from "../../../src/eth1/provider/eth1Provider"; import {getGoerliRpcUrl} from "../../testParams"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; const dbLocation = "./.__testdb"; @@ -103,8 +103,8 @@ describe("eth1 / Eth1Provider", function () { const periodStart = maxTimestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE; // Compute correct deposit root tree - const depositRootTree = ssz.phase0.DepositDataRootList.createTreeBackedFromStruct( - pyrmontDepositsDataRoot.map((root) => fromHexString(root)) as List + const depositRootTree = ssz.phase0.DepositDataRootList.toViewDU( + pyrmontDepositsDataRoot.map((root) => fromHexString(root)) ); const tbState = generateState( @@ -125,7 +125,7 @@ describe("eth1 / Eth1Provider", function () { config ); - const state = createCachedBeaconState(config, tbState); + const state = createCachedBeaconStateTest(tbState, config); const result = await eth1ForBlockProduction.getEth1DataAndDeposits(state); expect(result.eth1Data).to.deep.equal(latestEth1Data, "Wrong eth1Data for block production"); diff --git a/packages/lodestar/test/e2e/interop/genesisState.test.ts b/packages/lodestar/test/e2e/interop/genesisState.test.ts new file mode 100644 index 00000000000..19b8abb364c --- /dev/null +++ b/packages/lodestar/test/e2e/interop/genesisState.test.ts @@ -0,0 +1,92 @@ +import {expect} from "chai"; +import {toHexString} from "@chainsafe/ssz"; +import {LevelDbController} from "@chainsafe/lodestar-db"; +import {config} from "@chainsafe/lodestar-config/default"; +import {BeaconDb} from "../../../src"; +import {initDevState} from "../../../src/node/utils/state"; +import {testLogger} from "../../utils/logger"; +import {interopDeposits} from "../../../src/node/utils/interop/deposits"; +import {ssz} from "@chainsafe/lodestar-types"; + +describe("interop / initDevState", () => { + let db: BeaconDb; + const logger = testLogger(); + + before(async () => { + db = new BeaconDb({ + config, + controller: new LevelDbController({name: ".tmpdb"}, {logger}), + }); + await db.start(); + }); + + after(async () => { + await db.stop(); + }); + + it("Create interop deposits", () => { + const deposits = interopDeposits(config, ssz.phase0.DepositDataRootList.defaultViewDU(), 1); + + /* eslint-disable @typescript-eslint/naming-convention */ + expect(deposits.map((deposit) => ssz.phase0.Deposit.toJson(deposit))).to.deep.equal([ + { + proof: [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7", + "0x0100000000000000000000000000000000000000000000000000000000000000", + ], + data: { + pubkey: "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c", + withdrawal_credentials: "0x00fad2a6bfb0e7f1f0f45460944fbd8dfa7f37da06a4d13b3983cc90bb46963b", + amount: "32000000000", + signature: + "0xa95af8ff0f8c06af4d29aef05ce865f85f82df42b606008ec5b1bcb42b17ae47f4b78cdce1db31ce32d18f42a6b296b4014a2164981780e56b5a40d7723c27b8423173e58fa36f075078b177634f66351412b867c103f532aedd50bcd9b98446", + }, + }, + ]); + }); + + it("Create correct genesisState", async () => { + const validatorCount = 8; + const state = await initDevState(config, db, validatorCount, { + genesisTime: 1644000000, + eth1BlockHash: Buffer.alloc(32, 0xaa), + eth1Timestamp: 1644000000, + }); + + expect(toHexString(state.hashTreeRoot())).to.equal( + "0x3ef3bda2cee48ebdbb6f7a478046631bad3b5eeda3543e55d9dd39da230425bb", + "Wrong genesis state root" + ); + }); +}); diff --git a/packages/lodestar/test/e2e/keymanager/keymanager.test.ts b/packages/lodestar/test/e2e/keymanager/keymanager.test.ts index fa976e1fa70..827aa8dd385 100644 --- a/packages/lodestar/test/e2e/keymanager/keymanager.test.ts +++ b/packages/lodestar/test/e2e/keymanager/keymanager.test.ts @@ -9,7 +9,7 @@ import {chainConfig as chainConfigDef} from "@chainsafe/lodestar-config/default" import {HttpClient} from "@chainsafe/lodestar-api/src"; import {getClient} from "@chainsafe/lodestar-api/src/keymanager/client"; import {ISlashingProtection, Validator} from "@chainsafe/lodestar-validator"; -import {ByteVector, fromHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {WinstonLogger} from "@chainsafe/lodestar-utils"; import {ssz} from "@chainsafe/lodestar-types"; import {LogLevel, testLogger, TestLoggerOpts} from "../../utils/logger"; @@ -483,7 +483,7 @@ function dirContainFileWithPubkeyInFilename(dir: string, pubkeys: string[]): boo } // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -function createAttesterDuty(pubkey: ByteVector, slot: number, committeeIndex: number, validatorIndex: number) { +function createAttesterDuty(pubkey: Uint8Array, slot: number, committeeIndex: number, validatorIndex: number) { return { slot: slot, committeeIndex: committeeIndex, diff --git a/packages/lodestar/test/e2e/network/network.test.ts b/packages/lodestar/test/e2e/network/network.test.ts index ceffccd96e3..e12b75a03b3 100644 --- a/packages/lodestar/test/e2e/network/network.test.ts +++ b/packages/lodestar/test/e2e/network/network.test.ts @@ -163,7 +163,7 @@ describe("network", function () { isAggregator: false, }; - netB.metadata.attnets[subscription.subnet] = true; + netB.metadata.attnets.set(subscription.subnet, true); const connected = Promise.all([onPeerConnect(netA), onPeerConnect(netB)]); // Add subnets to B ENR diff --git a/packages/lodestar/test/e2e/network/peers/peerManager.test.ts b/packages/lodestar/test/e2e/network/peers/peerManager.test.ts index 5bf3e2ff712..98187223d5f 100644 --- a/packages/lodestar/test/e2e/network/peers/peerManager.test.ts +++ b/packages/lodestar/test/e2e/network/peers/peerManager.test.ts @@ -3,6 +3,7 @@ import {EventEmitter} from "events"; import sinon from "sinon"; import {expect} from "chai"; import {config} from "@chainsafe/lodestar-config/default"; +import {BitArray} from "@chainsafe/ssz"; import {IReqResp, ReqRespMethod} from "../../../../src/network/reqresp"; import {PeerRpcScoreStore, PeerManager} from "../../../../src/network/peers"; import {NetworkEvent, NetworkEventBus} from "../../../../src/network"; @@ -120,7 +121,7 @@ describe("network / peers / PeerManager", function () { } as Connection); const seqNumber = BigInt(2); - const metadata: phase0.Metadata = {seqNumber, attnets: []}; + const metadata: phase0.Metadata = {seqNumber, attnets: BitArray.fromBitLen(0)}; // Simulate peer1 responding with its metadata reqResp.metadata.resolves(metadata); diff --git a/packages/lodestar/test/e2e/network/reqresp.test.ts b/packages/lodestar/test/e2e/network/reqresp.test.ts index 1b3ce44c4df..b33b8b6e0ac 100644 --- a/packages/lodestar/test/e2e/network/reqresp.test.ts +++ b/packages/lodestar/test/e2e/network/reqresp.test.ts @@ -7,6 +7,7 @@ import {config} from "@chainsafe/lodestar-config/default"; import {sleep as _sleep} from "@chainsafe/lodestar-utils"; import {altair, phase0, ssz} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; +import {BitArray} from "@chainsafe/ssz"; import {createPeerId, IReqRespOptions, Network, prettyPrintPeerId} from "../../../src/network"; import {defaultNetworkOptions, INetworkOptions} from "../../../src/network/options"; import {Method, Encoding} from "../../../src/network/reqresp/types"; @@ -118,8 +119,8 @@ describe("network / ReqResp", function () { const [netA, netB] = await createAndConnectPeers(); // Modify the metadata to make the seqNumber non-zero - netB.metadata.attnets = []; - netB.metadata.attnets = []; + netB.metadata.attnets = BitArray.fromBitLen(0); + netB.metadata.attnets = BitArray.fromBitLen(0); const expectedPong = netB.metadata.seqNumber; expect(expectedPong.toString()).to.deep.equal("2", "seqNumber"); diff --git a/packages/lodestar/test/perf/api/impl/validator/attester.test.ts b/packages/lodestar/test/perf/api/impl/validator/attester.test.ts index 3589239f989..4d6ed8ca911 100644 --- a/packages/lodestar/test/perf/api/impl/validator/attester.test.ts +++ b/packages/lodestar/test/perf/api/impl/validator/attester.test.ts @@ -37,7 +37,7 @@ describe("api / impl / validator", () => { noThreshold: true, fn: () => { for (let i = 0; i < reqCount; i++) { - const pubkey = state.index2pubkey[i]; + const pubkey = state.epochCtx.index2pubkey[i]; pubkey.toBytes(PointFormat.compressed); } }, diff --git a/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index acb359bbee8..01166df9e9b 100644 --- a/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/lodestar/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -1,35 +1,38 @@ import {itBench} from "@dapplion/benchmark"; import {expect} from "chai"; import { - CachedBeaconStateAllForks, + CachedBeaconStateAltair, computeEpochAtSlot, computeStartSlotAtEpoch, getBlockRootAtSlot, } from "@chainsafe/lodestar-beacon-state-transition"; -import {AggregatedAttestationPool, flagIsTimelySource} from "../../../../src/chain/opPools/aggregatedAttestationPool"; -import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {List} from "@chainsafe/ssz"; +import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool"; +import {SLOTS_PER_EPOCH, TIMELY_SOURCE_FLAG_INDEX} from "@chainsafe/lodestar-params"; import {generatePerfTestCachedStateAltair} from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; -import {ssz} from "@chainsafe/lodestar-types"; +import {BitArray} from "@chainsafe/ssz"; + +/** Same to https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/specs/altair/beacon-chain.md#has_flag */ +const TIMELY_SOURCE = 1 << TIMELY_SOURCE_FLAG_INDEX; +function flagIsTimelySource(flag: number): boolean { + return (flag & TIMELY_SOURCE) === TIMELY_SOURCE; +} // Aug 11 2021 // getAttestationsForBlock // ✓ getAttestationsForBlock 4.410948 ops/s 226.7086 ms/op - 64 runs 51.8 s describe("getAttestationsForBlock", () => { - let originalState: CachedBeaconStateAllForks; + let originalState: CachedBeaconStateAltair; before(function () { this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow - originalState = (generatePerfTestCachedStateAltair({ - goBackOneSlot: true, - }) as unknown) as CachedBeaconStateAllForks; - const numPreviousEpochParticipation = originalState.previousEpochParticipation.persistent - .toArray() - .filter((flags) => flagIsTimelySource(flags)).length; - const numCurrentEpochParticipation = originalState.currentEpochParticipation.persistent - .toArray() - .filter((flags) => flagIsTimelySource(flags)).length; + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}); + + const previousEpochParticipationArr = originalState.previousEpochParticipation.getAll(); + const currentEpochParticipationArr = originalState.currentEpochParticipation.getAll(); + + const numPreviousEpochParticipation = previousEpochParticipationArr.filter(flagIsTimelySource).length; + const numCurrentEpochParticipation = currentEpochParticipationArr.filter(flagIsTimelySource).length; expect(numPreviousEpochParticipation).to.equal(250000, "Wrong numPreviousEpochParticipation"); expect(numCurrentEpochParticipation).to.equal(250000, "Wrong numCurrentEpochParticipation"); @@ -45,19 +48,19 @@ describe("getAttestationsForBlock", () => { }); }); -function getAggregatedAttestationPool(state: CachedBeaconStateAllForks): AggregatedAttestationPool { +function getAggregatedAttestationPool(state: CachedBeaconStateAltair): AggregatedAttestationPool { const pool = new AggregatedAttestationPool(); for (let epochSlot = 0; epochSlot < SLOTS_PER_EPOCH; epochSlot++) { const slot = state.slot - 1 - epochSlot; const epoch = computeEpochAtSlot(slot); - const committeeCount = state.getCommitteeCountPerSlot(epoch); + const committeeCount = state.epochCtx.getCommitteeCountPerSlot(epoch); const sourceCheckpoint = { epoch: state.currentJustifiedCheckpoint.epoch, - root: state.currentJustifiedCheckpoint.root.valueOf() as Uint8Array, + root: state.currentJustifiedCheckpoint.root, }; for (let committeeIndex = 0; committeeIndex < committeeCount; committeeIndex++) { const attestation = { - aggregationBits: Array.from({length: 64}, () => false) as List, + aggregationBits: BitArray.fromBitLen(64), data: { slot: slot, index: committeeIndex, @@ -71,10 +74,10 @@ function getAggregatedAttestationPool(state: CachedBeaconStateAllForks): Aggrega signature: Buffer.alloc(96), }; - const committee = state.getBeaconCommittee(slot, committeeIndex); + const committee = state.epochCtx.getBeaconCommittee(slot, committeeIndex); // all attestation has full participation so getAttestationsForBlock() has to do a lot of filter - // aggregate_and_proof messages are all TreeBacked - pool.add(ssz.phase0.Attestation.createTreeBackedFromStruct(attestation), committee, committee); + // aggregate_and_proof messages + pool.add(attestation, committee, committee); } } return pool; diff --git a/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts b/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts index 629d3714d3a..159bc8689b4 100644 --- a/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts +++ b/packages/lodestar/test/perf/chain/validation/aggregateAndProof.test.ts @@ -1,5 +1,4 @@ import {itBench} from "@dapplion/benchmark"; -import {ssz} from "@chainsafe/lodestar-types"; import {validateGossipAggregateAndProof} from "../../../../src/chain/validation"; import {generateTestCachedBeaconStateOnlyValidators} from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; import {getAggregateAndProofValidData} from "../../../utils/validationData/aggregateAndProof"; @@ -14,9 +13,8 @@ describe("validate gossip signedAggregateAndProof", () => { }); const aggStruct = signedAggregateAndProof; - const aggTreeBacked = ssz.phase0.SignedAggregateAndProof.createTreeBackedFromStruct(aggStruct); - for (const [id, agg] of Object.entries({struct: aggStruct, treeBacked: aggTreeBacked})) { + for (const [id, agg] of Object.entries({struct: aggStruct})) { itBench({ id: `validate gossip signedAggregateAndProof - ${id}`, beforeEach: () => chain.seenAggregators["validatorIndexesByEpoch"].clear(), diff --git a/packages/lodestar/test/perf/chain/validation/attestation.test.ts b/packages/lodestar/test/perf/chain/validation/attestation.test.ts index 29f3e3dc612..aece2d0a60a 100644 --- a/packages/lodestar/test/perf/chain/validation/attestation.test.ts +++ b/packages/lodestar/test/perf/chain/validation/attestation.test.ts @@ -1,5 +1,4 @@ import {itBench} from "@dapplion/benchmark"; -import {ssz} from "@chainsafe/lodestar-types"; import {validateGossipAttestation} from "../../../../src/chain/validation"; import {generateTestCachedBeaconStateOnlyValidators} from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; import {getAttestationValidData} from "../../../utils/validationData/attestation"; @@ -14,9 +13,8 @@ describe("validate gossip attestation", () => { }); const attStruct = attestation; - const attTreeBacked = ssz.phase0.Attestation.createTreeBackedFromStruct(attStruct); - for (const [id, att] of Object.entries({struct: attStruct, treeBacked: attTreeBacked})) { + for (const [id, att] of Object.entries({struct: attStruct})) { itBench({ id: `validate gossip attestation - ${id}`, beforeEach: () => chain.seenAttesters["validatorIndexesByEpoch"].clear(), diff --git a/packages/lodestar/test/perf/eth1/pickEth1Vote.test.ts b/packages/lodestar/test/perf/eth1/pickEth1Vote.test.ts index 3d604b01a27..0c72c1da7d6 100644 --- a/packages/lodestar/test/perf/eth1/pickEth1Vote.test.ts +++ b/packages/lodestar/test/perf/eth1/pickEth1Vote.test.ts @@ -1,7 +1,8 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {phase0, ssz} from "@chainsafe/lodestar-types"; import {fastSerializeEth1Data, pickEth1Vote} from "../../../src/eth1/utils/eth1Vote"; -import {ContainerType, ListType} from "@chainsafe/ssz"; +import {ContainerType, ListCompositeType} from "@chainsafe/ssz"; +import {newFilledArray, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; describe("eth1 / pickEth1Vote", () => { const ETH1_FOLLOW_DISTANCE_MAINNET = 2048; @@ -9,23 +10,23 @@ describe("eth1 / pickEth1Vote", () => { const SLOTS_PER_EPOCH_MAINNET = 32; const eth1DataVotesLimit = EPOCHS_PER_ETH1_VOTING_PERIOD_MAINNET * SLOTS_PER_EPOCH_MAINNET; - const stateMainnetType = new ContainerType({ - fields: { - eth1DataVotes: new ListType({elementType: ssz.phase0.Eth1Data, limit: eth1DataVotesLimit}), - }, + const stateMainnetType = new ContainerType({ + eth1DataVotes: new ListCompositeType(ssz.phase0.Eth1Data, eth1DataVotesLimit), }); - const stateNoVotes = stateMainnetType.defaultTreeBacked(); - const stateMaxVotes = stateMainnetType.defaultTreeBacked(); + const stateNoVotes = stateMainnetType.defaultViewDU(); + const stateMaxVotes = stateMainnetType.defaultViewDU(); - stateMaxVotes.eth1DataVotes = Array.from({length: eth1DataVotesLimit}, () => - ssz.phase0.Eth1Data.createTreeBackedFromStruct({ + // Must convert all instances to create a cache + stateMaxVotes.eth1DataVotes = ssz.phase0.Eth1DataVotes.toViewDU( + newFilledArray(eth1DataVotesLimit, { depositRoot: Buffer.alloc(32, 0xdd), // All votes are the same depositCount: 1e6, blockHash: Buffer.alloc(32, 0xdd), }) ); + stateMaxVotes.commit(); // votesToConsider range: // lte: periodStart - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE, @@ -38,11 +39,11 @@ describe("eth1 / pickEth1Vote", () => { })); itBench("pickEth1Vote - no votes", () => { - pickEth1Vote(stateNoVotes, votesToConsider); + pickEth1Vote((stateNoVotes as unknown) as BeaconStateAllForks, votesToConsider); }); itBench("pickEth1Vote - max votes", () => { - pickEth1Vote(stateMaxVotes, votesToConsider); + pickEth1Vote((stateMaxVotes as unknown) as BeaconStateAllForks, votesToConsider); }); }); @@ -63,7 +64,7 @@ describe("eth1 / pickEth1Vote serializers", () => { depositCount: 1e6, blockHash: Buffer.alloc(32, 0xdd), }; - const eth1DataTree = ssz.phase0.Eth1Data.createTreeBackedFromStruct(eth1DataValue); + const eth1DataTree = ssz.phase0.Eth1Data.toViewDU(eth1DataValue); itBench(`pickEth1Vote - Eth1Data hashTreeRoot value x${ETH1_FOLLOW_DISTANCE_MAINNET}`, () => { for (let i = 0; i < ETH1_FOLLOW_DISTANCE_MAINNET; i++) { @@ -75,9 +76,7 @@ describe("eth1 / pickEth1Vote serializers", () => { itBench({ id: `pickEth1Vote - Eth1Data hashTreeRoot tree x${ETH1_FOLLOW_DISTANCE_MAINNET}`, beforeEach: () => - Array.from({length: ETH1_FOLLOW_DISTANCE_MAINNET}, () => - ssz.phase0.Eth1Data.createTreeBackedFromStruct(eth1DataValue) - ), + Array.from({length: ETH1_FOLLOW_DISTANCE_MAINNET}, () => ssz.phase0.Eth1Data.toViewDU(eth1DataValue)), fn: (eth1DataTrees) => { for (let i = 0; i < eth1DataTrees.length; i++) { ssz.phase0.Eth1Data.hashTreeRoot(eth1DataTrees[i]); diff --git a/packages/lodestar/test/sim/threaded/types.ts b/packages/lodestar/test/sim/threaded/types.ts index b4a64f3b1d8..f52b3767b0f 100644 --- a/packages/lodestar/test/sim/threaded/types.ts +++ b/packages/lodestar/test/sim/threaded/types.ts @@ -2,7 +2,6 @@ import {IChainConfig} from "@chainsafe/lodestar-config"; import {RecursivePartial} from "@chainsafe/lodestar-utils"; import {ChainEvent} from "../../../src/chain"; import {IBeaconNodeOptions} from "../../../src/node/options"; -import {Json} from "@chainsafe/ssz"; export type NodeWorkerOptions = { params: Pick; @@ -26,5 +25,5 @@ export enum MessageEvent { } export type Message = - | {event: ChainEvent.justified; checkpoint: Json} + | {event: ChainEvent.justified; checkpoint: unknown} | {event: MessageEvent.NodeStarted; multiaddr: string}; diff --git a/packages/lodestar/test/spec/allForks/epochProcessing.ts b/packages/lodestar/test/spec/allForks/epochProcessing.ts index c718b117e15..b4caefe7ee9 100644 --- a/packages/lodestar/test/spec/allForks/epochProcessing.ts +++ b/packages/lodestar/test/spec/allForks/epochProcessing.ts @@ -3,17 +3,16 @@ import fs from "node:fs"; import { CachedBeaconStateAllForks, - allForks, EpochProcess, - createCachedBeaconState, + BeaconStateAllForks, beforeProcessEpoch, } from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import {ssz} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {getConfig} from "./util"; import {IBaseSpecTest} from "../type"; @@ -23,8 +22,8 @@ export type EpochProcessFn = (state: CachedBeaconStateAllForks, epochProcess: Ep * https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/epoch_processing/README.md */ type EpochProcessingStateTestCase = IBaseSpecTest & { - pre: allForks.BeaconState; - post: allForks.BeaconState; + pre: BeaconStateAllForks; + post: BeaconStateAllForks; }; /** @@ -39,18 +38,21 @@ export function epochProcessing(fork: ForkName, epochProcessFns: Record( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/epoch_processing/${testDir}`, join(rootDir, `${testDir}/pyspec_tests`), (testcase) => { - const stateTB = (testcase.pre as TreeBacked).clone(); - const state = createCachedBeaconState(getConfig(fork), stateTB); + const stateTB = testcase.pre.clone(); + const state = createCachedBeaconStateTest(stateTB, getConfig(fork)); + const epochProcess = beforeProcessEpoch(state); epochProcessFn(state, epochProcess); + state.commit(); + return state; }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, diff --git a/packages/lodestar/test/spec/allForks/finality.ts b/packages/lodestar/test/spec/allForks/finality.ts index 029897d28fb..460148618c4 100644 --- a/packages/lodestar/test/spec/allForks/finality.ts +++ b/packages/lodestar/test/spec/allForks/finality.ts @@ -1,49 +1,39 @@ import {join} from "node:path"; -import {TreeBacked} from "@chainsafe/ssz"; -import { - CachedBeaconStateAllForks, - allForks, - altair, - createCachedBeaconState, -} from "@chainsafe/lodestar-beacon-state-transition"; +import {allForks, altair, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import {bellatrix, ssz} from "@chainsafe/lodestar-types"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest, shouldVerify} from "../type"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {getConfig} from "./util"; import {generateBlocksSZZTypeMapping} from "./sanity"; /* eslint-disable @typescript-eslint/naming-convention */ export function finality(fork: ForkName): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/finality/finality`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/finality/finality/pyspec_tests`), (testcase) => { - let wrappedState = createCachedBeaconState( - getConfig(fork), - testcase.pre as TreeBacked - ) as CachedBeaconStateAllForks; + let state = createCachedBeaconStateTest(testcase.pre, getConfig(fork)); const verify = shouldVerify(testcase); for (let i = 0; i < testcase.meta.blocks_count; i++) { const signedBlock = testcase[`blocks_${i}`] as bellatrix.SignedBeaconBlock; - wrappedState = allForks.stateTransition( - wrappedState, - ssz[fork].SignedBeaconBlock.createTreeBackedFromStruct(signedBlock), - { - verifyStateRoot: false, - verifyProposer: verify, - verifySignatures: verify, - } - ); + state = allForks.stateTransition(state, signedBlock, { + verifyStateRoot: false, + verifyProposer: verify, + verifySignatures: verify, + }); } - return wrappedState; + + state.commit(); + return state; }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, @@ -72,6 +62,6 @@ interface IFinalityTestCase extends IBaseSpecTest { blocks_count: number; bls_setting: bigint; }; - pre: altair.BeaconState; - post?: altair.BeaconState; + pre: BeaconStateAllForks; + post?: BeaconStateAllForks; } diff --git a/packages/lodestar/test/spec/allForks/fork.ts b/packages/lodestar/test/spec/allForks/fork.ts index 41a822e8dd5..d3641c940e7 100644 --- a/packages/lodestar/test/spec/allForks/fork.ts +++ b/packages/lodestar/test/spec/allForks/fork.ts @@ -1,25 +1,25 @@ import {join} from "node:path"; -import {TreeBacked} from "@chainsafe/ssz"; -import {allForks, phase0, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {allForks, phase0, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; +import {createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; import {ssz} from "@chainsafe/lodestar-types"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest} from "../type"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; -import {createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; export function fork(forkConfig: Partial, pre: ForkName, fork: Exclude): void { const testConfig = createIChainForkConfig(forkConfig); - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/fork/fork`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/fork/fork/pyspec_tests`), (testcase) => { - const preState = createCachedBeaconState(testConfig, testcase.pre as TreeBacked); + const preState = createCachedBeaconStateTest(testcase.pre, testConfig); return allForks.upgradeStateByFork[fork](preState); }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[pre].BeaconState, post: ssz[fork].BeaconState, @@ -35,9 +35,9 @@ export function fork(forkConfig: Partial, pre: ForkName, fork: Exc ); } -type PostBeaconState = Exclude; +type PostBeaconState = Exclude; interface IUpgradeStateCase extends IBaseSpecTest { - pre: allForks.BeaconState; + pre: BeaconStateAllForks; post: PostBeaconState; } diff --git a/packages/lodestar/test/spec/allForks/forkChoice.ts b/packages/lodestar/test/spec/allForks/forkChoice.ts index b8a4d5fe5da..56adbbeedc6 100644 --- a/packages/lodestar/test/spec/allForks/forkChoice.ts +++ b/packages/lodestar/test/spec/allForks/forkChoice.ts @@ -1,7 +1,6 @@ import {join} from "node:path"; import {expect} from "chai"; import { - createCachedBeaconState, phase0, allForks, computeEpochAtSlot, @@ -10,6 +9,7 @@ import { getEffectiveBalanceIncrementsZeroInactive, computeStartSlotAtEpoch, EffectiveBalanceIncrements, + BeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; import {initializeForkChoice} from "@chainsafe/lodestar/src/chain/forkChoice"; @@ -24,12 +24,15 @@ import {CheckpointWithHex, ForkChoiceError, ForkChoiceErrorCode, IForkChoice} fr import {ssz, RootHex} from "@chainsafe/lodestar-types"; import {bnToNum} from "@chainsafe/lodestar-utils"; import {ACTIVE_PRESET, SLOTS_PER_EPOCH, ForkName} from "@chainsafe/lodestar-params"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; import {testLogger} from "../../utils/logger"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {getConfig} from "./util"; /* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/naming-convention */ + const ANCHOR_STATE_FILE_NAME = "anchor_state"; const ANCHOR_BLOCK_FILE_NAME = "anchor_block"; const BLOCK_FILE_NAME = "^(block)_([0-9a-zA-Z]+)$"; @@ -40,14 +43,13 @@ const logger = testLogger("spec-test"); export function forkChoiceTest(fork: ForkName): void { for (const testFolder of ["get_head", "on_block"]) { describeDirectorySpecTest( - `${ACTIVE_PRESET}/${fork}/fork_choice/get_head`, + `${ACTIVE_PRESET}/${fork}/fork_choice/${testFolder}`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/fork_choice/${testFolder}/pyspec_tests`), (testcase) => { const {steps, anchorState} = testcase; const currentSlot = anchorState.slot; const config = getConfig(fork); - const tbState = config.getForkTypes(currentSlot).BeaconState.createTreeBackedFromStruct(anchorState); - let state = createCachedBeaconState(config, tbState); + let state = createCachedBeaconStateTest(anchorState, config); const emitter = new ChainEventEmitter(); const forkchoice = initializeForkChoice(config, emitter, currentSlot, state, true); @@ -62,11 +64,14 @@ export function forkChoiceTest(fork: ForkName): void { for (const [i, step] of steps.entries()) { if (isTick(step)) { tickTime = bnToNum(step.tick); - forkchoice.updateTime(Math.floor(tickTime / config.SECONDS_PER_SLOT)); + const currentSlot = Math.floor(tickTime / config.SECONDS_PER_SLOT); + logger.debug("Step tick", {currentSlot, valid: Boolean(step.valid), time: tickTime}); + forkchoice.updateTime(currentSlot); } // attestation step else if (isAttestation(step)) { + logger.debug("Step attestation", {root: step.attestation, valid: Boolean(step.valid)}); const attestation = testcase.attestations.get(step.attestation); if (!attestation) throw Error(`No attestation ${step.attestation}`); forkchoice.onAttestation(state.epochCtx.getIndexedAttestation(attestation)); @@ -76,6 +81,13 @@ export function forkChoiceTest(fork: ForkName): void { else if (isBlock(step)) { const signedBlock = testcase.blocks.get(step.block); if (!signedBlock) throw Error(`No block ${step.block}`); + + // Log the BeaconBlock root instead of the SignedBeaconBlock root, forkchoice references BeaconBlock roots + const blockRoot = config + .getForkTypes(signedBlock.message.slot) + .BeaconBlock.hashTreeRoot(signedBlock.message); + logger.debug("Step block", {slot: signedBlock.message.slot, root: toHexString(blockRoot)}); + const preState = stateCache.get(toHexString(signedBlock.message.parentRoot)); if (!preState) { continue; @@ -83,6 +95,8 @@ export function forkChoiceTest(fork: ForkName): void { } const blockDelaySec = (tickTime - preState.genesisTime) % config.SECONDS_PER_SLOT; state = runStateTranstion(preState, signedBlock, forkchoice, checkpointStateCache, blockDelaySec); + // TODO: May be part of runStateTranstion, necessary to commit again? + state.commit(); cacheState(state, stateCache); } @@ -192,13 +206,13 @@ function runStateTranstion( postState = allForks.processSlots(postState, nextEpochSlot, null); cacheCheckpointState(postState, checkpointCache); } - preEpoch = postState.currentShuffling.epoch; + preEpoch = postState.epochCtx.epoch; postState = allForks.stateTransition(postState, signedBlock, { verifyStateRoot: true, verifyProposer: false, verifySignatures: false, }); - const postEpoch = postState.currentShuffling.epoch; + const postEpoch = postState.epochCtx.epoch; if (postEpoch > preEpoch) { cacheCheckpointState(postState, checkpointCache); } @@ -246,14 +260,13 @@ function runStateTranstion( } function cacheCheckpointState(checkpointState: CachedBeaconStateAllForks, checkpointCache: CheckpointStateCache): void { - const config = checkpointState.config; const slot = checkpointState.slot; if (slot % SLOTS_PER_EPOCH !== 0) { throw new Error(`Invalid checkpoint state slot ${checkpointState.slot}`); } const blockHeader = ssz.phase0.BeaconBlockHeader.clone(checkpointState.latestBlockHeader); if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { - blockHeader.stateRoot = config.getForkTypes(slot).BeaconState.hashTreeRoot(checkpointState); + blockHeader.stateRoot = checkpointState.hashTreeRoot(); } const cp: phase0.Checkpoint = { root: ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader), @@ -324,7 +337,7 @@ interface IForkChoiceTestCase { description?: string; bls_setting: BigInt; }; - anchorState: allForks.BeaconState; + anchorState: BeaconStateAllForks; anchorBlock: allForks.BeaconBlock; steps: Step[]; blocks: Map; diff --git a/packages/lodestar/test/spec/allForks/genesis.ts b/packages/lodestar/test/spec/allForks/genesis.ts index 7f4258ee19f..a80ae6f2b72 100644 --- a/packages/lodestar/test/spec/allForks/genesis.ts +++ b/packages/lodestar/test/spec/allForks/genesis.ts @@ -1,9 +1,13 @@ import {join} from "node:path"; import {expect} from "chai"; -import {phase0, Uint64, Root, ssz, allForks, bellatrix} from "@chainsafe/lodestar-types"; -import {TreeBacked} from "@chainsafe/ssz"; +import {phase0, Root, ssz, bellatrix, TimeSeconds} from "@chainsafe/lodestar-types"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import {initializeBeaconStateFromEth1, isValidGenesisState} from "@chainsafe/lodestar-beacon-state-transition"; +import { + BeaconStateAllForks, + createEmptyEpochContextImmutableData, + initializeBeaconStateFromEth1, + isValidGenesisState, +} from "@chainsafe/lodestar-beacon-state-transition"; import {bnToNum} from "@chainsafe/lodestar-utils"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; @@ -11,10 +15,13 @@ import {expectEqualBeaconState} from "../util"; import {IBaseSpecTest} from "../type"; import {getConfig} from "./util"; +// The aim of the genesis tests is to provide a baseline to test genesis-state initialization and test if the +// proposed genesis-validity conditions are working. + /* eslint-disable @typescript-eslint/naming-convention */ export function genesis(fork: ForkName): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/genesis/initialization`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/genesis/initialization/pyspec_tests`), (testcase) => { @@ -22,21 +29,33 @@ export function genesis(fork: ForkName): void { for (let i = 0; i < testcase.meta.deposits_count; i++) { deposits.push(testcase[`deposits_${i}`] as phase0.Deposit); } - let executionPayloadHeader: TreeBacked | undefined = undefined; - if (testcase["execution_payload_header"]) { - executionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.createTreeBackedFromStruct( - testcase["execution_payload_header"] - ); - } + + const config = getConfig(fork); + const immutableData = createEmptyEpochContextImmutableData(config, { + // TODO: Should the genesisValidatorsRoot be random here? + genesisValidatorsRoot: Buffer.alloc(32, 0), + }); + return initializeBeaconStateFromEth1( getConfig(fork), + immutableData, ssz.Root.fromJson((testcase.eth1 as IGenesisInitCase).eth1_block_hash), bnToNum((testcase.eth1 as IGenesisInitCase).eth1_timestamp), deposits, undefined, - executionPayloadHeader - ) as phase0.BeaconState; + testcase["execution_payload_header"] && + ssz.bellatrix.ExecutionPayloadHeader.toViewDU(testcase["execution_payload_header"]) + ); }, + // eth1.yaml + // ``` + // {eth1_block_hash: '0x1212121212121212121212121212121212121212121212121212121212121212', + // eth1_timestamp: 1578009600} + // ``` + // meta.yaml + // ``` + // {deposits_count: 64} + // ``` { inputTypes: {meta: InputType.YAML, eth1: InputType.YAML}, sszTypes: { @@ -54,6 +73,11 @@ export function genesis(fork: ForkName): void { } ); + interface IGenesisValidityTestCase extends IBaseSpecTest { + is_valid: boolean; + genesis: BeaconStateAllForks; + } + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/genesis/validity`, join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/genesis/validity/pyspec_tests`), @@ -65,8 +89,6 @@ export function genesis(fork: ForkName): void { is_valid: InputType.YAML, genesis: InputType.SSZ_SNAPPY, }, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore sszTypes: { genesis: ssz[fork].BeaconState, }, @@ -89,20 +111,15 @@ function generateDepositSSZTypeMapping(n: number): Record { const {proof: specTestProof, state} = testcase; - const stateTB = state as TreeBacked; - const stateRoot = stateTB.hashTreeRoot(); + const stateRoot = state.hashTreeRoot(); const leaf = fromHexString(specTestProof.leaf); const branch = specTestProof.branch.map((item) => fromHexString(item)); const leafIndex = Number(specTestProof.leaf_index); const depth = Math.floor(Math.log2(leafIndex)); const verified = verifyMerkleBranch(leaf, branch, depth, leafIndex % 2 ** depth, stateRoot); expect(verified, "cannot verify merkle branch").to.be.true; - const lodestarProof = stateTB.tree.getProof({ + + const lodestarProof = new Tree(state.node).getProof({ gindex: specTestProof.leaf_index, type: ProofType.single, }) as SingleProof; + return { leaf: toHexString(lodestarProof.leaf), leaf_index: lodestarProof.gindex, @@ -40,11 +42,9 @@ export function merkle(fork: ForkName): void { state: {type: InputType.SSZ_SNAPPY as const, treeBacked: true as const}, proof: InputType.YAML as const, }, - getSszTypes: () => { - return { - state: ssz[fork].BeaconState, - }; - }, + getSszTypes: () => ({ + state: ssz[fork].BeaconState, + }), timeout: 10000, getExpected: (testCase) => testCase.proof, expectFunc: (testCase, expected, actual) => { @@ -54,7 +54,7 @@ export function merkle(fork: ForkName): void { ); interface IMerkleTestCase extends IBaseSpecTest { - state: allForks.BeaconState; + state: BeaconStateAllForks; proof: IProof; } diff --git a/packages/lodestar/test/spec/allForks/operations.ts b/packages/lodestar/test/spec/allForks/operations.ts index d5f5fd2a568..766f24ed1a0 100644 --- a/packages/lodestar/test/spec/allForks/operations.ts +++ b/packages/lodestar/test/spec/allForks/operations.ts @@ -1,31 +1,29 @@ import fs from "node:fs"; import {join} from "node:path"; -import {CachedBeaconState, allForks, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; import {ssz} from "@chainsafe/lodestar-types"; -import {TreeBacked, Type} from "@chainsafe/ssz"; +import {Type} from "@chainsafe/ssz"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {IBaseSpecTest} from "../type"; import {getConfig} from "./util"; /* eslint-disable @typescript-eslint/naming-convention */ -export type BlockProcessFn = ( - state: CachedBeaconState, - testCase: any -) => void; +export type BlockProcessFn = (state: T, testCase: any) => void; -export type OperationsTestCase = IBaseSpecTest & { - pre: BeaconState; - post: BeaconState; +export type OperationsTestCase = IBaseSpecTest & { + pre: BeaconStateAllForks; + post: BeaconStateAllForks; execution: {execution_valid: boolean}; }; -export function operations( +export function operations( fork: ForkName, - operationFns: Record>, + operationFns: Record>, sszTypes?: Record> ): void { const rootDir = join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/operations`); @@ -40,17 +38,18 @@ export function operations( throw Error(`No operationFn for ${testDir}`); } - describeDirectorySpecTest, BeaconState>( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/operations/${testDir}`, join(rootDir, `${testDir}/pyspec_tests`), (testcase) => { - const stateTB = (testcase.pre as TreeBacked).clone(); - const state = createCachedBeaconState(getConfig(fork), stateTB); - operationFn(state, testcase); + const state = testcase.pre.clone(); + const cachedState = createCachedBeaconStateTest(state, getConfig(fork)); + operationFn(cachedState as T, testcase); + state.commit(); return state; }, { - inputTypes: {...inputTypeSszTreeBacked, execution: InputType.YAML}, + inputTypes: {...inputTypeSszTreeViewDU, execution: InputType.YAML}, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, diff --git a/packages/lodestar/test/spec/allForks/rewards.ts b/packages/lodestar/test/spec/allForks/rewards.ts index 270bd6d2fd7..1cc4a3617d3 100644 --- a/packages/lodestar/test/spec/allForks/rewards.ts +++ b/packages/lodestar/test/spec/allForks/rewards.ts @@ -5,17 +5,18 @@ import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import { altair, phase0, - allForks, CachedBeaconStatePhase0, - createCachedBeaconState, + BeaconStateAllForks, + BeaconStateAltair, beforeProcessEpoch, } from "@chainsafe/lodestar-beacon-state-transition"; -import {TreeBacked, VectorType} from "@chainsafe/ssz"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; +import {VectorCompositeType} from "@chainsafe/ssz"; import {ssz} from "@chainsafe/lodestar-types"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest} from "../type"; -import {inputTypeSszTreeBacked} from "../util"; +import {inputTypeSszTreeViewDU} from "../util"; import {getConfig} from "./util"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -29,10 +30,7 @@ export function rewards(fork: ForkName): void { } } -const Deltas = new VectorType({ - elementType: ssz.altair.BeaconState.fields.balances, - length: 2, -}); +const deltasType = new VectorCompositeType(ssz.phase0.Balances, 2); export function rewardsPhase0(fork: ForkName): void { const rootDir = join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/rewards`); @@ -42,19 +40,19 @@ export function rewardsPhase0(fork: ForkName): void { join(rootDir, `${testDir}/pyspec_tests`), (testcase) => { const config = getConfig(fork); - const wrappedState = createCachedBeaconState(config, testcase.pre as TreeBacked); + const wrappedState = createCachedBeaconStateTest(testcase.pre, config); const epochProcess = beforeProcessEpoch(wrappedState); return phase0.getAttestationDeltas(wrappedState as CachedBeaconStatePhase0, epochProcess); }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, - source_deltas: Deltas, - target_deltas: Deltas, - head_deltas: Deltas, - inclusion_delay_deltas: Deltas, - inactivity_penalty_deltas: Deltas, + source_deltas: deltasType, + target_deltas: deltasType, + head_deltas: deltasType, + inclusion_delay_deltas: deltasType, + inactivity_penalty_deltas: deltasType, }, timeout: 100000000, getExpected: (testCase) => @@ -81,7 +79,7 @@ export function rewardsAltair(fork: ForkName): void { join(rootDir, `${testDir}/pyspec_tests`), (testcase) => { const config = getConfig(fork); - const state = createCachedBeaconState(config, testcase.pre as TreeBacked); + const state = createCachedBeaconStateTest(testcase.pre as BeaconStateAltair, config); const epochProcess = beforeProcessEpoch(state); // To debug this test and get granular results you can tweak inputs to get more granular results // @@ -98,13 +96,13 @@ export function rewardsAltair(fork: ForkName): void { return altair.getRewardsAndPenalties(state, epochProcess); }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, - head_deltas: Deltas, - source_deltas: Deltas, - target_deltas: Deltas, - inactivity_penalty_deltas: Deltas, + head_deltas: deltasType, + source_deltas: deltasType, + target_deltas: deltasType, + inactivity_penalty_deltas: deltasType, }, getExpected: (testCase) => sumDeltas([ @@ -124,7 +122,7 @@ export function rewardsAltair(fork: ForkName): void { type Deltas = [number[], number[]]; interface RewardTestCasePhase0 extends IBaseSpecTest { - pre: altair.BeaconState; + pre: BeaconStateAllForks; source_deltas: Deltas; target_deltas: Deltas; head_deltas: Deltas; @@ -133,7 +131,7 @@ interface RewardTestCasePhase0 extends IBaseSpecTest { } interface RewardTestCaseAltair extends IBaseSpecTest { - pre: altair.BeaconState; + pre: BeaconStateAllForks; head_deltas: Deltas; source_deltas: Deltas; target_deltas: Deltas; diff --git a/packages/lodestar/test/spec/allForks/sanity.ts b/packages/lodestar/test/spec/allForks/sanity.ts index a28d00fff31..b7c689b598c 100644 --- a/packages/lodestar/test/spec/allForks/sanity.ts +++ b/packages/lodestar/test/spec/allForks/sanity.ts @@ -1,11 +1,11 @@ import {join} from "node:path"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import {allForks, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; -import {TreeBacked} from "@chainsafe/ssz"; +import {allForks, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {bellatrix, ssz} from "@chainsafe/lodestar-types"; import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {bnToNum} from "@chainsafe/lodestar-utils"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest, shouldVerify} from "../type"; import {getConfig} from "./util"; @@ -18,17 +18,19 @@ export function sanity(fork: ForkName): void { } export function sanitySlot(fork: ForkName): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/sanity/slots`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/sanity/slots/pyspec_tests`), (testcase) => { - const stateTB = (testcase.pre as TreeBacked).clone(); - const state = createCachedBeaconState(getConfig(fork), stateTB); + const stateTB = testcase.pre.clone(); + const state = createCachedBeaconStateTest(stateTB, getConfig(fork)); const postState = allForks.processSlots(state, state.slot + bnToNum(testcase.slots)); - return postState.type.createTreeBacked(postState.tree); + // TODO: May be part of runStateTranstion, necessary to commit again? + postState.commit(); + return postState; }, { - inputTypes: {...inputTypeSszTreeBacked, slots: InputType.YAML}, + inputTypes: {...inputTypeSszTreeViewDU, slots: InputType.YAML}, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, @@ -44,29 +46,25 @@ export function sanitySlot(fork: ForkName): void { } export function sanityBlock(fork: ForkName, testPath: string): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/sanity/blocks`, join(SPEC_TEST_LOCATION, testPath), (testcase) => { - const stateTB = testcase.pre as TreeBacked; - let wrappedState = createCachedBeaconState(getConfig(fork), stateTB); + const stateTB = testcase.pre; + let wrappedState = createCachedBeaconStateTest(stateTB, getConfig(fork)); const verify = shouldVerify(testcase); for (let i = 0; i < testcase.meta.blocks_count; i++) { const signedBlock = testcase[`blocks_${i}`] as bellatrix.SignedBeaconBlock; - wrappedState = allForks.stateTransition( - wrappedState, - ssz[fork].SignedBeaconBlock.createTreeBackedFromStruct(signedBlock), - { - verifyStateRoot: verify, - verifyProposer: verify, - verifySignatures: verify, - } - ); + wrappedState = allForks.stateTransition(wrappedState, signedBlock, { + verifyStateRoot: verify, + verifyProposer: verify, + verifySignatures: verify, + }); } return wrappedState; }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, @@ -98,12 +96,12 @@ interface IBlockSanityTestCase extends IBaseSpecTest { blocks_count: number; bls_setting: bigint; }; - pre: allForks.BeaconState; - post: allForks.BeaconState; + pre: BeaconStateAllForks; + post: BeaconStateAllForks; } interface IProcessSlotsTestCase extends IBaseSpecTest { - pre: allForks.BeaconState; - post?: allForks.BeaconState; + pre: BeaconStateAllForks; + post?: BeaconStateAllForks; slots: bigint; } diff --git a/packages/lodestar/test/spec/allForks/ssz_static.ts b/packages/lodestar/test/spec/allForks/ssz_static.ts index 8ac5e89eebf..a8df955874b 100644 --- a/packages/lodestar/test/spec/allForks/ssz_static.ts +++ b/packages/lodestar/test/spec/allForks/ssz_static.ts @@ -1,113 +1,65 @@ import fs from "node:fs"; import path from "node:path"; -import {describeDirectorySpecTest, InputType, safeType} from "@chainsafe/lodestar-spec-test-util"; -import {Bytes32, ssz} from "@chainsafe/lodestar-types"; -import {expect} from "chai"; -import {CompositeType, ContainerType, Type} from "@chainsafe/ssz"; -import {IBaseSSZStaticTestCase} from "../ssz/type"; +import {ssz} from "@chainsafe/lodestar-types"; +import {Type} from "@chainsafe/ssz"; +import {ACTIVE_PRESET, ForkName} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; -import {ACTIVE_PRESET, ForkName, PresetName} from "@chainsafe/lodestar-params"; +import {replaceUintTypeWithUintBigintType} from "../utils/replaceUintTypeWithUintBigintType"; +import {parseSszStaticTestcase} from "../utils/sszTestCaseParser"; +import {runValidSszTest} from "../utils/runValidSszTest"; -/* eslint-disable @typescript-eslint/naming-convention, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, no-console */ +// ssz_static +// | Attestation +// | case_0 +// | roots.yaml +// | serialized.ssz_snappy +// | value.yaml +// +// Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md + +/* eslint-disable + @typescript-eslint/naming-convention, + @typescript-eslint/no-unsafe-assignment, + @typescript-eslint/no-unsafe-call, + @typescript-eslint/no-unsafe-member-access, + no-console +*/ // eslint-disable-next-line type Types = Record>; -const extraTypes = { - Eth1Block: new ContainerType({ - fields: { - timestamp: ssz.Number64, - depositRoot: ssz.Root, - depositCount: ssz.Number64, - }, - }), -}; - export function sszStatic(fork: ForkName): void { const rootDir = path.join(SPEC_TEST_LOCATION, `tests/${ACTIVE_PRESET}/${fork}/ssz_static`); for (const typeName of fs.readdirSync(rootDir)) { - const type = - ((ssz[fork] as Types)[typeName] as Type | undefined) || - ((ssz.altair as Types)[typeName] as Type | undefined) || - ((ssz.phase0 as Types)[typeName] as Type | undefined) || - ((extraTypes as Types)[typeName] as Type | undefined); + /* eslint-disable @typescript-eslint/strict-boolean-expressions */ + const type = (ssz[fork] as Types)[typeName] || (ssz.altair as Types)[typeName] || (ssz.phase0 as Types)[typeName]; if (!type) { throw Error(`No type for ${typeName}`); } - const sszType = safeType(type) as CompositeType; - testStatic(typeName, sszType, fork, ACTIVE_PRESET); + testStatic(typeName, type, fork, ACTIVE_PRESET); + /* eslint-enable @typescript-eslint/strict-boolean-expressions */ } } -interface IResult { - root: Bytes32; - serialized: Uint8Array; -} - -function testStatic(typeName: string, sszType: CompositeType, forkName: ForkName, preset: PresetName): void { +function testStatic(typeName: string, sszType: Type, forkName: ForkName, preset: string): void { const typeDir = path.join(SPEC_TEST_LOCATION, `tests/${preset}/${forkName}/ssz_static/${typeName}`); for (const caseName of fs.readdirSync(typeDir)) { - describeDirectorySpecTest, IResult>( - `${preset}/${forkName}/ssz_static/${typeName}/${caseName}`, - path.join(typeDir, caseName), - (testcase) => { - //debugger; - const serialized = sszType.serialize(testcase.serialized); - const root = sszType.hashTreeRoot(testcase.serialized); - return { - serialized, - root, - }; - }, - { - inputTypes: { - roots: InputType.YAML, - serialized: InputType.SSZ_SNAPPY, - }, - sszTypes: { - serialized: sszType, - }, - getExpected: (testCase) => { - return { - root: Buffer.from(testCase.roots.root.replace("0x", ""), "hex"), - serialized: testCase.serialized_raw, - }; - }, - expectFunc: (testCase, expected, actual) => { - if (true && (!expected.serialized.equals(actual.serialized) || !expected.root.equals(actual.root))) { - console.log("testCase", testCase); - console.log("serialize expected", expected.serialized.toString("hex")); - console.log("serialize actual ", Buffer.from(actual.serialized).toString("hex")); - console.log("hashTreeRoot expected", expected.root.toString("hex")); - console.log("hashTreeRoot actual ", Buffer.from(actual.root).toString("hex")); - /* - const bbroot = Type.fields[2][1] - console.log("s bbroot hash", bbroot.hashTreeRoot(structural.beaconBlockRoot)) - console.log("t bbroot hash", bbroot.hashTreeRoot(tree.beaconBlockRoot)) - */ - // debugger; + describe(`${preset}/${forkName}/ssz_static/${typeName}/${caseName}`, () => { + const sszTypeNoUint = replaceUintTypeWithUintBigintType(sszType); + const caseDir = path.join(typeDir, caseName); + for (const testId of fs.readdirSync(caseDir)) { + it(testId, function () { + // Mainnet must deal with big full states and hash each one multiple times + if (preset === "mainnet") { + this.timeout(30 * 1000); } - const structural = sszType.deserialize(testCase.serialized_raw); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const tree = sszType.createTreeBackedFromBytes(testCase.serialized_raw); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const treeFromStructural = sszType.createTreeBackedFromStruct(structural); - expect(tree.serialize(), "tree serialization != structural serialization").to.deep.equal( - sszType.serialize(structural) - ); - expect(sszType.equals(tree, treeFromStructural), "tree != treeFromStructural"); - expect(expected.serialized.equals(sszType.serialize(structural))); - expect(expected.root.equals(sszType.hashTreeRoot(structural))); - expect(expected.serialized.equals(actual.serialized), "incorrect serialize").to.be.true; - expect(expected.root.equals(actual.root), "incorrect hashTreeRoot").to.be.true; - }, - //shouldSkip: (a, b, i) => i !== 0, - //shouldSkip: (a, b, i) => b !== 'case_10', + + const testData = parseSszStaticTestcase(path.join(caseDir, testId)); + runValidSszTest(sszTypeNoUint, testData); + }); } - ); + }); } } diff --git a/packages/lodestar/test/spec/allForks/transition.ts b/packages/lodestar/test/spec/allForks/transition.ts index 383664fd760..8b45a45d135 100644 --- a/packages/lodestar/test/spec/allForks/transition.ts +++ b/packages/lodestar/test/spec/allForks/transition.ts @@ -1,61 +1,40 @@ import {join} from "node:path"; -import {allForks, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {allForks, BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {ssz} from "@chainsafe/lodestar-types"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util"; import {createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; import {ForkName, ACTIVE_PRESET} from "@chainsafe/lodestar-params"; -import {TreeBacked} from "@chainsafe/ssz"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest} from "../type"; -import {expectEqualBeaconState, inputTypeSszTreeBacked} from "../util"; +import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../util"; import {bnToNum} from "@chainsafe/lodestar-utils"; - -type CreateTreeBackedSignedBlock = (block: allForks.SignedBeaconBlock) => TreeBacked; -const createTreeBackedSignedBlockByFork: Record = { - // Dummy placeholder Fn for phase0 - [ForkName.phase0]: ssz.phase0.SignedBeaconBlock.createTreeBackedFromStruct.bind( - ssz.phase0.SignedBeaconBlock - ) as CreateTreeBackedSignedBlock, - [ForkName.altair]: ssz.altair.SignedBeaconBlock.createTreeBackedFromStruct.bind( - ssz.altair.SignedBeaconBlock - ) as CreateTreeBackedSignedBlock, - [ForkName.bellatrix]: ssz.bellatrix.SignedBeaconBlock.createTreeBackedFromStruct.bind( - ssz.bellatrix.SignedBeaconBlock - ) as CreateTreeBackedSignedBlock, -}; +import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState"; export function transition( forkConfig: (forkEpoch: number) => Partial, pre: ForkName, fork: Exclude ): void { - describeDirectorySpecTest( + describeDirectorySpecTest( `${ACTIVE_PRESET}/${fork}/transition`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/${fork}/transition/core/pyspec_tests`), (testcase) => { const meta = testcase.meta; // testConfig is used here to load forkEpoch from meta.yaml const testConfig = createIChainForkConfig(forkConfig(bnToNum(meta.fork_epoch))); - let wrappedState = createCachedBeaconState(testConfig, testcase.pre as TreeBacked); + let state = createCachedBeaconStateTest(testcase.pre, testConfig); for (let i = 0; i < meta.blocks_count; i++) { - let tbSignedBlock: TreeBacked; - if (i <= meta.fork_block) { - const signedBlock = testcase[`blocks_${i}`] as allForks.SignedBeaconBlock; - tbSignedBlock = createTreeBackedSignedBlockByFork[pre](signedBlock); - } else { - const signedBlock = testcase[`blocks_${i}`] as allForks.SignedBeaconBlock; - tbSignedBlock = createTreeBackedSignedBlockByFork[fork](signedBlock); - } - wrappedState = allForks.stateTransition(wrappedState, tbSignedBlock, { + const signedBlock = testcase[`blocks_${i}`] as allForks.SignedBeaconBlock; + state = allForks.stateTransition(state, signedBlock, { verifyStateRoot: true, verifyProposer: false, verifySignatures: false, }); } - return wrappedState; + return state; }, { - inputTypes: inputTypeSszTreeBacked, + inputTypes: inputTypeSszTreeViewDU, getSszTypes: (meta: ITransitionTestCase["meta"]) => { return { pre: ssz[pre].BeaconState, @@ -100,6 +79,6 @@ interface ITransitionTestCase extends IBaseSpecTest { blocks_count: bigint; bls_setting?: bigint; }; - pre: allForks.BeaconState; - post: allForks.BeaconState; + pre: BeaconStateAllForks; + post: BeaconStateAllForks; } diff --git a/packages/lodestar/test/spec/altair/epoch_processing.test.ts b/packages/lodestar/test/spec/altair/epoch_processing.test.ts index 03924e049f8..0c1d765b9e2 100644 --- a/packages/lodestar/test/spec/altair/epoch_processing.test.ts +++ b/packages/lodestar/test/spec/altair/epoch_processing.test.ts @@ -1,5 +1,5 @@ import {allForks, altair} from "@chainsafe/lodestar-beacon-state-transition"; -import {processParticipationRecordUpdates} from "@chainsafe/lodestar-beacon-state-transition/src/phase0/epoch/processParticipationRecordUpdates"; +import {phase0} from "@chainsafe/lodestar-beacon-state-transition"; import {ForkName} from "@chainsafe/lodestar-params"; import {EpochProcessFn, epochProcessing} from "../allForks/epochProcessing"; @@ -12,7 +12,7 @@ epochProcessing(ForkName.altair, { inactivity_updates: altair.processInactivityUpdates as EpochProcessFn, justification_and_finalization: allForks.processJustificationAndFinalization, participation_flag_updates: altair.processParticipationFlagUpdates as EpochProcessFn, - participation_record_updates: (processParticipationRecordUpdates as unknown) as EpochProcessFn, + participation_record_updates: (phase0.processParticipationRecordUpdates as unknown) as EpochProcessFn, randao_mixes_reset: allForks.processRandaoMixesReset, registry_updates: allForks.processRegistryUpdates, rewards_and_penalties: altair.processRewardsAndPenalties as EpochProcessFn, diff --git a/packages/lodestar/test/spec/altair/operations.test.ts b/packages/lodestar/test/spec/altair/operations.test.ts index cacb20ad546..a8fa46774b0 100644 --- a/packages/lodestar/test/spec/altair/operations.test.ts +++ b/packages/lodestar/test/spec/altair/operations.test.ts @@ -1,4 +1,4 @@ -import {CachedBeaconStateAllForks, allForks, altair} from "@chainsafe/lodestar-beacon-state-transition"; +import {CachedBeaconStateAltair, allForks, altair} from "@chainsafe/lodestar-beacon-state-transition"; import {phase0, ssz} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; import {IBaseSpecTest, shouldVerify} from "../type"; @@ -7,20 +7,20 @@ import {operations, BlockProcessFn} from "../allForks/operations"; /* eslint-disable @typescript-eslint/naming-convention */ // Define above to re-use in sync_aggregate and sync_aggregate_random -const sync_aggregate: BlockProcessFn = ( +const sync_aggregate: BlockProcessFn = ( state, testCase: IBaseSpecTest & {sync_aggregate: altair.SyncAggregate} ) => { - const block = ssz.altair.BeaconBlock.defaultTreeBacked(); + const block = ssz.altair.BeaconBlock.defaultValue(); // processSyncAggregate() needs the full block to get the slot block.slot = state.slot; - block.body.syncAggregate = testCase["sync_aggregate"]; + block.body.syncAggregate = ssz.altair.SyncAggregate.toViewDU(testCase["sync_aggregate"]); altair.processSyncAggregate(state, block); }; -operations(ForkName.altair, { +operations(ForkName.altair, { attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { altair.processAttestations(state, [testCase.attestation]); }, @@ -30,7 +30,7 @@ operations(ForkName.altair, { }, block_header: (state, testCase: IBaseSpecTest & {block: altair.BeaconBlock}) => { - allForks.processBlockHeader(state as CachedBeaconStateAllForks, testCase.block); + allForks.processBlockHeader(state, testCase.block); }, deposit: (state, testCase: IBaseSpecTest & {deposit: phase0.Deposit}) => { diff --git a/packages/lodestar/test/spec/bellatrix/operations.test.ts b/packages/lodestar/test/spec/bellatrix/operations.test.ts index 44a715a803a..012fb92a301 100644 --- a/packages/lodestar/test/spec/bellatrix/operations.test.ts +++ b/packages/lodestar/test/spec/bellatrix/operations.test.ts @@ -16,22 +16,22 @@ import {processExecutionPayload} from "@chainsafe/lodestar-beacon-state-transiti /* eslint-disable @typescript-eslint/naming-convention */ // Define above to re-use in sync_aggregate and sync_aggregate_random -const sync_aggregate: BlockProcessFn = ( +const sync_aggregate: BlockProcessFn = ( state, testCase: IBaseSpecTest & {sync_aggregate: altair.SyncAggregate} ) => { - const block = ssz.altair.BeaconBlock.defaultTreeBacked(); + const block = ssz.altair.BeaconBlock.defaultValue(); // processSyncAggregate() needs the full block to get the slot block.slot = state.slot; - block.body.syncAggregate = testCase["sync_aggregate"]; + block.body.syncAggregate = ssz.altair.SyncAggregate.toViewDU(testCase["sync_aggregate"]); - altair.processSyncAggregate((state as unknown) as CachedBeaconStateAltair, block); + altair.processSyncAggregate((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, block); }; -operations(ForkName.bellatrix, { +operations(ForkName.bellatrix, { attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { - altair.processAttestations((state as unknown) as CachedBeaconStateAltair, [testCase.attestation]); + altair.processAttestations((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, [testCase.attestation]); }, attester_slashing: (state, testCase: IBaseSpecTest & {attester_slashing: phase0.AttesterSlashing}) => { @@ -39,11 +39,11 @@ operations(ForkName.bellatrix, { }, block_header: (state, testCase: IBaseSpecTest & {block: altair.BeaconBlock}) => { - allForks.processBlockHeader(state as CachedBeaconStateAllForks, testCase.block); + allForks.processBlockHeader(state, testCase.block); }, deposit: (state, testCase: IBaseSpecTest & {deposit: phase0.Deposit}) => { - altair.processDeposit((state as unknown) as CachedBeaconStateAltair, testCase.deposit); + altair.processDeposit((state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, testCase.deposit); }, proposer_slashing: (state, testCase: IBaseSpecTest & {proposer_slashing: phase0.ProposerSlashing}) => { @@ -54,15 +54,20 @@ operations(ForkName.bellatrix, { sync_aggregate_random: sync_aggregate, voluntary_exit: (state, testCase: IBaseSpecTest & {voluntary_exit: phase0.SignedVoluntaryExit}) => { - altair.processVoluntaryExit((state as unknown) as CachedBeaconStateAltair, testCase.voluntary_exit); + altair.processVoluntaryExit( + (state as CachedBeaconStateAllForks) as CachedBeaconStateAltair, + testCase.voluntary_exit + ); }, execution_payload: ( state, testCase: IBaseSpecTest & {execution_payload: bellatrix.ExecutionPayload; execution: {execution_valid: boolean}} ) => { - processExecutionPayload((state as unknown) as CachedBeaconStateBellatrix, testCase.execution_payload, { - notifyNewPayload: () => testCase.execution.execution_valid, - }); + processExecutionPayload( + (state as CachedBeaconStateAllForks) as CachedBeaconStateBellatrix, + testCase.execution_payload, + {notifyNewPayload: () => testCase.execution.execution_valid} + ); }, }); diff --git a/packages/lodestar/test/spec/phase0/operations.test.ts b/packages/lodestar/test/spec/phase0/operations.test.ts index ac0865d6223..c1cbe1ff29c 100644 --- a/packages/lodestar/test/spec/phase0/operations.test.ts +++ b/packages/lodestar/test/spec/phase0/operations.test.ts @@ -1,4 +1,4 @@ -import {CachedBeaconStateAllForks, allForks, phase0} from "@chainsafe/lodestar-beacon-state-transition"; +import {CachedBeaconStatePhase0, allForks, phase0} from "@chainsafe/lodestar-beacon-state-transition"; import {ForkName} from "@chainsafe/lodestar-params"; import {IBaseSpecTest, shouldVerify} from "../type"; import {operations} from "../allForks/operations"; @@ -6,7 +6,7 @@ import {operations} from "../allForks/operations"; /* eslint-disable @typescript-eslint/naming-convention */ /** Describe with which function to run each directory of tests */ -operations(ForkName.phase0, { +operations(ForkName.phase0, { attestation: (state, testCase: IBaseSpecTest & {attestation: phase0.Attestation}) => { phase0.processAttestation(state, testCase.attestation); }, @@ -16,7 +16,7 @@ operations(ForkName.phase0, { }, block_header: (state, testCase: IBaseSpecTest & {block: phase0.BeaconBlock}) => { - allForks.processBlockHeader(state as CachedBeaconStateAllForks, testCase.block); + allForks.processBlockHeader(state, testCase.block); }, deposit: (state, testCase: IBaseSpecTest & {deposit: phase0.Deposit}) => { diff --git a/packages/lodestar/test/spec/phase0/shuffling.test.ts b/packages/lodestar/test/spec/phase0/shuffling.test.ts index 62e8a456bed..e54a177ef21 100644 --- a/packages/lodestar/test/spec/phase0/shuffling.test.ts +++ b/packages/lodestar/test/spec/phase0/shuffling.test.ts @@ -1,31 +1,31 @@ import {join} from "node:path"; import {unshuffleList} from "@chainsafe/lodestar-beacon-state-transition"; import {describeDirectorySpecTest, InputType} from "@chainsafe/lodestar-spec-test-util"; -import {Uint64} from "@chainsafe/lodestar-types"; import {ACTIVE_PRESET} from "@chainsafe/lodestar-params"; import {SPEC_TEST_LOCATION} from "../specTestVersioning"; import {IBaseSpecTest} from "../type"; +import {bnToNum, fromHex} from "@chainsafe/lodestar-utils"; describeDirectorySpecTest( `${ACTIVE_PRESET}/phase0/shuffling/`, join(SPEC_TEST_LOCATION, `/tests/${ACTIVE_PRESET}/phase0/shuffling/core/shuffle`), (testcase) => { - const seed = Buffer.from(testcase.mapping.seed.replace("0x", ""), "hex"); - const output = Array.from({length: Number(testcase.mapping.count)}, (_, i) => i); + const seed = fromHex(testcase.mapping.seed); + const output = Array.from({length: bnToNum(testcase.mapping.count)}, (_, i) => i); unshuffleList(output, seed); return output; }, { inputTypes: {mapping: InputType.YAML}, timeout: 10000, - getExpected: (testCase) => testCase.mapping.mapping.map((value) => Number(value)), + getExpected: (testCase) => testCase.mapping.mapping.map((value) => bnToNum(value)), } ); interface IShufflingTestCase extends IBaseSpecTest { mapping: { seed: string; - count: Uint64; - mapping: Uint64[]; + count: bigint; + mapping: bigint[]; }; } diff --git a/packages/lodestar/test/spec/ssz/generic/index.test.ts b/packages/lodestar/test/spec/ssz/generic/index.test.ts index f1ca99be5df..7fd0755cf01 100644 --- a/packages/lodestar/test/spec/ssz/generic/index.test.ts +++ b/packages/lodestar/test/spec/ssz/generic/index.test.ts @@ -1,29 +1,13 @@ import {expect} from "chai"; import path from "node:path"; import fs from "node:fs"; -// eslint-disable-next-line no-restricted-imports -import {parseInvalidTestcase, parseValidTestcase} from "@chainsafe/lodestar-spec-test-util/lib/sszGeneric"; -import {CompositeType, isCompositeType, toHexString, Type} from "@chainsafe/ssz"; import {SPEC_TEST_LOCATION} from "../../specTestVersioning"; - -// Test types defined here +import {parseSszGenericValidTestcase, parseSszGenericInvalidTestcase} from "../../utils/sszTestCaseParser"; +import {runValidSszTest} from "../../utils/runValidSszTest"; import {getTestType} from "./types"; const rootGenericSszPath = path.join(SPEC_TEST_LOCATION, "tests", "general", "phase0", "ssz_generic"); -// ssz_generic -// | basic_vector -// | invalid -// | vec_bool_0 -// | serialized.ssz_snappy -// | valid -// | vec_bool_1_max -// | meta.yaml -// | serialized.ssz_snappy -// | value.yaml -// -// Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_generic/README.md - for (const testType of fs.readdirSync(rootGenericSszPath)) { const testTypePath = path.join(rootGenericSszPath, testType); @@ -31,8 +15,19 @@ for (const testType of fs.readdirSync(rootGenericSszPath)) { const invalidCasesPath = path.join(testTypePath, "invalid"); for (const invalidCase of fs.readdirSync(invalidCasesPath)) { it(invalidCase, () => { + // TODO: Strong type errors and assert that the entire it() throws known errors + if (invalidCase.endsWith("_0")) { + expect(() => getTestType(testType, invalidCase), "Must throw constructing type").to.throw(); + return; + } + const type = getTestType(testType, invalidCase); - const testData = parseInvalidTestcase(path.join(invalidCasesPath, invalidCase)); + const testData = parseSszGenericInvalidTestcase(path.join(invalidCasesPath, invalidCase)); + + /* eslint-disable no-console */ + if (process.env.DEBUG) { + console.log({serialized: Buffer.from(testData.serialized).toString("hex")}); + } // Unlike the valid suite, invalid encodings do not have any value or hash tree root. The serialized data // should simply not be decoded without raising an error. @@ -55,48 +50,13 @@ for (const testType of fs.readdirSync(rootGenericSszPath)) { it(validCase, () => { const type = getTestType(testType, validCase); - - const testData = parseValidTestcase(path.join(validCasesPath, validCase), type); - const testDataSerialized = toHexString(testData.serialized); - const testDataRoot = testData.root; - - const serialized = wrapErr(() => type.serialize(testData.value), "type.serialize()"); - const value = wrapErr(() => type.deserialize(testData.serialized), "type.deserialize()"); - const root = wrapErr(() => type.hashTreeRoot(testData.value), "type.hashTreeRoot()"); - const valueSerdes = wrapErr(() => type.deserialize(serialized), "type.deserialize(serialized)"); - - expect(valueSerdes).to.deep.equal(testData.value, "round trip serdes"); - expect(toHexString(serialized)).to.equal(testDataSerialized, "struct serialize"); - expect(value).to.deep.equal(testData.value, "struct deserialize"); - expect(toHexString(root)).to.equal(testDataRoot, "struct hashTreeRoot"); - - // If the type is composite, test tree-backed ops - // eslint-disable-next-line @typescript-eslint/no-explicit-any - if (!isCompositeType(type as Type)) return; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const compositeType = type as CompositeType; - - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const treebackedValue = compositeType.createTreeBackedFromStruct(testData.value); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const treeToStruct = compositeType.tree_convertToStruct(treebackedValue.tree); - - expect(treeToStruct).to.deep.equal(testData.value, "tree-backed to struct"); - expect(type.equals(testData.value, treebackedValue), "struct - tree-backed type.equals()").to.be.true; - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call - expect(toHexString(treebackedValue.serialize())).to.equal(testDataSerialized, "tree-backed serialize"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call - expect(toHexString(treebackedValue.hashTreeRoot())).to.equal(testDataRoot, "tree-backed hashTreeRoot"); + const testData = parseSszGenericValidTestcase(path.join(validCasesPath, validCase)); + runValidSszTest(type, { + root: testData.root, + serialized: testData.serialized, + jsonValue: testData.jsonValue, + }); }); } }); } - -function wrapErr(fn: () => T, prefix: string): T { - try { - return fn(); - } catch (e) { - (e as Error).message = `${prefix}: ${(e as Error).message}`; - throw e; - } -} diff --git a/packages/lodestar/test/spec/ssz/generic/types.ts b/packages/lodestar/test/spec/ssz/generic/types.ts index 153f5f84d9e..387e198867c 100644 --- a/packages/lodestar/test/spec/ssz/generic/types.ts +++ b/packages/lodestar/test/spec/ssz/generic/types.ts @@ -1,33 +1,39 @@ import { - BigIntUintType, - BitListType, + Type, + BooleanType, + UintBigintType, + UintNumberType, BitVectorType, - booleanType, - byteType, + BitListType, ContainerType, - ListType, - Type, - VectorType, + ListBasicType, + VectorBasicType, + VectorCompositeType, } from "@chainsafe/ssz"; +const bool = new BooleanType(); +const byte = new UintNumberType(1); +const uint8 = new UintNumberType(1); +const uint16 = new UintNumberType(2); +const uint32 = new UintNumberType(4); +const uint64 = new UintBigintType(8); +const uint128 = new UintBigintType(16); +const uint256 = new UintBigintType(32); + /* eslint-disable @typescript-eslint/naming-convention */ // class SingleFieldTestStruct(Container): // A: byte const SingleFieldTestStruct = new ContainerType({ - fields: { - a: byteType, - }, + A: byte, }); // class SmallTestStruct(Container): // A: uint16 // B: uint16 const SmallTestStruct = new ContainerType({ - fields: { - a: new BigIntUintType({byteLength: 16 / 8}), - b: new BigIntUintType({byteLength: 16 / 8}), - }, + A: uint16, + B: uint16, }); // class FixedTestStruct(Container): @@ -35,11 +41,9 @@ const SmallTestStruct = new ContainerType({ // B: uint64 // C: uint32 const FixedTestStruct = new ContainerType({ - fields: { - a: new BigIntUintType({byteLength: 8 / 8}), - b: new BigIntUintType({byteLength: 64 / 8}), - c: new BigIntUintType({byteLength: 32 / 8}), - }, + A: uint8, + B: uint64, + C: uint32, }); // class VarTestStruct(Container): @@ -47,11 +51,9 @@ const FixedTestStruct = new ContainerType({ // B: List[uint16, 1024] // C: uint8 const VarTestStruct = new ContainerType({ - fields: { - a: new BigIntUintType({byteLength: 16 / 8}), - b: new ListType({elementType: new BigIntUintType({byteLength: 16 / 8}), limit: 1024}), - c: new BigIntUintType({byteLength: 8 / 8}), - }, + A: uint16, + B: new ListBasicType(uint16, 1024), + C: uint8, }); // class ComplexTestStruct(Container): @@ -63,15 +65,13 @@ const VarTestStruct = new ContainerType({ // F: Vector[FixedTestStruct, 4] // G: Vector[VarTestStruct, 2] const ComplexTestStruct = new ContainerType({ - fields: { - a: new BigIntUintType({byteLength: 16 / 8}), - b: new ListType({elementType: new BigIntUintType({byteLength: 16 / 8}), limit: 128}), - c: new BigIntUintType({byteLength: 8 / 8}), - d: new BitListType({limit: 256}), - e: VarTestStruct, - f: new VectorType({elementType: FixedTestStruct, length: 4}), - g: new VectorType({elementType: VarTestStruct, length: 2}), - }, + A: uint16, + B: new ListBasicType(uint16, 128), + C: uint8, + D: new BitListType(256), + E: VarTestStruct, + F: new VectorCompositeType(FixedTestStruct, 4), + G: new VectorCompositeType(VarTestStruct, 2), }); // class BitsStruct(Container): @@ -81,13 +81,11 @@ const ComplexTestStruct = new ContainerType({ // D: Bitlist[6] // E: Bitvector[8] const BitsStruct = new ContainerType({ - fields: { - a: new BitListType({limit: 5}), - b: new BitVectorType({length: 2}), - c: new BitVectorType({length: 1}), - d: new BitListType({limit: 6}), - e: new BitVectorType({length: 8}), - }, + A: new BitListType(5), + B: new BitVectorType(2), + C: new BitVectorType(1), + D: new BitListType(6), + E: new BitVectorType(8), }); const containerTypes = { @@ -100,13 +98,13 @@ const containerTypes = { }; const vecElementTypes = { - bool: booleanType, - uint8: new BigIntUintType({byteLength: 8 / 8}), - uint16: new BigIntUintType({byteLength: 16 / 8}), - uint32: new BigIntUintType({byteLength: 32 / 8}), - uint64: new BigIntUintType({byteLength: 64 / 8}), - uint128: new BigIntUintType({byteLength: 128 / 8}), - uint256: new BigIntUintType({byteLength: 256 / 8}), + bool, + uint8, + uint16, + uint32, + uint64, + uint128, + uint256, }; export function getTestType(testType: string, testCase: string): Type { @@ -121,28 +119,29 @@ export function getTestType(testType: string, testCase: string): Type { if (elementType === undefined) throw Error(`No vecElementType for ${elementTypeStr}: '${testCase}'`); const length = parseInt(lengthStr); if (isNaN(length)) throw Error(`Bad length ${length}: '${testCase}'`); - return new VectorType({elementType, length}); + return new VectorBasicType(elementType, length); } // `bitlist_{limit}` // {limit}: the list limit, in bits, of the bitlist. case "bitlist": { // Consider case `bitlist_no_delimiter_empty` - const limit = testCase.includes("no_delimiter") ? 0 : parseSecondNum(testCase, "limit"); + // Set bitLen to a random big value. 0 is invalid and will throw at the constructor + const limit = testCase.includes("no_delimiter") ? 1024 : parseSecondNum(testCase, "limit"); // TODO: memoize - return new BitListType({limit}); + return new BitListType(limit); } // `bitvec_{length}` // {length}: the length, in bits, of the bitvector. case "bitvector": { // TODO: memoize - return new BitVectorType({length: parseSecondNum(testCase, "length")}); + return new BitVectorType(parseSecondNum(testCase, "length")); } // A boolean has no type variations. Instead, file names just plainly describe the contents for debugging. case "boolean": - return booleanType; + return bool; // {container name} // {container name}: Any of the container names listed below (excluding the `(Container)` python super type) @@ -158,7 +157,7 @@ export function getTestType(testType: string, testCase: string): Type { // {size}: the uint size: 8, 16, 32, 64, 128 or 256. case "uints": { // TODO: memoize - return new BigIntUintType({byteLength: parseSecondNum(testCase, "size") / 8}); + return new UintBigintType((parseSecondNum(testCase, "size") / 8) as 8); } default: diff --git a/packages/lodestar/test/spec/ssz/util/specTypes/uint.ts b/packages/lodestar/test/spec/ssz/util/specTypes/uint.ts deleted file mode 100644 index 3937c6ded7c..00000000000 --- a/packages/lodestar/test/spec/ssz/util/specTypes/uint.ts +++ /dev/null @@ -1,8 +0,0 @@ -import {IBaseCase} from "@chainsafe/lodestar-spec-test-util"; - -export interface IUintCase extends IBaseCase { - value: string; - ssz: string; - type: string; - valid: boolean; -} diff --git a/packages/lodestar/test/spec/util.ts b/packages/lodestar/test/spec/util.ts index 27ee0f188f8..bfa78c10dff 100644 --- a/packages/lodestar/test/spec/util.ts +++ b/packages/lodestar/test/spec/util.ts @@ -2,15 +2,19 @@ import {expect} from "chai"; import {allForks, ssz} from "@chainsafe/lodestar-types"; import {ForkName} from "@chainsafe/lodestar-params"; import {InputType} from "@chainsafe/lodestar-spec-test-util"; -import {ContainerType} from "@chainsafe/ssz"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; /** Compare each field in BeaconState to help debug failed test easier. */ export function expectEqualBeaconState( fork: ForkName, - expected: allForks.BeaconState, - actual: allForks.BeaconState + expectedView: BeaconStateAllForks, + actualView: BeaconStateAllForks ): void { - const stateType = ssz[fork].BeaconState as ContainerType; + // TODO: Is it cheaper to compare roots? Or maybe the serialized bytes? + const expected = expectedView.toValue(); + const actual = actualView.toValue(); + + const stateType = ssz[fork].BeaconState as allForks.AllForksSSZTypes["BeaconState"]; if (!stateType.equals(actual, expected)) { expect(stateType.toJson(actual)).to.deep.equal(stateType.toJson(expected)); throw Error("Wrong state"); @@ -18,8 +22,8 @@ export function expectEqualBeaconState( } /** Shortcut for commonly used inputType */ -export const inputTypeSszTreeBacked = { - pre: {type: InputType.SSZ_SNAPPY as const, treeBacked: true as const}, - post: {type: InputType.SSZ_SNAPPY as const, treeBacked: true as const}, +export const inputTypeSszTreeViewDU = { + pre: InputType.SSZ_SNAPPY, + post: InputType.SSZ_SNAPPY, meta: InputType.YAML as const, }; diff --git a/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts b/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts new file mode 100644 index 00000000000..57e7f3ac335 --- /dev/null +++ b/packages/lodestar/test/spec/utils/replaceUintTypeWithUintBigintType.ts @@ -0,0 +1,49 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment +, @typescript-eslint/no-explicit-any */ + +import { + Type, + UintNumberType, + UintBigintType, + ContainerType, + ListBasicType, + ListCompositeType, + VectorBasicType, + VectorCompositeType, +} from "@chainsafe/ssz"; + +/** + * Transform the type to something that is safe to deserialize + * + * This mainly entails making sure all numbers are bignumbers + */ +export function replaceUintTypeWithUintBigintType>(type: T): T { + if (type instanceof UintNumberType && type.byteLength === 8) { + return (new UintBigintType(type.byteLength) as unknown) as T; + } + + // For Container iterate and replace all sub properties + if (type instanceof ContainerType) { + const fields = {...type.fields}; + for (const key of Object.keys(fields) as (keyof typeof fields)[]) { + fields[key] = replaceUintTypeWithUintBigintType(fields[key]); + } + return (new ContainerType(fields, type.opts) as unknown) as T; + } + + // For List or vectors replace the subType + if (type instanceof ListBasicType) { + return (new ListBasicType(replaceUintTypeWithUintBigintType(type.elementType), type.limit) as unknown) as T; + } + if (type instanceof VectorBasicType) { + return (new VectorBasicType(replaceUintTypeWithUintBigintType(type.elementType), type.length) as unknown) as T; + } + if (type instanceof ListCompositeType) { + return (new ListCompositeType(replaceUintTypeWithUintBigintType(type.elementType), type.limit) as unknown) as T; + } + if (type instanceof VectorCompositeType) { + return (new VectorCompositeType(replaceUintTypeWithUintBigintType(type.elementType), type.length) as unknown) as T; + } + + return type; +} diff --git a/packages/lodestar/test/spec/utils/runValidSszTest.ts b/packages/lodestar/test/spec/utils/runValidSszTest.ts new file mode 100644 index 00000000000..1c7f9cb0ecc --- /dev/null +++ b/packages/lodestar/test/spec/utils/runValidSszTest.ts @@ -0,0 +1,188 @@ +import {expect} from "chai"; +import {Node} from "@chainsafe/persistent-merkle-tree"; +import {Type, CompositeType, fromHexString, toHexString} from "@chainsafe/ssz"; + +type ValidTestCaseData = { + root: string; + serialized: string | Uint8Array; + jsonValue: unknown; +}; + +/* eslint-disable no-console */ + +export function runValidSszTest(type: Type, testData: ValidTestCaseData): void { + const testDataRootHex = testData.root; + const testDataSerialized = + typeof testData.serialized === "string" ? fromHexString(testData.serialized) : testData.serialized; + const testDataSerializedStr = + typeof testData.serialized === "string" ? testData.serialized : toHexString(testData.serialized); + + if (process.env.RENDER_JSON) { + console.log( + JSON.stringify( + testData.jsonValue, + (key, value: unknown) => (typeof value === "bigint" ? value.toString() : value), + 2 + ) + ); + } + + if (process.env.RENDER_SERIALIZED) { + console.log("serialized", testDataSerializedStr); + } + + // JSON -> value - fromJson() + const testDataValue = wrapErr(() => type.fromJson(testData.jsonValue), "type.fromJson()"); + // Use our re-transformed to JSON to ensure the type is the safe + const testDataJson = wrapErr(() => type.toJson(testDataValue), "type.toJson()"); + + function assertBytes(bytes: Uint8Array, msg: string): void { + expect(toHexString(bytes)).to.equal(testDataSerializedStr, `Wrong serialized - ${msg}`); + } + + function assertValue(value: unknown, msg: string): void { + expect(type.toJson(value)).to.deep.equal(testDataJson, `Wrong json - ${msg}`); + } + + function assertRoot(root: Uint8Array, msg: string): void { + expect(toHexString(root)).to.equal(testDataRootHex, `Wrong root - ${msg}`); + } + + function assertNode(node: Node, msg: string): void { + expect(toHexString(node.root)).to.equal(testDataRootHex, `Wrong node - ${msg}`); + } + + { + // value - equals + const isEqual = wrapErr(() => type.equals(testDataValue, testDataValue), "type.equals()"); + expect(isEqual).to.equal(true, "Value is not equal to itself"); + } + + { + // value - clone + const isEqual = wrapErr(() => type.equals(type.clone(testDataValue), testDataValue), "type.clone()"); + expect(isEqual).to.equal(true, "Cloned value is not equal to itself"); + } + + // value -> bytes - serialize() + const serialized = wrapErr(() => type.serialize(testDataValue), "type.serialize()"); + assertBytes(serialized, "type.serialize()"); + + { + // bytes -> value - deserialize() + const value = wrapErr(() => type.deserialize(copy(testDataSerialized)), "type.deserialize()"); + assertValue(value, "type.deserialize()"); + } + + // To print the chunk roots of a type value + // + // $ RENDER_ROOTS=true ONLY_ID="4 arrays" ../../node_modules/.bin/mocha test/unit/byType/vector/valid.test.ts + // + // 0x0000000000000000000000000000000000000000000000000000000000000000 + if (process.env.RENDER_ROOTS) { + if (type.isBasic) { + console.log("ROOTS Basic", toHexString(type.serialize(testDataValue))); + } else { + const roots = (type as CompositeType)["getRoots"](testDataValue); + console.log( + "ROOTS Composite", + roots.map((root) => toHexString(root)) + ); + } + } + + { + // hashTreeRoot() + const root = wrapErr(() => type.hashTreeRoot(testDataValue), "type.hashTreeRoot()"); + assertRoot(root, "type.hashTreeRoot()"); + } + + // value -> tree - value_toTree() + const node = wrapErr(() => type.value_toTree(testDataValue), "type.value_toTree()"); + assertNode(node, "type.value_toTree()"); + + // To print a tree a single test you are debugging do + // + // $ RENDER_TREE=true ONLY_ID="4 arrays" ../../node_modules/.bin/mocha test/unit/byType/vector/valid.test.ts + // + // '1000' => '0x0000000000000000000000000000000000000000000000000000000000000000', + // '1001' => '0x0000000000000000000000000000000000000000000000000000000000000000', + // '1010' => '0x40e2010000000000000000000000000000000000000000000000000000000000', + // '1011' => '0xf1fb090000000000000000000000000000000000000000000000000000000000', + // '1100' => '0x4794030000000000000000000000000000000000000000000000000000000000', + // '1101' => '0xf8ad0b0000000000000000000000000000000000000000000000000000000000', + // '1110' => '0x4e46050000000000000000000000000000000000000000000000000000000000', + // '1111' => '0xff5f0d0000000000000000000000000000000000000000000000000000000000' + if (process.env.RENDER_TREE) { + renderTree(node); + } + + { + // tree -> bytes + const uint8Array = new Uint8Array(type.tree_serializedSize(node)); + const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); + type.tree_serializeToBytes({uint8Array, dataView}, 0, node); + assertBytes(uint8Array, "type.tree_serializeToBytes"); + } + + { + // tree -> value - + const value = wrapErr(() => type.tree_toValue(node), "type.tree_toValue()"); + assertValue(value, "type.tree_toValue()"); + } + + { + // bytes -> tree + const uint8Array = copy(testDataSerialized); + const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); + const node = type.tree_deserializeFromBytes({uint8Array, dataView}, 0, testDataSerialized.length); + assertNode(node, "type.tree_deserializeFromBytes()"); + } + + if (!type.isBasic) { + const compositeType = type as CompositeType; + + const view = compositeType.deserializeToView(copy(testDataSerialized)); + assertNode(compositeType.commitView(view), "deserializeToView"); + + const viewDU = compositeType.deserializeToViewDU(copy(testDataSerialized)); + assertNode(compositeType.commitViewDU(viewDU), "deserializeToViewDU"); + } +} + +function copy(buf: Uint8Array): Uint8Array { + const copy = new Uint8Array(buf.length); + copy.set(buf); + return copy; +} + +function wrapErr(fn: () => T, prefix: string): T { + try { + return fn(); + } catch (e) { + (e as Error).message = `${prefix}: ${(e as Error).message}`; + throw e; + } +} + +export function toJsonOrString(value: unknown): unknown { + if (typeof value === "number" || typeof value === "bigint") { + return value.toString(10); + } else { + return value; + } +} + +function renderTree(node: Node): void { + console.log(gatherLeafNodes(node)); +} + +export function gatherLeafNodes(node: Node, nodes = new Map(), gindex = "1"): Map { + if (node.isLeaf()) { + nodes.set(gindex, toHexString(node.root)); + } else { + gatherLeafNodes(node.left, nodes, gindex + "0"); + gatherLeafNodes(node.right, nodes, gindex + "1"); + } + return nodes; +} diff --git a/packages/lodestar/test/spec/utils/specTestTypes/beaconStateComparison.ts b/packages/lodestar/test/spec/utils/specTestTypes/beaconStateComparison.ts deleted file mode 100644 index d4069af35dc..00000000000 --- a/packages/lodestar/test/spec/utils/specTestTypes/beaconStateComparison.ts +++ /dev/null @@ -1,47 +0,0 @@ -import {IBaseCase} from "@chainsafe/lodestar-spec-test-util"; - -export interface IBeaconStateComparisonCase extends IBaseCase { - pre: any; - post: any; -} - -export interface IOperationsCase extends IBeaconStateComparisonCase { - // eslint-disable-next-line @typescript-eslint/naming-convention - bls_setting?: bigint; -} - -export interface IAttestationCase extends IOperationsCase { - attestation: any; -} - -export interface IAttesterSlashingCase extends IOperationsCase { - attesterSlashing: any; -} - -export interface IBlockHeaderCase extends IOperationsCase { - block: any; -} - -export interface IDepositCase extends IOperationsCase { - deposit: any; -} - -export interface IProposerSlashingCase extends IOperationsCase { - proposerSlashing: any; -} - -export interface ITransferCase extends IOperationsCase { - transfer: any; -} - -export interface IVoluntaryExitCase extends IOperationsCase { - voluntaryExit: any; -} - -export interface IBlockSanityCase extends IOperationsCase { - blocks: any[]; -} - -export interface ISlotSanityCase extends IOperationsCase { - slots: bigint; -} diff --git a/packages/lodestar/test/spec/utils/specTestTypes/genesis.ts b/packages/lodestar/test/spec/utils/specTestTypes/genesis.ts deleted file mode 100644 index 234bfcf68f7..00000000000 --- a/packages/lodestar/test/spec/utils/specTestTypes/genesis.ts +++ /dev/null @@ -1,13 +0,0 @@ -import {IBaseCase} from "@chainsafe/lodestar-spec-test-util"; - -export interface IGenesisInitCase extends IBaseCase { - state: any; - eth1BlockHash: any; - eth1Timestamp: any; - deposits: any; -} - -export interface IGenesisValidityCase extends IBaseCase { - genesis: any; - isValid: true; -} diff --git a/packages/lodestar/test/spec/utils/specTestTypes/shufflingCase.ts b/packages/lodestar/test/spec/utils/specTestTypes/shufflingCase.ts deleted file mode 100644 index 764b9694ce4..00000000000 --- a/packages/lodestar/test/spec/utils/specTestTypes/shufflingCase.ts +++ /dev/null @@ -1,7 +0,0 @@ -import {IBaseCase} from "@chainsafe/lodestar-spec-test-util"; - -export interface IShufflingCase extends IBaseCase { - seed: string; - count: bigint; - shuffled: bigint[]; -} diff --git a/packages/lodestar/test/spec/utils/specTestTypes/stateTestCase.ts b/packages/lodestar/test/spec/utils/specTestTypes/stateTestCase.ts deleted file mode 100644 index 3aa8fa1f4be..00000000000 --- a/packages/lodestar/test/spec/utils/specTestTypes/stateTestCase.ts +++ /dev/null @@ -1,7 +0,0 @@ -import {phase0} from "@chainsafe/lodestar-types"; -import {IBaseSpecTest} from "../../type"; - -export interface IStateTestCase extends IBaseSpecTest { - pre: phase0.BeaconState; - post: phase0.BeaconState; -} diff --git a/packages/lodestar/test/spec/utils/sszTestCaseParser.ts b/packages/lodestar/test/spec/utils/sszTestCaseParser.ts new file mode 100644 index 00000000000..d3c99a6a238 --- /dev/null +++ b/packages/lodestar/test/spec/utils/sszTestCaseParser.ts @@ -0,0 +1,92 @@ +import path from "node:path"; +import fs from "node:fs"; +import {uncompress} from "snappyjs"; +import {loadYaml} from "@chainsafe/lodestar-utils"; +import jsyaml from "js-yaml"; + +/* eslint-disable + @typescript-eslint/explicit-module-boundary-types, + @typescript-eslint/explicit-function-return-type +*/ + +export type ValidTestCaseData = { + root: string; + serialized: Uint8Array; + jsonValue: unknown; +}; + +/** + * ssz_static + * | Attestation + * | case_0 + * | roots.yaml + * | serialized.ssz_snappy + * | value.yaml + * + * Docs: https://github.com/ethereum/consensus-specs/blob/master/tests/formats/ssz_static/core.md + */ +export function parseSszStaticTestcase(dirpath: string): ValidTestCaseData { + return parseSszValidTestcase(dirpath, "roots.yaml"); +} + +/** + * ssz_generic + * | basic_vector + * | valid + * | vec_bool_1_max + * | meta.yaml + * | serialized.ssz_snappy + * | value.yaml + * + * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + */ +export function parseSszGenericValidTestcase(dirpath: string): ValidTestCaseData { + return parseSszValidTestcase(dirpath, "meta.yaml"); +} + +export function parseSszValidTestcase(dirpath: string, metaFilename: string): ValidTestCaseData { + // The root is stored in meta.yml as: + // root: 0xDEADBEEF + const metaStr = fs.readFileSync(path.join(dirpath, metaFilename), "utf8"); + const meta = jsyaml.load(metaStr) as {root: string}; + if (typeof meta.root !== "string") { + throw Error(`meta.root not a string: ${meta.root}\n${fs}`); + } + + // The serialized value is stored in serialized.ssz_snappy + const serialized = uncompress(fs.readFileSync(path.join(dirpath, "serialized.ssz_snappy"))); + + // The value is stored in value.yml + const yamlPath = path.join(dirpath, "value.yaml"); + const yamlStr = fs.readFileSync(yamlPath, "utf8"); + const jsonValue = readYamlNumbersAsStrings(yamlStr); + // type.fromJson(loadYamlFile(path.join(dirpath, "value.yaml")) as Json) as T; + + return { + root: meta.root, + serialized, + jsonValue, + }; +} + +/** + * ssz_generic + * | basic_vector + * | invalid + * | vec_bool_0 + * | serialized.ssz_snappy + * + * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + */ +export function parseSszGenericInvalidTestcase(dirpath: string) { + // The serialized value is stored in serialized.ssz_snappy + const serialized = uncompress(fs.readFileSync(path.join(dirpath, "serialized.ssz_snappy"))); + + return { + serialized, + }; +} + +export function readYamlNumbersAsStrings(yamlStr: string): unknown { + return loadYaml(yamlStr); +} diff --git a/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlock.test.ts b/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlock.test.ts index a3208eb7026..5877f617e32 100644 --- a/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlock.test.ts +++ b/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlock.test.ts @@ -1,6 +1,5 @@ import sinon from "sinon"; import * as blockUtils from "../../../../../../src/api/impl/beacon/blocks/utils"; -import {ssz} from "@chainsafe/lodestar-types"; import {expect, use} from "chai"; import chaiAsPromised from "chai-as-promised"; import {generateEmptySignedBlock} from "../../../../../utils/block"; @@ -31,6 +30,5 @@ describe("api - beacon - getBlock", function () { resolveBlockIdStub.withArgs(sinon.match.any, sinon.match.any, "head").resolves(generateEmptySignedBlock()); const {data: result} = await server.blockApi.getBlock("head"); expect(result).to.not.be.null; - expect(() => ssz.phase0.SignedBeaconBlock.assertValidValue(result)).to.not.throw(); }); }); diff --git a/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts b/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts index f916e61dcab..2163e56e3f5 100644 --- a/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts +++ b/packages/lodestar/test/unit/api/impl/beacon/blocks/getBlockHeaders.test.ts @@ -6,7 +6,6 @@ import { generateEmptySignedBlock, generateSignedBlock, } from "../../../../../utils/block"; -import deepmerge from "deepmerge"; import {expect} from "chai"; import {setupApiImplTestServer, ApiImplTestModules} from "../../index.test"; import {toHexString} from "@chainsafe/ssz"; @@ -31,7 +30,11 @@ describe("api - beacon - getBlockHeaders", function () { blockRoot: toHexString(ssz.phase0.BeaconBlock.hashTreeRoot(generateEmptyBlock())), }, ]); - server.dbStub.block.get.resolves(deepmerge(generateEmptySignedBlock(), {message: {slot: 3}})); + + const blockFromDb3 = generateEmptySignedBlock(); + blockFromDb3.message.slot = 3; + server.dbStub.block.get.resolves(blockFromDb3); + server.dbStub.blockArchive.get.resolves(null); const {data: blockHeaders} = await server.blockApi.getBlockHeaders({}); expect(blockHeaders).to.not.be.null; diff --git a/packages/lodestar/test/unit/api/impl/beacon/pool/pool.test.ts b/packages/lodestar/test/unit/api/impl/beacon/pool/pool.test.ts index fdef6b926ff..252ebd5df42 100644 --- a/packages/lodestar/test/unit/api/impl/beacon/pool/pool.test.ts +++ b/packages/lodestar/test/unit/api/impl/beacon/pool/pool.test.ts @@ -13,8 +13,7 @@ import * as attesterSlashingValidation from "../../../../../../src/chain/validat import * as proposerSlashingValidation from "../../../../../../src/chain/validation/proposerSlashing"; import * as voluntaryExitValidation from "../../../../../../src/chain/validation/voluntaryExit"; -import {phase0, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; +import {phase0} from "@chainsafe/lodestar-types"; import {Eth2Gossipsub} from "../../../../../../src/network/gossip"; import {generateEmptySignedBlockHeader} from "../../../../../utils/block"; import {setupApiImplTestServer} from "../../index.test"; @@ -87,12 +86,12 @@ describe("beacon pool api impl", function () { describe("submitPoolAttesterSlashing", function () { const atterterSlashing: phase0.AttesterSlashing = { attestation1: { - attestingIndices: [0] as List, + attestingIndices: [0], data: generateAttestationData(0, 1, 0, 1), signature: Buffer.alloc(96), }, attestation2: { - attestingIndices: [0] as List, + attestingIndices: [0], data: generateAttestationData(0, 1, 0, 1), signature: Buffer.alloc(96), }, diff --git a/packages/lodestar/test/unit/api/impl/beacon/state/stateValidators.test.ts b/packages/lodestar/test/unit/api/impl/beacon/state/stateValidators.test.ts deleted file mode 100644 index e8865b8c9df..00000000000 --- a/packages/lodestar/test/unit/api/impl/beacon/state/stateValidators.test.ts +++ /dev/null @@ -1,181 +0,0 @@ -import {config} from "@chainsafe/lodestar-config/default"; -import {CachedBeaconStateAllForks, PubkeyIndexMap} from "@chainsafe/lodestar-beacon-state-transition"; -import {List, toHexString} from "@chainsafe/ssz"; -import {expect, use} from "chai"; -import chaiAsPromised from "chai-as-promised"; -import sinon, {SinonStubbedInstance, SinonStubbedMember} from "sinon"; -import {getBeaconStateApi} from "../../../../../../src/api/impl/beacon/state"; -import * as stateApiUtils from "../../../../../../src/api/impl/beacon/state/utils"; -import {generateState} from "../../../../../utils/state"; -import {generateValidator, generateValidators} from "../../../../../utils/validator"; -import {BeaconChain} from "../../../../../../src/chain"; -import {StubbedBeaconDb} from "../../../../../utils/stub"; -import {setupApiImplTestServer, ApiImplTestModules} from "../../index.test"; - -use(chaiAsPromised); - -const validatorId1 = toHexString(Buffer.alloc(48, 1)); -const validatorId2 = toHexString(Buffer.alloc(48, 2)); - -describe("beacon api impl - state - validators", function () { - let resolveStateIdStub: SinonStubbedMember; - let toValidatorResponseStub: SinonStubbedMember; - let dbStub: StubbedBeaconDb; - let chainStub: SinonStubbedInstance; - let server: ApiImplTestModules; - - before(function () { - server = setupApiImplTestServer(); - }); - - beforeEach(function () { - resolveStateIdStub = server.sandbox.stub(stateApiUtils, "resolveStateId"); - toValidatorResponseStub = server.sandbox.stub(stateApiUtils, "toValidatorResponse"); - toValidatorResponseStub.returns({ - index: 1, - balance: 3200000, - status: "active_ongoing", - validator: generateValidator(), - }); - dbStub = server.dbStub; - chainStub = server.chainStub; - }); - - afterEach(function () { - server.sandbox.restore(); - }); - - describe("get validators", function () { - it("indices filter", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - chainStub.getHeadState.onCall(0).returns({ - pubkey2index: ({ - get: () => 0, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: validators} = await api.getStateValidators("someState", {id: [0, 1, 123]}); - expect(validators.length).to.equal(2); - }); - - // TODO: Make a normal test without stubs - it.skip("status filter", async function () { - const numValidators = 10; - resolveStateIdStub.resolves(generateState({validators: generateValidators(numValidators)})); - toValidatorResponseStub.onFirstCall().returns({ - index: 1, - balance: 3200000, - status: "exited_slashed", - validator: generateValidator(), - }); - for (let i = 0; i < 10; i++) { - chainStub.getHeadState.onCall(i).returns({ - pubkey2index: ({ - get: () => i, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - } - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: validators} = await api.getStateValidators("someState", { - statuses: ["pending_initialized"], - }); - expect(validators.length).to.equal(9); - }); - - it("statuses and indices filters", async function () { - const numValidators = 10; - const validators = generateValidators(numValidators); - validators[0].exitEpoch = 0; - resolveStateIdStub.resolves(generateState({validators})); - for (let i = 0; i < 10; i++) { - chainStub.getHeadState.onCall(i).returns({ - pubkey2index: ({ - get: () => i, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - } - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: stateValidators} = await api.getStateValidators("someState", { - id: [0, 1, 2, 123], - statuses: ["pending_initialized"], - }); - expect(stateValidators.length).to.equal(3); - }); - - it("success", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: validators} = await api.getStateValidators("someState"); - expect(validators.length).to.equal(10); - }); - }); - - describe("get validator", function () { - it("validator by index not found", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - await expect(api.getStateValidator("someState", 15)).to.be.rejectedWith("Validator not found"); - }); - it("validator by index found", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - expect(await api.getStateValidator("someState", 1)).to.not.be.null; - }); - it.skip("validator by root not found", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - chainStub.getHeadState.returns({ - pubkey2index: ({ - get: () => undefined, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - await expect(api.getStateValidator("someState", validatorId1)).to.be.rejectedWith("Validator not found"); - }); - it("validator by root found", async function () { - resolveStateIdStub.resolves(generateState({validators: generateValidators(10)})); - chainStub.getHeadState.returns({ - pubkey2index: ({ - get: () => 2, - } as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - expect(await api.getStateValidator("someState", validatorId1)).to.not.be.null; - }); - }); - - describe("get validators balances", function () { - it.skip("indices filters", async function () { - resolveStateIdStub.resolves( - generateState({ - validators: generateValidators(10), - balances: Array.from({length: 10}, () => 10) as List, - }) - ); - const pubkey2IndexStub = sinon.createStubInstance(PubkeyIndexMap); - pubkey2IndexStub.get.withArgs(validatorId1).returns(3); - pubkey2IndexStub.get.withArgs(validatorId2).returns(25); - chainStub.getHeadState.returns({ - pubkey2index: (pubkey2IndexStub as unknown) as PubkeyIndexMap, - } as CachedBeaconStateAllForks); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: balances} = await api.getStateValidatorBalances("somestate", [1, 24, validatorId1, validatorId2]); - expect(balances.length).to.equal(2); - expect(balances[0].index).to.equal(1); - expect(balances[1].index).to.equal(3); - }); - - it("no filters", async function () { - resolveStateIdStub.resolves( - generateState({ - validators: generateValidators(10), - balances: Array.from({length: 10}, () => 10) as List, - }) - ); - const api = getBeaconStateApi({config, db: dbStub, chain: chainStub}); - const {data: balances} = await api.getStateValidatorBalances("somestate"); - expect(balances.length).to.equal(10); - expect(balances[0].index).to.equal(0); - expect(balances[0].balance.toString()).to.equal("10"); - }); - }); -}); diff --git a/packages/lodestar/test/unit/api/impl/node/node.test.ts b/packages/lodestar/test/unit/api/impl/node/node.test.ts index e64eade155f..14de59bb1d2 100644 --- a/packages/lodestar/test/unit/api/impl/node/node.test.ts +++ b/packages/lodestar/test/unit/api/impl/node/node.test.ts @@ -3,6 +3,7 @@ import {Connection} from "libp2p"; import {getNodeApi} from "../../../../../src/api/impl/node"; import sinon, {SinonStubbedInstance} from "sinon"; +import {BitArray} from "@chainsafe/ssz"; import {createPeerId, INetwork, Network} from "../../../../../src/network"; import {BeaconSync, IBeaconSync} from "../../../../../src/sync"; import {createKeypairFromPeerId, ENR} from "@chainsafe/discv5/lib"; @@ -58,8 +59,8 @@ describe("node api implementation", function () { networkStub.metadata = { get json(): altair.Metadata { return { - attnets: [true], - syncnets: [], + attnets: BitArray.fromBoolArray([true]), + syncnets: BitArray.fromBitLen(0), seqNumber: BigInt(1), }; }, diff --git a/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts b/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts index 1b610967bb3..46efb30924a 100644 --- a/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts +++ b/packages/lodestar/test/unit/api/impl/validator/duties/proposer.test.ts @@ -2,7 +2,6 @@ import sinon, {SinonStubbedInstance} from "sinon"; import {use, expect} from "chai"; import chaiAsPromised from "chai-as-promised"; import {config} from "@chainsafe/lodestar-config/default"; -import {createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; import {ForkChoice} from "@chainsafe/lodestar-fork-choice"; import {IBeaconChain} from "../../../../../../src/chain"; @@ -10,13 +9,13 @@ import {LocalClock} from "../../../../../../src/chain/clock"; import {FAR_FUTURE_EPOCH} from "../../../../../../src/constants"; import {getValidatorApi} from "../../../../../../src/api/impl/validator"; import {ApiModules} from "../../../../../../src/api/impl/types"; -import {generateInitialMaxBalances} from "../../../../../utils/balances"; import {generateState} from "../../../../../utils/state"; import {IBeaconSync} from "../../../../../../src/sync"; import {generateValidators} from "../../../../../utils/validator"; import {StubbedBeaconDb} from "../../../../../utils/stub"; import {setupApiImplTestServer, ApiImplTestModules} from "../../index.test"; import {testLogger} from "../../../../../utils/logger"; +import {createCachedBeaconStateTest} from "../../../../../utils/cachedBeaconState"; import {ssz} from "@chainsafe/lodestar-types"; import {MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; @@ -66,11 +65,11 @@ describe("get proposers api impl", function () { activationEpoch: 0, exitEpoch: FAR_FUTURE_EPOCH, }), - balances: generateInitialMaxBalances(config, 25), + balances: Array.from({length: 25}, () => MAX_EFFECTIVE_BALANCE), }, config ); - const cachedState = createCachedBeaconState(config, state); + const cachedState = createCachedBeaconStateTest(state, config); chainStub.getHeadStateAtCurrentEpoch.resolves(cachedState); sinon.stub(cachedState.epochCtx, "getBeaconProposer").returns(1); const {data: result} = await api.getProposerDuties(0); diff --git a/packages/lodestar/test/unit/api/impl/validator/utils.test.ts b/packages/lodestar/test/unit/api/impl/validator/utils.test.ts index 907328f5ecb..5ea978a2350 100644 --- a/packages/lodestar/test/unit/api/impl/validator/utils.test.ts +++ b/packages/lodestar/test/unit/api/impl/validator/utils.test.ts @@ -1,23 +1,24 @@ import {expect} from "chai"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; -import {allForks, BLSPubkey, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; -import {getPubkeysForIndex, getPubkeysForIndices} from "../../../../../src/api/impl/validator/utils"; +import {toHexString} from "@chainsafe/ssz"; +import {BLSPubkey, ssz, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {getPubkeysForIndices} from "../../../../../src/api/impl/validator/utils"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; describe("api / impl / validator / utils", () => { const vc = 32; const pubkeys: BLSPubkey[] = []; const indexes: ValidatorIndex[] = []; - let state: TreeBacked; + let state: BeaconStateAllForks; before("Prepare state", () => { - state = ssz.phase0.BeaconState.defaultTreeBacked() as TreeBacked; + state = ssz.phase0.BeaconState.defaultViewDU(); const validator = ssz.phase0.Validator.defaultValue(); const validators = state.validators; for (let i = 0; i < vc; i++) { indexes.push(i); const pubkey = Buffer.alloc(48, i); pubkeys.push(pubkey); - validators.push({...validator, pubkey}); + validators.push(ssz.phase0.Validator.toViewDU({...validator, pubkey})); } }); @@ -25,11 +26,4 @@ describe("api / impl / validator / utils", () => { const pubkeysRes = getPubkeysForIndices(state.validators, indexes); expect(pubkeysRes.map(toHexString)).to.deep.equal(pubkeys.map(toHexString)); }); - - it("getPubkeysForIndex", () => { - for (const index of indexes) { - const pubkeyRes = getPubkeysForIndex(state.validators, index); - expect(toHexString(pubkeyRes)).to.deep.equal(toHexString(pubkeys[index]), `Wrong pubkey for index ${index}`); - } - }); }); diff --git a/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts b/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts index 15dc25094fd..e99058d8ed1 100644 --- a/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts +++ b/packages/lodestar/test/unit/chain/factory/block/blockAssembly.test.ts @@ -50,7 +50,7 @@ describe("block assembly", function () { const state = generateCachedState({slot: 1}); sinon.stub(state.epochCtx, "getBeaconProposer").returns(2); regenStub.getBlockSlotState.resolves(state); - beaconDB.depositDataRoot.getTreeBacked.resolves(ssz.phase0.DepositDataRootList.defaultTreeBacked()); + beaconDB.depositDataRoot.getDepositRootTreeAtIndex.resolves(ssz.phase0.DepositDataRootList.defaultViewDU()); assembleBodyStub.resolves(generateEmptyBlock().body); const eth1 = sandbox.createStubInstance(Eth1ForBlockProduction); diff --git a/packages/lodestar/test/unit/chain/factory/block/body.test.ts b/packages/lodestar/test/unit/chain/factory/block/body.test.ts index 4e9cbbdb988..95abd865f3c 100644 --- a/packages/lodestar/test/unit/chain/factory/block/body.test.ts +++ b/packages/lodestar/test/unit/chain/factory/block/body.test.ts @@ -51,7 +51,7 @@ describe("blockAssembly - body", function () { [generateEmptySignedVoluntaryExit()], ]); aggregatedAttestationPool.getAttestationsForBlock.returns([generateEmptyAttestation()]); - dbStub.depositDataRoot.getTreeBacked.resolves(ssz.phase0.DepositDataRootList.defaultTreeBacked()); + dbStub.depositDataRoot.getDepositRootTreeAtIndex.resolves(ssz.phase0.DepositDataRootList.defaultViewDU()); const result = await assembleBody(chain, generateCachedState(), { randaoReveal: Buffer.alloc(96, 0), diff --git a/packages/lodestar/test/unit/chain/forkChoice/forkChoice.test.ts b/packages/lodestar/test/unit/chain/forkChoice/forkChoice.test.ts index dd9ab11c2e6..da7574c5cb3 100644 --- a/packages/lodestar/test/unit/chain/forkChoice/forkChoice.test.ts +++ b/packages/lodestar/test/unit/chain/forkChoice/forkChoice.test.ts @@ -11,11 +11,12 @@ import { getTemporaryBlockHeader, phase0, CachedBeaconStateAllForks, - createCachedBeaconState, getEffectiveBalanceIncrementsZeroed, + BeaconStateAllForks, } from "@chainsafe/lodestar-beacon-state-transition"; import {expect} from "chai"; -import {List, toHexString, TreeBacked} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; +import {createCachedBeaconStateTest} from "../../../utils/cachedBeaconState"; import {generateValidators} from "../../../utils/validator"; describe("LodestarForkChoice", function () { @@ -29,7 +30,7 @@ describe("LodestarForkChoice", function () { exitEpoch: FAR_FUTURE_EPOCH, withdrawableEpoch: FAR_FUTURE_EPOCH, }), - balances: Array.from({length: 3}, () => 0) as List, + balances: Array.from({length: 3}, () => 0), // Jan 01 2020 genesisTime: 1577836800, }, @@ -47,7 +48,7 @@ describe("LodestarForkChoice", function () { let state: CachedBeaconStateAllForks; before(() => { - state = createCachedBeaconState(config, anchorState); + state = createCachedBeaconStateTest(anchorState, config); }); beforeEach(() => { @@ -69,7 +70,7 @@ describe("LodestarForkChoice", function () { targetBlock.message.parentRoot = finalizedRoot; // const targetState = runStateTransition(anchorState, targetBlock); - targetBlock.message.stateRoot = config.getForkTypes(targetState.slot).BeaconState.hashTreeRoot(targetState); + targetBlock.message.stateRoot = targetState.hashTreeRoot(); const {block: orphanedBlock, state: orphanedState} = makeChild({block: targetBlock, state: targetState}, 36); const {block: parentBlock, state: parentState} = makeChild({block: targetBlock, state: targetState}, 37); const {block: childBlock, state: childState} = makeChild({block: parentBlock, state: parentState}, 38); @@ -115,28 +116,34 @@ describe("LodestarForkChoice", function () { const block08 = generateSignedBlock({message: {slot: 8}}); block08.message.parentRoot = finalizedRoot; const state08 = runStateTransition(anchorState, block08); - block08.message.stateRoot = config.getForkTypes(state08.slot).BeaconState.hashTreeRoot(state08); + block08.message.stateRoot = state08.hashTreeRoot(); const {block: block12, state: state12} = makeChild({block: block08, state: state08}, 12); const {block: block16, state: state16} = makeChild({block: block12, state: state12}, 16); - state16.currentJustifiedCheckpoint = { + state16.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ root: ssz.phase0.BeaconBlock.hashTreeRoot(block08.message), epoch: 1, - }; + }); const {block: block20, state: state20} = makeChild({block: block16, state: state16}, 20); const {block: block24, state: state24} = makeChild({block: block20, state: state20}, 24); - state24.finalizedCheckpoint = {root: ssz.phase0.BeaconBlock.hashTreeRoot(block08.message), epoch: 1}; - state24.currentJustifiedCheckpoint = { + state24.finalizedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ + root: ssz.phase0.BeaconBlock.hashTreeRoot(block08.message), + epoch: 1, + }); + state24.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ root: ssz.phase0.BeaconBlock.hashTreeRoot(block16.message), epoch: 2, - }; + }); const {block: block28, state: state28} = makeChild({block: block24, state: state24}, 28); const {block: block32, state: state32} = makeChild({block: block28, state: state28}, 32); - state32.finalizedCheckpoint = {root: ssz.phase0.BeaconBlock.hashTreeRoot(block16.message), epoch: 2}; - state32.currentJustifiedCheckpoint = { + state32.finalizedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ + root: ssz.phase0.BeaconBlock.hashTreeRoot(block16.message), + epoch: 2, + }); + state32.currentJustifiedCheckpoint = ssz.phase0.Checkpoint.toViewDU({ root: ssz.phase0.BeaconBlock.hashTreeRoot(block24.message), epoch: 3, - }; + }); forkChoice.updateTime(128); forkChoice.onBlock(block08.message, state08, {justifiedBalances, blockDelaySec: 0}); @@ -184,7 +191,7 @@ describe("LodestarForkChoice", function () { const targetBlock = generateSignedBlock({message: {slot: 32}}); targetBlock.message.parentRoot = finalizedRoot; const targetState = runStateTransition(anchorState, targetBlock); - targetBlock.message.stateRoot = config.getForkTypes(targetState.slot).BeaconState.hashTreeRoot(targetState); + targetBlock.message.stateRoot = targetState.hashTreeRoot(); const {block: orphanedBlock, state: orphanedState} = makeChild({block: targetBlock, state: targetState}, 33); const {block: parentBlock, state: parentState} = makeChild({block: targetBlock, state: targetState}, 34); const {block: childBlock, state: childState} = makeChild({block: parentBlock, state: parentState}, 35); @@ -209,24 +216,23 @@ describe("LodestarForkChoice", function () { }); // lightweight state transtion function for this test -function runStateTransition( - preState: TreeBacked, - signedBlock: phase0.SignedBeaconBlock -): TreeBacked { +function runStateTransition(preState: BeaconStateAllForks, signedBlock: phase0.SignedBeaconBlock): BeaconStateAllForks { // Clone state because process slots and block are not pure const postState = preState.clone(); // Process slots (including those with no blocks) since block - allForks.processSlots(createCachedBeaconState(config, postState), signedBlock.message.slot); + allForks.processSlots(createCachedBeaconStateTest(postState, config), signedBlock.message.slot); // processBlock - postState.latestBlockHeader = getTemporaryBlockHeader(config, signedBlock.message); + postState.latestBlockHeader = ssz.phase0.BeaconBlockHeader.toViewDU( + getTemporaryBlockHeader(config, signedBlock.message) + ); return postState; } // create a child block/state from a parent block/state and a provided slot function makeChild( - parent: {block: phase0.SignedBeaconBlock; state: TreeBacked}, + parent: {block: phase0.SignedBeaconBlock; state: BeaconStateAllForks}, slot: Slot -): {block: phase0.SignedBeaconBlock; state: TreeBacked} { +): {block: phase0.SignedBeaconBlock; state: BeaconStateAllForks} { const childBlock = generateSignedBlock({message: {slot}}); const parentRoot = ssz.phase0.BeaconBlock.hashTreeRoot(parent.block.message); childBlock.message.parentRoot = parentRoot; @@ -241,7 +247,7 @@ export function createIndexedAttestation( validatorIndex: ValidatorIndex ): phase0.IndexedAttestation { return { - attestingIndices: [validatorIndex] as List, + attestingIndices: [validatorIndex], data: { slot: block.message.slot, index: 0, diff --git a/packages/lodestar/test/unit/chain/genesis/genesis.test.ts b/packages/lodestar/test/unit/chain/genesis/genesis.test.ts index bfaa42ad5d6..7d47d043552 100644 --- a/packages/lodestar/test/unit/chain/genesis/genesis.test.ts +++ b/packages/lodestar/test/unit/chain/genesis/genesis.test.ts @@ -2,7 +2,7 @@ import chai, {expect} from "chai"; import chaiAsPromised from "chai-as-promised"; import {SecretKey, PublicKey} from "@chainsafe/bls"; -import {DOMAIN_DEPOSIT} from "@chainsafe/lodestar-params"; +import {DOMAIN_DEPOSIT, MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; import {config} from "@chainsafe/lodestar-config/default"; import { computeDomain, @@ -121,7 +121,7 @@ function generateDeposit(index: ValidatorIndex, secretKey: SecretKey, publicKey: const depositMessage = { pubkey: publicKey.toBytes(), withdrawalCredentials: Buffer.alloc(32, index), - amount: 32e18, + amount: MAX_EFFECTIVE_BALANCE, }; const signingRoot = computeSigningRoot(ssz.phase0.DepositMessage, depositMessage, domain); const signature = secretKey.sign(signingRoot); diff --git a/packages/lodestar/test/unit/chain/lightclient/proof.test.ts b/packages/lodestar/test/unit/chain/lightclient/proof.test.ts index 5be711e846a..3221c068df8 100644 --- a/packages/lodestar/test/unit/chain/lightclient/proof.test.ts +++ b/packages/lodestar/test/unit/chain/lightclient/proof.test.ts @@ -1,8 +1,7 @@ +import {BeaconStateAltair} from "@chainsafe/lodestar-beacon-state-transition"; import {SYNC_COMMITTEE_SIZE} from "@chainsafe/lodestar-params"; import {altair, ssz} from "@chainsafe/lodestar-types"; -import {verifyMerkleBranch} from "@chainsafe/lodestar-utils"; -import {hash} from "@chainsafe/persistent-merkle-tree"; -import {TreeBacked} from "@chainsafe/ssz"; +import {verifyMerkleBranch, hash} from "@chainsafe/lodestar-utils"; import {expect} from "chai"; import {getNextSyncCommitteeBranch, getSyncCommitteesWitness} from "../../../../src/chain/lightClient/proofs"; @@ -11,17 +10,18 @@ const nextSyncCommitteeGindex = 55; const syncCommitteesGindex = 27; describe("chain / lightclient / proof", () => { - let state: TreeBacked; + let state: BeaconStateAltair; let stateRoot: Uint8Array; const currentSyncCommittee = fillSyncCommittee(Buffer.alloc(48, 0xbb)); const nextSyncCommittee = fillSyncCommittee(Buffer.alloc(48, 0xcc)); before("random state", () => { - state = ssz.altair.BeaconState.defaultTreeBacked(); - state.currentSyncCommittee = currentSyncCommittee; - state.nextSyncCommittee = nextSyncCommittee; - stateRoot = ssz.altair.BeaconState.hashTreeRoot(state); + state = ssz.altair.BeaconState.defaultViewDU(); + state.currentSyncCommittee = ssz.altair.SyncCommittee.toViewDU(currentSyncCommittee); + state.nextSyncCommittee = ssz.altair.SyncCommittee.toViewDU(nextSyncCommittee); + // Note: .hashTreeRoot() automatically commits() + stateRoot = state.hashTreeRoot(); }); it("SyncCommittees proof", () => { diff --git a/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index ba28dde8f68..9c1e25d6021 100644 --- a/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/lodestar/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -1,9 +1,9 @@ import {bls, SecretKey} from "@chainsafe/bls"; +import {BitArray} from "@chainsafe/ssz"; import {initBLS} from "@chainsafe/lodestar-cli/src/util"; import {createIChainForkConfig, defaultChainConfig} from "@chainsafe/lodestar-config"; import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; -import {List} from "@chainsafe/ssz"; import {expect} from "chai"; import {ssz, phase0} from "@chainsafe/lodestar-types"; import { @@ -14,6 +14,7 @@ import { import {InsertOutcome} from "../../../../src/chain/opPools/types"; import {generateAttestation, generateEmptyAttestation} from "../../../utils/attestation"; import {generateCachedState} from "../../../utils/state"; +import {renderBitArray} from "../../../utils/render"; import sinon from "sinon"; describe("AggregatedAttestationPool", function () { @@ -71,7 +72,14 @@ describe("MatchingDataAttestationGroup", function () { const attestationSeed = generateEmptyAttestation(); const attestationDataRoot = ssz.phase0.AttestationData.serialize(attestationSeed.data); let sk1: SecretKey; - const attestation1 = {...attestationSeed, ...{aggregationBits: [true, true, false] as List}}; + const attestation1: phase0.Attestation = { + ...attestationSeed, + ...{aggregationBits: BitArray.fromBoolArray([true, true, false])}, + }; + + function attestationFromBits(bitsBoolArr: boolean[]): phase0.Attestation { + return {...attestationSeed, ...{aggregationBits: BitArray.fromBoolArray(bitsBoolArr)}}; + } before(async () => { await initBLS(); @@ -88,7 +96,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("add - new data, getAttestations() return 2", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false, true] as List}}; + const attestation2 = attestationFromBits([true, false, true]); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([100, 300])}); expect(result).to.be.equal(InsertOutcome.NewData, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestations(); @@ -96,7 +104,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("add - new data, remove existing attestation, getAttestations() return 1", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, true, true] as List}}; + const attestation2 = attestationFromBits([true, true, true]); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set(committee)}); expect(result).to.be.equal(InsertOutcome.NewData, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestations(); @@ -104,7 +112,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("add - already known, getAttestations() return 1", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false, false] as List}}; + const attestation2 = attestationFromBits([true, false, false]); // attestingIndices is subset of an existing one const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([100])}); expect(result).to.be.equal(InsertOutcome.AlreadyKnown, "incorrect InsertOutcome"); @@ -113,19 +121,18 @@ describe("MatchingDataAttestationGroup", function () { }); it("add - aggregate into existing attestation, getAttestations() return 1", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [false, false, true] as List}}; + const attestation2 = attestationFromBits([false, false, true]); const sk2 = bls.SecretKey.fromBytes(Buffer.alloc(32, 2)); attestation2.signature = sk2.sign(attestationDataRoot).toBytes(); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([300])}); expect(result).to.be.equal(InsertOutcome.Aggregated, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestations(); expect(attestations.length).to.be.equal(1, "expect exactly 1 aggregated attestation"); - expect(attestations[0].aggregationBits).to.be.deep.equal([true, true, true], "incorrect aggregationBits"); - const aggregatedSignature = bls.Signature.fromBytes( - attestations[0].signature.valueOf() as Uint8Array, - undefined, - true + expect(renderBitArray(attestations[0].aggregationBits)).to.be.deep.equal( + renderBitArray(BitArray.fromBoolArray([true, true, true])), + "incorrect aggregationBits" ); + const aggregatedSignature = bls.Signature.fromBytes(attestations[0].signature, undefined, true); expect( aggregatedSignature.verifyAggregate([sk1.toPublicKey(), sk2.toPublicKey()], attestationDataRoot) ).to.be.equal(true, "invalid aggregated signature"); @@ -152,7 +159,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("getAttestationsForBlock - return 2", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false, true] as List}}; + const attestation2 = attestationFromBits([true, false, true]); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([100, 300])}); expect(result).to.be.equal(InsertOutcome.NewData, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestationsForBlock(new Set([200])); @@ -174,7 +181,7 @@ describe("MatchingDataAttestationGroup", function () { }); it("getAttestations", () => { - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false, true] as List}}; + const attestation2 = attestationFromBits([true, false, true]); const result = attestationGroup.add({attestation: attestation2, attestingIndices: new Set([100, 300])}); expect(result).to.be.equal(InsertOutcome.NewData, "incorrect InsertOutcome"); const attestations = attestationGroup.getAttestations(); @@ -184,8 +191,9 @@ describe("MatchingDataAttestationGroup", function () { describe("aggregateInto", function () { const attestationSeed = generateEmptyAttestation(); - const attestation1 = {...attestationSeed, ...{aggregationBits: [false, true] as List}}; - const attestation2 = {...attestationSeed, ...{aggregationBits: [true, false] as List}}; + const attestation1 = {...attestationSeed, ...{aggregationBits: BitArray.fromBoolArray([false, true])}}; + const attestation2 = {...attestationSeed, ...{aggregationBits: BitArray.fromBoolArray([true, false])}}; + const mergedBitArray = BitArray.fromBoolArray([true, true]); // = [false, true] + [true, false] const attestationDataRoot = ssz.phase0.AttestationData.serialize(attestationSeed.data); let sk1: SecretKey; let sk2: SecretKey; @@ -201,15 +209,21 @@ describe("aggregateInto", function () { const attWithIndex1 = {attestation: attestation1, attestingIndices: new Set([100])}; const attWithIndex2 = {attestation: attestation2, attestingIndices: new Set([200])}; aggregateInto(attWithIndex1, attWithIndex2); - expect(attWithIndex1.attestingIndices).to.be.deep.equal(new Set([100, 200]), "invalid aggregated attestingIndices"); - expect(attWithIndex1.attestation.aggregationBits).to.be.deep.equal([true, true], "invalid aggregationBits"); - const aggregatedSignature = bls.Signature.fromBytes( - attWithIndex1.attestation.signature.valueOf() as Uint8Array, - undefined, - true + expect(setToArr(attWithIndex1.attestingIndices)).to.be.deep.equal( + [100, 200], + "invalid aggregated attestingIndices" ); + expect(renderBitArray(attWithIndex1.attestation.aggregationBits)).to.be.deep.equal( + renderBitArray(mergedBitArray), + "invalid aggregationBits" + ); + const aggregatedSignature = bls.Signature.fromBytes(attWithIndex1.attestation.signature, undefined, true); expect( aggregatedSignature.verifyAggregate([sk1.toPublicKey(), sk2.toPublicKey()], attestationDataRoot) ).to.be.equal(true, "invalid aggregated signature"); }); }); + +function setToArr(set: Set): T[] { + return Array.from(set.values()); +} diff --git a/packages/lodestar/test/unit/chain/opPools/syncCommittee.test.ts b/packages/lodestar/test/unit/chain/opPools/syncCommittee.test.ts index af311f2fade..07a5cc1263a 100644 --- a/packages/lodestar/test/unit/chain/opPools/syncCommittee.test.ts +++ b/packages/lodestar/test/unit/chain/opPools/syncCommittee.test.ts @@ -49,12 +49,8 @@ describe("chain / opPools / SyncCommitteeMessagePool", function () { expect(contribution.subcommitteeIndex).to.be.equal(subcommitteeIndex); const newIndices = [...newIndicesInSubSyncCommittee, indexInSubcommittee]; const aggregationBits = contribution.aggregationBits; - for (let index = 0; index < aggregationBits.length; index++) { - if (newIndices.includes(index)) { - expect(aggregationBits[index]).to.be.true; - } else { - expect(aggregationBits[index]).to.be.false; - } + for (let index = 0; index < aggregationBits.bitLen; index++) { + expect(aggregationBits.get(index)).to.equal(newIndices.includes(index), `Wrong bit value index ${index}`); } } }); diff --git a/packages/lodestar/test/unit/chain/opPools/syncCommitteeContribution.test.ts b/packages/lodestar/test/unit/chain/opPools/syncCommitteeContribution.test.ts index 17fe4f10bbd..c47a6fc70a3 100644 --- a/packages/lodestar/test/unit/chain/opPools/syncCommitteeContribution.test.ts +++ b/packages/lodestar/test/unit/chain/opPools/syncCommitteeContribution.test.ts @@ -1,19 +1,20 @@ import {altair, ssz} from "@chainsafe/lodestar-types"; import {initBLS} from "@chainsafe/lodestar-cli/src/util"; import {newFilledArray} from "@chainsafe/lodestar-beacon-state-transition"; - +import bls, {SecretKey} from "@chainsafe/bls"; +import {BitArray} from "@chainsafe/ssz"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; import {expect} from "chai"; import { aggregate, - contributionToFast, replaceIfBetter, SyncContributionAndProofPool, SyncContributionFast, } from "../../../../src/chain/opPools/syncContributionAndProofPool"; import {generateContributionAndProof, generateEmptyContribution} from "../../../utils/contributionAndProof"; import {InsertOutcome} from "../../../../src/chain/opPools/types"; -import bls, {SecretKey} from "@chainsafe/bls"; +import {EMPTY_SIGNATURE} from "../../../../src/constants"; +import {renderBitArray} from "../../../utils/render"; describe("chain / opPools / SyncContributionAndProofPool", function () { let cache: SyncContributionAndProofPool; @@ -38,24 +39,26 @@ describe("chain / opPools / SyncContributionAndProofPool", function () { const aggregate = cache.getAggregate(slot, beaconBlockRoot); expect(ssz.altair.SyncAggregate.equals(aggregate, ssz.altair.SyncAggregate.defaultValue())).to.be.false; // TODO Test it's correct. Modify the contributions above so they have 1 bit set to true - expect(aggregate.syncCommitteeBits.length).to.be.equal(32); + expect(aggregate.syncCommitteeBits.bitLen).to.be.equal(32); }); }); describe("replaceIfBetter", function () { + const numParticipants = 2; let bestContribution: SyncContributionFast; // const subnetSize = Math.floor(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT); beforeEach(() => { bestContribution = { - syncSubcommitteeBits: [true, true, false, false, false, false, false, false], - numParticipants: 2, - syncSubcommitteeSignature: new Uint8Array(0), + syncSubcommitteeBits: BitArray.fromBoolArray([true, true, false, false, false, false, false, false]), + numParticipants, + syncSubcommitteeSignature: EMPTY_SIGNATURE, }; }); + it("less participants", () => { const contribution = generateEmptyContribution(); - contribution.aggregationBits[0] = true; - expect(replaceIfBetter(bestContribution, contribution, 0)).to.be.equal( + contribution.aggregationBits.set(0, true); + expect(replaceIfBetter(bestContribution, contribution, numParticipants - 1)).to.be.equal( InsertOutcome.NotBetterThan, "less participant item should not replace the best contribution" ); @@ -63,9 +66,7 @@ describe("replaceIfBetter", function () { it("same participants", () => { const contribution = generateEmptyContribution(); - contribution.aggregationBits[0] = true; - contribution.aggregationBits[7] = true; - expect(replaceIfBetter(bestContribution, contribution, 2)).to.be.equal( + expect(replaceIfBetter(bestContribution, contribution, numParticipants)).to.be.equal( InsertOutcome.NotBetterThan, "same participant item should not replace the best contribution" ); @@ -73,41 +74,17 @@ describe("replaceIfBetter", function () { it("more participants", () => { const contribution = generateEmptyContribution(); - contribution.aggregationBits[3] = true; - contribution.aggregationBits[4] = true; - contribution.aggregationBits[5] = true; - expect(replaceIfBetter(bestContribution, contribution, 3)).to.be.equal( + const numParticipantsNew = numParticipants + 1; + + expect(replaceIfBetter(bestContribution, contribution, numParticipantsNew)).to.be.equal( InsertOutcome.NewData, "more participant item should replace the best contribution" ); - expect(bestContribution.syncSubcommitteeBits).to.be.deep.equal( - [false, false, false, true, true, true, false, false], - "incorect subcommittees" - ); - expect(bestContribution.numParticipants).to.be.equal(3, "incorrect numParticipants"); - }); -}); - -describe("contributionToFast", function () { - let sk1: SecretKey; - before(async () => { - await initBLS(); - sk1 = bls.SecretKey.fromBytes(Buffer.alloc(32, 1)); - }); - - it("convert a contribution to SyncContributionFast", () => { - const contribution = generateEmptyContribution(); - contribution.aggregationBits[3] = true; - contribution.aggregationBits[4] = true; - contribution.aggregationBits[5] = true; - contribution.signature = sk1.sign(Buffer.alloc(32)).toBytes(); - const fast = contributionToFast(contribution, 3); - expect(fast.syncSubcommitteeBits).to.be.deep.equal( - [false, false, false, true, true, true, false, false], + expect(renderBitArray(bestContribution.syncSubcommitteeBits)).to.be.deep.equal( + renderBitArray(contribution.aggregationBits), "incorect subcommittees" ); - expect(fast.numParticipants).to.be.equal(3, "incorrect numParticipants"); - // no need to check sygnature + expect(bestContribution.numParticipants).to.be.equal(numParticipantsNew, "incorrect numParticipants"); }); }); @@ -130,7 +107,7 @@ describe("aggregate", function () { for (let subnet = 0; subnet < numSubnet; subnet++) { bestContributionBySubnet.set(subnet, { // first participation of each subnet is true - syncSubcommitteeBits: [true, false, false, false, false, false, false, false], + syncSubcommitteeBits: BitArray.fromBoolArray([true, false, false, false, false, false, false, false]), numParticipants: 1, syncSubcommitteeSignature: sks[subnet].sign(blockRoot).toBytes(), }); @@ -142,12 +119,15 @@ describe("aggregate", function () { // first participation of each subnet is true expectSyncCommittees[subnet * 8] = true; } - expect(syncAggregate.syncCommitteeBits).to.be.deep.equal(expectSyncCommittees, "incorrect sync committees"); + expect(renderBitArray(syncAggregate.syncCommitteeBits)).to.be.deep.equal( + renderBitArray(BitArray.fromBoolArray(expectSyncCommittees)), + "incorrect sync committees" + ); expect( bls.verifyAggregate( testSks.map((sk) => sk.toPublicKey().toBytes()), blockRoot, - syncAggregate.syncCommitteeSignature.valueOf() as Uint8Array + syncAggregate.syncCommitteeSignature ) ).to.be.equal(true, "invalid aggregated signature"); }); diff --git a/packages/lodestar/test/unit/chain/stateCache/stateContextCache.test.ts b/packages/lodestar/test/unit/chain/stateCache/stateContextCache.test.ts index 81fa49c7115..29e02aa5032 100644 --- a/packages/lodestar/test/unit/chain/stateCache/stateContextCache.test.ts +++ b/packages/lodestar/test/unit/chain/stateCache/stateContextCache.test.ts @@ -3,12 +3,13 @@ import {IEpochShuffling} from "@chainsafe/lodestar-beacon-state-transition"; import {StateContextCache} from "../../../../src/chain/stateCache"; import {generateCachedState} from "../../../utils/state"; import {ZERO_HASH} from "../../../../src/constants"; -import {ByteVector, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; +import {Root} from "@chainsafe/lodestar-types"; describe("StateContextCache", function () { let cache: StateContextCache; - let key1: ByteVector, key2: ByteVector; + let key1: Root, key2: Root; const shuffling: IEpochShuffling = { epoch: 0, activeIndices: [], diff --git a/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts b/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts index 186f91a0edd..b776cbef82f 100644 --- a/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts +++ b/packages/lodestar/test/unit/chain/validation/aggregateAndProof.test.ts @@ -95,8 +95,8 @@ describe("chain / validation / aggregateAndProof", () => { const {chain, signedAggregateAndProof} = getValidData(); // Unset all aggregationBits const {aggregationBits} = signedAggregateAndProof.message.aggregate; - for (let i = 0, len = aggregationBits.length; i < len; i++) { - aggregationBits[i] = false; + for (let i = 0, len = aggregationBits.bitLen; i < len; i++) { + aggregationBits.set(i, false); } await expectError(chain, signedAggregateAndProof, AttestationErrorCode.EMPTY_AGGREGATION_BITFIELD); @@ -141,8 +141,8 @@ describe("chain / validation / aggregateAndProof", () => { const bitIndex = 1; const {chain, signedAggregateAndProof} = getValidData({bitIndex}); // Change the bit index so the signature is validated against a different pubkey - signedAggregateAndProof.message.aggregate.aggregationBits[bitIndex] = false; - signedAggregateAndProof.message.aggregate.aggregationBits[bitIndex + 1] = true; + signedAggregateAndProof.message.aggregate.aggregationBits.set(bitIndex, false); + signedAggregateAndProof.message.aggregate.aggregationBits.set(bitIndex + 1, true); await expectError(chain, signedAggregateAndProof, AttestationErrorCode.INVALID_SIGNATURE); }); diff --git a/packages/lodestar/test/unit/chain/validation/attestation.test.ts b/packages/lodestar/test/unit/chain/validation/attestation.test.ts index eeca2567101..5e5c2e5cd1a 100644 --- a/packages/lodestar/test/unit/chain/validation/attestation.test.ts +++ b/packages/lodestar/test/unit/chain/validation/attestation.test.ts @@ -1,5 +1,6 @@ import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {phase0} from "@chainsafe/lodestar-types"; +import {BitArray} from "@chainsafe/ssz"; import {IBeaconChain} from "../../../../src/chain"; import {AttestationErrorCode} from "../../../../src/chain/errors"; import {validateGossipAttestation} from "../../../../src/chain/validation"; @@ -65,7 +66,7 @@ describe("chain / validation / attestation", () => { // Unset the single aggregationBits const bitIndex = 1; const {chain, attestation, subnet} = getValidData({bitIndex}); - attestation.aggregationBits[bitIndex] = false; + attestation.aggregationBits.set(bitIndex, false); await expectError(chain, attestation, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET); }); @@ -74,7 +75,7 @@ describe("chain / validation / attestation", () => { // Set an extra bit in the attestation const bitIndex = 1; const {chain, attestation, subnet} = getValidData({bitIndex}); - attestation.aggregationBits[bitIndex + 1] = true; + attestation.aggregationBits.set(bitIndex + 1, true); await expectError(chain, attestation, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET); }); @@ -98,7 +99,10 @@ describe("chain / validation / attestation", () => { it("WRONG_NUMBER_OF_AGGREGATION_BITS", async () => { const {chain, attestation, subnet} = getValidData(); // Increase the length of aggregationBits beyond the committee size - attestation.aggregationBits[attestation.aggregationBits.length] = false; + attestation.aggregationBits = new BitArray( + attestation.aggregationBits.uint8Array, + attestation.aggregationBits.bitLen + 1 + ); await expectError(chain, attestation, subnet, AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS); }); @@ -123,8 +127,8 @@ describe("chain / validation / attestation", () => { const bitIndex = 1; const {chain, attestation, subnet} = getValidData({bitIndex}); // Change the bit index so the signature is validated against a different pubkey - attestation.aggregationBits[bitIndex] = false; - attestation.aggregationBits[bitIndex + 1] = true; + attestation.aggregationBits.set(bitIndex, false); + attestation.aggregationBits.set(bitIndex + 1, true); await expectError(chain, attestation, subnet, AttestationErrorCode.INVALID_SIGNATURE); }); diff --git a/packages/lodestar/test/unit/chain/validation/attesterSlashing.test.ts b/packages/lodestar/test/unit/chain/validation/attesterSlashing.test.ts index 321cc301864..33a0db46135 100644 --- a/packages/lodestar/test/unit/chain/validation/attesterSlashing.test.ts +++ b/packages/lodestar/test/unit/chain/validation/attesterSlashing.test.ts @@ -11,7 +11,6 @@ import {validateGossipAttesterSlashing} from "../../../../src/chain/validation/a import {AttesterSlashingErrorCode} from "../../../../src/chain/errors/attesterSlashingError"; import {OpPool} from "../../../../src/chain/opPools"; import {expectRejectedWithLodestarError} from "../../../utils/errors"; -import {List} from "@chainsafe/ssz"; describe("GossipMessageValidator", () => { const sandbox = sinon.createSandbox(); @@ -59,12 +58,12 @@ describe("GossipMessageValidator", () => { attestation1: { data: attestationData, signature: Buffer.alloc(96, 0), - attestingIndices: [0] as List, + attestingIndices: [0], }, attestation2: { data: {...attestationData, slot: 1}, // Make it different so it's slashable signature: Buffer.alloc(96, 0), - attestingIndices: [0] as List, + attestingIndices: [0], }, }; diff --git a/packages/lodestar/test/unit/chain/validation/contributionAndProof.test.ts b/packages/lodestar/test/unit/chain/validation/contributionAndProof.test.ts index ff05c8df765..2b42ff61f5b 100644 --- a/packages/lodestar/test/unit/chain/validation/contributionAndProof.test.ts +++ b/packages/lodestar/test/unit/chain/validation/contributionAndProof.test.ts @@ -2,6 +2,7 @@ import {initBLS} from "@chainsafe/lodestar-cli/src/util"; import {defaultChainConfig} from "@chainsafe/lodestar-config"; import sinon from "sinon"; import {SinonStubbedInstance} from "sinon"; +import {BitArray} from "@chainsafe/ssz"; import {BeaconChain, IBeaconChain} from "../../../../src/chain"; import {LocalClock} from "../../../../src/chain/clock"; import {SyncCommitteeErrorCode} from "../../../../src/chain/errors/syncCommitteeError"; @@ -12,7 +13,7 @@ import {validateSyncCommitteeGossipContributionAndProof} from "../../../../src/c import * as syncCommitteeUtils from "@chainsafe/lodestar-beacon-state-transition/lib/util/aggregator"; import {SinonStubFn} from "../../../utils/types"; import {generateCachedStateWithPubkeys} from "../../../utils/state"; -import {SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; +import {SLOTS_PER_EPOCH, SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params"; import {createIChainForkConfig} from "@chainsafe/lodestar-config"; import {SeenContributionAndProof} from "../../../../src/chain/seenCache"; @@ -27,6 +28,8 @@ describe("Sync Committee Contribution And Proof validation", function () { const currentSlot = SLOTS_PER_EPOCH * (altairForkEpoch + 1); // eslint-disable-next-line @typescript-eslint/naming-convention const config = createIChainForkConfig(Object.assign({}, defaultChainConfig, {ALTAIR_FORK_EPOCH: altairForkEpoch})); + // all validators have same pubkey + const aggregatorIndex = 15; before(async function () { await initBLS(); @@ -37,7 +40,6 @@ describe("Sync Committee Contribution And Proof validation", function () { (chain as { seenContributionAndProof: SeenContributionAndProof; }).seenContributionAndProof = new SeenContributionAndProof(); - chain.getGenesisTime.returns(Math.floor(Date.now() / 1000)); clockStub = sandbox.createStubInstance(LocalClock); chain.clock = clockStub; clockStub.isCurrentSlotGivenGossipDisparity.returns(true); @@ -52,7 +54,7 @@ describe("Sync Committee Contribution And Proof validation", function () { clockStub.isCurrentSlotGivenGossipDisparity.returns(false); sandbox.stub(clockStub, "currentSlot").get(() => 100); - const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: 1}}); + const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: 1}, aggregatorIndex}); await expectRejectedWithLodestarError( validateSyncCommitteeGossipContributionAndProof(chain, signedContributionAndProof), SyncCommitteeErrorCode.NOT_CURRENT_SLOT @@ -62,6 +64,7 @@ describe("Sync Committee Contribution And Proof validation", function () { it("should throw error - subcommitteeIndex is not in allowed range", async function () { const signedContributionAndProof = generateSignedContributionAndProof({ contribution: {slot: currentSlot, subcommitteeIndex: 10000}, + aggregatorIndex, }); await expectRejectedWithLodestarError( @@ -71,7 +74,10 @@ describe("Sync Committee Contribution And Proof validation", function () { }); it("should throw error - there is same contribution with same aggregator and index and slot", async function () { - const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: currentSlot}}); + const signedContributionAndProof = generateSignedContributionAndProof({ + contribution: {slot: currentSlot}, + aggregatorIndex, + }); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); chain.getHeadState.returns(headState); chain.seenContributionAndProof.isKnown = () => true; @@ -82,7 +88,10 @@ describe("Sync Committee Contribution And Proof validation", function () { }); it("should throw error - no participant", async function () { - const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: currentSlot}}); + const signedContributionAndProof = generateSignedContributionAndProof({ + contribution: {slot: currentSlot}, + aggregatorIndex, + }); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); chain.getHeadState.returns(headState); isSyncCommitteeAggregatorStub.returns(false); @@ -94,7 +103,8 @@ describe("Sync Committee Contribution And Proof validation", function () { it("should throw error - invalid aggregator", async function () { const signedContributionAndProof = generateSignedContributionAndProof({ - contribution: {slot: currentSlot, aggregationBits: [true]}, + contribution: {slot: currentSlot, aggregationBits: BitArray.fromSingleBit(SYNC_COMMITTEE_SUBNET_SIZE, 0)}, + aggregatorIndex, }); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); chain.getHeadState.returns(headState); @@ -109,20 +119,24 @@ describe("Sync Committee Contribution And Proof validation", function () { * Skip this spec: [REJECT] The aggregator's validator index is within the current sync committee -- i.e. state.validators[contribution_and_proof.aggregator_index].pubkey in state.current_sync_committee.pubkeys. * because we check the aggregator index already and we always sync sync pubkeys with indices */ - it.skip("should throw error - aggregator index is not in sync committee", async function () { - const signedContributionAndProof = generateSignedContributionAndProof({contribution: {slot: currentSlot}}); + it("should throw error - aggregator index is not in sync committee", async function () { + const signedContributionAndProof = generateSignedContributionAndProof({ + contribution: {slot: currentSlot}, + aggregatorIndex: Infinity, + }); isSyncCommitteeAggregatorStub.returns(true); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); chain.getHeadState.returns(headState); await expectRejectedWithLodestarError( validateSyncCommitteeGossipContributionAndProof(chain, signedContributionAndProof), - SyncCommitteeErrorCode.AGGREGATOR_PUBKEY_UNKNOWN + SyncCommitteeErrorCode.VALIDATOR_NOT_IN_SYNC_COMMITTEE ); }); it("should throw error - invalid selection_proof signature", async function () { const signedContributionAndProof = generateSignedContributionAndProof({ - contribution: {slot: currentSlot, aggregationBits: [true]}, + contribution: {slot: currentSlot, aggregationBits: BitArray.fromSingleBit(SYNC_COMMITTEE_SUBNET_SIZE, 0)}, + aggregatorIndex, }); isSyncCommitteeAggregatorStub.returns(true); const headState = await generateCachedStateWithPubkeys({slot: currentSlot}, config, true); diff --git a/packages/lodestar/test/unit/chain/validation/syncCommittee.test.ts b/packages/lodestar/test/unit/chain/validation/syncCommittee.test.ts index efff892ee32..e249216cc55 100644 --- a/packages/lodestar/test/unit/chain/validation/syncCommittee.test.ts +++ b/packages/lodestar/test/unit/chain/validation/syncCommittee.test.ts @@ -42,7 +42,6 @@ describe("Sync Committee Signature validation", function () { (chain as { seenSyncCommitteeMessages: SeenSyncCommitteeMessages; }).seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); - chain.getGenesisTime.returns(Math.floor(Date.now() / 1000)); clockStub = sandbox.createStubInstance(LocalClock); chain.clock = clockStub; clockStub.isCurrentSlotGivenGossipDisparity.returns(true); diff --git a/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts b/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts index 65b8203a29e..6d0323a57d1 100644 --- a/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts +++ b/packages/lodestar/test/unit/chain/validation/voluntaryExit.test.ts @@ -3,7 +3,6 @@ import sinon, {SinonStubbedInstance} from "sinon"; import {config} from "@chainsafe/lodestar-config/default"; import { phase0, - createCachedBeaconState, CachedBeaconStateAllForks, computeEpochAtSlot, computeDomain, @@ -22,6 +21,7 @@ import {expectRejectedWithLodestarError} from "../../../utils/errors"; import {DOMAIN_VOLUNTARY_EXIT, FAR_FUTURE_EPOCH, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {PointFormat, SecretKey} from "@chainsafe/bls"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; +import {createCachedBeaconStateTest} from "../../../utils/cachedBeaconState"; describe("validate voluntary exit", () => { const sandbox = sinon.createSandbox(); @@ -33,13 +33,13 @@ describe("validate voluntary exit", () => { before(() => { const sk = SecretKey.fromKeygen(); - const stateEmpty = ssz.phase0.BeaconState.defaultTreeBacked(); + const stateEmpty = ssz.phase0.BeaconState.defaultValue(); // Validator has to be active for long enough stateEmpty.slot = config.SHARD_COMMITTEE_PERIOD * SLOTS_PER_EPOCH; // Add a validator that's active since genesis and ready to exit - stateEmpty.validators[0] = { + const validator = ssz.phase0.Validator.toViewDU({ pubkey: sk.toPublicKey().toBytes(PointFormat.compressed), withdrawalCredentials: Buffer.alloc(32, 0), effectiveBalance: 32e9, @@ -48,7 +48,8 @@ describe("validate voluntary exit", () => { activationEpoch: 0, exitEpoch: FAR_FUTURE_EPOCH, withdrawableEpoch: FAR_FUTURE_EPOCH, - }; + }); + stateEmpty.validators[0] = validator; const voluntaryExit = { epoch: 0, @@ -63,7 +64,7 @@ describe("validate voluntary exit", () => { signedVoluntaryExit = {message: voluntaryExit, signature: sk.sign(signingRoot).toBytes()}; const _state = generateState(stateEmpty, config); - state = createCachedBeaconState(createIBeaconConfig(config, _state.genesisValidatorsRoot), _state); + state = createCachedBeaconStateTest(_state, createIBeaconConfig(config, _state.genesisValidatorsRoot)); }); beforeEach(() => { diff --git a/packages/lodestar/test/unit/db/api/repositories/blockArchive.test.ts b/packages/lodestar/test/unit/db/api/repositories/blockArchive.test.ts index 58ff6dce107..cf96e6e356f 100644 --- a/packages/lodestar/test/unit/db/api/repositories/blockArchive.test.ts +++ b/packages/lodestar/test/unit/db/api/repositories/blockArchive.test.ts @@ -121,7 +121,7 @@ describe("block archive repository", function () { ).to.be.true; expect( spy.withArgs( - encodeKey(Bucket.index_blockArchiveParentRootIndex, block.message.parentRoot.valueOf() as Uint8Array), + encodeKey(Bucket.index_blockArchiveParentRootIndex, block.message.parentRoot), intToBytes(block.message.slot, 8, "be") ).calledOnce ).to.be.true; @@ -139,7 +139,7 @@ describe("block archive repository", function () { ).to.be.true; expect( spy.withArgs( - encodeKey(Bucket.index_blockArchiveParentRootIndex, blocks[0].message.parentRoot.valueOf() as Uint8Array), + encodeKey(Bucket.index_blockArchiveParentRootIndex, blocks[0].message.parentRoot), intToBytes(blocks[0].message.slot, 8, "be") ).calledTwice ).to.be.true; diff --git a/packages/lodestar/test/unit/db/api/repository.test.ts b/packages/lodestar/test/unit/db/api/repository.test.ts index c4239f251b2..504943b304e 100644 --- a/packages/lodestar/test/unit/db/api/repository.test.ts +++ b/packages/lodestar/test/unit/db/api/repository.test.ts @@ -17,11 +17,9 @@ interface TestType { } // eslint-disable-next-line @typescript-eslint/naming-convention -const TestSSZType = new ContainerType({ - fields: { - bool: ssz.Boolean, - bytes: ssz.Bytes32, - }, +const TestSSZType = new ContainerType({ + bool: ssz.Boolean, + bytes: ssz.Bytes32, }); class TestRepository extends Repository { @@ -104,19 +102,15 @@ describe("database repository", function () { it("should delete given items", async function () { await repository.batchDelete(["1", "2", "3"]); - expect( - controller.batchDelete.withArgs(sinon.match((criteria: ContainerType[]) => criteria.length === 3)) - .calledOnce - ).to.be.true; + expect(controller.batchDelete.withArgs(sinon.match((criteria: unknown[]) => criteria.length === 3)).calledOnce).to + .be.true; }); it("should delete given items by value", async function () { const item = {bool: true, bytes: Buffer.alloc(32)}; await repository.batchRemove([item, item]); - expect( - controller.batchDelete.withArgs(sinon.match((criteria: ContainerType[]) => criteria.length === 2)) - .calledOnce - ).to.be.true; + expect(controller.batchDelete.withArgs(sinon.match((criteria: unknown[]) => criteria.length === 2)).calledOnce).to + .be.true; }); it("should add multiple values", async function () { @@ -124,10 +118,8 @@ describe("database repository", function () { {bool: true, bytes: Buffer.alloc(32)}, {bool: false, bytes: Buffer.alloc(32)}, ]); - expect( - controller.batchPut.withArgs(sinon.match((criteria: ContainerType[]) => criteria.length === 2)) - .calledOnce - ).to.be.true; + expect(controller.batchPut.withArgs(sinon.match((criteria: unknown[]) => criteria.length === 2)).calledOnce).to.be + .true; }); it("should fetch values stream", async function () { diff --git a/packages/lodestar/test/unit/eth1/utils/deposits.test.ts b/packages/lodestar/test/unit/eth1/utils/deposits.test.ts index a65ae2d6f4f..c61546dd4ed 100644 --- a/packages/lodestar/test/unit/eth1/utils/deposits.test.ts +++ b/packages/lodestar/test/unit/eth1/utils/deposits.test.ts @@ -1,16 +1,15 @@ import chai, {expect} from "chai"; import chaiAsPromised from "chai-as-promised"; -import {Root, phase0, ssz} from "@chainsafe/lodestar-types"; -import {List, TreeBacked} from "@chainsafe/ssz"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {MAX_DEPOSITS} from "@chainsafe/lodestar-params"; import {verifyMerkleBranch} from "@chainsafe/lodestar-utils"; import {filterBy} from "../../../utils/db"; -import {getTreeAtIndex} from "../../../../src/util/tree"; import {Eth1ErrorCode} from "../../../../src/eth1/errors"; import {generateDepositData, generateDepositEvent} from "../../../utils/deposit"; import {generateState} from "../../../utils/state"; import {expectRejectedWithLodestarError} from "../../../utils/errors"; import {getDeposits, getDepositsWithProofs, DepositGetter} from "../../../../src/eth1/utils/deposits"; +import {DepositTree} from "../../../../src/db/repositories/depositDataRoot"; chai.use(chaiAsPromised); @@ -102,8 +101,8 @@ describe("eth1 / util / deposits", function () { describe("getDepositsWithProofs", () => { it("return empty array if no pending deposits", function () { - const initialValues = [Buffer.alloc(32)] as List; - const depositRootTree = ssz.phase0.DepositDataRootList.createTreeBackedFromStruct(initialValues); + const initialValues = [Buffer.alloc(32)]; + const depositRootTree = ssz.phase0.DepositDataRootList.toViewDU(initialValues); const depositCount = 0; const eth1Data = generateEth1Data(depositCount, depositRootTree); @@ -121,7 +120,7 @@ describe("eth1 / util / deposits", function () { }) ); - const depositRootTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); + const depositRootTree = ssz.phase0.DepositDataRootList.defaultViewDU(); for (const depositEvent of depositEvents) { depositRootTree.push(ssz.phase0.DepositData.hashTreeRoot(depositEvent.depositData)); } @@ -138,10 +137,10 @@ describe("eth1 / util / deposits", function () { expect( verifyMerkleBranch( ssz.phase0.DepositData.hashTreeRoot(deposit.data), - Array.from(deposit.proof).map((p) => p.valueOf() as Uint8Array), + Array.from(deposit.proof).map((p) => p), 33, index, - eth1Data.depositRoot.valueOf() as Uint8Array + eth1Data.depositRoot ), `Wrong merkle proof on deposit ${index}` ).to.be.true; @@ -150,10 +149,10 @@ describe("eth1 / util / deposits", function () { }); }); -function generateEth1Data(depositCount: number, depositRootTree?: TreeBacked>): phase0.Eth1Data { +function generateEth1Data(depositCount: number, depositRootTree?: DepositTree): phase0.Eth1Data { return { blockHash: Buffer.alloc(32), - depositRoot: depositRootTree ? getTreeAtIndex(depositRootTree, depositCount - 1).hashTreeRoot() : Buffer.alloc(32), + depositRoot: depositRootTree ? depositRootTree.sliceTo(depositCount - 1).hashTreeRoot() : Buffer.alloc(32), depositCount, }; } diff --git a/packages/lodestar/test/unit/eth1/utils/eth1Data.test.ts b/packages/lodestar/test/unit/eth1/utils/eth1Data.test.ts index 97523d212be..ee97e845b4f 100644 --- a/packages/lodestar/test/unit/eth1/utils/eth1Data.test.ts +++ b/packages/lodestar/test/unit/eth1/utils/eth1Data.test.ts @@ -2,17 +2,17 @@ import chai, {expect} from "chai"; import chaiAsPromised from "chai-as-promised"; import {pick} from "lodash"; import {Root, phase0, ssz} from "@chainsafe/lodestar-types"; -import {List, TreeBacked} from "@chainsafe/ssz"; +import {toHex} from "@chainsafe/lodestar-utils"; import {iteratorFromArray} from "../../../utils/interator"; -import {mapToObj} from "../../../utils/map"; import { getEth1DataForBlocks, getDepositsByBlockNumber, getDepositRootByDepositCount, } from "../../../../src/eth1/utils/eth1Data"; +import {Eth1Block} from "../../../../src/eth1/interface"; import {expectRejectedWithLodestarError} from "../../../utils/errors"; import {Eth1ErrorCode} from "../../../../src/eth1/errors"; -import {Eth1Block} from "../../../../src/eth1/interface"; +import {DepositTree} from "../../../../src/db/repositories/depositDataRoot"; chai.use(chaiAsPromised); @@ -21,7 +21,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { id: string; blocks: Eth1Block[]; deposits: phase0.DepositEvent[]; - depositRootTree: TreeBacked>; + depositRootTree: DepositTree; lastProcessedDepositBlockNumber: number; expectedEth1Data?: Partial[]; error?: Eth1ErrorCode; @@ -48,8 +48,8 @@ describe("eth1 / util / getEth1DataForBlocks", function () { const lastProcessedDepositBlockNumber = expectedEth1Data[expectedEth1Data.length - 1].blockNumber; // Pre-fill the depositTree with roots for all deposits - const depositRootTree = ssz.phase0.DepositDataRootList.createTreeBackedFromStruct( - Array.from({length: deposits[deposits.length - 1].index + 1}, (_, i) => Buffer.alloc(32, i)) as List + const depositRootTree = ssz.phase0.DepositDataRootList.toViewDU( + Array.from({length: deposits[deposits.length - 1].index + 1}, (_, i) => Buffer.alloc(32, i)) ); return { @@ -67,7 +67,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { id: "No deposits yet, should throw with NoDepositsForBlockRange", blocks: [getMockBlock({blockNumber: 0})], deposits: [], - depositRootTree: ssz.phase0.DepositDataRootList.defaultTreeBacked(), + depositRootTree: ssz.phase0.DepositDataRootList.defaultViewDU(), lastProcessedDepositBlockNumber: 0, error: Eth1ErrorCode.NO_DEPOSITS_FOR_BLOCK_RANGE, }; @@ -78,7 +78,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { id: "With deposits and no deposit roots, should throw with NotEnoughDepositRoots", blocks: [getMockBlock({blockNumber: 0})], deposits: [getMockDeposit({blockNumber: 0, index: 0})], - depositRootTree: ssz.phase0.DepositDataRootList.defaultTreeBacked(), + depositRootTree: ssz.phase0.DepositDataRootList.defaultViewDU(), lastProcessedDepositBlockNumber: 0, error: Eth1ErrorCode.NOT_ENOUGH_DEPOSIT_ROOTS, }; @@ -89,7 +89,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { id: "Empty case", blocks: [], deposits: [], - depositRootTree: ssz.phase0.DepositDataRootList.defaultTreeBacked(), + depositRootTree: ssz.phase0.DepositDataRootList.defaultViewDU(), lastProcessedDepositBlockNumber: 0, expectedEth1Data: [], }; @@ -207,12 +207,12 @@ describe("eth1 / util / getDepositRootByDepositCount", function () { interface ITestCase { id: string; depositCounts: number[]; - depositRootTree: TreeBacked>; + depositRootTree: DepositTree; expectedMap: Map; } const fullRootMap = new Map(); - const fullDepositRootTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); + const fullDepositRootTree = ssz.phase0.DepositDataRootList.defaultViewDU(); for (let i = 0; i < 10; i++) { fullDepositRootTree.push(Buffer.alloc(32, i)); fullRootMap.set(fullDepositRootTree.length, fullDepositRootTree.hashTreeRoot()); @@ -242,7 +242,7 @@ describe("eth1 / util / getDepositRootByDepositCount", function () { }; }, () => { - const emptyTree = ssz.phase0.DepositDataRootList.defaultTreeBacked(); + const emptyTree = ssz.phase0.DepositDataRootList.defaultViewDU(); return { id: "Empty case", depositCounts: [], @@ -256,11 +256,19 @@ describe("eth1 / util / getDepositRootByDepositCount", function () { const {id, depositCounts, depositRootTree, expectedMap} = testCase(); it(id, function () { const map = getDepositRootByDepositCount(depositCounts, depositRootTree); - expect(mapToObj(map)).to.deep.equal(mapToObj(expectedMap)); + expect(renderDepositRootByDepositCount(map)).to.deep.equal(renderDepositRootByDepositCount(expectedMap)); }); } }); +function renderDepositRootByDepositCount(map: Map): Record { + const data: Record = {}; + for (const [key, root] of Object.entries(map)) { + data[key] = toHex(root); + } + return data; +} + function getMockBlock({blockNumber}: {blockNumber: number}): Eth1Block { return { blockNumber, diff --git a/packages/lodestar/test/unit/eth1/utils/eth1Vote.test.ts b/packages/lodestar/test/unit/eth1/utils/eth1Vote.test.ts index 3038be9fa5c..2cad0289b79 100644 --- a/packages/lodestar/test/unit/eth1/utils/eth1Vote.test.ts +++ b/packages/lodestar/test/unit/eth1/utils/eth1Vote.test.ts @@ -1,8 +1,8 @@ import {expect} from "chai"; import {config} from "@chainsafe/lodestar-config/default"; -import {List, TreeBacked} from "@chainsafe/ssz"; -import {allForks, phase0, ssz} from "@chainsafe/lodestar-types"; +import {phase0, ssz} from "@chainsafe/lodestar-types"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {generateState} from "../../../utils/state"; import {filterBy} from "../../../utils/db"; import { @@ -82,10 +82,10 @@ describe("eth1 / util / eth1Vote", function () { for (const testCase of testCases) { const {id, eth1DataVotesInState, votesToConsider, expectedEth1Vote} = testCase(); - it(`get eth1 vote: ${id}`, async function () { - const state = generateState({slot: 5, eth1DataVotes: eth1DataVotesInState as List}); + it(id, async function () { + const state = generateState({slot: 5, eth1DataVotes: eth1DataVotesInState}); const eth1Vote = pickEth1Vote(state, votesToConsider); - expect(ssz.phase0.Eth1Data.equals(eth1Vote, expectedEth1Vote)).to.be.true; + expect(ssz.phase0.Eth1Data.toJson(eth1Vote)).to.deep.equal(ssz.phase0.Eth1Data.toJson(expectedEth1Vote)); }); } }); @@ -94,7 +94,7 @@ describe("eth1 / util / eth1Vote", function () { // Function array to scope votes in each test case defintion const testCases: (() => { id: string; - state: TreeBacked; + state: BeaconStateAllForks; eth1Datas: IEth1DataWithTimestamp[]; expectedVotesToConsider: phase0.Eth1Data[]; })[] = [ @@ -132,7 +132,10 @@ describe("eth1 / util / eth1Vote", function () { filterBy(eth1Datas, timestampRange, (eth1Data) => eth1Data.timestamp); const votesToConsider = await getEth1VotesToConsider(config, state, eth1DataGetter); - expect(votesToConsider).to.deep.equal(expectedVotesToConsider); + + expect(votesToConsider.map((eth1Data) => ssz.phase0.Eth1Data.toJson(eth1Data))).to.deep.equal( + expectedVotesToConsider.map((eth1Data) => ssz.phase0.Eth1Data.toJson(eth1Data)) + ); }); } }); @@ -161,7 +164,7 @@ function getEth1DataBlock(eth1DataBlock: Partial): IEth1 * @param config * @param state */ -function getTimestampInRange(config: IChainForkConfig, state: TreeBacked): number { +function getTimestampInRange(config: IChainForkConfig, state: BeaconStateAllForks): number { const {SECONDS_PER_ETH1_BLOCK, ETH1_FOLLOW_DISTANCE} = config; const periodStart = votingPeriodStartTime(config, state); return periodStart - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE; diff --git a/packages/lodestar/test/unit/metrics/utils.ts b/packages/lodestar/test/unit/metrics/utils.ts index d5d0473a254..d13f132850b 100644 --- a/packages/lodestar/test/unit/metrics/utils.ts +++ b/packages/lodestar/test/unit/metrics/utils.ts @@ -3,6 +3,6 @@ import {ssz} from "@chainsafe/lodestar-types"; import {createMetrics, IMetrics} from "../../../src/metrics"; export function createMetricsTest(): IMetrics { - const state = ssz.phase0.BeaconState.defaultValue(); + const state = ssz.phase0.BeaconState.defaultViewDU(); return createMetrics({enabled: true, timeout: 12000}, config, state); } diff --git a/packages/lodestar/test/unit/network/attestationService.test.ts b/packages/lodestar/test/unit/network/attestationService.test.ts index 77f7928a888..fa38138f217 100644 --- a/packages/lodestar/test/unit/network/attestationService.test.ts +++ b/packages/lodestar/test/unit/network/attestationService.test.ts @@ -1,4 +1,3 @@ -import {allForks} from "@chainsafe/lodestar-types"; import { ATTESTATION_SUBNET_COUNT, EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION, @@ -6,14 +5,13 @@ import { SLOTS_PER_EPOCH, } from "@chainsafe/lodestar-params"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; -import {getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; // eslint-disable-next-line no-restricted-imports import * as mathUtils from "@chainsafe/lodestar-utils/lib/math"; import * as shuffleUtils from "../../../src/util/shuffle"; import sinon, {SinonStubbedInstance} from "sinon"; import {MockBeaconChain} from "../../utils/mocks/chain/chain"; import {generateState} from "../../utils/state"; -import {TreeBacked} from "@chainsafe/ssz"; import {testLogger} from "../../utils/logger"; import {expect} from "chai"; import {SinonStubFn} from "../../utils/types"; @@ -39,7 +37,7 @@ describe("AttnetsService", function () { let metadata: MetadataController; let chain: IBeaconChain; - let state: allForks.BeaconState; + let state: BeaconStateAllForks; const logger = testLogger(); const subscription: CommitteeSubscription = { validatorIndex: 2021, @@ -66,7 +64,7 @@ describe("AttnetsService", function () { genesisTime: Math.floor(Date.now() / 1000), chainId: 0, networkId: BigInt(0), - state: state as TreeBacked, + state, config, }); // load getCurrentSlot first, vscode not able to debug without this diff --git a/packages/lodestar/test/unit/network/peers/priorization.test.ts b/packages/lodestar/test/unit/network/peers/priorization.test.ts index a7ffb1046ca..287cbb2fcbb 100644 --- a/packages/lodestar/test/unit/network/peers/priorization.test.ts +++ b/packages/lodestar/test/unit/network/peers/priorization.test.ts @@ -1,6 +1,8 @@ import {expect} from "chai"; import PeerId from "peer-id"; import {phase0, altair} from "@chainsafe/lodestar-types"; +import {BitArray} from "@chainsafe/ssz"; +import {ATTESTATION_SUBNET_COUNT} from "@chainsafe/lodestar-params"; import {prioritizePeers} from "../../../../src/network/peers/utils/prioritizePeers"; import {getAttnets} from "../../../utils/network"; import {RequestedSubnet} from "../../../../src/network/peers/utils"; @@ -14,6 +16,7 @@ describe("network / peers / priorization", () => { peer.toB58String = () => `peer-${i}`; peers.push(peer); } + const none = BitArray.fromBitLen(ATTESTATION_SUBNET_COUNT); const testCases: { id: string; @@ -38,7 +41,7 @@ describe("network / peers / priorization", () => { }, { id: "Don't request a subnet query when enough peers are connected to it", - connectedPeers: [{id: peers[0], syncnets: [], attnets: getAttnets([3]), score: 0}], + connectedPeers: [{id: peers[0], syncnets: none, attnets: getAttnets([3]), score: 0}], activeAttnets: [3], activeSyncnets: [], opts: {targetPeers: 1, maxPeers: 1}, @@ -52,10 +55,10 @@ describe("network / peers / priorization", () => { { id: "Disconnect worst peers without duty", connectedPeers: [ - {id: peers[0], syncnets: [], attnets: getAttnets([3]), score: 0}, - {id: peers[1], syncnets: [], attnets: [], score: 0}, - {id: peers[2], syncnets: [], attnets: [], score: -20}, - {id: peers[3], syncnets: [], attnets: [], score: -40}, + {id: peers[0], syncnets: none, attnets: getAttnets([3]), score: 0}, + {id: peers[1], syncnets: none, attnets: none, score: 0}, + {id: peers[2], syncnets: none, attnets: none, score: -20}, + {id: peers[3], syncnets: none, attnets: none, score: -40}, ], activeAttnets: [3], activeSyncnets: [], @@ -71,14 +74,14 @@ describe("network / peers / priorization", () => { { id: "Complete example: Disconnect peers and request a subnet query", connectedPeers: [ - {id: peers[0], syncnets: [], attnets: getAttnets([0, 1, 2]), score: 0}, - {id: peers[1], syncnets: [], attnets: getAttnets([0, 1, 2]), score: -10}, - {id: peers[2], syncnets: [], attnets: getAttnets([0, 1]), score: 0}, - {id: peers[3], syncnets: [], attnets: getAttnets([0]), score: -10}, - {id: peers[4], syncnets: [], attnets: getAttnets([2]), score: 0}, - {id: peers[5], syncnets: [], attnets: getAttnets([0, 2]), score: -20}, - {id: peers[6], syncnets: [], attnets: getAttnets([1, 2, 3]), score: 0}, - {id: peers[7], syncnets: [], attnets: getAttnets([1, 2]), score: -10}, + {id: peers[0], syncnets: none, attnets: getAttnets([0, 1, 2]), score: 0}, + {id: peers[1], syncnets: none, attnets: getAttnets([0, 1, 2]), score: -10}, + {id: peers[2], syncnets: none, attnets: getAttnets([0, 1]), score: 0}, + {id: peers[3], syncnets: none, attnets: getAttnets([0]), score: -10}, + {id: peers[4], syncnets: none, attnets: getAttnets([2]), score: 0}, + {id: peers[5], syncnets: none, attnets: getAttnets([0, 2]), score: -20}, + {id: peers[6], syncnets: none, attnets: getAttnets([1, 2, 3]), score: 0}, + {id: peers[7], syncnets: none, attnets: getAttnets([1, 2]), score: -10}, ], activeAttnets: [1, 3], activeSyncnets: [], diff --git a/packages/lodestar/test/unit/network/peers/utils/enrSubnets.test.ts b/packages/lodestar/test/unit/network/peers/utils/enrSubnets.test.ts index eee689c7d20..d602f29b75c 100644 --- a/packages/lodestar/test/unit/network/peers/utils/enrSubnets.test.ts +++ b/packages/lodestar/test/unit/network/peers/utils/enrSubnets.test.ts @@ -1,5 +1,7 @@ import {SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; import {ssz} from "@chainsafe/lodestar-types"; +import {toHex} from "@chainsafe/lodestar-utils"; +import {BitArray} from "@chainsafe/ssz"; import {expect} from "chai"; import {deserializeEnrSubnets} from "../../../../../src/network/peers/utils/enrSubnetsDeserialize"; @@ -17,7 +19,9 @@ describe("ENR syncnets", () => { it(`Deserialize syncnet ${bytes}`, () => { const bytesBuf = Buffer.from(bytes, "hex"); - expect(ssz.altair.SyncSubnets.deserialize(bytesBuf)).to.deep.equal(bools); + expect(toHex(ssz.altair.SyncSubnets.deserialize(bytesBuf).uint8Array)).to.deep.equal( + toHex(BitArray.fromBoolArray(bools).uint8Array) + ); expect( deserializeEnrSubnets(bytesBuf, SYNC_COMMITTEE_SUBNET_COUNT).slice(0, SYNC_COMMITTEE_SUBNET_COUNT) diff --git a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/decode.test.ts b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/decode.test.ts index 7c9c95d6d47..7428fba6fe8 100644 --- a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/decode.test.ts +++ b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/decode.test.ts @@ -51,13 +51,13 @@ describe("network / reqresp / sszSnappy / decode", () => { id: "if it read more than maxEncodedLen", type: ssz.phase0.Ping, error: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, - chunks: [Buffer.from(varint.encode(ssz.phase0.Ping.getMinSerializedLength())), Buffer.alloc(100)], + chunks: [Buffer.from(varint.encode(ssz.phase0.Ping.minSize)), Buffer.alloc(100)], }, { id: "if failed ssz snappy input malformed", type: ssz.phase0.Status, error: SszSnappyErrorCode.DECOMPRESSOR_ERROR, - chunks: [Buffer.from(varint.encode(ssz.phase0.Status.minSize())), Buffer.from("wrong snappy data")], + chunks: [Buffer.from(varint.encode(ssz.phase0.Status.minSize)), Buffer.from("wrong snappy data")], }, ]; diff --git a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts index 128e63f64cb..96974ddb150 100644 --- a/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts +++ b/packages/lodestar/test/unit/network/reqresp/encodingStrategies/sszSnappy/testData.ts @@ -1,4 +1,4 @@ -import {fromHexString, List} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {altair, phase0, ssz} from "@chainsafe/lodestar-types"; import {RequestOrIncomingResponseBody, RequestOrResponseType} from "../../../../../../src/network/reqresp/types"; @@ -57,11 +57,11 @@ export const sszSnappySignedBeaconBlockPhase0: ISszSnappyTestData, - attesterSlashings: ([] as phase0.AttesterSlashing[]) as List, - attestations: ([] as phase0.Attestation[]) as List, - deposits: ([] as phase0.Deposit[]) as List, - voluntaryExits: ([] as phase0.SignedVoluntaryExit[]) as List, + proposerSlashings: [], + attesterSlashings: [], + attestations: [], + deposits: [], + voluntaryExits: [], }, }, signature: Buffer.alloc(96, 0xda), diff --git a/packages/lodestar/test/unit/network/reqresp/utils.ts b/packages/lodestar/test/unit/network/reqresp/utils.ts index 623cdcc6a07..ddcd9c2ed12 100644 --- a/packages/lodestar/test/unit/network/reqresp/utils.ts +++ b/packages/lodestar/test/unit/network/reqresp/utils.ts @@ -1,5 +1,5 @@ import {Root, phase0} from "@chainsafe/lodestar-types"; -import {List, toHexString} from "@chainsafe/ssz"; +import {toHexString} from "@chainsafe/ssz"; import {expect} from "chai"; import {Libp2pStream} from "../../../../src/network"; import {generateEmptySignedBlock} from "../../../utils/block"; @@ -14,12 +14,12 @@ export function createStatus(): phase0.Status { }; } -export function generateRoots(count: number, offset = 0): List { +export function generateRoots(count: number, offset = 0): Root[] { const roots: Root[] = []; for (let i = 0; i < count; i++) { roots.push(Buffer.alloc(32, i + offset)); } - return roots as List; + return roots; } /** diff --git a/packages/lodestar/test/unit/sync/backfill/verify.test.ts b/packages/lodestar/test/unit/sync/backfill/verify.test.ts index c8edad0fa68..b3960fb4274 100644 --- a/packages/lodestar/test/unit/sync/backfill/verify.test.ts +++ b/packages/lodestar/test/unit/sync/backfill/verify.test.ts @@ -1,5 +1,4 @@ import {BackfillSyncErrorCode, BackfillSyncError} from "./../../../../src/sync/backfill/errors"; -import {Json} from "@chainsafe/ssz"; import {createIBeaconConfig} from "@chainsafe/lodestar-config"; import {config} from "@chainsafe/lodestar-config/default"; import {phase0, ssz} from "@chainsafe/lodestar-types"; @@ -45,9 +44,9 @@ describe("backfill sync - verify block sequence", function () { //first 4 mainnet blocks function getBlocks(): phase0.SignedBeaconBlock[] { - const json = JSON.parse(readFileSync(path.join(__dirname, "./blocks.json"), "utf-8")) as Json[]; + const json = JSON.parse(readFileSync(path.join(__dirname, "./blocks.json"), "utf-8")) as unknown[]; return json.map((b) => { - return ssz.phase0.SignedBeaconBlock.fromJson(b, {case: "snake"}); + return ssz.phase0.SignedBeaconBlock.fromJson(b); }); } }); diff --git a/packages/lodestar/test/unit/sync/unknownBlock.test.ts b/packages/lodestar/test/unit/sync/unknownBlock.test.ts index deac5ae081f..58870251bc6 100644 --- a/packages/lodestar/test/unit/sync/unknownBlock.test.ts +++ b/packages/lodestar/test/unit/sync/unknownBlock.test.ts @@ -16,9 +16,9 @@ describe("sync / UnknownBlockSync", () => { it("fetch and process multiple unknown block parents", async () => { const peer = getValidPeerId(); const peerIdStr = peer.toB58String(); - const blockA = ssz.phase0.SignedBeaconBlock.defaultTreeBacked(); - const blockB = ssz.phase0.SignedBeaconBlock.defaultTreeBacked(); - const blockC = ssz.phase0.SignedBeaconBlock.defaultTreeBacked(); + const blockA = ssz.phase0.SignedBeaconBlock.defaultValue(); + const blockB = ssz.phase0.SignedBeaconBlock.defaultValue(); + const blockC = ssz.phase0.SignedBeaconBlock.defaultValue(); blockA.message.slot = 1; blockB.message.slot = 2; blockC.message.slot = 3; diff --git a/packages/lodestar/test/utils/aggregationBits.ts b/packages/lodestar/test/utils/aggregationBits.ts deleted file mode 100644 index 0793e16ba0d..00000000000 --- a/packages/lodestar/test/utils/aggregationBits.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {List} from "@chainsafe/ssz"; - -/** Create a filled bits array with a single true bit */ -export function toSingleBit(len: number, index: number): List { - const bits = ([] as boolean[]) as List; - for (let i = 0; i < len; i++) bits[i] = false; - bits[index] = true; - return bits; -} diff --git a/packages/lodestar/test/utils/attestation.ts b/packages/lodestar/test/utils/attestation.ts index b0e9ce5c1c2..ba5d4b9a3f6 100644 --- a/packages/lodestar/test/utils/attestation.ts +++ b/packages/lodestar/test/utils/attestation.ts @@ -1,9 +1,9 @@ -import {List} from "@chainsafe/ssz"; import {CommitteeIndex, Epoch, Slot, phase0} from "@chainsafe/lodestar-types"; import crypto from "node:crypto"; import deepmerge from "deepmerge"; import {isPlainObject} from "@chainsafe/lodestar-utils"; import {RecursivePartial} from "@chainsafe/lodestar-utils"; +import {BitArray} from "@chainsafe/ssz"; /** * Generates a fake attestation data for test purposes. @@ -38,7 +38,7 @@ export function generateAttestationData( export function generateAttestation(override: RecursivePartial = {}): phase0.Attestation { return deepmerge>( { - aggregationBits: Array.from({length: 64}, () => false) as List, + aggregationBits: BitArray.fromBitLen(64), data: { slot: 0, index: 0, diff --git a/packages/lodestar/test/utils/balances.ts b/packages/lodestar/test/utils/balances.ts deleted file mode 100644 index 32f0b76a5c8..00000000000 --- a/packages/lodestar/test/utils/balances.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {List} from "@chainsafe/ssz"; -import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {MAX_EFFECTIVE_BALANCE} from "@chainsafe/lodestar-params"; - -export function generateInitialMaxBalances(config: IChainForkConfig, count?: number): List { - return Array.from({length: count ?? config.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT}, () => - Number(MAX_EFFECTIVE_BALANCE) - ) as List; -} diff --git a/packages/lodestar/test/utils/block.ts b/packages/lodestar/test/utils/block.ts index eb13d4004c6..638d3241cf4 100644 --- a/packages/lodestar/test/utils/block.ts +++ b/packages/lodestar/test/utils/block.ts @@ -2,7 +2,6 @@ import {ssz} from "@chainsafe/lodestar-types"; import {config as defaultConfig} from "@chainsafe/lodestar-config/default"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; import {allForks, phase0} from "@chainsafe/lodestar-types"; -import {List} from "@chainsafe/ssz"; import {IProtoBlock, ExecutionStatus} from "@chainsafe/lodestar-fork-choice"; import {isPlainObject} from "@chainsafe/lodestar-utils"; import {RecursivePartial} from "@chainsafe/lodestar-utils"; @@ -24,11 +23,11 @@ export function generateEmptyBlock(): phase0.BeaconBlock { depositCount: 0, }, graffiti: Buffer.alloc(32), - proposerSlashings: ([] as phase0.ProposerSlashing[]) as List, - attesterSlashings: ([] as phase0.AttesterSlashing[]) as List, - attestations: ([] as phase0.Attestation[]) as List, - deposits: ([] as phase0.Deposit[]) as List, - voluntaryExits: ([] as phase0.SignedVoluntaryExit[]) as List, + proposerSlashings: [], + attesterSlashings: [], + attestations: [], + deposits: [], + voluntaryExits: [], }, }; } diff --git a/packages/lodestar/test/utils/cachedBeaconState.ts b/packages/lodestar/test/utils/cachedBeaconState.ts new file mode 100644 index 00000000000..7cac0c6b5c8 --- /dev/null +++ b/packages/lodestar/test/utils/cachedBeaconState.ts @@ -0,0 +1,14 @@ +import { + BeaconStateAllForks, + BeaconStateCache, + createCachedBeaconState, + createEmptyEpochContextImmutableData, +} from "@chainsafe/lodestar-beacon-state-transition"; +import {IChainForkConfig} from "@chainsafe/lodestar-config"; + +export function createCachedBeaconStateTest( + state: T, + chainConfig: IChainForkConfig +): T & BeaconStateCache { + return createCachedBeaconState(state, createEmptyEpochContextImmutableData(chainConfig, state)); +} diff --git a/packages/lodestar/test/utils/contributionAndProof.ts b/packages/lodestar/test/utils/contributionAndProof.ts index 35efdd03fd7..0e5b8bb0a0c 100644 --- a/packages/lodestar/test/utils/contributionAndProof.ts +++ b/packages/lodestar/test/utils/contributionAndProof.ts @@ -2,12 +2,12 @@ import {EMPTY_SIGNATURE} from "@chainsafe/lodestar-beacon-state-transition"; import {SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params"; import {altair} from "@chainsafe/lodestar-types"; import {isPlainObject, RecursivePartial} from "@chainsafe/lodestar-utils"; -import {fromHexString, List} from "@chainsafe/ssz"; +import {BitArray, fromHexString} from "@chainsafe/ssz"; import deepmerge from "deepmerge"; export function generateEmptyContribution(): altair.SyncCommitteeContribution { return { - aggregationBits: Array.from({length: SYNC_COMMITTEE_SUBNET_SIZE}, () => false) as List, + aggregationBits: BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_SIZE), beaconBlockRoot: Buffer.alloc(32), signature: fromHexString( "99cb82bc69b4111d1a828963f0316ec9aa38c4e9e041a8afec86cd20dfe9a590999845bf01d4689f3bbe3df54e48695e081f1216027b577c7fccf6ab0a4fcc75faf8009c6b55e518478139f604f542d138ae3bc34bad01ee6002006d64c4ff82" diff --git a/packages/lodestar/test/utils/errors.ts b/packages/lodestar/test/utils/errors.ts index b949fc62f59..1cff5e68303 100644 --- a/packages/lodestar/test/utils/errors.ts +++ b/packages/lodestar/test/utils/errors.ts @@ -1,6 +1,5 @@ import {expect} from "chai"; import {LodestarError, mapValues} from "@chainsafe/lodestar-utils"; -import {Json} from "@chainsafe/ssz"; export function expectThrowsLodestarError(fn: () => void, expectedErr: LodestarError | string): void { try { @@ -49,7 +48,7 @@ export function expectLodestarError(err1: LodestarErro expect(errMeta1).to.deep.equal(errMeta2, "Wrong LodestarError metadata"); } -export function getErrorMetadata(err: LodestarError | Error | Json): Json { +export function getErrorMetadata(err: LodestarError | Error | unknown): unknown { if (err instanceof LodestarError) { return mapValues(err.getMetadata(), (value) => getErrorMetadata(value as any)); } else if (err instanceof Error) { diff --git a/packages/lodestar/test/utils/mocks/chain/chain.ts b/packages/lodestar/test/utils/mocks/chain/chain.ts index 0cf3420a6bc..c0eaf5c3a0e 100644 --- a/packages/lodestar/test/utils/mocks/chain/chain.ts +++ b/packages/lodestar/test/utils/mocks/chain/chain.ts @@ -1,10 +1,10 @@ import {AbortController} from "@chainsafe/abort-controller"; import sinon from "sinon"; -import {toHexString, TreeBacked} from "@chainsafe/ssz"; -import {allForks, Number64, Root, Slot, ssz, Uint16, Uint64} from "@chainsafe/lodestar-types"; +import {toHexString} from "@chainsafe/ssz"; +import {allForks, UintNum64, Root, Slot, ssz, Uint16, UintBn64} from "@chainsafe/lodestar-types"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; -import {CachedBeaconStateAllForks, createCachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition"; +import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {phase0} from "@chainsafe/lodestar-beacon-state-transition"; import {CheckpointWithHex, IForkChoice, IProtoBlock, ExecutionStatus} from "@chainsafe/lodestar-fork-choice"; @@ -36,19 +36,20 @@ import {ExecutionEngineDisabled} from "../../../../src/executionEngine"; import {ReqRespBlockResponse} from "../../../../src/network/reqresp/types"; import {testLogger} from "../../logger"; import {ReprocessController} from "../../../../src/chain/reprocess"; +import {createCachedBeaconStateTest} from "@chainsafe/lodestar-beacon-state-transition/test/utils/state"; /* eslint-disable @typescript-eslint/no-empty-function */ export interface IMockChainParams { - genesisTime?: Number64; + genesisTime?: UintNum64; chainId: Uint16; - networkId: Uint64; - state: TreeBacked; + networkId: UintBn64; + state: BeaconStateAllForks; config: IBeaconConfig; } export class MockBeaconChain implements IBeaconChain { - readonly genesisTime: Number64; + readonly genesisTime: UintNum64; readonly genesisValidatorsRoot: Root; readonly eth1 = new Eth1ForBlockProductionDisabled(); readonly executionEngine = new ExecutionEngineDisabled(); @@ -60,7 +61,7 @@ export class MockBeaconChain implements IBeaconChain { stateCache: StateContextCache; checkpointStateCache: CheckpointStateCache; chainId: Uint16; - networkId: Uint64; + networkId: UintBn64; clock: IBeaconClock; regen: IStateRegenerator; emitter: ChainEventEmitter; @@ -81,7 +82,7 @@ export class MockBeaconChain implements IBeaconChain { readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages(); readonly seenContributionAndProof = new SeenContributionAndProof(); - private state: TreeBacked; + private state: BeaconStateAllForks; private abortController: AbortController; constructor({genesisTime, chainId, networkId, state, config}: IMockChainParams) { @@ -126,11 +127,11 @@ export class MockBeaconChain implements IBeaconChain { } getHeadState(): CachedBeaconStateAllForks { - return createCachedBeaconState(this.config, this.state); + return createCachedBeaconStateTest(this.state, this.config); } async getHeadStateAtCurrentEpoch(): Promise { - return createCachedBeaconState(this.config, this.state); + return createCachedBeaconStateTest(this.state, this.config); } async getCanonicalBlockAtSlot(slot: Slot): Promise { @@ -147,10 +148,6 @@ export class MockBeaconChain implements IBeaconChain { })); } - getGenesisTime(): Number64 { - return Math.floor(Date.now() / 1000); - } - async receiveBlock(): Promise {} async processBlock(): Promise {} async processChainSegment(): Promise {} diff --git a/packages/lodestar/test/utils/network.ts b/packages/lodestar/test/utils/network.ts index 1eb403d1a7a..394cc886996 100644 --- a/packages/lodestar/test/utils/network.ts +++ b/packages/lodestar/test/utils/network.ts @@ -1,6 +1,7 @@ import PeerId from "peer-id"; import {Multiaddr} from "multiaddr"; import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params"; +import {BitArray} from "@chainsafe/ssz"; import {Network} from "../../src/network"; import {NodejsNode} from "../../src/network/nodejs"; import {createPeerId} from "../../src/network"; @@ -41,10 +42,10 @@ export function onPeerDisconnect(network: Network): Promise { /** * Generate valid filled attnets BitVector */ -export function getAttnets(subnetIds: number[] = []): boolean[] { - const attnets = new Array(ATTESTATION_SUBNET_COUNT).fill(false); +export function getAttnets(subnetIds: number[] = []): BitArray { + const attnets = BitArray.fromBitLen(ATTESTATION_SUBNET_COUNT); for (const subnetId of subnetIds) { - attnets[subnetId] = true; + attnets.set(subnetId, true); } return attnets; } @@ -52,10 +53,10 @@ export function getAttnets(subnetIds: number[] = []): boolean[] { /** * Generate valid filled syncnets BitVector */ -export function getSyncnets(subnetIds: number[] = []): boolean[] { - const attnets = new Array(SYNC_COMMITTEE_SUBNET_COUNT).fill(false); +export function getSyncnets(subnetIds: number[] = []): BitArray { + const syncnets = BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_COUNT); for (const subnetId of subnetIds) { - attnets[subnetId] = true; + syncnets.set(subnetId, true); } - return attnets; + return syncnets; } diff --git a/packages/lodestar/test/utils/node/beacon.ts b/packages/lodestar/test/utils/node/beacon.ts index 61721557387..3c4562a3662 100644 --- a/packages/lodestar/test/utils/node/beacon.ts +++ b/packages/lodestar/test/utils/node/beacon.ts @@ -6,6 +6,8 @@ import {config as minimalConfig} from "@chainsafe/lodestar-config/default"; import {createIBeaconConfig, createIChainForkConfig, IChainConfig} from "@chainsafe/lodestar-config"; import {ILogger, RecursivePartial} from "@chainsafe/lodestar-utils"; import {LevelDbController} from "@chainsafe/lodestar-db"; +import {phase0} from "@chainsafe/lodestar-types"; +import {BeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {BeaconNode} from "../../../src/node"; import {createNodeJsLibp2p} from "../../../src/network/nodejs"; import {createPeerId} from "../../../src/network"; @@ -16,8 +18,6 @@ import {defaultOptions} from "../../../src/node/options"; import {BeaconDb} from "../../../src/db"; import {testLogger} from "../logger"; import {InteropStateOpts} from "../../../src/node/utils/interop/state"; -import {TreeBacked} from "@chainsafe/ssz"; -import {allForks, phase0} from "@chainsafe/lodestar-types"; import {isPlainObject} from "@chainsafe/lodestar-utils"; export async function getDevBeaconNode( @@ -28,7 +28,7 @@ export async function getDevBeaconNode( logger?: ILogger; peerId?: PeerId; peerStoreDir?: string; - anchorState?: TreeBacked; + anchorState?: BeaconStateAllForks; wsCheckpoint?: phase0.Checkpoint; } & InteropStateOpts ): Promise { diff --git a/packages/lodestar/test/utils/node/simTest.ts b/packages/lodestar/test/utils/node/simTest.ts index 40316b43030..adb3e9a0f6a 100644 --- a/packages/lodestar/test/utils/node/simTest.ts +++ b/packages/lodestar/test/utils/node/simTest.ts @@ -11,11 +11,11 @@ import {SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@chainsafe/lodestar-pa import {Epoch, Slot} from "@chainsafe/lodestar-types"; import {Checkpoint} from "@chainsafe/lodestar-types/phase0"; import {ILogger, mapValues} from "@chainsafe/lodestar-utils"; +import {toHexString} from "@chainsafe/ssz"; import {BeaconNode} from "../../../src"; import {ChainEvent} from "../../../src/chain"; import {linspace} from "../../../src/util/numpy"; import {RegenCaller} from "../../../src/chain/regen"; -import {toHexString} from "@chainsafe/ssz"; /* eslint-disable no-console */ @@ -69,7 +69,7 @@ export function simTestInfoTracker(bn: BeaconNode, logger: ILogger): () => void // Recover the pre-epoch transition state, use any random caller for regen const checkpointState = await bn.chain.regen.getCheckpointState(checkpoint, RegenCaller.onForkChoiceFinalized); const lastSlot = computeStartSlotAtEpoch(checkpoint.epoch) - 1; - const lastStateRoot = checkpointState.stateRoots[lastSlot % SLOTS_PER_HISTORICAL_ROOT]; + const lastStateRoot = checkpointState.stateRoots.get(lastSlot % SLOTS_PER_HISTORICAL_ROOT); const lastState = await bn.chain.regen.getState(toHexString(lastStateRoot), RegenCaller.onForkChoiceFinalized); logParticipation(lastState); } @@ -91,7 +91,7 @@ export function simTestInfoTracker(bn: BeaconNode, logger: ILogger): () => void function sumAttestationBits(block: allForks.BeaconBlock): number { return Array.from(block.body.attestations).reduce( - (total, att) => total + Array.from(att.aggregationBits).filter(Boolean).length, + (total, att) => total + att.aggregationBits.getTrueBitIndexes().length, 0 ); } diff --git a/packages/lodestar/test/utils/render.ts b/packages/lodestar/test/utils/render.ts new file mode 100644 index 00000000000..9001bf0e6c0 --- /dev/null +++ b/packages/lodestar/test/utils/render.ts @@ -0,0 +1,5 @@ +import {BitArray} from "@chainsafe/ssz"; + +export function renderBitArray(bitArray: BitArray): string { + return Buffer.from(bitArray.uint8Array).toString("hex"); +} diff --git a/packages/lodestar/test/utils/state.ts b/packages/lodestar/test/utils/state.ts index 7bcb26cf7ef..5e7540ac220 100644 --- a/packages/lodestar/test/utils/state.ts +++ b/packages/lodestar/test/utils/state.ts @@ -1,8 +1,14 @@ import {config as minimalConfig} from "@chainsafe/lodestar-config/default"; -import {CachedBeaconStateAllForks, createCachedBeaconState, phase0} from "@chainsafe/lodestar-beacon-state-transition"; -import {List, TreeBacked} from "@chainsafe/ssz"; -import {allForks, altair, Root, ssz} from "@chainsafe/lodestar-types"; -import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import { + BeaconStateAllForks, + CachedBeaconStateAllForks, + createCachedBeaconState, + phase0, + PubkeyIndexMap, +} from "@chainsafe/lodestar-beacon-state-transition"; +import {BitArray} from "@chainsafe/ssz"; +import {allForks, altair, ssz} from "@chainsafe/lodestar-types"; +import {createIBeaconConfig} from "@chainsafe/lodestar-config"; import { EPOCHS_PER_HISTORICAL_VECTOR, EPOCHS_PER_SLASHINGS_VECTOR, @@ -23,13 +29,10 @@ import {initBLS} from "@chainsafe/lodestar-cli/src/util"; */ type TestBeaconState = Partial; -const phase0States = new Map>(); -const altairStates = new Map>(); - /** * Generate beaconState, by default it will generate a mostly empty state with "just enough" to be valid-ish * NOTE: All fields can be overridden through `opts`. - * should allow 1st test calling generateState more time since TreeBacked.createValue api is expensive. + * should allow 1st test calling generateState more time since creating a new instance is expensive. * * @param {TestBeaconState} opts * @param config @@ -40,7 +43,7 @@ export function generateState( config = minimalConfig, isAltair = false, withPubkey = false -): TreeBacked { +): BeaconStateAllForks { const validatorOpts = { activationEpoch: 0, effectiveBalance: MAX_EFFECTIVE_BALANCE, @@ -75,21 +78,21 @@ export function generateState( }, blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH), - historicalRoots: ([] as Root[]) as List, + historicalRoots: [], eth1Data: { depositRoot: Buffer.alloc(32), blockHash: Buffer.alloc(32), depositCount: 0, }, - eth1DataVotes: ([] as phase0.Eth1Data[]) as List, + eth1DataVotes: [], eth1DepositIndex: 0, - validators: validators as List, - balances: Array.from({length: numValidators}, () => MAX_EFFECTIVE_BALANCE) as List, + validators: validators, + balances: Array.from({length: numValidators}, () => MAX_EFFECTIVE_BALANCE), randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH), slashings: Array.from({length: EPOCHS_PER_SLASHINGS_VECTOR}, () => BigInt(0)), - previousEpochAttestations: ([] as phase0.PendingAttestation[]) as List, - currentEpochAttestations: ([] as phase0.PendingAttestation[]) as List, - justificationBits: Array.from({length: 4}, () => false), + previousEpochAttestations: [], + currentEpochAttestations: [], + justificationBits: BitArray.fromBitLen(4), previousJustifiedCheckpoint: { epoch: GENESIS_EPOCH, root: ZERO_HASH, @@ -102,16 +105,15 @@ export function generateState( epoch: GENESIS_EPOCH, root: ZERO_HASH, }, + ...opts, }; + if (isAltair) { const defaultAltairState: altair.BeaconState = { - ...ssz.altair.BeaconState.struct_defaultValue(), + ...ssz.altair.BeaconState.defaultValue(), ...defaultState, - previousEpochParticipation: [ - ...[0xff, 0xff], - ...Array.from({length: numValidators - 2}, () => 0), - ] as List, - currentEpochParticipation: [...[0xff, 0xff], ...Array.from({length: numValidators - 2}, () => 0)] as List, + previousEpochParticipation: [...[0xff, 0xff], ...Array.from({length: numValidators - 2}, () => 0)], + currentEpochParticipation: [...[0xff, 0xff], ...Array.from({length: numValidators - 2}, () => 0)], currentSyncCommittee: { pubkeys: Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => validators[i % validators.length].pubkey), aggregatePubkey: ssz.BLSPubkey.defaultValue(), @@ -122,26 +124,10 @@ export function generateState( }, }; - const state = - altairStates.get(config) ?? - (ssz.altair.BeaconState.createTreeBackedFromStruct(defaultAltairState) as TreeBacked); - altairStates.set(config, state); + return ssz.altair.BeaconState.toViewDU(defaultAltairState); } else { - const state = - phase0States.get(config) ?? - (ssz.phase0.BeaconState.createTreeBackedFromStruct(defaultState) as TreeBacked); - phase0States.set(config, state); - } - const resultState = (isAltair - ? altairStates.get(config)?.clone() - : phase0States.get(config)?.clone()) as TreeBacked; - - for (const key in opts) { - const newValue = opts[key as keyof TestBeaconState]; - // eslint-disable-next-line - resultState[key as keyof TestBeaconState] = (newValue as unknown) as any; + return ssz.phase0.BeaconState.toViewDU(defaultState); } - return resultState; } /** @@ -153,7 +139,12 @@ export function generateCachedState( isAltair = false ): CachedBeaconStateAllForks { const state = generateState(opts, config, isAltair); - return createCachedBeaconState(config, state); + return createCachedBeaconState(state, { + config: createIBeaconConfig(config, state.genesisValidatorsRoot), + // This is a performance test, there's no need to have a global shared cache of keys + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }); } /** @@ -166,6 +157,5 @@ export async function generateCachedStateWithPubkeys( ): Promise { // somehow this is called in the test but BLS isn't init await initBLS(); - const state = generateState(opts, config, isAltair, true); - return createCachedBeaconState(config, state); + return generateCachedState(opts, config, isAltair); } diff --git a/packages/lodestar/test/utils/stub/index.ts b/packages/lodestar/test/utils/stub/index.ts index 2096e154bfb..28d24d111b4 100644 --- a/packages/lodestar/test/utils/stub/index.ts +++ b/packages/lodestar/test/utils/stub/index.ts @@ -1,13 +1,6 @@ import {SinonStubbedInstance} from "sinon"; +import {IBeaconChain} from "../../../src/chain"; -import {IForkChoice} from "@chainsafe/lodestar-fork-choice"; -import {IBeaconChain, ChainEventEmitter} from "../../../src/chain"; - -interface IStubbedChain extends IBeaconChain { - forkChoice: SinonStubbedInstance; - emitter: SinonStubbedInstance; -} - -export type StubbedChain = IStubbedChain & SinonStubbedInstance; +export type StubbedChain = IBeaconChain & SinonStubbedInstance; export * from "./beaconDb"; diff --git a/packages/lodestar/test/utils/validationData/attestation.ts b/packages/lodestar/test/utils/validationData/attestation.ts index 3f74c909f85..bd0668edd40 100644 --- a/packages/lodestar/test/utils/validationData/attestation.ts +++ b/packages/lodestar/test/utils/validationData/attestation.ts @@ -1,5 +1,4 @@ import { - CachedBeaconStateAllForks, computeEpochAtSlot, computeSigningRoot, computeStartSlotAtEpoch, @@ -16,11 +15,9 @@ import { } from "@chainsafe/lodestar-beacon-state-transition/test/perf/util"; import {SeenAttesters} from "../../../src/chain/seenCache"; import {BlsSingleThreadVerifier} from "../../../src/chain/bls"; -import {computeSubnetForSlot} from "../../../src/chain/validation"; import {signCached} from "../cache"; import {ClockStatic} from "../clock"; -import {toSingleBit} from "../aggregationBits"; -import {toHexString} from "@chainsafe/ssz"; +import {BitArray, toHexString} from "@chainsafe/ssz"; import {config} from "@chainsafe/lodestar-config/default"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; @@ -77,9 +74,8 @@ export function getAttestationValidData( }, } as Partial) as IForkChoice; - const committeeIndices = state.getBeaconCommittee(attSlot, attIndex); + const committeeIndices = state.epochCtx.getBeaconCommittee(attSlot, attIndex); const validatorIndex = committeeIndices[bitIndex]; - const aggregationBits = toSingleBit(committeeIndices.length, bitIndex); const attestationData: phase0.AttestationData = { slot: attSlot, @@ -101,16 +97,16 @@ export function getAttestationValidData( const sk = getSecretKeyFromIndexCached(validatorIndex); const attestation: phase0.Attestation = { - aggregationBits, + aggregationBits: BitArray.fromSingleBit(committeeIndices.length, bitIndex), data: attestationData, signature: signCached(sk, signingRoot), }; - const subnet = computeSubnetForSlot(state, attSlot, attIndex); + const subnet = state.epochCtx.computeSubnetForSlot(attSlot, attIndex); // Add state to regen const regen = ({ - getState: async () => (state as unknown) as CachedBeaconStateAllForks, + getState: async () => state, } as Partial) as IStateRegenerator; const chain = ({ diff --git a/packages/lodestar/test/utils/validator.ts b/packages/lodestar/test/utils/validator.ts index 939951d6cc1..eec5760f3de 100644 --- a/packages/lodestar/test/utils/validator.ts +++ b/packages/lodestar/test/utils/validator.ts @@ -1,4 +1,4 @@ -import {fromHexString, List} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {phase0} from "@chainsafe/lodestar-types"; import {FAR_FUTURE_EPOCH} from "../../src/constants"; @@ -33,6 +33,6 @@ export function generateValidator(opts: Partial = {}): phase0. * @param opts * @returns {Validator[]} */ -export function generateValidators(n: number, opts?: Partial): List { - return Array.from({length: n}, () => generateValidator(opts)) as List; +export function generateValidators(n: number, opts?: Partial): phase0.Validator[] { + return Array.from({length: n}, () => generateValidator(opts)); } diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 266e2caa73d..5629dd6673f 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -46,7 +46,6 @@ ], "dependencies": { "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/ssz": "^0.8.20", "async-retry": "^1.3.3", "axios": "^0.21.0", "chai": "^4.2.0", diff --git a/packages/spec-test-util/src/index.ts b/packages/spec-test-util/src/index.ts index 157f29b3552..fb039991ef0 100644 --- a/packages/spec-test-util/src/index.ts +++ b/packages/spec-test-util/src/index.ts @@ -1,4 +1,3 @@ export * from "./downloadTests"; -export * from "./multi"; export * from "./single"; -export * from "./transform"; +export * from "./sszGeneric"; diff --git a/packages/spec-test-util/src/multi.ts b/packages/spec-test-util/src/multi.ts deleted file mode 100644 index 9d449b1a3cd..00000000000 --- a/packages/spec-test-util/src/multi.ts +++ /dev/null @@ -1,88 +0,0 @@ -import fs from "node:fs"; -import {loadYaml} from "@chainsafe/lodestar-utils"; -import {expect} from "chai"; - -/* eslint-disable - @typescript-eslint/no-unsafe-call, - @typescript-eslint/no-unsafe-member-access, - @typescript-eslint/no-unsafe-return, - @typescript-eslint/no-unsafe-assignment, - @typescript-eslint/no-explicit-any, - @typescript-eslint/no-unused-vars, - @typescript-eslint/naming-convention, - func-names */ - -export interface IBaseCase { - description: string; -} - -/** - * TestSpec - represent structure of yaml file containing spec test cases - * TestCase - single test case, usually under test_cases property in yaml file - */ -// eslint-disable-next-line @typescript-eslint/naming-convention -interface TestSpec { - title: string; - summary: string; - forks_timeline: string; - forks: string; - config: string; - runner: string; - handler: string; - test_cases: TestCase[]; -} - -/** - * Run yaml Ethereum Consensus bulk spec tests (m) for a certain function - * Compares actual vs expected for all test cases - * @param {string} testYamlPath - path to yaml spec test - * @param {Function} testFunc - function to use to generate output - * @param {Function} getInput - function to convert test case into input array - * @param {Function} getExpected - function to convert test case into a - * comparable expected output - * @param {Function} getActual - function to convert function output into - * comparable actual output - * @param {Function} shouldError - function to convert test case into a - * boolean, if the case should result in an error - * @param {Function} shouldSkip - function to convert test case into a boolean, - * if the case should be skipped - * @param {Function} expectFunc - function to run expectations against expected - * and actual output - * @param timeout - how long to wait before marking tests as failed (default 2000ms). Set to 0 to wait infinitely - */ -export function describeMultiSpec( - testYamlPath: string, - testFunc: (...args: any) => any, - getInput: (testCase: TestCase) => any, - getExpected: (testCase: TestCase) => any, - getActual: (result: any) => Result, - shouldError = (testCase: TestCase, index: number) => false, - shouldSkip = (testCase: TestCase, index: number) => false, - expectFunc = (testCase: TestCase, expect: any, expected: any, actual: any) => expect(actual).to.be.equal(expected), - timeout = 10 * 60 * 1000 -): void { - const testSpec = loadYaml>(fs.readFileSync(testYamlPath, "utf8")); - - const testSuiteName = `${testSpec.runner} - ${testSpec.handler} - ${testSpec.title} - ${testSpec.config}`; - - describe(testSuiteName, function () { - this.timeout(timeout); - for (const [index, testCase] of testSpec.test_cases.entries()) { - if (shouldSkip(testCase, index)) { - continue; - } - const description = index + (testCase.description ? " - " + testCase.description : ""); - it(description, function () { - const inputs = getInput(testCase); - if (shouldError(testCase, index)) { - expect(testFunc.bind(null, ...inputs)).to.throw(); - } else { - const result = testFunc(...inputs); - const actual = getActual(result); - const expected = getExpected(testCase); - expectFunc(testCase, expect, expected, actual); - } - }); - } - }); -} diff --git a/packages/spec-test-util/src/single.ts b/packages/spec-test-util/src/single.ts index 600300ce948..d725caa7583 100644 --- a/packages/spec-test-util/src/single.ts +++ b/packages/spec-test-util/src/single.ts @@ -1,7 +1,6 @@ import {expect} from "chai"; -import fs, {readdirSync, readFileSync, existsSync} from "node:fs"; +import fs from "node:fs"; import {basename, join, parse} from "node:path"; -import {Type, CompositeType} from "@chainsafe/ssz"; import {uncompress} from "snappyjs"; import {loadYaml} from "@chainsafe/lodestar-utils"; @@ -23,6 +22,12 @@ export type ExpandedInputType = { treeBacked: boolean; }; +type SszTypeGeneric = { + typeName: string; + deserialize: (bytes: Uint8Array) => unknown; + deserializeToViewDU?: (bytes: Uint8Array) => unknown; +}; + export function toExpandedInputType(inputType: InputType | ExpandedInputType): ExpandedInputType { if ((inputType as ExpandedInputType).type) { return inputType as ExpandedInputType; @@ -44,12 +49,12 @@ export interface ISpecTestOptions { */ inputTypes?: {[K in keyof NonNullable]?: InputType | ExpandedInputType}; - sszTypes?: Record>; + sszTypes?: Record; /** * Some tests need to access the test case in order to generate ssz types for each input file. */ - getSszTypes?: (meta: TestCase["meta"]) => Record>; + getSszTypes?: (meta: TestCase["meta"]) => Record; /** * loadInputFiles sometimes not create TestCase due to abnormal input file names. @@ -108,7 +113,8 @@ export function describeDirectorySpecTest this.timeout(options.timeout || "10 min"); } - const testCases = readdirSync(testCaseDirectoryPath) + const testCases = fs + .readdirSync(testCaseDirectoryPath) .map((name) => join(testCaseDirectoryPath, name)) .filter(isDirectory); @@ -129,7 +135,7 @@ function generateTestCase( // some tests require to load meta.yaml first in order to know respective ssz types. const metaFilePath = join(testCaseDirectoryPath, "meta.yaml"); let meta: TestCase["meta"] = undefined; - if (existsSync(metaFilePath)) { + if (fs.existsSync(metaFilePath)) { meta = loadYaml(fs.readFileSync(metaFilePath, "utf8")); } let testCase = loadInputFiles(testCaseDirectoryPath, options, meta); @@ -160,7 +166,7 @@ function loadInputFiles( meta?: TestCase["meta"] ): TestCase { const testCase: any = {}; - readdirSync(directory) + fs.readdirSync(directory) .map((name) => join(directory, name)) .filter((file) => { if (isDirectory(file)) { @@ -180,10 +186,10 @@ function loadInputFiles( testCase[inputName] = deserializeInputFile(file, inputName, inputType, options, meta); switch (inputType) { case InputType.SSZ: - testCase[`${inputName}_raw`] = readFileSync(file); + testCase[`${inputName}_raw`] = fs.readFileSync(file); break; case InputType.SSZ_SNAPPY: - testCase[`${inputName}_raw`] = uncompress(readFileSync(file)); + testCase[`${inputName}_raw`] = uncompress(fs.readFileSync(file)); break; } if (!options.inputProcessing) throw Error("inputProcessing is not defined"); @@ -217,11 +223,12 @@ function deserializeInputFile( } else if (inputType === InputType.SSZ || inputType === InputType.SSZ_SNAPPY) { const sszTypes = options.getSszTypes ? options.getSszTypes(meta) : options.sszTypes; if (!sszTypes) throw Error("sszTypes is not defined"); - let data = readFileSync(file); + let data = fs.readFileSync(file); if (inputType === InputType.SSZ_SNAPPY) { data = uncompress(data); } - let sszType: Type | undefined; + + let sszType: SszTypeGeneric | undefined; for (const key of Object.keys(sszTypes)) { // most tests configure with exact match // fork_choice tests configure with regex @@ -230,15 +237,23 @@ function deserializeInputFile( break; } } - if (sszType) { - if ((options.inputTypes?.[inputName as keyof TestCase] as ExpandedInputType).treeBacked) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - return (sszType as CompositeType).createTreeBackedFromBytes(data); - } else { - return sszType.deserialize(data); + + if (!sszType) { + throw Error("Cannot find ssz type for inputName " + inputName); + } + + // TODO: Refactor this to be typesafe + if (sszType.typeName === "BeaconState") { + if (!sszType.deserializeToViewDU) { + throw Error("BeaconState type has no deserializeToViewDU method"); } + return sszType.deserializeToViewDU(data); } else { - throw Error("Cannot find ssz type for inputName " + inputName); + return sszType.deserialize(data); } } } + +export function loadYamlFile(path: string): Record { + return loadYaml(fs.readFileSync(path, "utf8")); +} diff --git a/packages/spec-test-util/src/sszGeneric.ts b/packages/spec-test-util/src/sszGeneric.ts index 8fb4f5337ab..a40b5faa811 100644 --- a/packages/spec-test-util/src/sszGeneric.ts +++ b/packages/spec-test-util/src/sszGeneric.ts @@ -1,65 +1,63 @@ -import path, {join} from "node:path"; -import fs, {readFileSync, readdirSync} from "node:fs"; -import {Json, Type} from "@chainsafe/ssz"; -import {loadYaml, objectToExpectedCase} from "@chainsafe/lodestar-utils"; +import path from "node:path"; +import fs from "node:fs"; +import {loadYaml} from "@chainsafe/lodestar-utils"; import {uncompress} from "snappyjs"; -export interface IValidTestcase { - root: string; - serialized: Uint8Array; - value: T; -} +/* eslint-disable + @typescript-eslint/explicit-module-boundary-types, + @typescript-eslint/explicit-function-return-type +*/ -export interface IInvalidTestcase { - path: string; +export type ValidTestCaseData = { + root: string; serialized: Uint8Array; -} + jsonValue: unknown; +}; -export function parseValidTestcase(dirpath: string, type: Type): IValidTestcase { +export function parseSszValidTestcase(dirpath: string, metaFilename: string): ValidTestCaseData { // The root is stored in meta.yml as: // root: 0xDEADBEEF - const metaStr = fs.readFileSync(path.join(dirpath, "meta.yaml"), "utf8"); - const meta = loadYaml<{root: string}>(metaStr); + const metaStr = fs.readFileSync(path.join(dirpath, metaFilename), "utf8"); + const meta = loadYaml(metaStr) as {root: string}; if (typeof meta.root !== "string") { throw Error(`meta.root not a string: ${meta.root}\n${fs}`); } + // The serialized value is stored in serialized.ssz_snappy - const serialized = uncompress(readFileSync(join(dirpath, "serialized.ssz_snappy"))); + const serialized = uncompress(fs.readFileSync(path.join(dirpath, "serialized.ssz_snappy"))); // The value is stored in value.yml - const yamlSnake = loadYaml(fs.readFileSync(join(dirpath, "value.yaml"), "utf8")); - const yamlCamel = objectToExpectedCase(yamlSnake, "camel"); - const value = type.fromJson(yamlCamel as Json); + const yamlPath = path.join(dirpath, "value.yaml"); + const yamlStrNumbersAsNumbers = fs.readFileSync(yamlPath, "utf8"); + const jsonValue = readYamlNumbersAsStrings(yamlStrNumbersAsNumbers); + + // type.fromJson(loadYamlFile(path.join(dirpath, "value.yaml")) as Json) as T; return { root: meta.root, serialized, - value, + jsonValue, }; } -export function parseInvalidTestcase(path: string): IInvalidTestcase { +/** + * ssz_generic + * | basic_vector + * | invalid + * | vec_bool_0 + * | serialized.ssz_snappy + * + * Docs: https://github.com/ethereum/eth2.0-specs/blob/master/tests/formats/ssz_generic/README.md + */ +export function parseSszGenericInvalidTestcase(dirpath: string) { // The serialized value is stored in serialized.ssz_snappy - const serialized = uncompress(readFileSync(join(path, "serialized.ssz_snappy"))); + const serialized = uncompress(fs.readFileSync(path.join(dirpath, "serialized.ssz_snappy"))); return { - path, serialized, }; } -export function getValidTestcases(path: string, prefix: string, type: Type): IValidTestcase[] { - const subdirs = readdirSync(path); - return subdirs - .filter((dir) => dir.includes(prefix)) - .map((d) => join(path, d)) - .map((p) => parseValidTestcase(p, type)) as IValidTestcase[]; -} - -export function getInvalidTestcases(path: string, prefix: string): IInvalidTestcase[] { - const subdirs = readdirSync(path); - return subdirs - .filter((dir) => dir.includes(prefix)) - .map((d) => join(path, d)) - .map(parseInvalidTestcase); +export function readYamlNumbersAsStrings(yamlStr: string): unknown { + return loadYaml(yamlStr); } diff --git a/packages/spec-test-util/src/transform.ts b/packages/spec-test-util/src/transform.ts deleted file mode 100644 index 1bba4c0082f..00000000000 --- a/packages/spec-test-util/src/transform.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment -, @typescript-eslint/no-explicit-any */ -import {Type, UintType, BigIntUintType, byteType, isCompositeType} from "@chainsafe/ssz"; - -/** - * Transform the type to something that is safe to deserialize - * - * This mainly entails making sure all numbers are bignumbers - */ -export function safeType(type: Type): Type { - if (type === byteType) { - return type; - } else if (!isCompositeType(type)) { - if ((type as UintType).byteLength) { - return new BigIntUintType({byteLength: (type as UintType).byteLength}); - } else { - return type; - } - } else { - const props = Object.getOwnPropertyDescriptors(type) as any; - if (props.elementType) { - if (props.elementType.byteLength !== 1) { - props.elementType.value = safeType(props.elementType.value); - } - } - if (props.fields) { - props.fields.value = {...props.fields.value}; - for (const fieldName of Object.keys(props.fields.value)) { - props.fields.value[fieldName] = safeType(props.fields.value[fieldName]); - } - } - const newtype = Object.create(Object.getPrototypeOf(type), props); - return newtype as Type; - } -} diff --git a/packages/spec-test-util/test/e2e/_test_files/multi/bulk.yml b/packages/spec-test-util/test/e2e/_test_files/multi/bulk.yml deleted file mode 100644 index 42bf48f37fc..00000000000 --- a/packages/spec-test-util/test/e2e/_test_files/multi/bulk.yml +++ /dev/null @@ -1,14 +0,0 @@ -title: Bulk test -summary: simulates multiple test cases in one file -forks_timeline: mainnet -forks: [phase0] -config: mainnet -runner: bulkRunner -handler: test -test_cases: - - input: "0xb53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f" - description: "test1" - output: "0xb53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f" - - input: "0xb53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f62" - description: "test2" - output: "0xb53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f62" diff --git a/packages/spec-test-util/test/e2e/multi/index.test.ts b/packages/spec-test-util/test/e2e/multi/index.test.ts deleted file mode 100644 index e4a1fe9943e..00000000000 --- a/packages/spec-test-util/test/e2e/multi/index.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -import {IBaseCase, describeMultiSpec} from "../../../src"; -import path from "node:path"; - -interface IBulkTestCase extends IBaseCase { - input: string; - output: string; -} - -describeMultiSpec( - path.join(__dirname, "../_test_files/multi/bulk.yml"), - (input) => input, - (testCase) => [testCase.input], - (testCase) => testCase.output, - (result) => result -); diff --git a/packages/spec-test-util/test/e2e/single/index.test.ts b/packages/spec-test-util/test/e2e/single/index.test.ts index 34934c72ea5..89aaefbf303 100644 --- a/packages/spec-test-util/test/e2e/single/index.test.ts +++ b/packages/spec-test-util/test/e2e/single/index.test.ts @@ -1,11 +1,10 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ -import fs, {unlinkSync, writeFileSync} from "node:fs"; +import {unlinkSync, writeFileSync} from "node:fs"; import {join} from "node:path"; -import {ContainerType, Type, Json} from "@chainsafe/ssz"; +import {ContainerType, Type} from "@chainsafe/ssz"; import {ssz} from "@chainsafe/lodestar-types"; -import {describeDirectorySpecTest, InputType} from "../../../src/single"; -import {loadYaml} from "@chainsafe/lodestar-utils"; +import {describeDirectorySpecTest, InputType, loadYamlFile} from "../../../src/single"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -22,18 +21,16 @@ export interface ISimpleCase extends Iterable { }; } -const inputSchema = new ContainerType({ - fields: { - test: ssz.Boolean, - number: ssz.Number64, - }, +const sampleContainerType = new ContainerType({ + test: ssz.Boolean, + number: ssz.UintNum64, }); before(() => { - yamlToSSZ(join(__dirname, "../_test_files/single/case0/input.yaml"), inputSchema); - yamlToSSZ(join(__dirname, "../_test_files/single/case0/output.yaml"), ssz.Number64); - yamlToSSZ(join(__dirname, "../_test_files/single/case1/input.yaml"), inputSchema); - yamlToSSZ(join(__dirname, "../_test_files/single/case1/output.yaml"), ssz.Number64); + yamlToSSZ(join(__dirname, "../_test_files/single/case0/input.yaml"), sampleContainerType); + yamlToSSZ(join(__dirname, "../_test_files/single/case0/output.yaml"), ssz.UintNum64); + yamlToSSZ(join(__dirname, "../_test_files/single/case1/input.yaml"), sampleContainerType); + yamlToSSZ(join(__dirname, "../_test_files/single/case1/output.yaml"), ssz.UintNum64); }); after(() => { @@ -55,8 +52,8 @@ describeDirectorySpecTest( output: InputType.YAML, }, sszTypes: { - input: inputSchema, - output: ssz.Number64, + input: sampleContainerType, + output: ssz.UintNum64, }, shouldError: (testCase) => !testCase.input.test, getExpected: (testCase) => testCase.output, @@ -64,10 +61,6 @@ describeDirectorySpecTest( ); function yamlToSSZ(file: string, sszSchema: Type): void { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const input: any = sszSchema.fromJson(loadYaml(fs.readFileSync(file, "utf8"))); - if (input.number) { - input.number = Number(input.number); - } + const input = sszSchema.fromJson(loadYamlFile(file)) as {test: boolean; number: number}; writeFileSync(file.replace(".yaml", ".ssz"), sszSchema.serialize(input)); } diff --git a/packages/types/package.json b/packages/types/package.json index c1d82d813c6..fa02b544ef7 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -36,7 +36,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/lodestar-params": "^0.36.0", - "@chainsafe/ssz": "^0.8.20" + "@chainsafe/ssz": "^0.9.0" }, "keywords": [ "ethereum", diff --git a/packages/types/src/allForks/sszTypes.ts b/packages/types/src/allForks/sszTypes.ts index 0cfff5e34e6..616be35633d 100644 --- a/packages/types/src/allForks/sszTypes.ts +++ b/packages/types/src/allForks/sszTypes.ts @@ -1,7 +1,3 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {ForkName} from "@chainsafe/lodestar-params"; - -import {AllForksSSZTypes} from "./types"; import {ssz as phase0} from "../phase0"; import {ssz as altair} from "../altair"; import {ssz as bellatrix} from "../bellatrix"; @@ -10,26 +6,26 @@ import {ssz as bellatrix} from "../bellatrix"; * Index the ssz types that differ by fork * A record of AllForksSSZTypes indexed by fork */ -export const allForks: {[K in ForkName]: AllForksSSZTypes} = { +export const allForks = { phase0: { - BeaconBlockBody: phase0.BeaconBlockBody as AllForksSSZTypes["BeaconBlockBody"], - BeaconBlock: phase0.BeaconBlock as AllForksSSZTypes["BeaconBlock"], - SignedBeaconBlock: phase0.SignedBeaconBlock as AllForksSSZTypes["SignedBeaconBlock"], - BeaconState: phase0.BeaconState as AllForksSSZTypes["BeaconState"], + BeaconBlockBody: phase0.BeaconBlockBody, + BeaconBlock: phase0.BeaconBlock, + SignedBeaconBlock: phase0.SignedBeaconBlock, + BeaconState: phase0.BeaconState, Metadata: phase0.Metadata, }, altair: { - BeaconBlockBody: altair.BeaconBlockBody as AllForksSSZTypes["BeaconBlockBody"], - BeaconBlock: altair.BeaconBlock as AllForksSSZTypes["BeaconBlock"], - SignedBeaconBlock: altair.SignedBeaconBlock as AllForksSSZTypes["SignedBeaconBlock"], - BeaconState: altair.BeaconState as AllForksSSZTypes["BeaconState"], - Metadata: altair.Metadata as AllForksSSZTypes["Metadata"], + BeaconBlockBody: altair.BeaconBlockBody, + BeaconBlock: altair.BeaconBlock, + SignedBeaconBlock: altair.SignedBeaconBlock, + BeaconState: altair.BeaconState, + Metadata: altair.Metadata, }, bellatrix: { - BeaconBlockBody: bellatrix.BeaconBlockBody as AllForksSSZTypes["BeaconBlockBody"], - BeaconBlock: bellatrix.BeaconBlock as AllForksSSZTypes["BeaconBlock"], - SignedBeaconBlock: bellatrix.SignedBeaconBlock as AllForksSSZTypes["SignedBeaconBlock"], - BeaconState: bellatrix.BeaconState as AllForksSSZTypes["BeaconState"], - Metadata: altair.Metadata as AllForksSSZTypes["Metadata"], + BeaconBlockBody: bellatrix.BeaconBlockBody, + BeaconBlock: bellatrix.BeaconBlock, + SignedBeaconBlock: bellatrix.SignedBeaconBlock, + BeaconState: bellatrix.BeaconState, + Metadata: altair.Metadata, }, }; diff --git a/packages/types/src/allForks/types.ts b/packages/types/src/allForks/types.ts index 8c2a2d14785..ad3a74a71ad 100644 --- a/packages/types/src/allForks/types.ts +++ b/packages/types/src/allForks/types.ts @@ -1,8 +1,10 @@ -import {ContainerType} from "@chainsafe/ssz"; - +import {CompositeType, ContainerType, ValueOf, CompositeView, CompositeViewDU} from "@chainsafe/ssz"; import {ts as phase0} from "../phase0"; import {ts as altair} from "../altair"; import {ts as bellatrix} from "../bellatrix"; +import {ssz as phase0Ssz} from "../phase0"; +import {ssz as altairSsz} from "../altair"; +import {ssz as bellatrixSsz} from "../bellatrix"; // Re-export union types for types that are _known_ to differ @@ -24,12 +26,46 @@ export type AllForksTypes = { }; /** - * SSZ Types known to change between forks + * An AllForks type must accept as any parameter the UNION of all fork types. + * The generic argument of `AllForksTypeOf` must be the union of the fork types: + * + * + * For example, `allForks.BeaconState.defaultValue()` must return + * ``` + * phase0.BeaconState | altair.BeaconState | bellatrix.BeaconState + * ``` + * + * And `allForks.BeaconState.serialize()` must accept as parameter + * ``` + * phase0.BeaconState | altair.BeaconState | bellatrix.BeaconState + * ``` + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AllForksTypeOf> = CompositeType< + ValueOf, + CompositeView, + CompositeViewDU +>; + +/** + * SSZ Types known to change between forks. + * + * Re-wrapping a union of fields in a new ContainerType allows to pass a generic block to .serialize() + * - .serialize() requires a value with ONLY the common fork fields + * - .deserialize() and ValueOf return a value with ONLY the general fork fields */ export type AllForksSSZTypes = { - BeaconBlockBody: ContainerType; - BeaconBlock: ContainerType; - SignedBeaconBlock: ContainerType; - BeaconState: ContainerType; - Metadata: ContainerType; + BeaconBlockBody: AllForksTypeOf< + typeof phase0Ssz.BeaconBlockBody | typeof altairSsz.BeaconBlockBody | typeof bellatrixSsz.BeaconBlockBody + >; + BeaconBlock: AllForksTypeOf< + typeof phase0Ssz.BeaconBlock | typeof altairSsz.BeaconBlock | typeof bellatrixSsz.BeaconBlock + >; + SignedBeaconBlock: AllForksTypeOf< + typeof phase0Ssz.SignedBeaconBlock | typeof altairSsz.SignedBeaconBlock | typeof bellatrixSsz.SignedBeaconBlock + >; + BeaconState: AllForksTypeOf< + typeof phase0Ssz.BeaconState | typeof altairSsz.BeaconState | typeof bellatrixSsz.BeaconState + >; + Metadata: AllForksTypeOf; }; diff --git a/packages/types/src/altair/sszTypes.ts b/packages/types/src/altair/sszTypes.ts index 7e52fdfa0e1..9c5eeb106d5 100644 --- a/packages/types/src/altair/sszTypes.ts +++ b/packages/types/src/altair/sszTypes.ts @@ -1,6 +1,5 @@ -import {BitVectorType, ContainerType, VectorType, ListType, RootType, Vector} from "@chainsafe/ssz"; +import {BitVectorType, ContainerType, ListBasicType, ListCompositeType, VectorCompositeType} from "@chainsafe/ssz"; import { - JUSTIFICATION_BITS_LENGTH, FINALIZED_ROOT_DEPTH, NEXT_SYNC_COMMITTEE_DEPTH, SYNC_COMMITTEE_SUBNET_COUNT, @@ -8,22 +7,19 @@ import { SLOTS_PER_HISTORICAL_ROOT, HISTORICAL_ROOTS_LIMIT, VALIDATOR_REGISTRY_LIMIT, - EPOCHS_PER_HISTORICAL_VECTOR, - EPOCHS_PER_SLASHINGS_VECTOR, + EPOCHS_PER_SYNC_COMMITTEE_PERIOD, + SLOTS_PER_EPOCH, } from "@chainsafe/lodestar-params"; -import {Root} from "../primitive/types"; -import {ssz as phase0Ssz, ts as phase0Types} from "../phase0"; -import {ssz as primitiveSsz} from "../primitive"; -import {LazyVariable} from "../utils/lazyVar"; -import * as altair from "./types"; +import * as phase0Ssz from "../phase0/sszTypes"; +import * as primitiveSsz from "../primitive/sszTypes"; const { Bytes32, - Number64, + UintNum64, + UintBn64, Slot, SubcommitteeIndex, ValidatorIndex, - Gwei, Root, Version, BLSPubkey, @@ -31,176 +27,127 @@ const { ParticipationFlags, } = primitiveSsz; -// So the expandedRoots can be referenced, and break the circular dependency -const typesRef = new LazyVariable<{ - BeaconBlock: ContainerType; - BeaconState: ContainerType; -}>(); +export const SyncSubnets = new BitVectorType(SYNC_COMMITTEE_SUBNET_COUNT); -export const SyncSubnets = new BitVectorType({ - length: SYNC_COMMITTEE_SUBNET_COUNT, -}); - -export const Metadata = new ContainerType({ - fields: { - ...phase0Ssz.Metadata.fields, +export const Metadata = new ContainerType( + { + seqNumber: UintBn64, + attnets: phase0Ssz.AttestationSubnets, syncnets: SyncSubnets, }, - // New keys are strictly appended, phase0 key order is preserved - casingMap: { - ...phase0Ssz.Metadata.casingMap, - syncnets: "syncnets", - }, -}); + {typeName: "Metadata", jsonCase: "eth2"} +); -export const SyncCommittee = new ContainerType({ - fields: { - pubkeys: new VectorType({elementType: BLSPubkey, length: SYNC_COMMITTEE_SIZE}), +export const SyncCommittee = new ContainerType( + { + pubkeys: new VectorCompositeType(BLSPubkey, SYNC_COMMITTEE_SIZE), aggregatePubkey: BLSPubkey, }, - casingMap: { - pubkeys: "pubkeys", - aggregatePubkey: "aggregate_pubkey", - }, -}); + {typeName: "SyncCommittee", jsonCase: "eth2"} +); -export const SyncCommitteeMessage = new ContainerType({ - fields: { +export const SyncCommitteeMessage = new ContainerType( + { slot: Slot, beaconBlockRoot: Root, validatorIndex: ValidatorIndex, signature: BLSSignature, }, - casingMap: { - slot: "slot", - beaconBlockRoot: "beacon_block_root", - validatorIndex: "validator_index", - signature: "signature", - }, -}); + {typeName: "SyncCommitteeMessage", jsonCase: "eth2"} +); -export const SyncCommitteeContribution = new ContainerType({ - fields: { +export const SyncCommitteeContribution = new ContainerType( + { slot: Slot, beaconBlockRoot: Root, subcommitteeIndex: SubcommitteeIndex, - aggregationBits: new BitVectorType({length: SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT}), + aggregationBits: new BitVectorType(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT), signature: BLSSignature, }, - casingMap: { - slot: "slot", - beaconBlockRoot: "beacon_block_root", - subcommitteeIndex: "subcommittee_index", - aggregationBits: "aggregation_bits", - signature: "signature", - }, -}); + {typeName: "SyncCommitteeContribution", jsonCase: "eth2"} +); -export const ContributionAndProof = new ContainerType({ - fields: { +export const ContributionAndProof = new ContainerType( + { aggregatorIndex: ValidatorIndex, contribution: SyncCommitteeContribution, selectionProof: BLSSignature, }, - casingMap: { - aggregatorIndex: "aggregator_index", - contribution: "contribution", - selectionProof: "selection_proof", - }, -}); + {typeName: "ContributionAndProof", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedContributionAndProof = new ContainerType({ - fields: { +export const SignedContributionAndProof = new ContainerType( + { message: ContributionAndProof, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedContributionAndProof", jsonCase: "eth2"} +); -export const SyncAggregatorSelectionData = new ContainerType({ - fields: { +export const SyncAggregatorSelectionData = new ContainerType( + { slot: Slot, subcommitteeIndex: SubcommitteeIndex, }, - casingMap: { - slot: "slot", - subcommitteeIndex: "subcommittee_index", - }, -}); + {typeName: "SyncAggregatorSelectionData", jsonCase: "eth2"} +); -export const SyncCommitteeBits = new BitVectorType({ - length: SYNC_COMMITTEE_SIZE, -}); +export const SyncCommitteeBits = new BitVectorType(SYNC_COMMITTEE_SIZE); -export const SyncAggregate = new ContainerType({ - fields: { +export const SyncAggregate = new ContainerType( + { syncCommitteeBits: SyncCommitteeBits, syncCommitteeSignature: BLSSignature, }, - casingMap: { - syncCommitteeBits: "sync_committee_bits", - syncCommitteeSignature: "sync_committee_signature", - }, -}); + {typeName: "SyncCommitteeBits", jsonCase: "eth2"} +); -// Re-declare with the new expanded type -export const HistoricalBlockRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); +export const HistoricalBlockRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); +export const HistoricalStateRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); -export const HistoricalStateRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconState}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); - -export const HistoricalBatch = new ContainerType({ - fields: { +export const HistoricalBatch = new ContainerType( + { blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, }, - casingMap: phase0Ssz.HistoricalBatch.casingMap, -}); + {typeName: "HistoricalBatch", jsonCase: "eth2"} +); -export const BeaconBlockBody = new ContainerType({ - fields: { +export const BeaconBlockBody = new ContainerType( + { ...phase0Ssz.BeaconBlockBody.fields, syncAggregate: SyncAggregate, }, - casingMap: { - ...phase0Ssz.BeaconBlockBody.casingMap, - syncAggregate: "sync_aggregate", - }, -}); + {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const BeaconBlock = new ContainerType({ - fields: { +export const BeaconBlock = new ContainerType( + { slot: Slot, proposerIndex: ValidatorIndex, - // Reclare expandedType() with altair block and altair state - parentRoot: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - stateRoot: new RootType({expandedType: () => typesRef.get().BeaconState}), + parentRoot: Root, + stateRoot: Root, body: BeaconBlockBody, }, - casingMap: phase0Ssz.BeaconBlock.casingMap, -}); + {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedBeaconBlock = new ContainerType({ - fields: { +export const SignedBeaconBlock = new ContainerType( + { message: BeaconBlock, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedBeaconBlock", jsonCase: "eth2"} +); -export const EpochParticipation = new ListType({elementType: ParticipationFlags, limit: VALIDATOR_REGISTRY_LIMIT}); -export const InactivityScores = new ListType({elementType: Number64, limit: VALIDATOR_REGISTRY_LIMIT}); +export const EpochParticipation = new ListBasicType(ParticipationFlags, VALIDATOR_REGISTRY_LIMIT); +export const InactivityScores = new ListBasicType(UintNum64, VALIDATOR_REGISTRY_LIMIT); // we don't reuse phase0.BeaconState fields since we need to replace some keys // and we cannot keep order doing that -export const BeaconState = new ContainerType({ - fields: { - genesisTime: Number64, +export const BeaconState = new ContainerType( + { + genesisTime: UintNum64, genesisValidatorsRoot: Root, slot: Slot, fork: phase0Ssz.Fork, @@ -208,25 +155,22 @@ export const BeaconState = new ContainerType({ latestBlockHeader: phase0Ssz.BeaconBlockHeader, blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, - historicalRoots: new ListType({ - elementType: new RootType({expandedType: HistoricalBatch}), - limit: HISTORICAL_ROOTS_LIMIT, - }), + historicalRoots: new ListCompositeType(Root, HISTORICAL_ROOTS_LIMIT), // Eth1 eth1Data: phase0Ssz.Eth1Data, eth1DataVotes: phase0Ssz.Eth1DataVotes, - eth1DepositIndex: Number64, + eth1DepositIndex: UintNum64, // Registry - validators: new ListType({elementType: phase0Ssz.Validator, limit: VALIDATOR_REGISTRY_LIMIT}), - balances: new ListType({elementType: Number64, limit: VALIDATOR_REGISTRY_LIMIT}), - randaoMixes: new VectorType({elementType: Bytes32, length: EPOCHS_PER_HISTORICAL_VECTOR}), + validators: phase0Ssz.Validators, + balances: phase0Ssz.Balances, + randaoMixes: phase0Ssz.RandaoMixes, // Slashings - slashings: new VectorType({elementType: Gwei, length: EPOCHS_PER_SLASHINGS_VECTOR}), + slashings: phase0Ssz.Slashings, // Participation previousEpochParticipation: EpochParticipation, currentEpochParticipation: EpochParticipation, // Finality - justificationBits: new BitVectorType({length: JUSTIFICATION_BITS_LENGTH}), + justificationBits: phase0Ssz.JustificationBits, previousJustifiedCheckpoint: phase0Ssz.Checkpoint, currentJustifiedCheckpoint: phase0Ssz.Checkpoint, finalizedCheckpoint: phase0Ssz.Checkpoint, @@ -236,37 +180,35 @@ export const BeaconState = new ContainerType({ currentSyncCommittee: SyncCommittee, nextSyncCommittee: SyncCommittee, }, - casingMap: { - ...phase0Ssz.BeaconState.casingMap, - inactivityScores: "inactivity_scores", - currentSyncCommittee: "current_sync_committee", - nextSyncCommittee: "next_sync_committee", + {typeName: "BeaconState", jsonCase: "eth2"} +); + +export const LightClientSnapshot = new ContainerType( + { + header: phase0Ssz.BeaconBlockHeader, + currentSyncCommittee: SyncCommittee, + nextSyncCommittee: SyncCommittee, }, -}); + {typeName: "LightClientSnapshot", jsonCase: "eth2"} +); -export const LightClientUpdate = new ContainerType({ - fields: { +export const LightClientUpdate = new ContainerType( + { attestedHeader: phase0Ssz.BeaconBlockHeader, nextSyncCommittee: SyncCommittee, - nextSyncCommitteeBranch: new VectorType({ - elementType: Bytes32, - length: NEXT_SYNC_COMMITTEE_DEPTH, - }), + nextSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH), finalizedHeader: phase0Ssz.BeaconBlockHeader, - finalityBranch: new VectorType({elementType: Bytes32, length: FINALIZED_ROOT_DEPTH}), + finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH), syncAggregate: SyncAggregate, forkVersion: Version, }, - casingMap: { - attestedHeader: "attested_header", - nextSyncCommittee: "next_sync_committee", - nextSyncCommitteeBranch: "next_sync_committee_branch", - finalizedHeader: "finalized_header", - finalityBranch: "finality_branch", - syncAggregate: "sync_aggregate", - forkVersion: "fork_version", - }, -}); + {typeName: "LightClientUpdate", jsonCase: "eth2"} +); -// MUST set typesRef here, otherwise expandedType() calls will throw -typesRef.set({BeaconBlock, BeaconState}); +export const LightClientStore = new ContainerType( + { + snapshot: LightClientSnapshot, + validUpdates: new ListCompositeType(LightClientUpdate, EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH), + }, + {typeName: "LightClientStore", jsonCase: "eth2"} +); diff --git a/packages/types/src/altair/types.ts b/packages/types/src/altair/types.ts index 98e4d3087ef..6507c6b146f 100644 --- a/packages/types/src/altair/types.ts +++ b/packages/types/src/altair/types.ts @@ -1,126 +1,19 @@ -import {BitVector, List, BitList, Vector} from "@chainsafe/ssz"; -import { - Number64, - Uint64, - ParticipationFlags, - Bytes32, - BLSSignature, - Version, - BLSPubkey, - Slot, - Root, - ValidatorIndex, - SubcommitteeIndex, -} from "../primitive/types"; -import * as phase0 from "../phase0/types"; - -export type SyncSubnets = BitVector; - -export interface Metadata { - seqNumber: Uint64; - attnets: phase0.AttestationSubnets; - syncnets: SyncSubnets; -} - -export interface SyncCommittee { - pubkeys: Vector; - aggregatePubkey: BLSPubkey; -} - -export interface SyncCommitteeMessage { - slot: Slot; - beaconBlockRoot: Root; - validatorIndex: ValidatorIndex; - signature: BLSSignature; -} - -export interface SyncCommitteeContribution { - slot: Slot; - beaconBlockRoot: Root; - subcommitteeIndex: SubcommitteeIndex; - aggregationBits: BitList; - signature: BLSSignature; -} - -export interface ContributionAndProof { - aggregatorIndex: ValidatorIndex; - contribution: SyncCommitteeContribution; - selectionProof: BLSSignature; -} - -export interface SignedContributionAndProof { - message: ContributionAndProof; - signature: BLSSignature; -} - -export interface SyncAggregatorSelectionData { - slot: Slot; - subcommitteeIndex: SubcommitteeIndex; -} - -export interface SyncAggregate { - syncCommitteeBits: BitVector; - syncCommitteeSignature: BLSSignature; -} - -export interface BeaconBlockBody extends phase0.BeaconBlockBody { - syncAggregate: SyncAggregate; -} - -export interface BeaconBlock extends phase0.BeaconBlock { - body: BeaconBlockBody; -} - -export interface SignedBeaconBlock extends phase0.SignedBeaconBlock { - message: BeaconBlock; -} - -export interface BeaconState - extends Omit { - // Participation - previousEpochParticipation: List; - currentEpochParticipation: List; - // Inactivity - inactivityScores: List; - // Sync - currentSyncCommittee: SyncCommittee; - nextSyncCommittee: SyncCommittee; -} - -/** - * Spec v1.0.1 - */ -export interface LightClientUpdate { - /** The beacon block header that is attested to by the sync committee */ - attestedHeader: phase0.BeaconBlockHeader; - /** Next sync committee corresponding to the header */ - nextSyncCommittee: SyncCommittee; - nextSyncCommitteeBranch: Vector; - /** The finalized beacon block header attested to by Merkle branch */ - finalizedHeader: phase0.BeaconBlockHeader; - finalityBranch: Vector; - /** Sync committee aggregate signature */ - syncAggregate: SyncAggregate; - /** Fork version for the aggregate signature */ - forkVersion: Version; -} - -/** - * Spec v1.0.1 - */ -export interface LightClientStore { - /** Beacon block header that is finalized */ - finalizedHeader: phase0.BeaconBlockHeader; - /** Sync committees corresponding to the header */ - currentSyncCommittee: SyncCommittee; - nextSyncCommittee: SyncCommittee; - /** Best available header to switch finalized head to if we see nothing else */ - bestValidUpdate?: LightClientUpdate; - /** Most recent available reasonably-safe header */ - optimisticHeader: phase0.BeaconBlockHeader; - /** Max number of active participants in a sync committee (used to calculate - * safety threshold) - */ - previousMaxActiveParticipants: Uint64; - currentMaxActiveParticipants: Uint64; -} +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes"; + +export type SyncSubnets = ValueOf; +export type Metadata = ValueOf; +export type SyncCommittee = ValueOf; +export type SyncCommitteeMessage = ValueOf; +export type SyncCommitteeContribution = ValueOf; +export type ContributionAndProof = ValueOf; +export type SignedContributionAndProof = ValueOf; +export type SyncAggregatorSelectionData = ValueOf; +export type SyncAggregate = ValueOf; +export type BeaconBlockBody = ValueOf; +export type BeaconBlock = ValueOf; +export type SignedBeaconBlock = ValueOf; +export type BeaconState = ValueOf; +export type LightClientSnapshot = ValueOf; +export type LightClientUpdate = ValueOf; +export type LightClientStore = ValueOf; diff --git a/packages/types/src/bellatrix/sszTypes.ts b/packages/types/src/bellatrix/sszTypes.ts index 301b7019a5c..724db286f96 100644 --- a/packages/types/src/bellatrix/sszTypes.ts +++ b/packages/types/src/bellatrix/sszTypes.ts @@ -1,4 +1,7 @@ -import {byteType, ByteVectorType, ContainerType, List, ListType, RootType, Vector, VectorType} from "@chainsafe/ssz"; +import {ByteListType, ByteVectorType, ContainerType, ListCompositeType, VectorCompositeType} from "@chainsafe/ssz"; +import {ssz as primitiveSsz} from "../primitive"; +import {ssz as phase0Ssz} from "../phase0"; +import {ssz as altairSsz} from "../altair"; import { BYTES_PER_LOGS_BLOOM, HISTORICAL_ROOTS_LIMIT, @@ -7,188 +10,111 @@ import { MAX_EXTRA_DATA_BYTES, SLOTS_PER_HISTORICAL_ROOT, } from "@chainsafe/lodestar-params"; -import {ssz as primitiveSsz, ts as primitive} from "../primitive"; -import {ssz as phase0Ssz, ts as phase0} from "../phase0"; -import {ssz as altairSsz} from "../altair"; -import * as bellatrix from "./types"; -import {LazyVariable} from "../utils/lazyVar"; -import {Uint256} from "../primitive/sszTypes"; - -const {Bytes20, Bytes32, Number64, Slot, ValidatorIndex, Root, BLSSignature, Uint8} = primitiveSsz; -// So the expandedRoots can be referenced, and break the circular dependency -const typesRef = new LazyVariable<{ - BeaconBlock: ContainerType; - BeaconState: ContainerType; -}>(); +const {Bytes20, Bytes32, UintNum64, Slot, ValidatorIndex, Root, BLSSignature, UintBn256: Uint256} = primitiveSsz; /** * ByteList[MAX_BYTES_PER_TRANSACTION] * * Spec v1.0.1 */ -export const Transaction = new ListType({elementType: byteType, limit: MAX_BYTES_PER_TRANSACTION}); +export const Transaction = new ByteListType(MAX_BYTES_PER_TRANSACTION); -export const Transactions = new ListType>({ - elementType: Transaction, - limit: MAX_TRANSACTIONS_PER_PAYLOAD, -}); +/** + * Union[OpaqueTransaction] + * + * Spec v1.0.1 + */ +export const Transactions = new ListCompositeType(Transaction, MAX_TRANSACTIONS_PER_PAYLOAD); const executionPayloadFields = { parentHash: Root, feeRecipient: Bytes20, stateRoot: Bytes32, receiptsRoot: Bytes32, - logsBloom: new ByteVectorType({length: BYTES_PER_LOGS_BLOOM}), + logsBloom: new ByteVectorType(BYTES_PER_LOGS_BLOOM), prevRandao: Bytes32, - blockNumber: Number64, - gasLimit: Number64, - gasUsed: Number64, - timestamp: Number64, + blockNumber: UintNum64, + gasLimit: UintNum64, + gasUsed: UintNum64, + timestamp: UintNum64, // TODO: if there is perf issue, consider making ByteListType - extraData: new ListType({limit: MAX_EXTRA_DATA_BYTES, elementType: Uint8}), + extraData: new ByteListType(MAX_EXTRA_DATA_BYTES), baseFeePerGas: Uint256, // Extra payload fields blockHash: Root, }; -const executionPayloadCasingMap = { - parentHash: "parent_hash", - feeRecipient: "fee_recipient", - stateRoot: "state_root", - receiptsRoot: "receipts_root", - logsBloom: "logs_bloom", - prevRandao: "prev_randao", - blockNumber: "block_number", - gasLimit: "gas_limit", - gasUsed: "gas_used", - timestamp: "timestamp", - extraData: "extra_data", - baseFeePerGas: "base_fee_per_gas", - blockHash: "block_hash", -}; -/** - * ```python - * class ExecutionPayload(Container): - * # Execution block header fields - * parent_hash: Hash32 - * coinbase: Bytes20 # 'beneficiary' in the yellow paper - * state_root: Bytes32 - * receipt_root: Bytes32 # 'receipts root' in the yellow paper - * logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] - * prevRandao: Bytes32 # 'difficulty' in the yellow paper - * block_number: uint64 # 'number' in the yellow paper - * gas_limit: uint64 - * gas_used: uint64 - * timestamp: uint64 - * extra_data: ByteList[MAX_EXTRA_DATA_BYTES] - * base_fee_per_gas: Bytes32 # base fee introduced in EIP-1559, little-endian serialized - * # Extra payload fields - * block_hash: Hash32 # Hash of execution block - * transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] - * ``` - * - * Spec v1.0.1 - */ -export const ExecutionPayload = new ContainerType({ - fields: { +export const ExecutionPayload = new ContainerType( + { ...executionPayloadFields, transactions: Transactions, }, - casingMap: { - ...executionPayloadCasingMap, - transactions: "transactions", - }, -}); + {typeName: "ExecutionPayload", jsonCase: "eth2"} +); -/** - * ```python - * class ExecutionPayload(Container): - * ... - * transactions_root: Root - * ``` - * - * Spec v1.0.1 - */ -export const ExecutionPayloadHeader = new ContainerType({ - fields: { +export const ExecutionPayloadHeader = new ContainerType( + { ...executionPayloadFields, transactionsRoot: Root, }, - casingMap: { - ...executionPayloadCasingMap, - transactionsRoot: "transactions_root", - }, -}); + {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} +); -export const BeaconBlockBody = new ContainerType({ - fields: { +export const BeaconBlockBody = new ContainerType( + { ...altairSsz.BeaconBlockBody.fields, executionPayload: ExecutionPayload, }, - casingMap: { - ...altairSsz.BeaconBlockBody.casingMap, - executionPayload: "execution_payload", - }, -}); + {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const BeaconBlock = new ContainerType({ - fields: { +export const BeaconBlock = new ContainerType( + { slot: Slot, proposerIndex: ValidatorIndex, // Reclare expandedType() with altair block and altair state - parentRoot: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - stateRoot: new RootType({expandedType: () => typesRef.get().BeaconState}), + parentRoot: Root, + stateRoot: Root, body: BeaconBlockBody, }, - casingMap: altairSsz.BeaconBlock.casingMap, -}); + {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedBeaconBlock = new ContainerType({ - fields: { +export const SignedBeaconBlock = new ContainerType( + { message: BeaconBlock, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedBeaconBlock", jsonCase: "eth2"} +); -export const PowBlock = new ContainerType({ - fields: { +export const PowBlock = new ContainerType( + { blockHash: Root, parentHash: Root, totalDifficulty: Uint256, }, - casingMap: { - blockHash: "block_hash", - parentHash: "parent_hash", - totalDifficulty: "total_difficulty", - }, -}); + {typeName: "PowBlock", jsonCase: "eth2"} +); // Re-declare with the new expanded type -export const HistoricalBlockRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); - -export const HistoricalStateRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconState}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); +export const HistoricalBlockRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); +export const HistoricalStateRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); -export const HistoricalBatch = new ContainerType({ - fields: { +export const HistoricalBatch = new ContainerType( + { blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, }, - casingMap: phase0Ssz.HistoricalBatch.casingMap, -}); + {typeName: "HistoricalBatch", jsonCase: "eth2"} +); // we don't reuse phase0.BeaconState fields since we need to replace some keys // and we cannot keep order doing that -export const BeaconState = new ContainerType({ - fields: { - genesisTime: Number64, +export const BeaconState = new ContainerType( + { + genesisTime: UintNum64, genesisValidatorsRoot: Root, slot: primitiveSsz.Slot, fork: phase0Ssz.Fork, @@ -196,14 +122,11 @@ export const BeaconState = new ContainerType({ latestBlockHeader: phase0Ssz.BeaconBlockHeader, blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, - historicalRoots: new ListType({ - elementType: new RootType({expandedType: HistoricalBatch}), - limit: HISTORICAL_ROOTS_LIMIT, - }), + historicalRoots: new ListCompositeType(Root, HISTORICAL_ROOTS_LIMIT), // Eth1 eth1Data: phase0Ssz.Eth1Data, eth1DataVotes: phase0Ssz.Eth1DataVotes, - eth1DepositIndex: Number64, + eth1DepositIndex: UintNum64, // Registry validators: phase0Ssz.Validators, balances: phase0Ssz.Balances, @@ -224,13 +147,7 @@ export const BeaconState = new ContainerType({ currentSyncCommittee: altairSsz.SyncCommittee, nextSyncCommittee: altairSsz.SyncCommittee, // Execution - latestExecutionPayloadHeader: ExecutionPayloadHeader, // [New in Bellatrix] + latestExecutionPayloadHeader: ExecutionPayloadHeader, // [New in Merge] }, - casingMap: { - ...altairSsz.BeaconState.casingMap, - latestExecutionPayloadHeader: "latest_execution_payload_header", - }, -}); - -// MUST set typesRef here, otherwise expandedType() calls will throw -typesRef.set({BeaconBlock, BeaconState}); + {typeName: "BeaconState", jsonCase: "eth2"} +); diff --git a/packages/types/src/bellatrix/types.ts b/packages/types/src/bellatrix/types.ts index 1c859fb11ac..c7bbc384070 100644 --- a/packages/types/src/bellatrix/types.ts +++ b/packages/types/src/bellatrix/types.ts @@ -1,52 +1,11 @@ -import * as altair from "../altair/types"; -import {Root, Bytes32, Number64, ExecutionAddress, Uint256} from "../primitive/types"; - -export type Transaction = Uint8Array; - -type ExecutionPayloadFields = { - // Execution block header fields - parentHash: Root; - feeRecipient: ExecutionAddress; - stateRoot: Bytes32; - receiptsRoot: Bytes32; - logsBloom: Uint8Array; - prevRandao: Bytes32; - blockNumber: number; - gasLimit: Number64; - gasUsed: Number64; - timestamp: Number64; - extraData: Uint8Array; - baseFeePerGas: Uint256; - // Extra payload fields - blockHash: Bytes32; -}; - -export type ExecutionPayload = ExecutionPayloadFields & { - transactions: Transaction[]; -}; - -export type ExecutionPayloadHeader = ExecutionPayloadFields & { - transactionsRoot: Root; -}; - -export interface BeaconBlockBody extends altair.BeaconBlockBody { - executionPayload: ExecutionPayload; -} - -export interface BeaconBlock extends altair.BeaconBlock { - body: BeaconBlockBody; -} - -export interface SignedBeaconBlock extends altair.SignedBeaconBlock { - message: BeaconBlock; -} - -export interface BeaconState extends altair.BeaconState { - latestExecutionPayloadHeader: ExecutionPayloadHeader; -} - -export type PowBlock = { - blockHash: Root; - parentHash: Root; - totalDifficulty: bigint; -}; +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes"; + +export type Transaction = ValueOf; +export type ExecutionPayload = ValueOf; +export type ExecutionPayloadHeader = ValueOf; +export type BeaconBlockBody = ValueOf; +export type BeaconBlock = ValueOf; +export type SignedBeaconBlock = ValueOf; +export type BeaconState = ValueOf; +export type PowBlock = ValueOf; diff --git a/packages/types/src/phase0/sszTypes.ts b/packages/types/src/phase0/sszTypes.ts index e3305229651..467c07156a3 100644 --- a/packages/types/src/phase0/sszTypes.ts +++ b/packages/types/src/phase0/sszTypes.ts @@ -1,13 +1,12 @@ import { BitListType, BitVectorType, - ContainerLeafNodeStructType, ContainerType, - List, - ListType, - RootType, - Vector, - VectorType, + ContainerNodeStructType, + ListBasicType, + ListCompositeType, + VectorBasicType, + VectorCompositeType, } from "@chainsafe/ssz"; import { ATTESTATION_SUBNET_COUNT, @@ -28,15 +27,13 @@ import { SLOTS_PER_HISTORICAL_ROOT, VALIDATOR_REGISTRY_LIMIT, } from "@chainsafe/lodestar-params"; -import {ssz as primitiveSsz, ts as primitiveTs} from "../primitive"; -import {LazyVariable} from "../utils/lazyVar"; -import * as phase0 from "./types"; +import * as primitiveSsz from "../primitive/sszTypes"; const { Boolean, Bytes32, - Number64, - Uint64, + UintNum64, + UintBn64, Slot, Epoch, CommitteeIndex, @@ -50,412 +47,314 @@ const { Domain, } = primitiveSsz; -// So the expandedRoots can be referenced, and break the circular dependency -const typesRef = new LazyVariable<{ - BeaconBlock: ContainerType; - BeaconState: ContainerType; -}>(); - // Misc types // ========== -export const AttestationSubnets = new BitVectorType({ - length: ATTESTATION_SUBNET_COUNT, -}); +export const AttestationSubnets = new BitVectorType(ATTESTATION_SUBNET_COUNT); -export const BeaconBlockHeader = new ContainerType({ - fields: { +export const BeaconBlockHeader = new ContainerType( + { slot: Slot, proposerIndex: ValidatorIndex, parentRoot: Root, stateRoot: Root, bodyRoot: Root, }, - casingMap: { - slot: "slot", - proposerIndex: "proposer_index", - parentRoot: "parent_root", - stateRoot: "state_root", - bodyRoot: "body_root", - }, -}); + {typeName: "BeaconBlockHeader", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedBeaconBlockHeader = new ContainerType({ - fields: { +export const SignedBeaconBlockHeader = new ContainerType( + { message: BeaconBlockHeader, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedBeaconBlockHeader", jsonCase: "eth2"} +); -export const Checkpoint = new ContainerType({ - fields: { +export const Checkpoint = new ContainerType( + { epoch: Epoch, root: Root, }, - expectedCase: "notransform", -}); + {typeName: "Checkpoint", jsonCase: "eth2"} +); -export const CommitteeBits = new BitListType({ - limit: MAX_VALIDATORS_PER_COMMITTEE, -}); +export const CommitteeBits = new BitListType(MAX_VALIDATORS_PER_COMMITTEE); -export const CommitteeIndices = new ListType>({ - elementType: ValidatorIndex, - limit: MAX_VALIDATORS_PER_COMMITTEE, -}); +export const CommitteeIndices = new ListBasicType(ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE); -export const DepositMessage = new ContainerType({ - fields: { +export const DepositMessage = new ContainerType( + { pubkey: BLSPubkey, withdrawalCredentials: Bytes32, - amount: Number64, - }, - casingMap: { - pubkey: "pubkey", - withdrawalCredentials: "withdrawal_credentials", - amount: "amount", + amount: UintNum64, }, -}); + {typeName: "DepositMessage", jsonCase: "eth2"} +); -export const DepositData = new ContainerType({ - fields: { - // Fields order is strickly preserved - ...DepositMessage.fields, +export const DepositData = new ContainerType( + { + pubkey: BLSPubkey, + withdrawalCredentials: Bytes32, + amount: UintNum64, signature: BLSSignature, }, - casingMap: { - ...DepositMessage.casingMap, - signature: "signature", - }, -}); + {typeName: "DepositData", jsonCase: "eth2"} +); -export const DepositDataRootList = new ListType>({ - elementType: new RootType({expandedType: DepositData}), - limit: 2 ** DEPOSIT_CONTRACT_TREE_DEPTH, -}); +export const DepositDataRootList = new ListCompositeType(Root, 2 ** DEPOSIT_CONTRACT_TREE_DEPTH); -export const DepositEvent = new ContainerType({ - fields: { +export const DepositEvent = new ContainerType( + { depositData: DepositData, - blockNumber: Number64, - index: Number64, - }, - // Custom type, not in the consensus specs - casingMap: { - depositData: "deposit_data", - blockNumber: "block_number", - index: "index", + blockNumber: UintNum64, + index: UintNum64, }, -}); + {typeName: "DepositEvent", jsonCase: "eth2"} +); -export const Eth1Data = new ContainerType({ - fields: { +export const Eth1Data = new ContainerType( + { depositRoot: Root, - depositCount: Number64, + depositCount: UintNum64, blockHash: Bytes32, }, - casingMap: { - depositRoot: "deposit_root", - depositCount: "deposit_count", - blockHash: "block_hash", - }, -}); + {typeName: "Eth1Data", jsonCase: "eth2"} +); -export const Eth1DataVotes = new ListType({ - elementType: Eth1Data, - limit: EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH, -}); +export const Eth1DataVotes = new ListCompositeType(Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH); -export const Eth1DataOrdered = new ContainerType({ - fields: { - // Fields order is strickly preserved - ...Eth1Data.fields, - blockNumber: Number64, +export const Eth1DataOrdered = new ContainerType( + { + depositRoot: Root, + depositCount: UintNum64, + blockHash: Bytes32, + blockNumber: UintNum64, }, - // Custom type, not in the consensus specs - casingMap: { - ...Eth1Data.casingMap, - blockNumber: "block_number", + {typeName: "Eth1DataOrdered", jsonCase: "eth2"} +); + +/** Spec'ed but only used in lodestar as a type */ +export const Eth1Block = new ContainerType( + { + timestamp: UintNum64, + depositRoot: Root, + depositCount: UintNum64, }, -}); + {typeName: "Eth1Block", jsonCase: "eth2"} +); -export const Fork = new ContainerType({ - fields: { +export const Fork = new ContainerType( + { previousVersion: Version, currentVersion: Version, epoch: Epoch, }, - casingMap: { - previousVersion: "previous_version", - currentVersion: "current_version", - epoch: "epoch", - }, -}); + {typeName: "Fork", jsonCase: "eth2"} +); -export const ForkData = new ContainerType({ - fields: { +export const ForkData = new ContainerType( + { currentVersion: Version, genesisValidatorsRoot: Root, }, - casingMap: { - currentVersion: "current_version", - genesisValidatorsRoot: "genesis_validators_root", - }, -}); + {typeName: "ForkData", jsonCase: "eth2"} +); -export const ENRForkID = new ContainerType({ - fields: { +export const ENRForkID = new ContainerType( + { forkDigest: ForkDigest, nextForkVersion: Version, nextForkEpoch: Epoch, }, - casingMap: { - forkDigest: "fork_digest", - nextForkVersion: "next_fork_version", - nextForkEpoch: "next_fork_epoch", - }, -}); + {typeName: "ENRForkID", jsonCase: "eth2"} +); -export const HistoricalBlockRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); +export const HistoricalBlockRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); +export const HistoricalStateRoots = new VectorCompositeType(Root, SLOTS_PER_HISTORICAL_ROOT); -export const HistoricalStateRoots = new VectorType>({ - elementType: new RootType({expandedType: () => typesRef.get().BeaconState}), - length: SLOTS_PER_HISTORICAL_ROOT, -}); - -export const HistoricalBatch = new ContainerType({ - fields: { +export const HistoricalBatch = new ContainerType( + { blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, }, - casingMap: { - blockRoots: "block_roots", - stateRoots: "state_roots", + {typeName: "HistoricalBatch", jsonCase: "eth2"} +); + +/** + * Non-spec'ed helper type to allow efficient hashing in epoch transition. + * This type is like a 'Header' of HistoricalBatch where its fields are hashed. + */ +export const HistoricalBatchRoots = new ContainerType( + { + blockRoots: Root, // Hashed HistoricalBlockRoots + stateRoots: Root, // Hashed HistoricalStateRoots }, -}); + {typeName: "HistoricalBatchRoots", jsonCase: "eth2"} +); -export const Validator = new ContainerLeafNodeStructType({ - fields: { +export const ValidatorContainer = new ContainerType( + { pubkey: BLSPubkey, withdrawalCredentials: Bytes32, - effectiveBalance: Number64, + effectiveBalance: UintNum64, slashed: Boolean, activationEligibilityEpoch: Epoch, activationEpoch: Epoch, exitEpoch: Epoch, withdrawableEpoch: Epoch, }, - casingMap: { - pubkey: "pubkey", - withdrawalCredentials: "withdrawal_credentials", - effectiveBalance: "effective_balance", - slashed: "slashed", - activationEligibilityEpoch: "activation_eligibility_epoch", - activationEpoch: "activation_epoch", - exitEpoch: "exit_epoch", - withdrawableEpoch: "withdrawable_epoch", - }, -}); + {typeName: "Validator", jsonCase: "eth2"} +); + +export const ValidatorNodeStruct = new ContainerNodeStructType(ValidatorContainer.fields, ValidatorContainer.opts); +// The main Validator type is the 'ContainerNodeStructType' version +export const Validator = ValidatorNodeStruct; // Export as stand-alone for direct tree optimizations -export const Validators = new ListType({elementType: Validator, limit: VALIDATOR_REGISTRY_LIMIT}); -export const Balances = new ListType({elementType: Number64, limit: VALIDATOR_REGISTRY_LIMIT}); -export const RandaoMixes = new VectorType({elementType: Bytes32, length: EPOCHS_PER_HISTORICAL_VECTOR}); -export const Slashings = new VectorType({elementType: Gwei, length: EPOCHS_PER_SLASHINGS_VECTOR}); -export const JustificationBits = new BitVectorType({length: JUSTIFICATION_BITS_LENGTH}); +export const Validators = new ListCompositeType(ValidatorNodeStruct, VALIDATOR_REGISTRY_LIMIT); +export const Balances = new ListBasicType(UintNum64, VALIDATOR_REGISTRY_LIMIT); +export const RandaoMixes = new VectorCompositeType(Bytes32, EPOCHS_PER_HISTORICAL_VECTOR); +export const Slashings = new VectorBasicType(Gwei, EPOCHS_PER_SLASHINGS_VECTOR); +export const JustificationBits = new BitVectorType(JUSTIFICATION_BITS_LENGTH); // Misc dependants -export const AttestationData = new ContainerType({ - fields: { +export const AttestationData = new ContainerType( + { slot: Slot, index: CommitteeIndex, beaconBlockRoot: Root, source: Checkpoint, target: Checkpoint, }, - casingMap: { - slot: "slot", - index: "index", - beaconBlockRoot: "beacon_block_root", - source: "source", - target: "target", - }, -}); + {typeName: "AttestationData", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const IndexedAttestation = new ContainerType({ - fields: { +export const IndexedAttestation = new ContainerType( + { attestingIndices: CommitteeIndices, data: AttestationData, signature: BLSSignature, }, - casingMap: { - attestingIndices: "attesting_indices", - data: "data", - signature: "signature", - }, -}); + {typeName: "IndexedAttestation", jsonCase: "eth2"} +); -export const PendingAttestation = new ContainerType({ - fields: { +export const PendingAttestation = new ContainerType( + { aggregationBits: CommitteeBits, data: AttestationData, inclusionDelay: Slot, proposerIndex: ValidatorIndex, }, - casingMap: { - aggregationBits: "aggregation_bits", - data: "data", - inclusionDelay: "inclusion_delay", - proposerIndex: "proposer_index", - }, -}); + {typeName: "PendingAttestation", jsonCase: "eth2"} +); -export const SigningData = new ContainerType({ - fields: { +export const SigningData = new ContainerType( + { objectRoot: Root, domain: Domain, }, - casingMap: { - objectRoot: "object_root", - domain: "domain", - }, -}); + {typeName: "SigningData", jsonCase: "eth2"} +); // Operations types // ================ -export const Attestation = new ContainerType({ - fields: { +export const Attestation = new ContainerType( + { aggregationBits: CommitteeBits, data: AttestationData, signature: BLSSignature, }, - casingMap: { - aggregationBits: "aggregation_bits", - data: "data", - signature: "signature", - }, -}); + {typeName: "Attestation", jsonCase: "eth2"} +); -export const AttesterSlashing = new ContainerType({ - fields: { +export const AttesterSlashing = new ContainerType( + { attestation1: IndexedAttestation, attestation2: IndexedAttestation, }, - // Declaration time casingMap for toJson/fromJson for container <=> json data - casingMap: { - attestation1: "attestation_1", - attestation2: "attestation_2", - }, -}); + {typeName: "AttesterSlashing", jsonCase: "eth2"} +); -export const Deposit = new ContainerType({ - fields: { - proof: new VectorType({elementType: Bytes32, length: DEPOSIT_CONTRACT_TREE_DEPTH + 1}), +export const Deposit = new ContainerType( + { + proof: new VectorCompositeType(Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH + 1), data: DepositData, }, - expectedCase: "notransform", -}); + {typeName: "Deposit", jsonCase: "eth2"} +); -export const ProposerSlashing = new ContainerType({ - fields: { +export const ProposerSlashing = new ContainerType( + { signedHeader1: SignedBeaconBlockHeader, signedHeader2: SignedBeaconBlockHeader, }, - // Declaration time casingMap for toJson/fromJson for container <=> json data - casingMap: { - signedHeader1: "signed_header_1", - signedHeader2: "signed_header_2", - }, -}); + {typeName: "ProposerSlashing", jsonCase: "eth2"} +); -export const VoluntaryExit = new ContainerType({ - fields: { +export const VoluntaryExit = new ContainerType( + { epoch: Epoch, validatorIndex: ValidatorIndex, }, - casingMap: { - epoch: "epoch", - validatorIndex: "validator_index", - }, -}); + {typeName: "VoluntaryExit", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedVoluntaryExit = new ContainerType({ - fields: { +export const SignedVoluntaryExit = new ContainerType( + { message: VoluntaryExit, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedVoluntaryExit", jsonCase: "eth2"} +); // Block types // =========== -export const BeaconBlockBody = new ContainerType({ - fields: { +export const BeaconBlockBody = new ContainerType( + { randaoReveal: BLSSignature, eth1Data: Eth1Data, graffiti: Bytes32, - proposerSlashings: new ListType({elementType: ProposerSlashing, limit: MAX_PROPOSER_SLASHINGS}), - attesterSlashings: new ListType({elementType: AttesterSlashing, limit: MAX_ATTESTER_SLASHINGS}), - attestations: new ListType({elementType: Attestation, limit: MAX_ATTESTATIONS}), - deposits: new ListType({elementType: Deposit, limit: MAX_DEPOSITS}), - voluntaryExits: new ListType({elementType: SignedVoluntaryExit, limit: MAX_VOLUNTARY_EXITS}), - }, - casingMap: { - randaoReveal: "randao_reveal", - eth1Data: "eth1_data", - graffiti: "graffiti", - proposerSlashings: "proposer_slashings", - attesterSlashings: "attester_slashings", - attestations: "attestations", - deposits: "deposits", - voluntaryExits: "voluntary_exits", - }, -}); - -export const BeaconBlock = new ContainerType({ - fields: { + proposerSlashings: new ListCompositeType(ProposerSlashing, MAX_PROPOSER_SLASHINGS), + attesterSlashings: new ListCompositeType(AttesterSlashing, MAX_ATTESTER_SLASHINGS), + attestations: new ListCompositeType(Attestation, MAX_ATTESTATIONS), + deposits: new ListCompositeType(Deposit, MAX_DEPOSITS), + voluntaryExits: new ListCompositeType(SignedVoluntaryExit, MAX_VOLUNTARY_EXITS), + }, + {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const BeaconBlock = new ContainerType( + { slot: Slot, proposerIndex: ValidatorIndex, - parentRoot: new RootType({expandedType: () => typesRef.get().BeaconBlock}), - stateRoot: new RootType({expandedType: () => typesRef.get().BeaconState}), + parentRoot: Root, + stateRoot: Root, body: BeaconBlockBody, }, - casingMap: { - slot: "slot", - proposerIndex: "proposer_index", - parentRoot: "parent_root", - stateRoot: "state_root", - body: "body", - }, -}); + {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedBeaconBlock = new ContainerType({ - fields: { +export const SignedBeaconBlock = new ContainerType( + { message: BeaconBlock, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedBeaconBlock", jsonCase: "eth2"} +); // State types // =========== -export const EpochAttestations = new ListType>({ - elementType: PendingAttestation, - limit: MAX_ATTESTATIONS * SLOTS_PER_EPOCH, -}); +export const EpochAttestations = new ListCompositeType(PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH); -export const BeaconState = new ContainerType({ - fields: { +export const BeaconState = new ContainerType( + { // Misc - genesisTime: Number64, + genesisTime: UintNum64, genesisValidatorsRoot: Root, slot: Slot, fork: Fork, @@ -463,14 +362,11 @@ export const BeaconState = new ContainerType({ latestBlockHeader: BeaconBlockHeader, blockRoots: HistoricalBlockRoots, stateRoots: HistoricalStateRoots, - historicalRoots: new ListType({ - elementType: new RootType({expandedType: HistoricalBatch}), - limit: HISTORICAL_ROOTS_LIMIT, - }), + historicalRoots: new ListCompositeType(Root, HISTORICAL_ROOTS_LIMIT), // Eth1 eth1Data: Eth1Data, eth1DataVotes: Eth1DataVotes, - eth1DepositIndex: Number64, + eth1DepositIndex: UintNum64, // Registry validators: Validators, balances: Balances, @@ -486,135 +382,83 @@ export const BeaconState = new ContainerType({ currentJustifiedCheckpoint: Checkpoint, finalizedCheckpoint: Checkpoint, }, - casingMap: { - genesisTime: "genesis_time", - genesisValidatorsRoot: "genesis_validators_root", - slot: "slot", - fork: "fork", - latestBlockHeader: "latest_block_header", - blockRoots: "block_roots", - stateRoots: "state_roots", - historicalRoots: "historical_roots", - eth1Data: "eth1_data", - eth1DataVotes: "eth1_data_votes", - eth1DepositIndex: "eth1_deposit_index", - validators: "validators", - balances: "balances", - randaoMixes: "randao_mixes", - slashings: "slashings", - previousEpochAttestations: "previous_epoch_attestations", - currentEpochAttestations: "current_epoch_attestations", - justificationBits: "justification_bits", - previousJustifiedCheckpoint: "previous_justified_checkpoint", - currentJustifiedCheckpoint: "current_justified_checkpoint", - finalizedCheckpoint: "finalized_checkpoint", - }, -}); + {typeName: "BeaconState", jsonCase: "eth2"} +); // Validator types // =============== -export const CommitteeAssignment = new ContainerType({ - fields: { +export const CommitteeAssignment = new ContainerType( + { validators: CommitteeIndices, committeeIndex: CommitteeIndex, slot: Slot, }, - // Custom type, not in the consensus specs - casingMap: { - validators: "validators", - committeeIndex: "committee_index", - slot: "slot", - }, -}); + {typeName: "CommitteeAssignment", jsonCase: "eth2"} +); -export const AggregateAndProof = new ContainerType({ - fields: { +export const AggregateAndProof = new ContainerType( + { aggregatorIndex: ValidatorIndex, aggregate: Attestation, selectionProof: BLSSignature, }, - casingMap: { - aggregatorIndex: "aggregator_index", - aggregate: "aggregate", - selectionProof: "selection_proof", - }, -}); + {typeName: "AggregateAndProof", jsonCase: "eth2", cachePermanentRootStruct: true} +); -export const SignedAggregateAndProof = new ContainerType({ - fields: { +export const SignedAggregateAndProof = new ContainerType( + { message: AggregateAndProof, signature: BLSSignature, }, - expectedCase: "notransform", -}); + {typeName: "SignedAggregateAndProof", jsonCase: "eth2"} +); // ReqResp types // ============= -export const Status = new ContainerType({ - fields: { +export const Status = new ContainerType( + { forkDigest: ForkDigest, finalizedRoot: Root, finalizedEpoch: Epoch, headRoot: Root, headSlot: Slot, }, - casingMap: { - forkDigest: "fork_digest", - finalizedRoot: "finalized_root", - finalizedEpoch: "finalized_epoch", - headRoot: "head_root", - headSlot: "head_slot", - }, -}); + {typeName: "Status", jsonCase: "eth2"} +); -export const Goodbye = Uint64; +export const Goodbye = UintBn64; -export const Ping = Uint64; +export const Ping = UintBn64; -export const Metadata = new ContainerType({ - fields: { - seqNumber: Uint64, +export const Metadata = new ContainerType( + { + seqNumber: UintBn64, attnets: AttestationSubnets, }, - casingMap: { - seqNumber: "seq_number", - attnets: "attnets", - }, -}); + {typeName: "Metadata", jsonCase: "eth2"} +); -export const BeaconBlocksByRangeRequest = new ContainerType({ - fields: { +export const BeaconBlocksByRangeRequest = new ContainerType( + { startSlot: Slot, - count: Number64, - step: Number64, - }, - casingMap: { - startSlot: "start_slot", - count: "count", - step: "step", + count: UintNum64, + step: UintNum64, }, -}); + {typeName: "BeaconBlocksByRangeRequest", jsonCase: "eth2"} +); -export const BeaconBlocksByRootRequest = new ListType({elementType: Root, limit: MAX_REQUEST_BLOCKS}); +export const BeaconBlocksByRootRequest = new ListCompositeType(Root, MAX_REQUEST_BLOCKS); // Api types // ========= -export const Genesis = new ContainerType({ - fields: { +export const Genesis = new ContainerType( + { genesisValidatorsRoot: Root, - genesisTime: Uint64, + genesisTime: UintNum64, genesisForkVersion: Version, }, - // From beacon-apis - casingMap: { - genesisValidatorsRoot: "genesis_validators_root", - genesisTime: "genesis_time", - genesisForkVersion: "genesis_fork_version", - }, -}); - -// MUST set typesRef here, otherwise expandedType() calls will throw -typesRef.set({BeaconBlock, BeaconState}); + {typeName: "Genesis", jsonCase: "eth2"} +); diff --git a/packages/types/src/phase0/types.ts b/packages/types/src/phase0/types.ts index e0312a42c6c..62299692067 100644 --- a/packages/types/src/phase0/types.ts +++ b/packages/types/src/phase0/types.ts @@ -1,334 +1,42 @@ -import {BitList, List, Vector, BitVector} from "@chainsafe/ssz"; -import { - BLSPubkey, - BLSSignature, - Epoch, - Root, - Number64, - Slot, - ValidatorIndex, - Version, - CommitteeIndex, - Bytes32, - Domain, - ForkDigest, - Gwei, - Uint64, -} from "../primitive/types"; - -export type AttestationSubnets = BitVector; - -export interface BeaconBlockHeader { - slot: Slot; - proposerIndex: ValidatorIndex; - parentRoot: Root; - stateRoot: Root; - bodyRoot: Root; -} - -export interface SignedBeaconBlockHeader { - message: BeaconBlockHeader; - signature: BLSSignature; -} - -export interface Checkpoint { - epoch: Epoch; - root: Root; -} - -export interface DepositMessage { - // BLS pubkey - pubkey: BLSPubkey; - // Withdrawal credentials - withdrawalCredentials: Bytes32; - // Amount in Gwei - amount: Number64; -} - -export interface DepositData { - // BLS pubkey - pubkey: BLSPubkey; - // Withdrawal credentials - withdrawalCredentials: Bytes32; - // Amount in Gwei - amount: Number64; - // Signing over DepositMessage - signature: BLSSignature; -} - -export interface DepositEvent { - depositData: DepositData; - /// The block number of the log that included this `DepositData`. - blockNumber: Number64; - /// The index included with the deposit log. - index: Number64; -} - -export interface Eth1Data { - // Root of the deposit tree - depositRoot: Root; - // Total number of deposits - depositCount: Number64; - // Block hash - blockHash: Bytes32; -} - -export interface Eth1DataOrdered { - // block number for this eth1 data block hash - blockNumber: Number64; - // Root of the deposit tree - depositRoot: Root; - // Total number of deposits - depositCount: Number64; - // Block hash - blockHash: Bytes32; -} - -export interface Fork { - // Previous fork version - previousVersion: Version; - // Current fork version - currentVersion: Version; - // Fork epoch number - epoch: Epoch; -} - -export interface ForkData { - // Current fork version - currentVersion: Version; - // root of genesis validator list - genesisValidatorsRoot: Root; -} - -export interface ENRForkID { - // Current fork digest - forkDigest: ForkDigest; - // next planned fork versin - nextForkVersion: Version; - // next fork epoch - nextForkEpoch: Epoch; -} - -export interface HistoricalBatch { - // Block roots - blockRoots: Vector; - // State roots - stateRoots: Vector; -} - -export interface Validator { - // BLS public key - pubkey: BLSPubkey; - // Commitment to pubkey for withdrawals - withdrawalCredentials: Bytes32; - // Balance at stake - effectiveBalance: Number64; - // Was the validator slashed - slashed: boolean; - // When criteria for activation were met - activationEligibilityEpoch: Epoch; - // Epoch when validator activated - activationEpoch: Epoch; - // Epoch when validator exited - exitEpoch: Epoch; - // When validator can withdraw or transfer funds - withdrawableEpoch: Epoch; -} - -export interface AttestationData { - slot: Slot; - index: CommitteeIndex; - // LMD GHOST vote - beaconBlockRoot: Root; - // FFG vote - source: Checkpoint; - target: Checkpoint; -} - -export interface IndexedAttestation { - // Validator Indices - attestingIndices: List; - // Attestation Data - data: AttestationData; - // Aggregate signature - signature: BLSSignature; -} - -export interface PendingAttestation { - // Attester aggregation bitfield - aggregationBits: BitList; - // Attestation data - data: AttestationData; - // Inclusion delay - inclusionDelay: Slot; - // Proposer index - proposerIndex: ValidatorIndex; -} - -export interface SigningData { - objectRoot: Root; - domain: Domain; -} - -export interface Eth1Block { - // Use blockHash to be consistent with the Eth1Data type - blockHash: Bytes32; - // Use blockNumber to be consistent with DepositEvent type - blockNumber: Number64; - timestamp: Number64; -} - -export interface Attestation { - // Attester participation bitfield - aggregationBits: BitList; - // Attestation data - data: AttestationData; - // BLS aggregate signature - signature: BLSSignature; -} - -export interface AttesterSlashing { - // First attestation - attestation1: IndexedAttestation; - // Second attestation - attestation2: IndexedAttestation; -} - -export interface Deposit { - // Branch in the deposit tree - proof: Vector; - // Deposit data - data: DepositData; -} - -export interface ProposerSlashing { - // First block header - signedHeader1: SignedBeaconBlockHeader; - // Second block header - signedHeader2: SignedBeaconBlockHeader; -} - -export interface VoluntaryExit { - // Minimum epoch for processing exit - epoch: Epoch; - // Index of the exiting validator - validatorIndex: ValidatorIndex; -} - -export interface SignedVoluntaryExit { - message: VoluntaryExit; - // Validator signature - signature: BLSSignature; -} - -export interface BeaconBlockBody { - randaoReveal: BLSSignature; - eth1Data: Eth1Data; - graffiti: Bytes32; - proposerSlashings: List; - attesterSlashings: List; - attestations: List; - deposits: List; - voluntaryExits: List; -} - -export interface BeaconBlock { - // Header - slot: Slot; - proposerIndex: ValidatorIndex; - parentRoot: Root; - stateRoot: Root; - body: BeaconBlockBody; -} - -export interface SignedBeaconBlock { - message: BeaconBlock; - signature: BLSSignature; -} - -export interface BeaconState { - // Misc - genesisTime: Number64; - genesisValidatorsRoot: Root; - slot: Slot; - fork: Fork; // For versioning hard forks - - // History - latestBlockHeader: BeaconBlockHeader; - blockRoots: Vector; - stateRoots: Vector; - historicalRoots: List; - - // Eth1 - eth1Data: Eth1Data; - eth1DataVotes: List; - eth1DepositIndex: Number64; - - // Registry - validators: List; - balances: List; - - // Shuffling - randaoMixes: Vector; - - // Slashings - slashings: Vector; // Balances penalized at every withdrawal period - - // Attestations - previousEpochAttestations: List; - currentEpochAttestations: List; - - // Finality - justificationBits: BitVector; - previousJustifiedCheckpoint: Checkpoint; - currentJustifiedCheckpoint: Checkpoint; - finalizedCheckpoint: Checkpoint; -} - -export interface CommitteeAssignment { - validators: List; - committeeIndex: CommitteeIndex; - slot: Slot; -} - -export interface AggregateAndProof { - aggregatorIndex: ValidatorIndex; - aggregate: Attestation; - selectionProof: BLSSignature; -} - -export interface SignedAggregateAndProof { - message: AggregateAndProof; - signature: BLSSignature; -} - -export interface Status { - forkDigest: ForkDigest; - finalizedRoot: Root; - finalizedEpoch: Epoch; - headRoot: Root; - headSlot: Slot; -} - -export type Goodbye = Uint64; - -export type Ping = Uint64; - -export interface Metadata { - seqNumber: Uint64; - attnets: AttestationSubnets; -} - -export interface BeaconBlocksByRangeRequest { - startSlot: Slot; - count: Number64; - step: Number64; -} - -export type BeaconBlocksByRootRequest = List; - -export interface Genesis { - genesisTime: Uint64; - genesisValidatorsRoot: Root; - genesisForkVersion: Version; -} +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes"; + +export type AttestationSubnets = ValueOf; +export type BeaconBlockHeader = ValueOf; +export type SignedBeaconBlockHeader = ValueOf; +export type Checkpoint = ValueOf; +export type DepositMessage = ValueOf; +export type DepositData = ValueOf; +export type DepositEvent = ValueOf; +export type Eth1Data = ValueOf; +export type Eth1DataOrdered = ValueOf; +export type Eth1Block = ValueOf; +export type Fork = ValueOf; +export type ForkData = ValueOf; +export type ENRForkID = ValueOf; +export type HistoricalBatch = ValueOf; +export type Validator = ValueOf; +export type AttestationData = ValueOf; +export type IndexedAttestation = ValueOf; +export type PendingAttestation = ValueOf; +export type SigningData = ValueOf; +export type Attestation = ValueOf; +export type AttesterSlashing = ValueOf; +export type Deposit = ValueOf; +export type ProposerSlashing = ValueOf; +export type VoluntaryExit = ValueOf; +export type SignedVoluntaryExit = ValueOf; +export type BeaconBlockBody = ValueOf; +export type BeaconBlock = ValueOf; +export type SignedBeaconBlock = ValueOf; +export type BeaconState = ValueOf; +export type CommitteeAssignment = ValueOf; +export type AggregateAndProof = ValueOf; +export type SignedAggregateAndProof = ValueOf; +export type Status = ValueOf; +export type Goodbye = ValueOf; +export type Ping = ValueOf; +export type Metadata = ValueOf; +export type BeaconBlocksByRangeRequest = ValueOf; +export type BeaconBlocksByRootRequest = ValueOf; +export type Genesis = ValueOf; diff --git a/packages/types/src/primitive/sszTypes.ts b/packages/types/src/primitive/sszTypes.ts index db41e0d8db2..95e73c47849 100644 --- a/packages/types/src/primitive/sszTypes.ts +++ b/packages/types/src/primitive/sszTypes.ts @@ -1,46 +1,37 @@ -import { - byteType, - booleanType, - ByteVectorType, - BigIntUintType, - number32Type, - NumberUintType, - RootType, - Number64UintType, -} from "@chainsafe/ssz"; +import {ByteVectorType, UintNumberType, UintBigintType, BooleanType} from "@chainsafe/ssz"; -export const Boolean = booleanType; -export const Bytes4 = new ByteVectorType({length: 4}); -export const Bytes8 = new ByteVectorType({length: 8}); -export const Bytes20 = new ByteVectorType({length: 20}); -export const Bytes32 = new ByteVectorType({length: 32}); -export const Bytes48 = new ByteVectorType({length: 48}); -export const Bytes96 = new ByteVectorType({length: 96}); -export const Uint8 = byteType; -export const Uint16 = new NumberUintType({byteLength: 2}); -export const Uint32 = number32Type; -export const Number64 = new Number64UintType(); -export const Uint64 = new BigIntUintType({byteLength: 8}); -export const Uint128 = new BigIntUintType({byteLength: 16}); -export const Uint256 = new BigIntUintType({byteLength: 32}); +export const Boolean = new BooleanType(); +export const Byte = new UintNumberType(1); +export const Bytes4 = new ByteVectorType(4); +export const Bytes8 = new ByteVectorType(8); +export const Bytes20 = new ByteVectorType(20); +export const Bytes32 = new ByteVectorType(32); +export const Bytes48 = new ByteVectorType(48); +export const Bytes96 = new ByteVectorType(96); +export const Uint8 = new UintNumberType(1); +export const Uint16 = new UintNumberType(2); +export const Uint32 = new UintNumberType(4); +export const UintNum64 = new UintNumberType(8); +export const UintNumInf64 = new UintNumberType(8, {clipInfinity: true}); +export const UintBn64 = new UintBigintType(8); +export const UintBn128 = new UintBigintType(16); +export const UintBn256 = new UintBigintType(32); // Custom types, defined for type hinting and readability -export const Slot = Number64; -export const Epoch = Number64; -export const CommitteeIndex = Number64; -export const SubcommitteeIndex = Number64; -export const ValidatorIndex = Number64; -export const Gwei = Uint64; -export const Root = new RootType({ - expandedType: () => { - throw new Error("Generic Root type has no expanded type"); - }, -}); +export const Slot = UintNumInf64; +export const Epoch = UintNumInf64; +export const SyncPeriod = UintNum64; +export const CommitteeIndex = UintNum64; +export const SubcommitteeIndex = UintNum64; +export const ValidatorIndex = UintNum64; +export const Gwei = UintBn64; +export const Root = new ByteVectorType(32); + export const Version = Bytes4; export const DomainType = Bytes4; export const ForkDigest = Bytes4; export const BLSPubkey = Bytes48; export const BLSSignature = Bytes96; export const Domain = Bytes32; -export const ParticipationFlags = Uint8; +export const ParticipationFlags = new UintNumberType(1, {setBitwiseOR: true}); export const ExecutionAddress = Bytes20; diff --git a/packages/types/src/primitive/types.ts b/packages/types/src/primitive/types.ts index 6a9d275c165..619f1915abe 100644 --- a/packages/types/src/primitive/types.ts +++ b/packages/types/src/primitive/types.ts @@ -1,32 +1,34 @@ -import {ByteVector} from "@chainsafe/ssz"; +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes"; // Each type exported here contains both a compile-time type // (a typescript interface) and a run-time ssz type (a javascript variable) // For more information, see ./index.ts -export type Bytes4 = ByteVector; -export type Bytes8 = ByteVector; -export type Bytes20 = ByteVector; -export type Bytes32 = ByteVector; -export type Bytes48 = ByteVector; -export type Bytes96 = ByteVector; -export type Uint8 = number; -export type Uint16 = number; -export type Uint32 = number; -export type Number64 = number; -export type Uint64 = bigint; -export type Uint128 = bigint; -export type Uint256 = bigint; +export type Bytes4 = ValueOf; +export type Bytes8 = ValueOf; +export type Bytes20 = ValueOf; +export type Bytes32 = ValueOf; +export type Bytes48 = ValueOf; +export type Bytes96 = ValueOf; +export type Uint8 = ValueOf; +export type Uint16 = ValueOf; +export type Uint32 = ValueOf; +export type UintNum64 = ValueOf; +export type UintNumInf64 = ValueOf; +export type UintBn64 = ValueOf; +export type UintBn128 = ValueOf; +export type UintBn256 = ValueOf; // Custom types, defined for type hinting and readability -export type Slot = Number64; -export type Epoch = Number64; -export type SyncPeriod = Number64; -export type CommitteeIndex = Number64; -export type SubcommitteeIndex = Number64; -export type ValidatorIndex = Number64; -export type Gwei = Uint64; +export type Slot = UintNumInf64; +export type Epoch = UintNumInf64; +export type SyncPeriod = UintNum64; +export type CommitteeIndex = UintNum64; +export type SubcommitteeIndex = UintNum64; +export type ValidatorIndex = UintNum64; +export type Gwei = UintBn64; export type Root = Bytes32; export type Version = Bytes4; export type DomainType = Bytes4; @@ -40,3 +42,5 @@ export type ExecutionAddress = Bytes20; /** Common non-spec type to represent roots as strings */ export type RootHex = string; +/** Non-spec type to signal time is represented in seconds */ +export type TimeSeconds = number; diff --git a/packages/types/src/utils/StringType.ts b/packages/types/src/utils/StringType.ts index f94ff80bc5e..98481ed7e08 100644 --- a/packages/types/src/utils/StringType.ts +++ b/packages/types/src/utils/StringType.ts @@ -3,36 +3,53 @@ import {BasicType} from "@chainsafe/ssz"; /* eslint-disable @typescript-eslint/naming-convention */ export class StringType extends BasicType { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - struct_getSerializedLength(data?: string): number { - throw new Error("unsupported ssz operation"); - } + readonly typeName = "string"; + byteLength = 0; + fixedSize = 0; + minSize = 0; + maxSize = 0; - struct_convertToJson(value: T): string { - return value; + defaultValue(): T { + return "" as T; } - struct_convertFromJson(data: string): T { - return data as T; - } + // Serialization + deserialization - struct_assertValidValue(data: unknown): data is T { - throw new Error("unsupported ssz operation"); + value_serializeToBytes(): number { + throw Error("Not supported in String type"); } - - serialize(): Uint8Array { - throw new Error("unsupported ssz type for serialization"); + value_deserializeFromBytes(): T { + throw Error("Not supported in String type"); + } + tree_serializeToBytes(): number { + throw Error("Not supported in String type"); + } + tree_deserializeFromBytes(): never { + throw Error("Not supported in String type"); } - struct_serializeToBytes(): number { - throw new Error("unsupported ssz type for serialization"); + // Fast tree opts + + tree_getFromNode(): T { + throw Error("Not supported in String type"); + } + tree_setToNode(): void { + throw Error("Not supported in String type"); + } + tree_getFromPackedNode(): T { + throw Error("Not supported in String type"); } + tree_setToPackedNode(): void { + throw Error("Not supported in String type"); + } + + // JSON - struct_deserializeFromBytes(): T { - throw new Error("unsupported ssz operation"); + fromJson(json: unknown): T { + return json as T; } - struct_defaultValue(): T { - return "something" as T; + toJson(value: T): unknown { + return value; } } diff --git a/packages/types/src/utils/lazyVar.ts b/packages/types/src/utils/lazyVar.ts deleted file mode 100644 index 9378501d095..00000000000 --- a/packages/types/src/utils/lazyVar.ts +++ /dev/null @@ -1,12 +0,0 @@ -export class LazyVariable { - private var: {set: false} | {set: true; value: T} = {set: false}; - - get(): T { - if (!this.var.set) throw Error("variable not set"); - return this.var.value; - } - - set(value: T): void { - this.var = {set: true, value}; - } -} diff --git a/packages/types/test/unit/constants.test.ts b/packages/types/test/unit/constants.test.ts index 67540b4bc4a..1ab21a18ac3 100644 --- a/packages/types/test/unit/constants.test.ts +++ b/packages/types/test/unit/constants.test.ts @@ -9,11 +9,11 @@ import {expect} from "chai"; // guarantee that these constants are correct. describe("Lightclient pre-computed constants", () => { - const FINALIZED_ROOT_GINDEX = Number(ssz.altair.BeaconState.getPathGindex(["finalizedCheckpoint", "root"])); + const FINALIZED_ROOT_GINDEX = bnToNum(ssz.altair.BeaconState.getPathInfo(["finalizedCheckpoint", "root"]).gindex); const FINALIZED_ROOT_DEPTH = floorlog2(FINALIZED_ROOT_GINDEX); const FINALIZED_ROOT_INDEX = FINALIZED_ROOT_GINDEX % 2 ** FINALIZED_ROOT_DEPTH; - const NEXT_SYNC_COMMITTEE_GINDEX = Number(ssz.altair.BeaconState.getPathGindex(["nextSyncCommittee"])); + const NEXT_SYNC_COMMITTEE_GINDEX = bnToNum(ssz.altair.BeaconState.getPathInfo(["nextSyncCommittee"]).gindex); const NEXT_SYNC_COMMITTEE_DEPTH = floorlog2(NEXT_SYNC_COMMITTEE_GINDEX); const NEXT_SYNC_COMMITTEE_INDEX = NEXT_SYNC_COMMITTEE_GINDEX % 2 ** NEXT_SYNC_COMMITTEE_DEPTH; @@ -36,3 +36,8 @@ describe("Lightclient pre-computed constants", () => { function floorlog2(num: number): number { return Math.floor(Math.log2(num)); } + +/** Type safe wrapper for Number constructor that takes 'any' */ +function bnToNum(bn: bigint): number { + return Number(bn); +} diff --git a/packages/types/test/unit/ssz.test.ts b/packages/types/test/unit/ssz.test.ts index 5ebf0bc0679..d738f47e653 100644 --- a/packages/types/test/unit/ssz.test.ts +++ b/packages/types/test/unit/ssz.test.ts @@ -3,8 +3,8 @@ import {ssz} from "../../src"; describe("size", function () { it("should calculate correct minSize and maxSize", () => { - const minSize = ssz.phase0.BeaconState.minSize(); - const maxSize = ssz.phase0.BeaconState.maxSize(); + const minSize = ssz.phase0.BeaconState.minSize; + const maxSize = ssz.phase0.BeaconState.maxSize; // https://gist.github.com/protolambda/db75c7faa1e94f2464787a480e5d613e expect(minSize).to.be.equal(2687377); expect(maxSize).to.be.equal(141837543039377); @@ -18,17 +18,12 @@ describe("container serialization/deserialization field casing(s)", function () attestation2: ssz.phase0.IndexedAttestation.defaultValue(), }; const json = { - attestation_1: ssz.phase0.IndexedAttestation.toJson(test.attestation1, {case: "snake"}), - attestation_2: ssz.phase0.IndexedAttestation.toJson(test.attestation2, {case: "snake"}), + attestation_1: ssz.phase0.IndexedAttestation.toJson(test.attestation1), + attestation_2: ssz.phase0.IndexedAttestation.toJson(test.attestation2), }; - const result = ssz.phase0.AttesterSlashing.fromJson(json, { - case: "snake", - }); - expect(ssz.phase0.AttesterSlashing.equals(test, result)).to.be.true; - const back = ssz.phase0.AttesterSlashing.toJson(result, { - case: "snake", - }); + const result = ssz.phase0.AttesterSlashing.fromJson(json); + const back = ssz.phase0.AttesterSlashing.toJson(result); expect(back).to.be.deep.equal(json); }); @@ -38,17 +33,12 @@ describe("container serialization/deserialization field casing(s)", function () signedHeader2: ssz.phase0.SignedBeaconBlockHeader.defaultValue(), }; const json = { - signed_header_1: ssz.phase0.SignedBeaconBlockHeader.toJson(test.signedHeader1, {case: "snake"}), - signed_header_2: ssz.phase0.SignedBeaconBlockHeader.toJson(test.signedHeader2, {case: "snake"}), + signed_header_1: ssz.phase0.SignedBeaconBlockHeader.toJson(test.signedHeader1), + signed_header_2: ssz.phase0.SignedBeaconBlockHeader.toJson(test.signedHeader2), }; - const result = ssz.phase0.ProposerSlashing.fromJson(json, { - case: "snake", - }); - expect(ssz.phase0.ProposerSlashing.equals(test, result)).to.be.true; - const back = ssz.phase0.ProposerSlashing.toJson(result, { - case: "snake", - }); + const result = ssz.phase0.ProposerSlashing.fromJson(json); + const back = ssz.phase0.ProposerSlashing.toJson(result); expect(back).to.be.deep.equal(json); }); }); diff --git a/packages/utils/package.json b/packages/utils/package.json index e146c6b1f0b..429d4878bb0 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -37,7 +37,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/abort-controller": "^3.0.1", - "@chainsafe/as-sha256": "^0.2.4", + "@chainsafe/as-sha256": "^0.3.0", "any-signal": "2.1.2", "bigint-buffer": "^1.1.5", "case": "^1.6.3", diff --git a/packages/utils/src/bytes.ts b/packages/utils/src/bytes.ts index f60d4ba35d1..5395c4315d2 100644 --- a/packages/utils/src/bytes.ts +++ b/packages/utils/src/bytes.ts @@ -46,7 +46,7 @@ export function bytesToBigInt(value: Uint8Array, endianness: Endianness = "le"): throw new Error("endianness must be either 'le' or 'be'"); } -export function toHex(buffer: Parameters[0]): string { +export function toHex(buffer: Uint8Array | Parameters[0]): string { if (Buffer.isBuffer(buffer)) { return "0x" + buffer.toString("hex"); } else if (buffer instanceof Uint8Array) { diff --git a/packages/utils/src/verifyMerkleBranch.ts b/packages/utils/src/verifyMerkleBranch.ts index 45299b7201f..e109813dd29 100644 --- a/packages/utils/src/verifyMerkleBranch.ts +++ b/packages/utils/src/verifyMerkleBranch.ts @@ -1,7 +1,7 @@ -import SHA256 from "@chainsafe/as-sha256"; +import {digest, digest64} from "@chainsafe/as-sha256"; export function hash(...inputs: Uint8Array[]): Uint8Array { - return SHA256.digest(Buffer.concat(inputs)); + return digest(Buffer.concat(inputs)); } /** @@ -18,9 +18,9 @@ export function verifyMerkleBranch( let value = leaf; for (let i = 0; i < depth; i++) { if (Math.floor(index / 2 ** i) % 2) { - value = SHA256.digest64(Buffer.concat([proof[i], value])); + value = digest64(Buffer.concat([proof[i], value])); } else { - value = SHA256.digest64(Buffer.concat([value, proof[i]])); + value = digest64(Buffer.concat([value, proof[i]])); } } return Buffer.from(value).equals(root); diff --git a/packages/validator/package.json b/packages/validator/package.json index 6349ff33433..1f96cbf9a11 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -54,7 +54,7 @@ "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", - "@chainsafe/ssz": "^0.8.20", + "@chainsafe/ssz": "^0.9.0", "bigint-buffer": "^1.1.5", "cross-fetch": "^3.1.4", "strict-event-emitter-types": "^2.0.0" diff --git a/packages/validator/src/repositories/metaDataRepository.ts b/packages/validator/src/repositories/metaDataRepository.ts index 39159ec46fb..0bbba61941b 100644 --- a/packages/validator/src/repositories/metaDataRepository.ts +++ b/packages/validator/src/repositories/metaDataRepository.ts @@ -1,5 +1,5 @@ import {Bucket, encodeKey, IDatabaseApiOptions} from "@chainsafe/lodestar-db"; -import {Root, Uint64} from "@chainsafe/lodestar-types"; +import {Root, UintNum64} from "@chainsafe/lodestar-types"; import {ssz} from "@chainsafe/lodestar-types"; import {LodestarValidatorDatabaseController} from "../types"; @@ -22,19 +22,16 @@ export class MetaDataRepository { } async setGenesisValidatorsRoot(genesisValidatorsRoot: Root): Promise { - await this.db.put( - this.encodeKey(GENESIS_VALIDATORS_ROOT), - Buffer.from(genesisValidatorsRoot.valueOf() as Uint8Array) - ); + await this.db.put(this.encodeKey(GENESIS_VALIDATORS_ROOT), Buffer.from(genesisValidatorsRoot)); } - async getGenesisTime(): Promise { + async getGenesisTime(): Promise { const bytes = await this.db.get(this.encodeKey(GENESIS_TIME)); - return bytes ? ssz.Uint64.deserialize(bytes) : null; + return bytes ? ssz.UintNum64.deserialize(bytes) : null; } - async setGenesisTime(genesisTime: Uint64): Promise { - await this.db.put(this.encodeKey(GENESIS_TIME), Buffer.from(ssz.Uint64.serialize(genesisTime))); + async setGenesisTime(genesisTime: UintNum64): Promise { + await this.db.put(this.encodeKey(GENESIS_TIME), Buffer.from(ssz.UintNum64.serialize(genesisTime))); } private encodeKey(key: Uint8Array): Uint8Array { diff --git a/packages/validator/src/services/utils.ts b/packages/validator/src/services/utils.ts index b307d6ff94c..8fc436cca2d 100644 --- a/packages/validator/src/services/utils.ts +++ b/packages/validator/src/services/utils.ts @@ -9,10 +9,6 @@ export type SubcommitteeDuty = { selectionProof: SyncSelectionProof["selectionProof"]; }; -export function getAggregationBits(committeeLength: number, validatorIndexInCommittee: number): boolean[] { - return Array.from({length: committeeLength}, (_, i) => i === validatorIndexInCommittee); -} - export function groupAttDutiesByCommitteeIndex(duties: AttDutyAndProof[]): Map { const dutiesByCommitteeIndex = new Map(); diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 8d2fbfcb833..0f17ba637a4 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -28,11 +28,10 @@ import { ValidatorIndex, ssz, } from "@chainsafe/lodestar-types"; -import {fromHexString, List, toHexString} from "@chainsafe/ssz"; +import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; import {routes} from "@chainsafe/lodestar-api"; import {Interchange, InterchangeFormatVersion, ISlashingProtection} from "../slashingProtection"; import {PubkeyHex} from "../types"; -import {getAggregationBits} from "./utils"; import {externalSignerPostSignature} from "../util/externalSignerClient"; export enum SignerType { @@ -166,7 +165,7 @@ export class ValidatorStore { }); return { - aggregationBits: getAggregationBits(duty.committeeLength, duty.validatorCommitteeIndex) as List, + aggregationBits: BitArray.fromSingleBit(duty.committeeLength, duty.validatorCommitteeIndex), data: attestationData, signature: await this.getSignature(duty.pubkey, signingRoot), }; @@ -246,7 +245,7 @@ export class ValidatorStore { const domain = this.config.getDomain(DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, slot); const signingData: altair.SyncAggregatorSelectionData = { slot, - subcommitteeIndex: subcommitteeIndex, + subcommitteeIndex, }; const signingRoot = computeSigningRoot(ssz.altair.SyncAggregatorSelectionData, signingData, domain); diff --git a/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts b/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts index 3a0d4f880f0..08361770523 100644 --- a/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts +++ b/packages/validator/src/slashingProtection/attestation/attestationByTargetRepository.ts @@ -18,19 +18,11 @@ export class AttestationByTargetRepository { constructor(opts: IDatabaseApiOptions) { this.db = opts.controller; - this.type = new ContainerType({ - fields: { - sourceEpoch: ssz.Epoch, - targetEpoch: ssz.Epoch, - signingRoot: ssz.Root, - }, - // Custom type, not in the consensus specs - casingMap: { - sourceEpoch: "source_epoch", - targetEpoch: "target_epoch", - signingRoot: "signing_root", - }, - }); + this.type = new ContainerType({ + sourceEpoch: ssz.Epoch, + targetEpoch: ssz.Epoch, + signingRoot: ssz.Root, + }); // casing doesn't matter } async getAll(pubkey: BLSPubkey, limit?: number): Promise { diff --git a/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts b/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts index 536fedce087..dd9daee6abf 100644 --- a/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts +++ b/packages/validator/src/slashingProtection/attestation/attestationLowerBoundRepository.ts @@ -21,17 +21,10 @@ export class AttestationLowerBoundRepository { constructor(opts: IDatabaseApiOptions) { this.db = opts.controller; - this.type = new ContainerType({ - fields: { - minSourceEpoch: ssz.Epoch, - minTargetEpoch: ssz.Epoch, - }, - // Custom type, not in the consensus specs - casingMap: { - minSourceEpoch: "min_source_epoch", - minTargetEpoch: "min_target_epoch", - }, - }); + this.type = new ContainerType({ + minSourceEpoch: ssz.Epoch, + minTargetEpoch: ssz.Epoch, + }); // casing doesn't matter } async get(pubkey: BLSPubkey): Promise { diff --git a/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts b/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts index c825ac39438..35624fc831b 100644 --- a/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts +++ b/packages/validator/src/slashingProtection/block/blockBySlotRepository.ts @@ -18,17 +18,10 @@ export class BlockBySlotRepository { constructor(opts: IDatabaseApiOptions) { this.db = opts.controller; - this.type = new ContainerType({ - fields: { - slot: ssz.Slot, - signingRoot: ssz.Root, - }, - // Custom type, not in the consensus specs - casingMap: { - slot: "slot", - signingRoot: "signing_root", - }, - }); + this.type = new ContainerType({ + slot: ssz.Slot, + signingRoot: ssz.Root, + }); // casing doesn't matter } async getAll(pubkey: BLSPubkey, limit?: number): Promise { diff --git a/packages/validator/src/slashingProtection/utils.ts b/packages/validator/src/slashingProtection/utils.ts index 0ada547cca3..b34812987e2 100644 --- a/packages/validator/src/slashingProtection/utils.ts +++ b/packages/validator/src/slashingProtection/utils.ts @@ -1,5 +1,5 @@ import {Epoch, Root, ssz} from "@chainsafe/lodestar-types"; -import {fromHexString, toHexString, Vector} from "@chainsafe/ssz"; +import {fromHexString, toHexString} from "@chainsafe/ssz"; export const blsPubkeyLen = 48; export const ZERO_ROOT = ssz.Root.defaultValue(); @@ -31,7 +31,7 @@ export function minEpoch(epochs: Epoch[]): Epoch | null { return epochs.length > 0 ? Math.min(...epochs) : null; } -export function uniqueVectorArr(buffers: Vector[]): Vector[] { +export function uniqueVectorArr(buffers: Uint8Array[]): Uint8Array[] { const bufferStr = new Set(); return buffers.filter((buffer) => { const str = toHexString(buffer); diff --git a/packages/validator/src/util/clock.ts b/packages/validator/src/util/clock.ts index 0dcf2301468..b37eb0c2e86 100644 --- a/packages/validator/src/util/clock.ts +++ b/packages/validator/src/util/clock.ts @@ -2,7 +2,7 @@ import {AbortSignal} from "@chainsafe/abort-controller"; import {ErrorAborted, ILogger, isErrorAborted, sleep} from "@chainsafe/lodestar-utils"; import {GENESIS_SLOT, SLOTS_PER_EPOCH} from "@chainsafe/lodestar-params"; import {IChainForkConfig} from "@chainsafe/lodestar-config"; -import {Epoch, Number64, Slot} from "@chainsafe/lodestar-types"; +import {Epoch, Slot, TimeSeconds} from "@chainsafe/lodestar-types"; import {computeEpochAtSlot, getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; type RunEveryFn = (slot: Slot, signal: AbortSignal) => Promise; @@ -111,7 +111,7 @@ export class Clock implements IClock { /** * Same to the spec but we use Math.round instead of Math.floor. */ -export function getCurrentSlotAround(config: IChainForkConfig, genesisTime: Number64): Slot { +export function getCurrentSlotAround(config: IChainForkConfig, genesisTime: TimeSeconds): Slot { const diffInSeconds = Date.now() / 1000 - genesisTime; const slotsSinceGenesis = Math.round(diffInSeconds / config.SECONDS_PER_SLOT); return GENESIS_SLOT + slotsSinceGenesis; diff --git a/packages/validator/test/unit/services/utils.test.ts b/packages/validator/test/unit/services/utils.test.ts deleted file mode 100644 index 543e45a361c..00000000000 --- a/packages/validator/test/unit/services/utils.test.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {getAggregationBits} from "../../../src/services/utils"; -import {expect} from "chai"; - -describe("getAggregationBits", function () { - it("should return correct bits", function () { - const bits = getAggregationBits(4, 3); - expect(bits).to.deep.equal([false, false, false, true]); - }); -}); diff --git a/yarn.lock b/yarn.lock index c4c863caed7..fd67721761a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -59,11 +59,6 @@ resolved "https://registry.yarnpkg.com/@actions/io/-/io-1.1.1.tgz#4a157406309e212ab27ed3ae30e8c1d641686a66" integrity sha512-Qi4JoKXjmE0O67wAOH6y0n26QXhMKMFo7GD/4IXNVcrtLjUlGjGuVys6pQgwF3ArfGTQu0XpqaNr0YhED2RaRA== -"@assemblyscript/loader@^0.9.2": - version "0.9.4" - resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.9.4.tgz#a483c54c1253656bb33babd464e3154a173e1577" - integrity sha512-HazVq9zwTVwGmqdwYzu7WyQ6FQVZ7SwET0KKQuKm55jD0IfUpZgN0OPIiZG3zV1iSrVYcN0bdwLRXI/VNCYsUA== - "@azure/abort-controller@^1.0.0": version "1.0.4" resolved "https://registry.yarnpkg.com/@azure/abort-controller/-/abort-controller-1.0.4.tgz#fd3c4d46c8ed67aace42498c8e2270960250eafd" @@ -385,13 +380,10 @@ dependencies: event-target-shim "^5.0.0" -"@chainsafe/as-sha256@^0.2.3", "@chainsafe/as-sha256@^0.2.4": - version "0.2.4" - resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.2.4.tgz#17e78d32c71c836160bb55fd79c3daad93d9abf0" - integrity sha512-rYfIOaQm0OlFcHdJFUu5VyYOA1HVeQXxOivUsawBjd7WXc3lMQ0bXMfCgN50gPPLWT92G4ioZ0EZz8RnH+YT/g== - dependencies: - "@assemblyscript/loader" "^0.9.2" - buffer "^5.4.3" +"@chainsafe/as-sha256@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.3.0.tgz#ba7d4b85a77c51a50c071856ea6ead4066583b74" + integrity sha512-X0Yaz1H9ByEelnrG2Q76aEmLmlsjZe8905n6Eez8wmV2E1TvBnY2AFYYy2U52dkaGrWrG4MHYKLo4NJcSqkFeg== "@chainsafe/bls-hd-key@^0.2.0": version "0.2.1" @@ -489,12 +481,12 @@ protobufjs "^6.11.2" uint8arrays "^3.0.0" -"@chainsafe/persistent-merkle-tree@^0.3.7": - version "0.3.7" - resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.3.7.tgz#d257674fcacf1770e54df1e7e476bb55efcb3ed4" - integrity sha512-UXXzvCN8P4eQWCsaTaHRrNa+2sTwY3NB0Vf+uWgM4cU1qf8LOsTU7aU2CeXFAGJag5W/xEQfVGNh463kXTYAgg== +"@chainsafe/persistent-merkle-tree@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.4.0.tgz#249abef9623d45889d71f0f7e807d3aa93472d21" + integrity sha512-xf/DfSAAeAnFOIlNG+m0IvaVK3ccc47vSLEPi0IgbOslXWa540Zxrr33PHbByKI4OhKBp+Zi58WHZAe5O/s2nQ== dependencies: - "@chainsafe/as-sha256" "^0.2.3" + "@chainsafe/as-sha256" "^0.3.0" "@chainsafe/persistent-ts@^0.19.1": version "0.19.1" @@ -513,13 +505,13 @@ buffer-from "^1.1.1" snappy "^6.3.5" -"@chainsafe/ssz@^0.8.20": - version "0.8.20" - resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.8.20.tgz#2c8e70e173a2540c8e4e0bad7c44fd836b8b256b" - integrity sha512-SjD/GqkByWbC5JC2W4cvaFjok6kfZeZDLLzr9k0Kn4sIOZPBfuRTumvV6ihqgn7ItOFpZRXMz0u4DgH+lr28WQ== +"@chainsafe/ssz@^0.9.0": + version "0.9.0" + resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.9.0.tgz#27e40fa3316ef8736bbae898bdf90c1dce4a7045" + integrity sha512-QTLxxcknbiFrpgK2FplAxqxAyi0AGDNoAyxaUUJGdG0mXU4WWIKKLXtYATAXOHVruTzKZeydHatkrpKyYRdyjw== dependencies: - "@chainsafe/as-sha256" "^0.2.4" - "@chainsafe/persistent-merkle-tree" "^0.3.7" + "@chainsafe/as-sha256" "^0.3.0" + "@chainsafe/persistent-merkle-tree" "^0.4.0" case "^1.6.3" "@cspotcode/source-map-consumer@0.8.0":