Move blob data availability and validation context out from state tra…

…nsition

rebase fixes

packages/beacon-node/src/api/impl/beacon/state/utils.ts

@@ -8,7 +8,6 @@ import {
   createEmptyEpochContextImmutableData,
   PubkeyIndexMap,
   ExecutionPayloadStatus,
-  DataAvailableStatus,
 } from "@lodestar/state-transition";
 import {BLSPubkey, phase0} from "@lodestar/types";
 import {stateTransition, processSlots} from "@lodestar/state-transition";
@@ -254,7 +253,6 @@ async function getFinalizedState(
     state = stateTransition(state, block, {
       // Replaying finalized blocks, all data is considered valid
       executionPayloadStatus: ExecutionPayloadStatus.valid,
-      dataAvailableStatus: DataAvailableStatus.available,
       verifyStateRoot: false,
       verifyProposer: false,
       verifySignatures: false,

packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts

@@ -1,11 +1,12 @@
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {ChainForkConfig} from "@lodestar/config";
-import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
-import {Slot} from "@lodestar/types";
+import {IForkChoice, ProtoBlock, DataAvailableStatus} from "@lodestar/fork-choice";
+import {Slot, deneb} from "@lodestar/types";
 import {toHexString} from "@lodestar/utils";
 import {BeaconClock} from "../clock/interface.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
-import {BlockInput, ImportBlockOpts} from "./types.js";
+import {validateBlobsSidecar} from "../validation/blobsSidecar.js";
+import {BlockInput, BlockInputType, ImportBlockOpts} from "./types.js";
 
 /**
  * Verifies some early cheap sanity checks on the block before running the full state transition.
@@ -23,7 +24,11 @@ export function verifyBlocksSanityChecks(
   chain: {forkChoice: IForkChoice; clock: BeaconClock; config: ChainForkConfig},
   blocks: BlockInput[],
   opts: ImportBlockOpts
-): {relevantBlocks: BlockInput[]; parentSlots: Slot[]; parentBlock: ProtoBlock | null} {
+): {
+  relevantBlocks: BlockInput[];
+  parentSlots: Slot[];
+  parentBlock: ProtoBlock | null;
+} {
   if (blocks.length === 0) {
     throw Error("Empty partiallyVerifiedBlocks");
   }
@@ -57,6 +62,10 @@ export function verifyBlocksSanityChecks(
       }
     }
 
+    // Validate status of only not yet finalized blocks, we don't need yet to propogate the status
+    // as it is not used upstream anywhere
+    maybeValidateBlobs(chain.config, blockInput, opts);
+
     let parentBlockSlot: Slot;
 
     if (relevantBlocks.length > 0) {
@@ -105,3 +114,30 @@ export function verifyBlocksSanityChecks(
 
   return {relevantBlocks, parentSlots, parentBlock};
 }
+
+function maybeValidateBlobs(
+  config: ChainForkConfig,
+  blockInput: BlockInput,
+  opts: ImportBlockOpts
+): DataAvailableStatus {
+  // TODO Deneb: Make switch verify it's exhaustive
+  switch (blockInput.type) {
+    case BlockInputType.postDeneb: {
+      if (opts.validBlobsSidecar) {
+        return DataAvailableStatus.available;
+      }
+
+      const {block, blobs} = blockInput;
+      const blockSlot = block.message.slot;
+      const {blobKzgCommitments} = (block as deneb.SignedBeaconBlock).message.body;
+      const beaconBlockRoot = config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message);
+      // TODO Deneb: This function throws un-typed errors
+      validateBlobsSidecar(blockSlot, beaconBlockRoot, blobKzgCommitments, blobs);
+
+      return DataAvailableStatus.available;
+    }
+
+    case BlockInputType.preDeneb:
+      return DataAvailableStatus.preDeneb;
+  }
+}

packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -1,18 +1,10 @@
-import {
-  CachedBeaconStateAllForks,
-  stateTransition,
-  ExecutionPayloadStatus,
-  DataAvailableStatus,
-} from "@lodestar/state-transition";
-import {deneb} from "@lodestar/types";
+import {CachedBeaconStateAllForks, stateTransition, ExecutionPayloadStatus} from "@lodestar/state-transition";
 import {ErrorAborted, Logger, sleep} from "@lodestar/utils";
-import {ChainForkConfig} from "@lodestar/config";
 import {Metrics} from "../../metrics/index.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";
 import {byteArrayEquals} from "../../util/bytes.js";
-import {validateBlobsSidecar} from "../validation/blobsSidecar.js";
-import {BlockInput, BlockInputType, ImportBlockOpts} from "./types.js";
+import {BlockInput, ImportBlockOpts} from "./types.js";
 
 /**
  * Verifies 1 or more blocks are fully valid running the full state transition; from a linear sequence of blocks.
@@ -30,7 +22,6 @@ export async function verifyBlocksStateTransitionOnly(
   signal: AbortSignal,
   opts: BlockProcessOpts & ImportBlockOpts
 ): Promise<{postStates: CachedBeaconStateAllForks[]; proposerBalanceDeltas: number[]}> {
-  const {config} = preState0;
   const postStates: CachedBeaconStateAllForks[] = [];
   const proposerBalanceDeltas: number[] = [];
 
@@ -42,7 +33,6 @@ export async function verifyBlocksStateTransitionOnly(
     // TODO Deneb: Is the best place here to call validateBlobsSidecar()?
     // TODO Deneb: Gossip may already call validateBlobsSidecar, add some flag to de-dup from here
     // TODO Deneb: For sync if this function is expensive, consider adding sleep(0) if metrics show it
-    const dataAvailableStatus = maybeValidateBlobs(config, blocks[i], opts);
 
     // STFN - per_slot_processing() + per_block_processing()
     // NOTE: `regen.getPreState()` should have dialed forward the state already caching checkpoint states
@@ -54,8 +44,6 @@ export async function verifyBlocksStateTransitionOnly(
         // NOTE: Assume valid for now while sending payload to execution engine in parallel
         // Latter verifyBlocksInEpoch() will make sure that payload is indeed valid
         executionPayloadStatus: ExecutionPayloadStatus.valid,
-        // TODO Deneb: Data is validated above for
-        dataAvailableStatus,
         // false because it's verified below with better error typing
         verifyStateRoot: false,
         // if block is trusted don't verify proposer or op signature
@@ -107,30 +95,3 @@ export async function verifyBlocksStateTransitionOnly(
 
   return {postStates, proposerBalanceDeltas};
 }
-
-function maybeValidateBlobs(
-  config: ChainForkConfig,
-  blockInput: BlockInput,
-  opts: ImportBlockOpts
-): DataAvailableStatus {
-  // TODO Deneb: Make switch verify it's exhaustive
-  switch (blockInput.type) {
-    case BlockInputType.postDeneb: {
-      if (opts.validBlobsSidecar) {
-        return DataAvailableStatus.available;
-      }
-
-      const {block, blobs} = blockInput;
-      const blockSlot = block.message.slot;
-      const {blobKzgCommitments} = (block as deneb.SignedBeaconBlock).message.body;
-      const beaconBlockRoot = config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message);
-      // TODO Deneb: This function throws un-typed errors
-      validateBlobsSidecar(blockSlot, beaconBlockRoot, blobKzgCommitments, blobs);
-
-      return DataAvailableStatus.available;
-    }
-
-    case BlockInputType.preDeneb:
-      return DataAvailableStatus.preDeneb;
-  }
-}

packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts

@@ -1,9 +1,4 @@
-import {
-  CachedBeaconStateAllForks,
-  DataAvailableStatus,
-  ExecutionPayloadStatus,
-  stateTransition,
-} from "@lodestar/state-transition";
+import {CachedBeaconStateAllForks, ExecutionPayloadStatus, stateTransition} from "@lodestar/state-transition";
 import {allForks, Root} from "@lodestar/types";
 import {ZERO_HASH} from "../../constants/index.js";
 import {Metrics} from "../../metrics/index.js";
@@ -30,8 +25,6 @@ export function computeNewStateRoot(
     {
       // ExecutionPayloadStatus.valid: Assume payload valid, it has been produced by a trusted EL
       executionPayloadStatus: ExecutionPayloadStatus.valid,
-      // DataAvailableStatus.available: Assume the blobs to be available, have just been produced by trusted EL
-      dataAvailableStatus: DataAvailableStatus.available,
       // verifyStateRoot: false  | the root in the block is zero-ed, it's being computed here
       verifyStateRoot: false,
       // verifyProposer: false   | as the block signature is zero-ed

packages/beacon-node/src/chain/regen/regen.ts

@@ -3,7 +3,6 @@ import {
   CachedBeaconStateAllForks,
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
-  DataAvailableStatus,
   ExecutionPayloadStatus,
   processSlots,
   stateTransition,
@@ -190,7 +189,6 @@ export class StateRegenerator implements IStateRegenerator {
           {
             // Replay previously imported blocks, assume valid and available
             executionPayloadStatus: ExecutionPayloadStatus.valid,
-            dataAvailableStatus: DataAvailableStatus.available,
             verifyStateRoot: false,
             verifyProposer: false,
             verifySignatures: false,

packages/beacon-node/test/spec/presets/finality.ts

@@ -1,9 +1,4 @@
-import {
-  BeaconStateAllForks,
-  DataAvailableStatus,
-  ExecutionPayloadStatus,
-  stateTransition,
-} from "@lodestar/state-transition";
+import {BeaconStateAllForks, ExecutionPayloadStatus, stateTransition} from "@lodestar/state-transition";
 import {altair, bellatrix, ssz} from "@lodestar/types";
 import {ForkName} from "@lodestar/params";
 import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js";
@@ -25,7 +20,6 @@ export const finality: TestRunnerFn<FinalityTestCase, BeaconStateAllForks> = (fo
         state = stateTransition(state, signedBlock, {
           // TODO DENEB: Should assume valid and available for this test?
           executionPayloadStatus: ExecutionPayloadStatus.valid,
-          dataAvailableStatus: DataAvailableStatus.available,
           verifyStateRoot: false,
           verifyProposer: verify,
           verifySignatures: verify,

packages/beacon-node/test/spec/presets/operations.ts

@@ -3,7 +3,6 @@ import {
   CachedBeaconStateAllForks,
   CachedBeaconStateBellatrix,
   CachedBeaconStateCapella,
-  DataAvailableStatus,
   ExecutionPayloadStatus,
   getBlockRootAtSlot,
 } from "@lodestar/state-transition";
@@ -79,8 +78,6 @@ const operationFns: Record<string, BlockProcessFn<CachedBeaconStateAllForks>> =
         executionPayloadStatus: testCase.execution.execution_valid
           ? ExecutionPayloadStatus.valid
           : ExecutionPayloadStatus.invalid,
-        // TODO Deneb: Make this value dynamic on fork Deneb
-        dataAvailableStatus: DataAvailableStatus.preDeneb,
       }
     );
   },

packages/beacon-node/test/spec/presets/sanity.ts

@@ -1,11 +1,5 @@
 import {InputType} from "@lodestar/spec-test-util";
-import {
-  BeaconStateAllForks,
-  DataAvailableStatus,
-  ExecutionPayloadStatus,
-  processSlots,
-  stateTransition,
-} from "@lodestar/state-transition";
+import {BeaconStateAllForks, ExecutionPayloadStatus, processSlots, stateTransition} from "@lodestar/state-transition";
 import {allForks, deneb, ssz} from "@lodestar/types";
 import {ForkName} from "@lodestar/params";
 import {bnToNum} from "@lodestar/utils";
@@ -66,7 +60,6 @@ export const sanityBlocks: TestRunnerFn<SanityBlocksTestCase, BeaconStateAllFork
         wrappedState = stateTransition(wrappedState, signedBlock, {
           // TODO DENEB: Should assume valid and available for this test?
           executionPayloadStatus: ExecutionPayloadStatus.valid,
-          dataAvailableStatus: DataAvailableStatus.available,
           verifyStateRoot: verify,
           verifyProposer: verify,
           verifySignatures: verify,

packages/beacon-node/test/spec/presets/transition.ts

@@ -1,9 +1,4 @@
-import {
-  BeaconStateAllForks,
-  DataAvailableStatus,
-  ExecutionPayloadStatus,
-  stateTransition,
-} from "@lodestar/state-transition";
+import {BeaconStateAllForks, ExecutionPayloadStatus, stateTransition} from "@lodestar/state-transition";
 import {allForks, ssz} from "@lodestar/types";
 import {createChainForkConfig, ChainConfig} from "@lodestar/config";
 import {ForkName} from "@lodestar/params";
@@ -54,7 +49,6 @@ export const transition =
           state = stateTransition(state, signedBlock, {
             // TODO DENEB: Should assume valid and available for this test?
             executionPayloadStatus: ExecutionPayloadStatus.valid,
-            dataAvailableStatus: DataAvailableStatus.available,
             verifyStateRoot: true,
             verifyProposer: false,
             verifySignatures: false,

packages/fork-choice/src/index.ts

@@ -5,6 +5,7 @@ export {
   ExecutionStatus,
   MaybeValidExecutionStatus,
   BlockExecution,
+  DataAvailableStatus,
   LVHValidResponse,
   LVHInvalidResponse,
 } from "./protoArray/interface.js";

packages/fork-choice/src/protoArray/interface.ts

@@ -20,6 +20,12 @@ export enum ExecutionStatus {
   Invalid = "Invalid",
 }
 
+export enum DataAvailableStatus {
+  preDeneb = "preDeneb",
+  notAvailable = "notAvailable",
+  available = "available",
+}
+
 export type LVHValidResponse = {
   executionStatus: ExecutionStatus.Valid;
   latestValidExecHash: RootHex;

packages/state-transition/src/block/externalData.ts

@@ -13,13 +13,6 @@ export enum ExecutionPayloadStatus {
   valid = "valid",
 }
 
-export enum DataAvailableStatus {
-  preDeneb = "preDeneb",
-  notAvailable = "notAvailable",
-  available = "available",
-}
-
 export interface BlockExternalData {
   executionPayloadStatus: ExecutionPayloadStatus;
-  dataAvailableStatus: DataAvailableStatus;
 }

packages/state-transition/src/block/index.ts

@@ -9,7 +9,7 @@ import {processEth1Data} from "./processEth1Data.js";
 import {processOperations} from "./processOperations.js";
 import {processRandao} from "./processRandao.js";
 import {processBlobKzgCommitments} from "./processBlobKzgCommitments.js";
-import {BlockExternalData, DataAvailableStatus} from "./externalData.js";
+import {BlockExternalData} from "./externalData.js";
 import {processWithdrawals} from "./processWithdrawals.js";
 import {ProcessBlockOpts} from "./types.js";
 
@@ -63,18 +63,5 @@ export function processBlock(
 
   if (fork >= ForkSeq.deneb) {
     processBlobKzgCommitments(block.body as deneb.BeaconBlockBody);
-
-    // New in Deneb, note: Can sync optimistically without this condition, see note on `is_data_available`
-    // NOTE: Ommitted and should be verified beforehand
-
-    // assert is_data_available(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments)
-    switch (externalData.dataAvailableStatus) {
-      case DataAvailableStatus.preDeneb:
-        throw Error("dataAvailableStatus preDeneb");
-      case DataAvailableStatus.notAvailable:
-        throw Error("dataAvailableStatus notAvailable");
-      case DataAvailableStatus.available:
-        break; // ok
-    }
   }
 }

packages/state-transition/src/index.ts

@@ -47,7 +47,7 @@ export {isValidVoluntaryExit} from "./block/processVoluntaryExit.js";
 export {isValidBlsToExecutionChange} from "./block/processBlsToExecutionChange.js";
 export {assertValidProposerSlashing} from "./block/processProposerSlashing.js";
 export {assertValidAttesterSlashing} from "./block/processAttesterSlashing.js";
-export {ExecutionPayloadStatus, DataAvailableStatus, BlockExternalData} from "./block/externalData.js";
+export {ExecutionPayloadStatus, BlockExternalData} from "./block/externalData.js";
 
 // BeaconChain, to prepare new blocks
 export {becomesNewEth1Data} from "./block/processEth1Data.js";

packages/state-transition/src/stateTransition.ts

@@ -21,7 +21,7 @@ import {
 } from "./slot/index.js";
 import {processBlock} from "./block/index.js";
 import {processEpoch} from "./epoch/index.js";
-import {BlockExternalData, DataAvailableStatus, ExecutionPayloadStatus} from "./block/externalData.js";
+import {BlockExternalData, ExecutionPayloadStatus} from "./block/externalData.js";
 import {ProcessBlockOpts} from "./block/types.js";
 
 // Multifork capable state transition
@@ -45,7 +45,6 @@ export function stateTransition(
   options: StateTransitionOpts = {
     // TODO DENEB: Review what default values make sense
     executionPayloadStatus: ExecutionPayloadStatus.valid,
-    dataAvailableStatus: DataAvailableStatus.available,
   },
   metrics?: BeaconStateTransitionMetrics | null
 ): CachedBeaconStateAllForks {

packages/state-transition/test/perf/block/processBlockAltair.test.ts

@@ -10,12 +10,7 @@ import {
   PresetName,
   SYNC_COMMITTEE_SIZE,
 } from "@lodestar/params";
-import {
-  CachedBeaconStateAltair,
-  DataAvailableStatus,
-  ExecutionPayloadStatus,
-  stateTransition,
-} from "../../../src/index.js";
+import {CachedBeaconStateAltair, ExecutionPayloadStatus, stateTransition} from "../../../src/index.js";
 import {cachedStateAltairPopulateCaches, generatePerfTestCachedStateAltair, perfStateId} from "../util.js";
 import {StateBlock} from "../types.js";
 import {BlockAltairOpts, getBlockAltair} from "./util.js";
@@ -126,7 +121,6 @@ describe("altair processBlock", () => {
         fn: ({state, block}) => {
           const postState = stateTransition(state, block, {
             executionPayloadStatus: ExecutionPayloadStatus.valid,
-            dataAvailableStatus: DataAvailableStatus.available,
             verifyProposer: false,
             verifySignatures: false,
             verifyStateRoot: false,