Free the blobs

fix the types rejig the new constants in params add comment for cleanup update reqresp fix api package rename blobs repo commit the wip modifications further appropriate renaming further references update further reference updates continue refac fix reqresp build further refac further refac fix api fix db interface fix beacondb alloc build fix api improve blob verificaion correct validation call fixes fix the produce block/blobs flow reduce diff blob gossip validation update validations cleanup block vali reduce diff handle gossip of block and blob fix test for timebeing modify publishing flow fix import flow onsidecarbyrange fix and some type fixes fix sidecars by root prune blockinput cache fix kzg interface small renaming interface rename fix fetch blockmaybeblobs by range test fix build lint issues for now c-kzg version fix FullOrBlindedBlobSidecar changes fix tests complete the blob publishing flow fix test get the single node run functional get the gossip blob flow working fix peer syncing using req/resp fix sidecar by root check refactor blobsidecars hotdb and remove archive add blob gossip validation flow fix topic fix the validation condition add blob validation and test various sync modes fix tests rebase fixes enable deneb spec tests make blobsbyroot multi block fixes cleanup defunt builder endpoint archive blobs post finalization uptill the blob window serve finalized blobs within the blob prune window fix test fix test lookup in archive as well cleanup and improvements rebase fx Add 4844 sim test and override the field elements per blob update image add blob test add test run in package start unknown sync and range sync finalize the sims change the signing flow fix test types fix tests fix test lint update tx type and corresponding ethereumjs image update c-kzg and use blobs bundle proof fix test fix test merge getblobsbundle into getpayloadv3 update images fix genesis config rebase fixes fix test update images fix tests lint fix the sidecar request count limit fix test lint rebase fixes cleanup fix unit tests update kzg to big endian devnet 6 integration fix passing setup arg update path fix tests fix blobs sidecar by range response cleanup blob import vals reduce diff
ChainSafe · Jun 19, 2023 · 27fdd86 · 27fdd86
1 parent 9a8a6f4
commit 27fdd86
Show file tree

Hide file tree

Showing 53 changed files with 985 additions and 547 deletions.
diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml
@@ -17,8 +17,9 @@ env:
   NETHERMIND_IMAGE: nethermind/nethermind:1.14.3
   MERGEMOCK_IMAGE: g11tech/mergemock:latest
   GETH_WITHDRAWALS_IMAGE: g11tech/geth:withdrawalsfeb8
-  ETHEREUMJS_WITHDRAWALS_IMAGE: g11tech/ethereumjs:feb8
+  ETHEREUMJS_WITHDRAWALS_IMAGE: g11tech/ethereumjs:blobs-b6b63
   NETHERMIND_WITHDRAWALS_IMAGE: nethermindeth/nethermind:withdrawals_yolo
+  ETHEREUMJS_BLOBS_IMAGE: g11tech/ethereumjs:blobs-b6b63
 
 jobs:
   sim-merge-tests:
@@ -127,6 +128,17 @@ jobs:
       #     EL_BINARY_DIR: ${{ env.NETHERMIND_WITHDRAWALS_IMAGE }}
       #     EL_SCRIPT_DIR: netherminddocker
 
+      - name: Pull ethereumjs blobs
+        run: docker pull $ETHEREUMJS_BLOBS_IMAGE
+
+      - name: Test Lodestar <> ethereumjs blobs
+        run: yarn test:sim:blobs
+        working-directory: packages/beacon-node
+        env:
+          EL_BINARY_DIR: ${{ env.ETHEREUMJS_BLOBS_IMAGE }}
+          EL_SCRIPT_DIR: ethereumjsdocker
+          DEV_RUN: true
+
       - name: Upload debug log test files
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json
@@ -85,6 +85,7 @@
     "test:sim:merge-interop": "mocha 'test/sim/merge-interop.test.ts'",
     "test:sim:mergemock": "mocha 'test/sim/mergemock.test.ts'",
     "test:sim:withdrawals": "mocha 'test/sim/withdrawal-interop.test.ts'",
+    "test:sim:blobs": "mocha 'test/sim/4844-interop.test.ts'",
     "download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts",
     "check-spec-tests": "mocha test/spec/checkCoverage.ts",
     "test:spec-bls-general": "mocha --config .mocharc.spec.cjs 'test/spec/bls/**/*.test.ts' 'test/spec/general/**/*.test.ts'",

diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts
@@ -4,13 +4,7 @@ import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {sleep} from "@lodestar/utils";
 import {allForks, deneb} from "@lodestar/types";
 import {fromHexString, toHexString} from "@chainsafe/ssz";
-import {
-  BlockSource,
-  getBlockInput,
-  ImportBlockOpts,
-  BlockInput,
-  blobSidecarsToBlobsSidecar,
-} from "../../../../chain/blocks/types.js";
+import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput} from "../../../../chain/blocks/types.js";
 import {promiseAllMaybeAsync} from "../../../../util/promises.js";
 import {isOptimisticBlock} from "../../../../util/forkChoice.js";
 import {BlockError, BlockErrorCode} from "../../../../chain/errors/index.js";
@@ -213,12 +207,7 @@ export function getBeaconBlockApi({
           config,
           signedBlock,
           BlockSource.api,
-          // The blobsSidecar will be replaced in the followup PRs with just blobs
-          blobSidecarsToBlobsSidecar(
-            config,
-            signedBlock,
-            signedBlobs.map((sblob) => sblob.message)
-          )
+          signedBlobs.map((sblob) => sblob.message)
         );
       } else {
         signedBlock = signedBlockOrContents;

diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts
@@ -1,10 +1,10 @@
 import {CachedBeaconStateAllForks, computeEpochAtSlot, DataAvailableStatus} from "@lodestar/state-transition";
 import {MaybeValidExecutionStatus} from "@lodestar/fork-choice";
-import {allForks, deneb, Slot, WithOptionalBytes} from "@lodestar/types";
+import {allForks, deneb, Slot, WithOptionalBytes, RootHex} from "@lodestar/types";
 import {ForkSeq, MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
-
-import {ckzg} from "../../util/kzg.js";
+import {toHexString} from "@chainsafe/ssz";
+import {pruneSetToMax} from "@lodestar/utils";
 
 export enum BlockInputType {
   preDeneb = "preDeneb",
@@ -21,7 +21,7 @@ export enum BlockSource {
 
 export type BlockInput =
   | {type: BlockInputType.preDeneb; block: allForks.SignedBeaconBlock; source: BlockSource}
-  | {type: BlockInputType.postDeneb; block: allForks.SignedBeaconBlock; source: BlockSource; blobs: deneb.BlobsSidecar};
+  | {type: BlockInputType.postDeneb; block: allForks.SignedBeaconBlock; source: BlockSource; blobs: deneb.BlobSidecars};
 
 export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean {
   return (
@@ -31,26 +31,87 @@ export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clo
   );
 }
 
-// TODO DENEB: a helper function to convert blobSidecars to blobsSidecar, to be cleanup on BlockInput
-// migration
-export function blobSidecarsToBlobsSidecar(
-  config: ChainForkConfig,
-  signedBlock: allForks.SignedBeaconBlock,
-  blobSidecars: deneb.BlobSidecars
-): deneb.BlobsSidecar {
-  const beaconBlockSlot = signedBlock.message.slot;
-  const beaconBlockRoot = config.getForkTypes(beaconBlockSlot).BeaconBlock.hashTreeRoot(signedBlock.message);
-  const blobs = blobSidecars.map(({blob}) => blob);
-  const blobsSidecar = {
-    beaconBlockRoot,
-    beaconBlockSlot,
-    blobs,
-    kzgAggregatedProof: ckzg.computeAggregateKzgProof(blobs),
-  };
-  return blobsSidecar;
+export enum GossipedInputType {
+  block = "block",
+  blob = "blob",
 }
+type GossipedBlockInput =
+  | {type: GossipedInputType.block; signedBlock: allForks.SignedBeaconBlock}
+  | {type: GossipedInputType.blob; signedBlob: deneb.SignedBlobSidecar};
+type BlockInputCacheType = {block?: allForks.SignedBeaconBlock; blobs: Map<number, deneb.BlobSidecar>};
+
+const MAX_GOSSIPINPUT_CACHE = 5;
 
 export const getBlockInput = {
+  blockInputCache: new Map<RootHex, BlockInputCacheType>(),
+
+  getFullBlockInput(
+    config: ChainForkConfig,
+    gossipedInput: GossipedBlockInput
+  ):
+    | {blockInput: BlockInput; blockInputMeta: {pending: null; haveBlobs: number; expectedBlobs: number}}
+    | {blockInput: null; blockInputMeta: {pending: GossipedInputType.block; haveBlobs: number; expectedBlobs: null}}
+    | {blockInput: null; blockInputMeta: {pending: GossipedInputType.blob; haveBlobs: number; expectedBlobs: number}} {
+    let blockHex;
+    let blockCache;
+    if (gossipedInput.type === GossipedInputType.block) {
+      const {signedBlock} = gossipedInput;
+      blockHex = toHexString(
+        config.getForkTypes(signedBlock.message.slot).BeaconBlock.hashTreeRoot(signedBlock.message)
+      );
+      blockCache = this.blockInputCache.get(blockHex) ?? {blobs: new Map<number, deneb.BlobSidecar>()};
+      blockCache.block = signedBlock;
+    } else {
+      const {signedBlob} = gossipedInput;
+      blockHex = toHexString(signedBlob.message.blockRoot);
+      blockCache = this.blockInputCache.get(blockHex);
+
+      // If a new entry is going to be inserted, prune out old ones
+      if (blockCache === undefined) {
+        pruneSetToMax(this.blockInputCache, MAX_GOSSIPINPUT_CACHE);
+        blockCache = {blobs: new Map<number, deneb.BlobSidecar>()};
+      }
+      // TODO: freetheblobs check if its the same blob or a duplicate and throw/take actions
+      blockCache.blobs.set(signedBlob.message.index, signedBlob.message);
+    }
+    this.blockInputCache.set(blockHex, blockCache);
+    const {block: signedBlock} = blockCache;
+    if (signedBlock !== undefined) {
+      const {blobKzgCommitments} = (signedBlock as deneb.SignedBeaconBlock).message.body;
+      if (blobKzgCommitments.length < blockCache.blobs.size) {
+        throw Error(`Received more blobs=${blockCache.blobs.size} than commitments=${blobKzgCommitments.length}`);
+      }
+      if (blobKzgCommitments.length === blockCache.blobs.size) {
+        const blobSidecars = [];
+        for (let index = 0; index < blobKzgCommitments.length; index++) {
+          const blobSidecar = blockCache.blobs.get(index);
+          if (blobSidecar === undefined) {
+            throw Error("Missing blobSidecar");
+          }
+          blobSidecars.push(blobSidecar);
+        }
+        return {
+          blockInput: getBlockInput.postDeneb(config, signedBlock, BlockSource.gossip, blobSidecars),
+          blockInputMeta: {pending: null, haveBlobs: blockCache.blobs.size, expectedBlobs: blobKzgCommitments.length},
+        };
+      } else {
+        return {
+          blockInput: null,
+          blockInputMeta: {
+            pending: GossipedInputType.blob,
+            haveBlobs: blockCache.blobs.size,
+            expectedBlobs: blobKzgCommitments.length,
+          },
+        };
+      }
+    } else {
+      return {
+        blockInput: null,
+        blockInputMeta: {pending: GossipedInputType.block, haveBlobs: blockCache.blobs.size, expectedBlobs: null},
+      };
+    }
+  },
+
   preDeneb(config: ChainForkConfig, block: allForks.SignedBeaconBlock, source: BlockSource): BlockInput {
     if (config.getForkSeq(block.message.slot) >= ForkSeq.deneb) {
       throw Error(`Post Deneb block slot ${block.message.slot}`);
@@ -66,7 +127,7 @@ export const getBlockInput = {
     config: ChainForkConfig,
     block: allForks.SignedBeaconBlock,
     source: BlockSource,
-    blobs: deneb.BlobsSidecar
+    blobs: deneb.BlobSidecars
   ): BlockInput {
     if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) {
       throw Error(`Pre Deneb block slot ${block.message.slot}`);
@@ -118,8 +179,8 @@ export type ImportBlockOpts = {
    * Metadata: `true` if all the signatures including the proposer signature have been verified
    */
   validSignatures?: boolean;
-  /** Set to true if already run `validateBlobsSidecar()` sucessfully on the blobs */
-  validBlobsSidecar?: boolean;
+  /** Set to true if already run `validateBlobSidecars()` sucessfully on the blobs */
+  validBlobSidecars?: boolean;
   /** Seen timestamp seconds */
   seenTimestampSec?: number;
   /** Set to true if persist block right at verification time */

diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts
@@ -5,7 +5,7 @@ import {Slot, deneb, WithOptionalBytes} from "@lodestar/types";
 import {toHexString} from "@lodestar/utils";
 import {IClock} from "../../util/clock.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
-import {validateBlobsSidecar} from "../validation/blobsSidecar.js";
+import {validateBlobSidecars} from "../validation/blobSidecar.js";
 import {BlockInput, BlockInputType, ImportBlockOpts} from "./types.js";
 
 /**
@@ -126,7 +126,7 @@ function maybeValidateBlobs(
   // TODO Deneb: Make switch verify it's exhaustive
   switch (blockInput.type) {
     case BlockInputType.postDeneb: {
-      if (opts.validBlobsSidecar) {
+      if (opts.validBlobSidecars) {
         return DataAvailableStatus.available;
       }
 
@@ -135,7 +135,7 @@ function maybeValidateBlobs(
       const {blobKzgCommitments} = (block as deneb.SignedBeaconBlock).message.body;
       const beaconBlockRoot = config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message);
       // TODO Deneb: This function throws un-typed errors
-      validateBlobsSidecar(blockSlot, beaconBlockRoot, blobKzgCommitments, blobs);
+      validateBlobSidecars(blockSlot, beaconBlockRoot, blobKzgCommitments, blobs);
 
       return DataAvailableStatus.available;
     }

diff --git a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts
@@ -1,4 +1,4 @@
-import {WithOptionalBytes, allForks, deneb} from "@lodestar/types";
+import {WithOptionalBytes} from "@lodestar/types";
 import {toHex} from "@lodestar/utils";
 import {BeaconChain} from "../chain.js";
 import {BlockInput, BlockInputType} from "./types.js";
@@ -34,13 +34,13 @@ export async function writeBlockInputToDb(
     });
 
     if (type === BlockInputType.postDeneb) {
-      const {blobs} = blockInput;
+      const {blobs: blobSidecars} = blockInput;
       // NOTE: Old blobs are pruned on archive
-      fnPromises.push(this.db.blobsSidecar.add(blobs));
-      this.logger.debug("Persist blobsSidecar to hot DB", {
-        blobsLen: blobs.blobs.length,
-        slot: blobs.beaconBlockSlot,
-        root: toHex(blobs.beaconBlockRoot),
+      fnPromises.push(this.db.blobSidecars.add({blockRoot, slot: block.message.slot, blobSidecars}));
+      this.logger.debug("Persisted blobSidecars to hot DB", {
+        blobsLen: blobSidecars.length,
+        slot: block.message.slot,
+        root: blockRootHex,
       });
     }
   }
@@ -55,27 +55,26 @@ export async function removeEagerlyPersistedBlockInputs(
   this: BeaconChain,
   blockInputs: WithOptionalBytes<BlockInput>[]
 ): Promise<void> {
-  const blockToRemove: allForks.SignedBeaconBlock[] = [];
-  const blobsToRemove: deneb.BlobsSidecar[] = [];
+  const blockToRemove = [];
+  const blobsToRemove = [];
 
   for (const blockInput of blockInputs) {
     const {block, type} = blockInput;
-    const blockRoot = toHex(this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message));
-    if (!this.forkChoice.hasBlockHex(blockRoot)) {
+    const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
+    const blockRootHex = toHex(blockRoot);
+    if (!this.forkChoice.hasBlockHex(blockRootHex)) {
       blockToRemove.push(block);
 
       if (type === BlockInputType.postDeneb) {
-        blobsToRemove.push(blockInput.blobs);
-        this.db.blobsSidecar.remove(blockInput.blobs).catch((e) => {
-          this.logger.verbose("Error removing eagerly imported blobsSidecar", {blockRoot}, e);
-        });
+        const blobSidecars = blockInput.blobs;
+        blobsToRemove.push({blockRoot, slot: block.message.slot, blobSidecars});
       }
     }
   }
 
   await Promise.all([
     // TODO: Batch DB operations not with Promise.all but with level db ops
     this.db.block.batchRemove(blockToRemove),
-    this.db.blobsSidecar.batchRemove(blobsToRemove),
+    this.db.blobSidecars.batchRemove(blobsToRemove),
   ]);
 }
diff --git a/...ode/src/chain/errors/blobsSidecarError.ts → ...node/src/chain/errors/blobSidecarError.ts b/...ode/src/chain/errors/blobsSidecarError.ts → ...node/src/chain/errors/blobSidecarError.ts
@@ -1,7 +1,8 @@
 import {Slot} from "@lodestar/types";
 import {GossipActionError} from "./gossipValidation.js";
 
-export enum BlobsSidecarErrorCode {
+export enum BlobSidecarErrorCode {
+  INVALID_INDEX = "BLOBS_SIDECAR_ERROR_INVALID_INDEX",
   /** !bls.KeyValidate(block.body.blob_kzg_commitments[i]) */
   INVALID_KZG = "BLOBS_SIDECAR_ERROR_INVALID_KZG",
   /** !verify_kzg_commitments_against_transactions(block.body.execution_payload.transactions, block.body.blob_kzg_commitments) */
@@ -14,11 +15,12 @@ export enum BlobsSidecarErrorCode {
   INVALID_KZG_PROOF = "BLOBS_SIDECAR_ERROR_INVALID_KZG_PROOF",
 }
 
-export type BlobsSidecarErrorType =
-  | {code: BlobsSidecarErrorCode.INVALID_KZG; kzgIdx: number}
-  | {code: BlobsSidecarErrorCode.INVALID_KZG_TXS}
-  | {code: BlobsSidecarErrorCode.INCORRECT_SLOT; blockSlot: Slot; blobSlot: Slot}
-  | {code: BlobsSidecarErrorCode.INVALID_BLOB; blobIdx: number}
-  | {code: BlobsSidecarErrorCode.INVALID_KZG_PROOF};
+export type BlobSidecarErrorType =
+  | {code: BlobSidecarErrorCode.INVALID_INDEX; blobIdx: number; gossipIndex: number}
+  | {code: BlobSidecarErrorCode.INVALID_KZG; blobIdx: number}
+  | {code: BlobSidecarErrorCode.INVALID_KZG_TXS}
+  | {code: BlobSidecarErrorCode.INCORRECT_SLOT; blockSlot: Slot; blobSlot: Slot; blobIdx: number}
+  | {code: BlobSidecarErrorCode.INVALID_BLOB; blobIdx: number}
+  | {code: BlobSidecarErrorCode.INVALID_KZG_PROOF; blobIdx: number};
 
-export class BlobsSidecarError extends GossipActionError<BlobsSidecarErrorType> {}
+export class BlobSidecarError extends GossipActionError<BlobSidecarErrorType> {}
diff --git a/packages/beacon-node/src/chain/errors/index.ts b/packages/beacon-node/src/chain/errors/index.ts
@@ -1,6 +1,6 @@
 export * from "./attestationError.js";
 export * from "./attesterSlashingError.js";
-export * from "./blobsSidecarError.js";
+export * from "./blobSidecarError.js";
 export * from "./blockError.js";
 export * from "./gossipValidation.js";
 export * from "./proposerSlashingError.js";

diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts
@@ -9,6 +9,7 @@ export enum RegenCaller {
   processBlock = "processBlock",
   produceBlock = "produceBlock",
   validateGossipBlock = "validateGossipBlock",
+  validateGossipBlob = "validateGossipBlob",
   precomputeEpoch = "precomputeEpoch",
   produceAttestationData = "produceAttestationData",
   processBlocksInEpoch = "processBlocksInEpoch",

diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts
@@ -3,8 +3,8 @@ import {
   CachedBeaconStateAllForks,
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
-  DataAvailableStatus,
   ExecutionPayloadStatus,
+  DataAvailableStatus,
   processSlots,
   stateTransition,
 } from "@lodestar/state-transition";