Persist invalid SSZ objects only if enabled

packages/cli/src/cmds/beacon/options.ts

@@ -98,7 +98,7 @@ export const beaconPathsOptions: ICliCommandOptions<IBeaconPaths> = {
   },
 
   persistInvalidSszObjectsDir: {
-    description: "Directory to persist invalid ssz objects",
+    description: "Enable and specify a directory to persist invalid ssz objects",
     defaultDescription: defaultBeaconPaths.persistInvalidSszObjectsDir,
     hidden: true,
     type: "string",

packages/lodestar/src/api/impl/beacon/pool/index.ts

@@ -1,7 +1,6 @@
 import {routes} from "@chainsafe/lodestar-api";
 import {Epoch, ssz} from "@chainsafe/lodestar-types";
 import {SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params";
-import {toHexString} from "@chainsafe/ssz";
 import {validateGossipAttestation} from "../../../../chain/validation/index.js";
 import {validateGossipAttesterSlashing} from "../../../../chain/validation/attesterSlashing.js";
 import {validateGossipProposerSlashing} from "../../../../chain/validation/proposerSlashing.js";
@@ -60,12 +59,7 @@ export function getBeaconPoolApi({
               e as Error
             );
             if (e instanceof AttestationError && e.action === GossipAction.REJECT) {
-              const archivedPath = chain.persistInvalidSszObject(
-                "attestation",
-                ssz.phase0.Attestation.serialize(attestation),
-                toHexString(ssz.phase0.Attestation.hashTreeRoot(attestation))
-              );
-              logger.debug("Submitted invalid attestation was written to", archivedPath);
+              chain.persistInvalidSszValue(ssz.phase0.Attestation, attestation, "api_reject");
             }
           }
         })
@@ -148,12 +142,7 @@ export function getBeaconPoolApi({
               e as Error
             );
             if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) {
-              const archivedPath = chain.persistInvalidSszObject(
-                "syncCommittee",
-                ssz.altair.SyncCommitteeMessage.serialize(signature),
-                toHexString(ssz.altair.SyncCommitteeMessage.hashTreeRoot(signature))
-              );
-              logger.debug("The submitted sync committee message was written to", archivedPath);
+              chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, signature, "api_reject");
             }
           }
         })

packages/lodestar/src/api/impl/validator/index.ts

@@ -16,7 +16,7 @@ import {
 import {Root, Slot, ValidatorIndex, ssz} from "@chainsafe/lodestar-types";
 import {ExecutionStatus} from "@chainsafe/lodestar-fork-choice";
 
-import {fromHexString, toHexString} from "@chainsafe/ssz";
+import {fromHexString} from "@chainsafe/ssz";
 import {assembleBlock} from "../../../chain/factory/block/index.js";
 import {AttestationError, AttestationErrorCode, GossipAction, SyncCommitteeError} from "../../../chain/errors/index.js";
 import {validateGossipAggregateAndProof} from "../../../chain/validation/index.js";
@@ -466,12 +466,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}:
               e as Error
             );
             if (e instanceof AttestationError && e.action === GossipAction.REJECT) {
-              const archivedPath = chain.persistInvalidSszObject(
-                "signedAggregatedAndProof",
-                ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof),
-                toHexString(ssz.phase0.SignedAggregateAndProof.hashTreeRoot(signedAggregateAndProof))
-              );
-              logger.debug("The submitted signed aggregate and proof was written to", archivedPath);
+              chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "api_reject");
             }
           }
         })
@@ -517,12 +512,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}:
               e as Error
             );
             if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) {
-              const archivedPath = chain.persistInvalidSszObject(
-                "contributionAndProof",
-                ssz.altair.SignedContributionAndProof.serialize(contributionAndProof),
-                toHexString(ssz.altair.SignedContributionAndProof.hashTreeRoot(contributionAndProof))
-              );
-              logger.debug("The submitted contribution adn proof was written to", archivedPath);
+              chain.persistInvalidSszValue(ssz.altair.SignedContributionAndProof, contributionAndProof, "api_reject");
             }
           }
         })

packages/lodestar/src/chain/chain.ts

@@ -2,7 +2,7 @@
  * @module chain
  */
 
-import fs from "node:fs";
+import path from "node:path";
 import {
   BeaconStateAllForks,
   CachedBeaconStateAllForks,
@@ -14,19 +14,20 @@ import {
 import {IBeaconConfig} from "@chainsafe/lodestar-config";
 import {IForkChoice} from "@chainsafe/lodestar-fork-choice";
 import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch} from "@chainsafe/lodestar-types";
-import {ILogger} from "@chainsafe/lodestar-utils";
-import {fromHexString} from "@chainsafe/ssz";
+import {ILogger, toHex} from "@chainsafe/lodestar-utils";
+import {CompositeTypeAny, fromHexString, TreeView, Type} from "@chainsafe/ssz";
 import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
 import {IBeaconDb} from "../db/index.js";
 import {IMetrics} from "../metrics/index.js";
 import {IEth1ForBlockProduction} from "../eth1/index.js";
 import {IExecutionEngine} from "../executionEngine/index.js";
+import {ensureDir, writeIfNotExist} from "../util/file.js";
 import {CheckpointStateCache, StateContextCache} from "./stateCache/index.js";
 import {BlockProcessor, PartiallyVerifiedBlockFlags} from "./blocks/index.js";
 import {IBeaconClock, LocalClock} from "./clock/index.js";
 import {ChainEventEmitter} from "./emitter.js";
 import {handleChainEvents} from "./eventHandlers.js";
-import {IBeaconChain, SSZObjectType, ProposerPreparationData} from "./interface.js";
+import {IBeaconChain, ProposerPreparationData} from "./interface.js";
 import {IChainOptions} from "./options.js";
 import {IStateRegenerator, QueuedStateRegenerator, RegenCaller} from "./regen/index.js";
 import {initializeForkChoice} from "./forkChoice/index.js";
@@ -302,24 +303,43 @@ export class BeaconChain implements IBeaconChain {
     return this.reprocessController.waitForBlockOfAttestation(slot, root);
   }
 
-  persistInvalidSszObject(type: SSZObjectType, bytes: Uint8Array, suffix = ""): string | null {
+  persistInvalidSszValue<T>(type: Type<T>, sszObject: T, suffix?: string): void {
+    if (this.opts.persistInvalidSszObjects) {
+      void this.persistInvalidSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix);
+    }
+  }
+
+  persistInvalidSszView(view: TreeView<CompositeTypeAny>, suffix?: string): void {
+    if (this.opts.persistInvalidSszObjects) {
+      void this.persistInvalidSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix);
+    }
+  }
+
+  private async persistInvalidSszObject(
+    typeName: string,
+    bytes: Uint8Array,
+    root: Uint8Array,
+    suffix?: string
+  ): Promise<void> {
+    if (!this.opts.persistInvalidSszObjects) {
+      return;
+    }
+
     const now = new Date();
     // yyyy-MM-dd
-    const date = now.toISOString().split("T")[0];
+    const dateStr = now.toISOString().split("T")[0];
+
     // by default store to lodestar_archive of current dir
-    const byDate = this.opts.persistInvalidSszObjectsDir
-      ? `${this.opts.persistInvalidSszObjectsDir}/${date}`
-      : `invalidSszObjects/${date}`;
-    if (!fs.existsSync(byDate)) {
-      fs.mkdirSync(byDate, {recursive: true});
-    }
-    const fileName = `${byDate}/${type}_${suffix}.ssz`;
+    const dirpath = path.join(this.opts.persistInvalidSszObjectsDir ?? "invalid_ssz_objects", dateStr);
+    const filepath = path.join(dirpath, `${typeName}_${toHex(root)}.ssz`);
+
+    await ensureDir(dirpath);
+
     // as of Feb 17 2022 there are a lot of duplicate files stored with different date suffixes
     // remove date suffixes in file name, and check duplicate to avoid redundant persistence
-    if (!fs.existsSync(fileName)) {
-      fs.writeFileSync(fileName, bytes);
-    }
-    return fileName;
+    await writeIfNotExist(filepath, bytes);
+
+    this.logger.debug("Persisted invalid ssz object", {id: suffix, filepath});
   }
 
   async updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise<void> {

packages/lodestar/src/chain/eventHandlers.ts

@@ -231,37 +231,19 @@ export async function onErrorBlock(this: BeaconChain, err: BlockError): Promise<
     const {signedBlock} = err;
     const blockSlot = signedBlock.message.slot;
     const {state} = err.type;
-    const blockPath = this.persistInvalidSszObject(
-      "signedBlock",
-      this.config.getForkTypes(blockSlot).SignedBeaconBlock.serialize(signedBlock),
-      `${blockSlot}_invalid_signature`
-    );
-    const statePath = this.persistInvalidSszObject("state", state.serialize(), `${state.slot}_invalid_signature`);
-    this.logger.debug("Invalid signature block and state were written to disc", {blockPath, statePath});
+    const forkTypes = this.config.getForkTypes(blockSlot);
+    this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `${blockSlot}_invalid_signature`);
+    this.persistInvalidSszView(state, `${state.slot}_invalid_signature`);
   } else if (err.type.code === BlockErrorCode.INVALID_STATE_ROOT) {
     const {signedBlock} = err;
     const blockSlot = signedBlock.message.slot;
     const {preState, postState} = err.type;
+    const forkTypes = this.config.getForkTypes(blockSlot);
     const invalidRoot = toHexString(postState.hashTreeRoot());
-    const blockPath = this.persistInvalidSszObject(
-      "signedBlock",
-      this.config.getForkTypes(blockSlot).SignedBeaconBlock.serialize(signedBlock),
-      `${blockSlot}_invalid_state_root_${invalidRoot}`
-    );
-    const preStatePath = this.persistInvalidSszObject(
-      "state",
-      preState.serialize(),
-      `${blockSlot}_invalid_state_root_preState_${invalidRoot}`
-    );
-    const postStatePath = this.persistInvalidSszObject(
-      "state",
-      postState.serialize(),
-      `${blockSlot}_invalid_state_root_postState_${invalidRoot}`
-    );
-    this.logger.debug("Invalid state root block and states were written to disc", {
-      blockPath,
-      preStatePath,
-      postStatePath,
-    });
+
+    const suffix = `slot_${blockSlot}_invalid_state_root_${invalidRoot}`;
+    this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, suffix);
+    this.persistInvalidSszView(preState, `${suffix}_preState`);
+    this.persistInvalidSszView(postState, `${suffix}_postState`);
   }
 }

packages/lodestar/src/chain/interface.ts

@@ -2,6 +2,7 @@ import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch} from "@chainsaf
 import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition";
 import {IForkChoice} from "@chainsafe/lodestar-fork-choice";
 import {IBeaconConfig} from "@chainsafe/lodestar-config";
+import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz";
 
 import {IEth1ForBlockProduction} from "../eth1/index.js";
 import {IExecutionEngine} from "../executionEngine/index.js";
@@ -102,10 +103,11 @@ export interface IBeaconChain {
 
   waitForBlockOfAttestation(slot: Slot, root: RootHex): Promise<boolean>;
 
-  /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */
-  persistInvalidSszObject(type: SSZObjectType, bytes: Uint8Array, suffix: string): string | null;
-
   updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise<void>;
+
+  persistInvalidSszValue<T>(type: Type<T>, sszObject: T | Uint8Array, suffix?: string): void;
+  /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */
+  persistInvalidSszView(view: TreeView<CompositeTypeAny>, suffix?: string): void;
 }
 
 export type SSZObjectType =

packages/lodestar/src/chain/options.ts

@@ -30,8 +30,6 @@ export const defaultChainOptions: IChainOptions = {
   blsVerifyAllMainThread: false,
   blsVerifyAllMultiThread: false,
   disableBlsBatchVerify: false,
-  persistInvalidSszObjects: true,
-  persistInvalidSszObjectsDir: "",
   proposerBoostEnabled: true,
   safeSlotsToImportOptimistically: SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY,
   defaultFeeRecipient: defaultDefaultFeeRecipient,

packages/lodestar/src/network/gossip/handlers/index.ts

@@ -75,7 +75,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
   return {
     [GossipType.beacon_block]: async (signedBlock, topic, peerIdStr, seenTimestampSec) => {
       const slot = signedBlock.message.slot;
-      const blockHex = prettyBytes(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message));
+      const forkTypes = config.getForkTypes(slot);
+      const blockHex = prettyBytes(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message));
       const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
       logger.verbose("Received gossip block", {
         slot: slot,
@@ -96,12 +97,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
         }
 
         if (e instanceof BlockGossipError && e.action === GossipAction.REJECT) {
-          const archivedPath = chain.persistInvalidSszObject(
-            "signedBlock",
-            config.getForkTypes(slot).SignedBeaconBlock.serialize(signedBlock),
-            `gossip_slot_${slot}`
-          );
-          logger.debug("The invalid gossip block was written to", archivedPath);
+          chain.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `gossip_reject_slot_${slot}`);
         }
 
         throw e;
@@ -151,12 +147,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
         validationResult = await validateGossipAggregateAndProofRetryUnknownRoot(chain, signedAggregateAndProof);
       } catch (e) {
         if (e instanceof AttestationError && e.action === GossipAction.REJECT) {
-          const archivedPath = chain.persistInvalidSszObject(
-            "signedAggregatedAndProof",
-            ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof),
-            toHexString(ssz.phase0.SignedAggregateAndProof.hashTreeRoot(signedAggregateAndProof))
-          );
-          logger.debug("The invalid gossip aggregate and proof was written to", archivedPath, e);
+          chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "gossip_reject");
         }
         throw e;
       }
@@ -191,12 +182,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
         validationResult = await validateGossipAttestationRetryUnknownRoot(chain, attestation, subnet);
       } catch (e) {
         if (e instanceof AttestationError && e.action === GossipAction.REJECT) {
-          const archivedPath = chain.persistInvalidSszObject(
-            "attestation",
-            ssz.phase0.Attestation.serialize(attestation),
-            toHexString(ssz.phase0.Attestation.hashTreeRoot(attestation))
-          );
-          logger.debug("The invalid gossip attestation was written to", archivedPath);
+          chain.persistInvalidSszValue(ssz.phase0.Attestation, attestation, "gossip_reject");
         }
         throw e;
       }
@@ -268,12 +254,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
         contributionAndProof
       ).catch((e) => {
         if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) {
-          const archivedPath = chain.persistInvalidSszObject(
-            "contributionAndProof",
-            ssz.altair.SignedContributionAndProof.serialize(contributionAndProof),
-            toHexString(ssz.altair.SignedContributionAndProof.hashTreeRoot(contributionAndProof))
-          );
-          logger.debug("The invalid gossip contribution and proof was written to", archivedPath);
+          chain.persistInvalidSszValue(ssz.altair.SignedContributionAndProof, contributionAndProof, "gossip_reject");
         }
         throw e;
       });
@@ -293,12 +274,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
         indexInSubcommittee = (await validateGossipSyncCommittee(chain, syncCommittee, subnet)).indexInSubcommittee;
       } catch (e) {
         if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) {
-          const archivedPath = chain.persistInvalidSszObject(
-            "syncCommittee",
-            ssz.altair.SyncCommitteeMessage.serialize(syncCommittee),
-            toHexString(ssz.altair.SyncCommitteeMessage.hashTreeRoot(syncCommittee))
-          );
-          logger.debug("The invalid gossip sync committee was written to", archivedPath);
+          chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, syncCommittee, "gossip_reject");
         }
         throw e;
       }

packages/lodestar/src/util/file.ts

@@ -4,19 +4,29 @@
 
 import fs from "node:fs";
 import path from "node:path";
+import {promisify} from "node:util";
 
-/**
- * Recursively ensures directory exists by creating any missing directories
- * @param {string} filePath
- */
-export function ensureDirectoryExistence(filePath: string): boolean {
-  const dirname = path.dirname(filePath);
-  if (fs.existsSync(dirname)) {
+/** Ensure a directory exists */
+export async function ensureDir(path: string): Promise<void> {
+  try {
+    await promisify(fs.stat)(path);
+  } catch (_) {
+    // not exists
+    await promisify(fs.mkdir)(path, {recursive: true});
+  }
+}
+
+/** Write data to a file if it does not exist */
+export async function writeIfNotExist(filepath: string, bytes: Uint8Array): Promise<boolean> {
+  try {
+    await promisify(fs.stat)(filepath);
+    return false;
+    // file exists, do nothing
+  } catch (_) {
+    // not exists
+    await promisify(fs.writeFile)(filepath, bytes);
     return true;
   }
-  ensureDirectoryExistence(dirname);
-  fs.mkdirSync(dirname);
-  return true;
 }
 
 export function rmDir(dir: string): void {