ChainSafe · dapplion · Aug 30, 2022 · Aug 27, 2022 · Aug 27, 2022 · Aug 27, 2022
@@ -202,6 +202,15 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}:
     try {
       notWhileSyncing();
       await waitForSlot(slot); // Must never request for a future slot > currentSlot
+      // Error early for builder if builder flow not active
+      if (type === BlockType.Blinded) {
+        if (!chain.executionBuilder) {
+          throw Error("Execution builder not set");
+        }
+        if (!chain.executionBuilder.status) {
+          throw Error("Execution builder disabled");
+        }
+      }
 
       // Process the queued attestations in the forkchoice for correct head estimation
       // forkChoice.updateTime() might have already been called by the onSlot clock

@@ -16,6 +16,8 @@ import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch, ValidatorIndex}
 import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {ILogger, toHex} from "@lodestar/utils";
 import {CompositeTypeAny, fromHexString, TreeView, Type} from "@chainsafe/ssz";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+
 import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
 import {IBeaconDb} from "../db/index.js";
 import {IMetrics} from "../metrics/index.js";
@@ -113,6 +115,9 @@ export class BeaconChain implements IBeaconChain {
   private successfulExchangeTransition = false;
   private readonly exchangeTransitionConfigurationEverySlots: number;
 
+  private readonly faultInspectionWindow: number;
+  private readonly allowedFaults: number;
+
   constructor(
     opts: IChainOptions,
     {
@@ -154,6 +159,24 @@ export class BeaconChain implements IBeaconChain {
     // Align to a multiple of SECONDS_PER_SLOT for nicer logs
     this.exchangeTransitionConfigurationEverySlots = Math.floor(60 / this.config.SECONDS_PER_SLOT);
 
+    /**
+     * Beacon clients select randomized values from the following ranges when initializing
+     * the circuit breaker (so at boot time and once for each unique boot).
+     *
+     * ALLOWED_FAULTS: between 1 and SLOTS_PER_EPOCH // 2
+     * FAULT_INSPECTION_WINDOW: between SLOTS_PER_EPOCH and 2 * SLOTS_PER_EPOCH
+     *
+     */
+    this.faultInspectionWindow = Math.max(
+      opts.faultInspectionWindow ?? SLOTS_PER_EPOCH + Math.floor(Math.random() * SLOTS_PER_EPOCH),
+      SLOTS_PER_EPOCH
+    );
+    // allowedFaults should be < faultInspectionWindow, limiting them to faultInspectionWindow/2
+    this.allowedFaults = Math.min(
+      opts.allowedFaults ?? Math.floor(this.faultInspectionWindow / 2),
+      Math.floor(this.faultInspectionWindow / 2)
+    );
+
     const signal = this.abortController.signal;
     const emitter = new ChainEventEmitter();
     // by default, verify signatures on both main threads and worker threads
@@ -610,4 +633,32 @@ export class BeaconChain implements IBeaconChain {
       this.beaconProposerCache.add(epoch, proposer);
     });
   }
+
+  updateBuilderStatus(clockSlot: Slot): void {
+    const executionBuilder = this.executionBuilder;
+    if (executionBuilder) {
+      const slotsPresent = this.forkChoice.getSlotsPresent(clockSlot - this.faultInspectionWindow);
+      const previousStatus = executionBuilder.status;
+      const shouldEnable = slotsPresent >= this.faultInspectionWindow - this.allowedFaults;
+
+      executionBuilder.updateStatus(shouldEnable);
+      // The status changed we should log
+      const status = executionBuilder.status;
+      if (status !== previousStatus) {
+        this.logger.info("Execution builder status updated", {
+          status,
+          slotsPresent,
+          window: this.faultInspectionWindow,
+          allowedFaults: this.allowedFaults,
+        });
+      } else {
+        this.logger.verbose("Execution builder status", {
+          status,
+          slotsPresent,
+          window: this.faultInspectionWindow,
+          allowedFaults: this.allowedFaults,
+        });
+      }
+    }
+  }
 }
@@ -120,6 +120,7 @@ export interface IBeaconChain {
   persistInvalidSszValue<T>(type: Type<T>, sszObject: T | Uint8Array, suffix?: string): void;
   /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */
   persistInvalidSszView(view: TreeView<CompositeTypeAny>, suffix?: string): void;
+  updateBuilderStatus(clockSlot: Slot): void;
 }
 
 export type SSZObjectType =

@@ -16,6 +16,10 @@ export type IChainOptions = BlockProcessOpts &
     skipCreateStateCacheIfAvailable?: boolean;
     defaultFeeRecipient: string;
     maxSkipSlots?: number;
+    /** Window to inspect missed slots for enabling/disabling builder circuit breaker */
+    faultInspectionWindow?: number;
+    /** Number of missed slots allowed in the faultInspectionWindow for builder circuit*/
+    allowedFaults?: number;
   };
 
 export type BlockProcessOpts = {

@@ -120,6 +120,14 @@ export class PrepareNextSlotScheduler {
         const proposerIndex = prepareState.epochCtx.getBeaconProposer(prepareSlot);
         const feeRecipient = this.chain.beaconProposerCache.get(proposerIndex);
         if (feeRecipient) {
+          // Update the builder status, if enabled shoot an api call to check status
+          this.chain.updateBuilderStatus(clockSlot);
+          if (this.chain.executionBuilder?.status) {
+            this.chain.executionBuilder.checkStatus().catch((e) => {
+              this.logger.error("Builder disabled as the check status api failed", {prepareSlot}, e as Error);
+            });
+          }
+
           const preparationTime =
             computeTimeAtSlot(this.config, prepareSlot, this.chain.genesisTime) - Date.now() / 1000;
           this.metrics?.blockPayload.payloadAdvancePrepTime.observe(preparationTime);

@@ -22,6 +22,8 @@ export const defaultExecutionBuilderHttpOpts: ExecutionBuilderHttpOpts = {
 export class ExecutionBuilderHttp implements IExecutionBuilder {
   readonly api: BuilderApi;
   readonly issueLocalFcUForBlockProduction?: boolean;
+  // Builder needs to be explicity enabled using updateStatus
+  status = false;
 
   constructor(opts: ExecutionBuilderHttpOpts, config: IChainForkConfig) {
     const baseUrl = opts.urls[0];
@@ -30,6 +32,20 @@ export class ExecutionBuilderHttp implements IExecutionBuilder {
     this.issueLocalFcUForBlockProduction = opts.issueLocalFcUForBlockProduction;
   }
 
+  updateStatus(shouldEnable: boolean): void {
+    this.status = shouldEnable;
+  }
+
+  async checkStatus(): Promise<void> {
+    try {
+      await this.api.checkStatus();
+    } catch (e) {
+      // Disable if the status was enabled
+      this.status = false;
+      throw e;
+    }
+  }
+
   async registerValidator(registrations: bellatrix.SignedValidatorRegistrationV1[]): Promise<void> {
     return this.api.registerValidator(registrations);
   }

@@ -7,6 +7,9 @@ export interface IExecutionBuilder {
    * fetch
    */
   readonly issueLocalFcUForBlockProduction?: boolean;
+  status: boolean;
+  updateStatus(shouldEnable: boolean): void;
+  checkStatus(): Promise<void>;
   registerValidator(registrations: bellatrix.SignedValidatorRegistrationV1[]): Promise<void>;
   getPayloadHeader(slot: Slot, parentHash: Root, proposerPubKey: BLSPubkey): Promise<bellatrix.ExecutionPayloadHeader>;
   submitSignedBlindedBlock(signedBlock: bellatrix.SignedBlindedBeaconBlock): Promise<bellatrix.SignedBeaconBlock>;

@@ -6,6 +6,7 @@ import {WinstonLogger} from "@lodestar/utils";
 import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {IChainForkConfig} from "@lodestar/config";
 import {BeaconChain, ChainEventEmitter} from "../../../src/chain/index.js";
+import {IBeaconChain} from "../../../src/chain/interface.js";
 import {LocalClock} from "../../../src/chain/clock/index.js";
 import {PrepareNextSlotScheduler} from "../../../src/chain/prepareNextSlot.js";
 import {StateRegenerator} from "../../../src/chain/regen/index.js";
@@ -27,13 +28,15 @@ describe("PrepareNextSlot scheduler", () => {
   let loggerStub: SinonStubbedInstance<WinstonLogger> & WinstonLogger;
   let beaconProposerCacheStub: SinonStubbedInstance<BeaconProposerCache> & BeaconProposerCache;
   let getForkSeqStub: SinonStubFn<typeof config["getForkSeq"]>;
+  let updateBuilderStatus: SinonStubFn<IBeaconChain["updateBuilderStatus"]>;
   let executionEngineStub: SinonStubbedInstance<ExecutionEngineHttp> & ExecutionEngineHttp;
 
   beforeEach(() => {
     sandbox.useFakeTimers();
     const chainStub = sandbox.createStubInstance(BeaconChain) as StubbedChainMutable<
       "clock" | "forkChoice" | "emitter" | "regen"
     >;
+    updateBuilderStatus = chainStub.updateBuilderStatus;
     const clockStub = sandbox.createStubInstance(LocalClock) as SinonStubbedInstance<LocalClock> & LocalClock;
     chainStub.clock = clockStub;
     forkChoiceStub = sandbox.createStubInstance(ForkChoice) as SinonStubbedInstance<ForkChoice> & ForkChoice;
@@ -135,6 +138,7 @@ describe("PrepareNextSlot scheduler", () => {
     forkChoiceStub.updateHead.returns({slot: SLOTS_PER_EPOCH - 3} as ProtoBlock);
     forkChoiceStub.getJustifiedBlock.returns({} as ProtoBlock);
     forkChoiceStub.getFinalizedBlock.returns({} as ProtoBlock);
+    updateBuilderStatus.returns(void 0);
     const state = generateCachedBellatrixState();
     regenStub.getBlockSlotState.resolves(state);
     beaconProposerCacheStub.get.returns("0x fee recipient address");
@@ -147,6 +151,7 @@ describe("PrepareNextSlot scheduler", () => {
 
     expect(forkChoiceStub.updateHead.called, "expect forkChoice.updateHead to be called").to.equal(true);
     expect(regenStub.getBlockSlotState.called, "expect regen.getBlockSlotState to be called").to.equal(true);
+    expect(updateBuilderStatus.called, "expect updateBuilderStatus to be called").to.be.equal(true);
     expect(forkChoiceStub.getJustifiedBlock.called, "expect forkChoice.getJustifiedBlock to be called").to.equal(true);
     expect(forkChoiceStub.getFinalizedBlock.called, "expect forkChoice.getFinalizedBlock to be called").to.equal(true);
     expect(executionEngineStub.notifyForkchoiceUpdate.calledOnce, "expect CL call notifyForkchoiceUpdate").to.equal(

@@ -210,6 +210,7 @@ export class MockBeaconChain implements IBeaconChain {
   }
 
   async updateBeaconProposerData(): Promise<void> {}
+  updateBuilderStatus(): void {}
 }
 
 function mockForkChoice(): IForkChoice {
@@ -250,6 +251,7 @@ function mockForkChoice(): IForkChoice {
     getTime: () => 0,
     hasBlock: () => true,
     hasBlockHex: () => true,
+    getSlotsPresent: () => 0,
     getBlock: () => block,
     getBlockHex: () => block,
     getFinalizedBlock: () => block,

@@ -236,6 +236,15 @@ export class ForkChoice implements IForkChoice {
     return (this.head = headNode);
   }
 
+  /**
+   * An iteration over protoArray to get present slots, to be called pre-emptively
+   * from prepareNextSlot to prevent delay on produceBlindedBlock
+   * @param windowStart is the slot after which (excluding) to provide present slots
+   */
+  getSlotsPresent(windowStart: number): number {
+    return this.protoArray.nodes.filter((node) => node.slot > windowStart).length;
+  }
+
   /** Very expensive function, iterates the entire ProtoArray. Called only in debug API */
   getHeads(): ProtoBlock[] {
     return this.protoArray.nodes.filter((node) => node.bestChild === undefined);

@@ -106,6 +106,7 @@ export interface IForkChoice {
    */
   hasBlock(blockRoot: Root): boolean;
   hasBlockHex(blockRoot: RootHex): boolean;
+  getSlotsPresent(windowStart: number): number;
   /**
    * Returns a `ProtoBlock` if the block is known **and** a descendant of the finalized root.
    */