test: introduce beacon api test ignore list

packages/api/src/beacon/routes/beacon/pool.ts

@@ -63,7 +63,7 @@ export type Api = {
    * @returns any Successful response
    * @throws ApiError
    */
-  getPoolBlsToExecutionChanges(): Promise<
+  getPoolBLSToExecutionChanges(): Promise<
     ApiClientResponse<{[HttpStatusCode.OK]: {data: capella.SignedBLSToExecutionChange[]}}>
   >;
 
@@ -123,7 +123,7 @@ export type Api = {
    * @returns any BLSToExecutionChange is stored in node and broadcasted to network
    * @throws ApiError
    */
-  submitPoolBlsToExecutionChange(
+  submitPoolBLSToExecutionChange(
     blsToExecutionChange: capella.SignedBLSToExecutionChange[]
   ): Promise<ApiClientResponse<{[HttpStatusCode.OK]: void}, HttpStatusCode.BAD_REQUEST>>;
 
@@ -143,12 +143,12 @@ export const routesData: RoutesData<Api> = {
   getPoolAttesterSlashings: {url: "/eth/v1/beacon/pool/attester_slashings", method: "GET"},
   getPoolProposerSlashings: {url: "/eth/v1/beacon/pool/proposer_slashings", method: "GET"},
   getPoolVoluntaryExits: {url: "/eth/v1/beacon/pool/voluntary_exits", method: "GET"},
-  getPoolBlsToExecutionChanges: {url: "/eth/v1/beacon/pool/bls_to_execution_changes", method: "GET"},
+  getPoolBLSToExecutionChanges: {url: "/eth/v1/beacon/pool/bls_to_execution_changes", method: "GET"},
   submitPoolAttestations: {url: "/eth/v1/beacon/pool/attestations", method: "POST"},
   submitPoolAttesterSlashings: {url: "/eth/v1/beacon/pool/attester_slashings", method: "POST"},
   submitPoolProposerSlashings: {url: "/eth/v1/beacon/pool/proposer_slashings", method: "POST"},
   submitPoolVoluntaryExit: {url: "/eth/v1/beacon/pool/voluntary_exits", method: "POST"},
-  submitPoolBlsToExecutionChange: {url: "/eth/v1/beacon/pool/bls_to_execution_changes", method: "POST"},
+  submitPoolBLSToExecutionChange: {url: "/eth/v1/beacon/pool/bls_to_execution_changes", method: "POST"},
   submitPoolSyncCommitteeSignatures: {url: "/eth/v1/beacon/pool/sync_committees", method: "POST"},
 };
 
@@ -158,12 +158,12 @@ export type ReqTypes = {
   getPoolAttesterSlashings: ReqEmpty;
   getPoolProposerSlashings: ReqEmpty;
   getPoolVoluntaryExits: ReqEmpty;
-  getPoolBlsToExecutionChanges: ReqEmpty;
+  getPoolBLSToExecutionChanges: ReqEmpty;
   submitPoolAttestations: {body: unknown};
   submitPoolAttesterSlashings: {body: unknown};
   submitPoolProposerSlashings: {body: unknown};
   submitPoolVoluntaryExit: {body: unknown};
-  submitPoolBlsToExecutionChange: {body: unknown};
+  submitPoolBLSToExecutionChange: {body: unknown};
   submitPoolSyncCommitteeSignatures: {body: unknown};
 };
 
@@ -177,12 +177,12 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
     getPoolAttesterSlashings: reqEmpty,
     getPoolProposerSlashings: reqEmpty,
     getPoolVoluntaryExits: reqEmpty,
-    getPoolBlsToExecutionChanges: reqEmpty,
+    getPoolBLSToExecutionChanges: reqEmpty,
     submitPoolAttestations: reqOnlyBody(ArrayOf(ssz.phase0.Attestation), Schema.ObjectArray),
     submitPoolAttesterSlashings: reqOnlyBody(ssz.phase0.AttesterSlashing, Schema.Object),
     submitPoolProposerSlashings: reqOnlyBody(ssz.phase0.ProposerSlashing, Schema.Object),
     submitPoolVoluntaryExit: reqOnlyBody(ssz.phase0.SignedVoluntaryExit, Schema.Object),
-    submitPoolBlsToExecutionChange: reqOnlyBody(ArrayOf(ssz.capella.SignedBLSToExecutionChange), Schema.ObjectArray),
+    submitPoolBLSToExecutionChange: reqOnlyBody(ArrayOf(ssz.capella.SignedBLSToExecutionChange), Schema.ObjectArray),
     submitPoolSyncCommitteeSignatures: reqOnlyBody(ArrayOf(ssz.altair.SyncCommitteeMessage), Schema.ObjectArray),
   };
 }
@@ -193,6 +193,6 @@ export function getReturnTypes(): ReturnTypes<Api> {
     getPoolAttesterSlashings: ContainerData(ArrayOf(ssz.phase0.AttesterSlashing)),
     getPoolProposerSlashings: ContainerData(ArrayOf(ssz.phase0.ProposerSlashing)),
     getPoolVoluntaryExits: ContainerData(ArrayOf(ssz.phase0.SignedVoluntaryExit)),
-    getPoolBlsToExecutionChanges: ContainerData(ArrayOf(ssz.capella.SignedBLSToExecutionChange)),
+    getPoolBLSToExecutionChanges: ContainerData(ArrayOf(ssz.capella.SignedBLSToExecutionChange)),
   };
 }

packages/api/src/beacon/routes/lightclient.ts

@@ -24,7 +24,7 @@ export type Api = {
    * - Has most bits
    * - Oldest update
    */
-  getUpdates(
+  getLightClientUpdatesByRange(
     startPeriod: SyncPeriod,
     count: number
   ): Promise<
@@ -39,15 +39,15 @@ export type Api = {
    * Returns the latest optimistic head update available. Clients should use the SSE type `light_client_optimistic_update`
    * unless to get the very first head update after syncing, or if SSE are not supported by the server.
    */
-  getOptimisticUpdate(): Promise<
+  getLightClientOptimisticUpdate(): Promise<
     ApiClientResponse<{
       [HttpStatusCode.OK]: {
         version: ForkName;
         data: allForks.LightClientOptimisticUpdate;
       };
     }>
   >;
-  getFinalityUpdate(): Promise<
+  getLightClientFinalityUpdate(): Promise<
     ApiClientResponse<{
       [HttpStatusCode.OK]: {
         version: ForkName;
@@ -60,7 +60,7 @@ export type Api = {
    * The trusted block root should be fetched with similar means to a weak subjectivity checkpoint.
    * Only block roots for checkpoints are guaranteed to be available.
    */
-  getBootstrap(blockRoot: string): Promise<
+  getLightClientBootstrap(blockRoot: string): Promise<
     ApiClientResponse<{
       [HttpStatusCode.OK]: {
         version: ForkName;
@@ -71,7 +71,7 @@ export type Api = {
   /**
    * Returns an array of sync committee hashes based on the provided period and count
    */
-  getCommitteeRoot(
+  getLightClientCommitteeRoot(
     startPeriod: SyncPeriod,
     count: number
   ): Promise<
@@ -87,39 +87,39 @@ export type Api = {
  * Define javascript values for each route
  */
 export const routesData: RoutesData<Api> = {
-  getUpdates: {url: "/eth/v1/beacon/light_client/updates", method: "GET"},
-  getOptimisticUpdate: {url: "/eth/v1/beacon/light_client/optimistic_update", method: "GET"},
-  getFinalityUpdate: {url: "/eth/v1/beacon/light_client/finality_update", method: "GET"},
-  getBootstrap: {url: "/eth/v1/beacon/light_client/bootstrap/{block_root}", method: "GET"},
-  getCommitteeRoot: {url: "/eth/v0/beacon/light_client/committee_root", method: "GET"},
+  getLightClientUpdatesByRange: {url: "/eth/v1/beacon/light_client/updates", method: "GET"},
+  getLightClientOptimisticUpdate: {url: "/eth/v1/beacon/light_client/optimistic_update", method: "GET"},
+  getLightClientFinalityUpdate: {url: "/eth/v1/beacon/light_client/finality_update", method: "GET"},
+  getLightClientBootstrap: {url: "/eth/v1/beacon/light_client/bootstrap/{block_root}", method: "GET"},
+  getLightClientCommitteeRoot: {url: "/eth/v0/beacon/light_client/committee_root", method: "GET"},
 };
 
 /* eslint-disable @typescript-eslint/naming-convention */
 export type ReqTypes = {
-  getUpdates: {query: {start_period: number; count: number}};
-  getOptimisticUpdate: ReqEmpty;
-  getFinalityUpdate: ReqEmpty;
-  getBootstrap: {params: {block_root: string}};
-  getCommitteeRoot: {query: {start_period: number; count: number}};
+  getLightClientUpdatesByRange: {query: {start_period: number; count: number}};
+  getLightClientOptimisticUpdate: ReqEmpty;
+  getLightClientFinalityUpdate: ReqEmpty;
+  getLightClientBootstrap: {params: {block_root: string}};
+  getLightClientCommitteeRoot: {query: {start_period: number; count: number}};
 };
 
 export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
   return {
-    getUpdates: {
+    getLightClientUpdatesByRange: {
       writeReq: (start_period, count) => ({query: {start_period, count}}),
       parseReq: ({query}) => [query.start_period, query.count],
       schema: {query: {start_period: Schema.UintRequired, count: Schema.UintRequired}},
     },
 
-    getOptimisticUpdate: reqEmpty,
-    getFinalityUpdate: reqEmpty,
+    getLightClientOptimisticUpdate: reqEmpty,
+    getLightClientFinalityUpdate: reqEmpty,
 
-    getBootstrap: {
+    getLightClientBootstrap: {
       writeReq: (block_root) => ({params: {block_root}}),
       parseReq: ({params}) => [params.block_root],
       schema: {params: {block_root: Schema.StringRequired}},
     },
-    getCommitteeRoot: {
+    getLightClientCommitteeRoot: {
       writeReq: (start_period, count) => ({query: {start_period, count}}),
       parseReq: ({query}) => [query.start_period, query.count],
       schema: {query: {start_period: Schema.UintRequired, count: Schema.UintRequired}},
@@ -132,27 +132,27 @@ export function getReturnTypes(): ReturnTypes<Api> {
   const VersionedUpdate = WithVersion((fork: ForkName) =>
     isForkLightClient(fork) ? ssz.allForksLightClient[fork].LightClientUpdate : ssz.altair.LightClientUpdate
   );
-  const getUpdates = {
+  const getLightClientUpdatesByRange = {
     toJson: (updates: {version: ForkName; data: allForks.LightClientUpdate}[]) =>
       updates.map((data) => VersionedUpdate.toJson(data)),
     fromJson: (updates: unknown[]) => updates.map((data) => VersionedUpdate.fromJson(data)),
   };
 
   return {
-    getUpdates,
-    getOptimisticUpdate: WithVersion((fork: ForkName) =>
+    getLightClientUpdatesByRange,
+    getLightClientOptimisticUpdate: WithVersion((fork: ForkName) =>
       isForkLightClient(fork)
         ? ssz.allForksLightClient[fork].LightClientOptimisticUpdate
         : ssz.altair.LightClientOptimisticUpdate
     ),
-    getFinalityUpdate: WithVersion((fork: ForkName) =>
+    getLightClientFinalityUpdate: WithVersion((fork: ForkName) =>
       isForkLightClient(fork)
         ? ssz.allForksLightClient[fork].LightClientFinalityUpdate
         : ssz.altair.LightClientFinalityUpdate
     ),
-    getBootstrap: WithVersion((fork: ForkName) =>
+    getLightClientBootstrap: WithVersion((fork: ForkName) =>
       isForkLightClient(fork) ? ssz.allForksLightClient[fork].LightClientBootstrap : ssz.altair.LightClientBootstrap
     ),
-    getCommitteeRoot: ContainerData(ArrayOf(ssz.Root)),
+    getLightClientCommitteeRoot: ContainerData(ArrayOf(ssz.Root)),
   };
 }

packages/api/test/unit/beacon/oapiSpec.test.ts

@@ -23,7 +23,7 @@ import {testData as validatorTestData} from "./testData/validator.js";
 // eslint-disable-next-line @typescript-eslint/naming-convention
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
-const version = "v2.3.0";
+const version = "v2.4.0";
 const openApiFile: OpenApiFile = {
   url: `https://github.com/ethereum/beacon-APIs/releases/download/${version}/beacon-node-oapi.json`,
   filepath: path.join(__dirname, "../../../oapi-schemas/beacon-node-oapi.json"),
@@ -84,11 +84,83 @@ const testDatas = {
   ...validatorTestData,
 };
 
+const ignoredOperationsIds = [
+  /*
+   #5693
+   missing finalized
+   */
+  "getStateRoot",
+  "getStateFork",
+  "getStateFinalityCheckpoints",
+  "getStateValidators",
+  "getStateValidator",
+  "getStateValidatorBalances",
+  "getEpochCommittees",
+  "getEpochSyncCommittees",
+  "getStateRandao",
+  "getBlockHeaders",
+  "getBlockHeader",
+  "getBlockV2",
+  "getBlockRoot",
+  "getBlockAttestations",
+  "getStateV2",
+
+  /* missing route */
+  /* #5694 */
+  "getSyncCommitteeRewards",
+  "getBlockRewards",
+  "getAttestationsRewards",
+  "getDepositSnapshot", // Won't fix for now, see https://github.com/ChainSafe/lodestar/issues/5697
+  "getBlindedBlock", // #5699
+  "getNextWithdrawals", // #5696
+  "getDebugForkChoice", // #5700
+
+  /* 
+   #6168
+   /query/syncing_status - must be integer
+   */
+  "getHealth",
+
+  /* 
+   #4638 
+   /query - must have required property 'skip_randao_verification'
+   */
+  "produceBlockV2",
+  "produceBlindedBlock",
+];
+
 const openApiJson = await fetchOpenApiSpec(openApiFile);
-runTestCheckAgainstSpec(openApiJson, routesData, reqSerializers, returnTypes, testDatas, {
-  // TODO: Investigate why schema validation fails otherwise
-  routesDropOneOf: ["produceBlockV2", "produceBlindedBlock", "publishBlindedBlock"],
-});
+runTestCheckAgainstSpec(
+  openApiJson,
+  routesData,
+  reqSerializers,
+  returnTypes,
+  testDatas,
+  {
+    // TODO: Investigate why schema validation fails otherwise
+    routesDropOneOf: ["produceBlockV2", "produceBlindedBlock", "publishBlindedBlock"],
+  },
+  ignoredOperationsIds
+);
+
+const ignoredTopics = [
+  /*
+   #6167
+   eventTestData[bls_to_execution_change] does not match spec's example
+   */
+  "bls_to_execution_change",
+  /*
+   #6170
+   Error: Invalid slot=0 fork=phase0 for lightclient fork types
+  */
+  "light_client_finality_update",
+  "light_client_optimistic_update",
+  /*
+   https://github.com/ethereum/beacon-APIs/pull/379
+   SyntaxError: Unexpected non-whitespace character after JSON at position 629 (line 1 column 630)
+  */
+  "payload_attributes",
+];
 
 // eventstream types are defined as comments in the description of "examples".
 // The function runTestCheckAgainstSpec() can't handle those, so the custom code before:
@@ -113,7 +185,9 @@ describe("eventstream event data", () => {
   const eventSerdes = routes.events.getEventSerdes(config);
   const knownTopics = new Set<string>(Object.values(routes.events.eventTypes));
 
-  for (const [topic, {value}] of Object.entries(eventstreamExamples ?? {})) {
+  for (const [topic, {value}] of Object.entries(eventstreamExamples ?? {}).filter(
+    ([topic]) => !ignoredTopics.includes(topic)
+  )) {
     it(topic, () => {
       if (!knownTopics.has(topic)) {
         throw Error(`topic ${topic} not implemented`);

packages/api/test/unit/beacon/testData/beacon.ts

@@ -93,7 +93,7 @@ export const testData: GenericServerTestCases<Api> = {
     args: [],
     res: {data: [ssz.phase0.SignedVoluntaryExit.defaultValue()]},
   },
-  getPoolBlsToExecutionChanges: {
+  getPoolBLSToExecutionChanges: {
     args: [],
     res: {data: [ssz.capella.SignedBLSToExecutionChange.defaultValue()]},
   },
@@ -113,7 +113,7 @@ export const testData: GenericServerTestCases<Api> = {
     args: [ssz.phase0.SignedVoluntaryExit.defaultValue()],
     res: undefined,
   },
-  submitPoolBlsToExecutionChange: {
+  submitPoolBLSToExecutionChange: {
     args: [[ssz.capella.SignedBLSToExecutionChange.defaultValue()]],
     res: undefined,
   },

packages/api/test/unit/beacon/testData/lightclient.ts

@@ -12,15 +12,15 @@ const header = ssz.altair.LightClientHeader.defaultValue();
 const signatureSlot = ssz.Slot.defaultValue();
 
 export const testData: GenericServerTestCases<Api> = {
-  getUpdates: {
+  getLightClientUpdatesByRange: {
     args: [1, 2],
     res: [{version: ForkName.bellatrix, data: lightClientUpdate}],
   },
-  getOptimisticUpdate: {
+  getLightClientOptimisticUpdate: {
     args: [],
     res: {version: ForkName.bellatrix, data: {syncAggregate, attestedHeader: header, signatureSlot}},
   },
-  getFinalityUpdate: {
+  getLightClientFinalityUpdate: {
     args: [],
     res: {
       version: ForkName.bellatrix,
@@ -33,7 +33,7 @@ export const testData: GenericServerTestCases<Api> = {
       },
     },
   },
-  getBootstrap: {
+  getLightClientBootstrap: {
     args: [toHexString(root)],
     res: {
       version: ForkName.bellatrix,
@@ -44,7 +44,7 @@ export const testData: GenericServerTestCases<Api> = {
       },
     },
   },
-  getCommitteeRoot: {
+  getLightClientCommitteeRoot: {
     args: [1, 2],
     res: {data: [Buffer.alloc(32, 0), Buffer.alloc(32, 1)]},
   },

packages/api/test/utils/checkAgainstSpec.ts

@@ -25,11 +25,14 @@ export function runTestCheckAgainstSpec(
   reqSerializers: Record<string, ReqSerializer<any, any>>,
   returnTypes: Record<string, ReturnTypes<any>[string]>,
   testDatas: Record<string, GenericServerTestCases<any>[string]>,
-  opts?: ParseOpenApiSpecOpts
+  opts?: ParseOpenApiSpecOpts,
+  filteredOperationsIds: string[] = []
 ): void {
   const openApiSpec = parseOpenApiSpec(openApiJson, opts);
 
-  for (const [operationId, routeSpec] of openApiSpec.entries()) {
+  for (const [operationId, routeSpec] of [...openApiSpec.entries()].filter(
+    ([operationId]) => !filteredOperationsIds.includes(operationId)
+  )) {
     describe(operationId, () => {
       const {requestSchema, responseOkSchema} = routeSpec;
       const routeId = operationId;