Add compact proofs API (#5155)

* Add compact proofs to API

* Fix type errors

* Fix lint errors

* Fix test

* PR review

packages/api/package.json

@@ -68,8 +68,8 @@
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
-    "@chainsafe/persistent-merkle-tree": "^0.4.2",
-    "@chainsafe/ssz": "^0.9.2",
+    "@chainsafe/persistent-merkle-tree": "^0.5.0",
+    "@chainsafe/ssz": "^0.10.1",
     "@lodestar/config": "^1.4.3",
     "@lodestar/params": "^1.4.3",
     "@lodestar/types": "^1.4.3",

packages/api/src/beacon/client/proof.ts

@@ -1,5 +1,5 @@
 import {IChainForkConfig} from "@lodestar/config";
-import {deserializeProof} from "@chainsafe/persistent-merkle-tree";
+import {CompactMultiProof, ProofType} from "@chainsafe/persistent-merkle-tree";
 import {Api, ReqTypes, routesData, getReqSerializers} from "../routes/proof.js";
 import {IHttpClient, getFetchOptsSerializers, HttpError} from "../../utils/client/index.js";
 import {HttpStatusCode} from "../../utils/client/httpStatusCode.js";
@@ -14,10 +14,45 @@ export function getClient(_config: IChainForkConfig, httpClient: IHttpClient): A
   const fetchOptsSerializers = getFetchOptsSerializers<Api, ReqTypes>(routesData, reqSerializers);
 
   return {
-    async getStateProof(stateId, paths) {
+    async getStateProof(stateId, descriptor) {
       try {
-        const res = await httpClient.arrayBuffer(fetchOptsSerializers.getStateProof(stateId, paths));
-        const proof = deserializeProof(new Uint8Array(res.body));
+        const res = await httpClient.arrayBuffer(fetchOptsSerializers.getStateProof(stateId, descriptor));
+        // reuse the response ArrayBuffer
+        if (!Number.isInteger(res.body.byteLength / 32)) {
+          throw new Error("Invalid proof data: Length not divisible by 32");
+        }
+
+        const proof: CompactMultiProof = {
+          type: ProofType.compactMulti,
+          descriptor,
+          leaves: Array.from({length: res.body.byteLength / 32}, (_, i) => new Uint8Array(res.body, i * 32, 32)),
+        };
+
+        return {ok: true, response: {data: proof}, status: HttpStatusCode.OK};
+      } catch (err) {
+        if (err instanceof HttpError) {
+          return {
+            ok: false,
+            error: {code: err.status, message: err.message, operationId: "proof.getStateProof"},
+            status: err.status,
+          };
+        }
+        throw err;
+      }
+    },
+    async getBlockProof(blockId, descriptor) {
+      try {
+        const res = await httpClient.arrayBuffer(fetchOptsSerializers.getBlockProof(blockId, descriptor));
+        // reuse the response ArrayBuffer
+        if (!Number.isInteger(res.body.byteLength / 32)) {
+          throw new Error("Invalid proof data: Length not divisible by 32");
+        }
+
+        const proof: CompactMultiProof = {
+          type: ProofType.compactMulti,
+          descriptor,
+          leaves: Array.from({length: res.body.byteLength / 32}, (_, i) => new Uint8Array(res.body, i * 32, 32)),
+        };
 
         return {ok: true, response: {data: proof}, status: HttpStatusCode.OK};
       } catch (err) {

packages/api/src/beacon/routes/proof.ts

@@ -1,20 +1,27 @@
-import {JsonPath} from "@chainsafe/ssz";
 import {Proof} from "@chainsafe/persistent-merkle-tree";
+import {fromHexString, toHexString} from "@chainsafe/ssz";
 import {ReturnTypes, RoutesData, Schema, sameType, ReqSerializers} from "../../utils/index.js";
-import {queryParseProofPathsArr, querySerializeProofPathsArr} from "../../utils/serdes.js";
 import {HttpStatusCode} from "../../utils/client/httpStatusCode.js";
 import {ApiClientResponse} from "../../interfaces.js";
 
 // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes
 
 export type Api = {
   /**
-   * Returns a multiproof of `jsonPaths` at the requested `stateId`.
+   * Returns a multiproof of `descriptor` at the requested `stateId`.
    * The requested `stateId` may not be available. Regular nodes only keep recent states in memory.
    */
   getStateProof(
     stateId: string,
-    jsonPaths: JsonPath[]
+    descriptor: Uint8Array
+  ): Promise<ApiClientResponse<{[HttpStatusCode.OK]: {data: Proof}}>>;
+  /**
+   * Returns a multiproof of `descriptor` at the requested `blockId`.
+   * The requested `blockId` may not be available. Regular nodes only keep recent states in memory.
+   */
+  getBlockProof(
+    blockId: string,
+    descriptor: Uint8Array
   ): Promise<ApiClientResponse<{[HttpStatusCode.OK]: {data: Proof}}>>;
 };
 
@@ -23,19 +30,26 @@ export type Api = {
  */
 export const routesData: RoutesData<Api> = {
   getStateProof: {url: "/eth/v0/beacon/proof/state/{state_id}", method: "GET"},
+  getBlockProof: {url: "/eth/v0/beacon/proof/block/{block_id}", method: "GET"},
 };
 
 /* eslint-disable @typescript-eslint/naming-convention */
 export type ReqTypes = {
-  getStateProof: {params: {state_id: string}; query: {paths: string[]}};
+  getStateProof: {params: {state_id: string}; query: {format: string}};
+  getBlockProof: {params: {block_id: string}; query: {format: string}};
 };
 
 export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
   return {
     getStateProof: {
-      writeReq: (state_id, paths) => ({params: {state_id}, query: {paths: querySerializeProofPathsArr(paths)}}),
-      parseReq: ({params, query}) => [params.state_id, queryParseProofPathsArr(query.paths)],
-      schema: {params: {state_id: Schema.StringRequired}, body: Schema.AnyArray},
+      writeReq: (state_id, descriptor) => ({params: {state_id}, query: {format: toHexString(descriptor)}}),
+      parseReq: ({params, query}) => [params.state_id, fromHexString(query.format)],
+      schema: {params: {state_id: Schema.StringRequired}, query: {format: Schema.StringRequired}},
+    },
+    getBlockProof: {
+      writeReq: (block_id, descriptor) => ({params: {block_id}, query: {format: toHexString(descriptor)}}),
+      parseReq: ({params, query}) => [params.block_id, fromHexString(query.format)],
+      schema: {params: {block_id: Schema.StringRequired}, query: {format: Schema.StringRequired}},
     },
   };
 }
@@ -44,5 +58,6 @@ export function getReturnTypes(): ReturnTypes<Api> {
   return {
     // Just sent the proof JSON as-is
     getStateProof: sameType(),
+    getBlockProof: sameType(),
   };
 }

packages/api/src/beacon/server/proof.ts

@@ -1,5 +1,5 @@
 import {IChainForkConfig} from "@lodestar/config";
-import {serializeProof} from "@chainsafe/persistent-merkle-tree";
+import {CompactMultiProof} from "@chainsafe/persistent-merkle-tree";
 import {Api, ReqTypes, routesData, getReturnTypes, getReqSerializers} from "../routes/proof.js";
 import {ServerRoutes, getGenericJsonServer} from "../../utils/server/index.js";
 import {ServerApi} from "../../interfaces.js";
@@ -19,8 +19,27 @@ export function getRoutes(config: IChainForkConfig, api: ServerApi<Api>): Server
       handler: async (req) => {
         const args = reqSerializers.getStateProof.parseReq(req);
         const {data} = await api.getStateProof(...args);
+        const leaves = (data as CompactMultiProof).leaves;
+        const response = new Uint8Array(32 * leaves.length);
+        for (let i = 0; i < leaves.length; i++) {
+          response.set(leaves[i], i * 32);
+        }
         // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer
-        return Buffer.from(serializeProof(data));
+        return Buffer.from(response);
+      },
+    },
+    getBlockProof: {
+      ...serverRoutes.getBlockProof,
+      handler: async (req) => {
+        const args = reqSerializers.getBlockProof.parseReq(req);
+        const {data} = await api.getBlockProof(...args);
+        const leaves = (data as CompactMultiProof).leaves;
+        const response = new Uint8Array(32 * leaves.length);
+        for (let i = 0; i < leaves.length; i++) {
+          response.set(leaves[i], i * 32);
+        }
+        // Fastify 3.x.x will automatically add header `Content-Type: application/octet-stream` if Buffer
+        return Buffer.from(response);
       },
     },
   };

packages/api/test/unit/beacon/testData/proofs.ts

@@ -3,30 +3,33 @@ import {Api} from "../../../../src/beacon/routes/proof.js";
 import {GenericServerTestCases} from "../../../utils/genericServerTest.js";
 
 const root = Uint8Array.from(Buffer.alloc(32, 1));
+const descriptor = Uint8Array.from([0, 0, 0, 0]);
 
 export const testData: GenericServerTestCases<Api> = {
   getStateProof: {
-    args: [
-      "head",
-      [
-        // ["validator", 0, "balance"],
-        ["finalized_checkpoint", 0, "root", 12000],
-      ],
-    ],
+    args: ["head", descriptor],
     res: {
       data: {
-        type: ProofType.treeOffset,
-        offsets: [1, 2, 3],
+        type: ProofType.compactMulti,
+        descriptor,
         leaves: [root, root, root, root],
       },
     },
-    /* eslint-disable quotes */
     query: {
-      paths: [
-        // '["validator",0,"balance"]',
-        '["finalized_checkpoint",0,"root",12000]',
-      ],
+      format: "0x00000000",
+    },
+  },
+  getBlockProof: {
+    args: ["head", descriptor],
+    res: {
+      data: {
+        type: ProofType.compactMulti,
+        descriptor,
+        leaves: [root, root, root, root],
+      },
+    },
+    query: {
+      format: "0x00000000",
     },
-    /* eslint-enable quotes */
   },
 };

packages/beacon-node/package.json

@@ -101,9 +101,9 @@
     "@chainsafe/discv5": "^3.0.0",
     "@chainsafe/libp2p-gossipsub": "^6.1.0",
     "@chainsafe/libp2p-noise": "^11.0.0",
-    "@chainsafe/persistent-merkle-tree": "^0.4.2",
+    "@chainsafe/persistent-merkle-tree": "^0.5.0",
     "@chainsafe/snappy-stream": "^5.1.2",
-    "@chainsafe/ssz": "^0.9.2",
+    "@chainsafe/ssz": "^0.10.1",
     "@chainsafe/threads": "^1.10.0",
     "@ethersproject/abi": "^5.0.0",
     "@libp2p/bootstrap": "^6.0.0",

packages/beacon-node/src/api/impl/proof/index.ts

@@ -1,7 +1,8 @@
 import {routes, ServerApi} from "@lodestar/api";
-import {ProofType, Tree} from "@chainsafe/persistent-merkle-tree";
+import {createProof, ProofType} from "@chainsafe/persistent-merkle-tree";
 import {ApiModules} from "../types.js";
 import {resolveStateId} from "../beacon/state/utils.js";
+import {resolveBlockId} from "../beacon/blocks/utils.js";
 import {IApiOptions} from "../../options.js";
 
 export function getProofApi(
@@ -13,29 +14,36 @@ export function getProofApi(
   const maxGindicesInProof = opts.maxGindicesInProof ?? 512;
 
   return {
-    async getStateProof(stateId, jsonPaths) {
+    async getStateProof(stateId, descriptor) {
+      // descriptor.length / 2 is a rough approximation of # of gindices
+      if (descriptor.length / 2 > maxGindicesInProof) {
+        throw new Error("Requested proof is too large.");
+      }
+
       const {state} = await resolveStateId(config, chain, db, stateId);
 
       // Commit any changes before computing the state root. In normal cases the state should have no changes here
       state.commit();
       const stateNode = state.node;
-      const tree = new Tree(stateNode);
 
-      const gindexes = state.type.tree_createProofGindexes(stateNode, jsonPaths);
-      // TODO: Is it necessary to de-duplicate?
-      //       It's not a problem if we overcount gindexes
-      const gindicesSet = new Set(gindexes);
+      const data = createProof(stateNode, {type: ProofType.compactMulti, descriptor});
 
-      if (gindicesSet.size > maxGindicesInProof) {
+      return {data};
+    },
+    async getBlockProof(blockId, descriptor) {
+      // descriptor.length / 2 is a rough approximation of # of gindices
+      if (descriptor.length / 2 > maxGindicesInProof) {
         throw new Error("Requested proof is too large.");
       }
 
-      return {
-        data: tree.getProof({
-          type: ProofType.treeOffset,
-          gindices: Array.from(gindicesSet),
-        }),
-      };
+      const {block} = await resolveBlockId(chain.forkChoice, db, blockId);
+
+      // Commit any changes before computing the state root. In normal cases the state should have no changes here
+      const blockNode = config.getForkTypes(block.message.slot).BeaconBlock.toView(block.message).node;
+
+      const data = createProof(blockNode, {type: ProofType.compactMulti, descriptor});
+
+      return {data};
     },
   };
 }

packages/beacon-node/test/e2e/chain/lightclient.test.ts

@@ -2,7 +2,7 @@ import {expect} from "chai";
 import {IChainConfig} from "@lodestar/config";
 import {ssz, altair} from "@lodestar/types";
 import {JsonPath, toHexString, fromHexString} from "@chainsafe/ssz";
-import {TreeOffsetProof} from "@chainsafe/persistent-merkle-tree";
+import {computeDescriptor, TreeOffsetProof} from "@chainsafe/persistent-merkle-tree";
 import {TimestampFormatCode} from "@lodestar/utils";
 import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {Lightclient} from "@lodestar/light-client";
@@ -194,7 +194,9 @@ async function getHeadStateProof(
 ): Promise<{proof: TreeOffsetProof; header: altair.LightClientHeader}> {
   const header = lightclient.getHead();
   const stateId = toHexString(header.beacon.stateRoot);
-  const res = await api.proof.getStateProof(stateId, paths);
+  const gindices = paths.map((path) => ssz.bellatrix.BeaconState.getPathInfo(path).gindex);
+  const descriptor = computeDescriptor(gindices);
+  const res = await api.proof.getStateProof(stateId, descriptor);
   ApiError.assert(res);
   return {
     proof: res.response.data as TreeOffsetProof,

packages/cli/package.json

@@ -59,7 +59,7 @@
     "@chainsafe/bls-keystore": "^2.0.0",
     "@chainsafe/blst": "^0.2.8",
     "@chainsafe/discv5": "^3.0.0",
-    "@chainsafe/ssz": "^0.9.2",
+    "@chainsafe/ssz": "^0.10.1",
     "@libp2p/peer-id-factory": "^2.0.1",
     "@lodestar/api": "^1.4.3",
     "@lodestar/beacon-node": "^1.4.3",

packages/config/package.json

@@ -65,7 +65,7 @@
     "blockchain"
   ],
   "dependencies": {
-    "@chainsafe/ssz": "^0.9.2",
+    "@chainsafe/ssz": "^0.10.1",
     "@lodestar/params": "^1.4.3",
     "@lodestar/types": "^1.4.3"
   }

packages/db/package.json

@@ -38,7 +38,7 @@
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
-    "@chainsafe/ssz": "^0.9.2",
+    "@chainsafe/ssz": "^0.10.1",
     "@lodestar/config": "^1.4.3",
     "@lodestar/utils": "^1.4.3",
     "@types/levelup": "^4.3.3",

packages/fork-choice/package.json

@@ -39,7 +39,7 @@
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
-    "@chainsafe/ssz": "^0.9.2",
+    "@chainsafe/ssz": "^0.10.1",
     "@lodestar/config": "^1.4.3",
     "@lodestar/params": "^1.4.3",
     "@lodestar/state-transition": "^1.4.3",

packages/light-client/package.json

@@ -64,8 +64,8 @@
   },
   "dependencies": {
     "@chainsafe/bls": "7.1.1",
-    "@chainsafe/persistent-merkle-tree": "^0.4.2",
-    "@chainsafe/ssz": "^0.9.2",
+    "@chainsafe/persistent-merkle-tree": "^0.5.0",
+    "@chainsafe/ssz": "^0.10.1",
     "@lodestar/api": "^1.4.3",
     "@lodestar/config": "^1.4.3",
     "@lodestar/params": "^1.4.3",

packages/light-client/test/mocks/LightclientServerApiMock.ts

@@ -1,7 +1,6 @@
 import {concat} from "uint8arrays";
 import {digest} from "@chainsafe/as-sha256";
-import {Proof} from "@chainsafe/persistent-merkle-tree";
-import {JsonPath} from "@chainsafe/ssz";
+import {createProof, Proof, ProofType} from "@chainsafe/persistent-merkle-tree";
 import {routes, ServerApi} from "@lodestar/api";
 import {altair, RootHex, SyncPeriod} from "@lodestar/types";
 import {notNullish} from "@lodestar/utils";
@@ -11,10 +10,14 @@ import {BeaconStateAltair} from "../utils/types.js";
 export class ProofServerApiMock implements ServerApi<routes.proof.Api> {
   readonly states = new Map<RootHex, BeaconStateAltair>();
 
-  async getStateProof(stateId: string, paths: JsonPath[]): Promise<{data: Proof}> {
+  async getStateProof(stateId: string, descriptor: Uint8Array): Promise<{data: Proof}> {
     const state = this.states.get(stateId);
     if (!state) throw Error(`stateId ${stateId} not available`);
-    return {data: state.createProof(paths)};
+    return {data: createProof(state.node, {type: ProofType.compactMulti, descriptor})};
+  }
+
+  async getBlockProof(blockId: string, _descriptor: Uint8Array): Promise<{data: Proof}> {
+    throw Error(`blockId ${blockId} not available`);
   }
 }