From c1b9b207c1c38be9a31135d90f1d0e6118d11087 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Sun, 14 Jul 2024 14:16:18 +0700 Subject: [PATCH 1/3] feat: implement hashInto() api --- packages/as-sha256/src/hashObject.ts | 112 +++++++++++---------- packages/as-sha256/src/index.ts | 62 ++++++++++-- packages/as-sha256/test/perf/index.test.ts | 8 +- packages/as-sha256/test/perf/simd.test.ts | 17 ++-- packages/as-sha256/test/unit/index.test.ts | 6 +- packages/as-sha256/test/unit/simd.test.ts | 21 +++- 6 files changed, 153 insertions(+), 73 deletions(-) diff --git a/packages/as-sha256/src/hashObject.ts b/packages/as-sha256/src/hashObject.ts index 18645a82..48efd519 100644 --- a/packages/as-sha256/src/hashObject.ts +++ b/packages/as-sha256/src/hashObject.ts @@ -98,95 +98,103 @@ export function hashObjectToByteArray(obj: HashObject, byteArr: Uint8Array, offs * This function contains multiple same procedures but we intentionally * do it step by step to improve performance a bit. **/ -export function byteArrayToHashObject(byteArr: Uint8Array): HashObject { +export function byteArrayToHashObject(byteArr: Uint8Array, offset: number): HashObject { + const result: HashObject = { + h0: 0, + h1: 0, + h2: 0, + h3: 0, + h4: 0, + h5: 0, + h6: 0, + h7: 0, + }; + + byteArrayIntoHashObject(byteArr, offset, result); + return result; +} + +/** + * Same to above but this set result to the output param to save memory. + */ +export function byteArrayIntoHashObject(byteArr: Uint8Array, offset: number, output: HashObject): void { let tmp = 0; - tmp |= byteArr[3] & 0xff; + tmp |= byteArr[3 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[2] & 0xff; + tmp |= byteArr[2 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[1] & 0xff; + tmp |= byteArr[1 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[0] & 0xff; - const h0 = tmp; + tmp |= byteArr[0 + offset] & 0xff; + output.h0 = tmp; tmp = 0; - tmp |= byteArr[7] & 0xff; + tmp |= byteArr[7 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[6] & 0xff; + tmp |= byteArr[6 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[5] & 0xff; + tmp |= byteArr[5 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[4] & 0xff; - const h1 = tmp; + tmp |= byteArr[4 + offset] & 0xff; + output.h1 = tmp; tmp = 0; - tmp |= byteArr[11] & 0xff; + tmp |= byteArr[11 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[10] & 0xff; + tmp |= byteArr[10 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[9] & 0xff; + tmp |= byteArr[9 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[8] & 0xff; - const h2 = tmp; + tmp |= byteArr[8 + offset] & 0xff; + output.h2 = tmp; tmp = 0; - tmp |= byteArr[15] & 0xff; + tmp |= byteArr[15 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[14] & 0xff; + tmp |= byteArr[14 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[13] & 0xff; + tmp |= byteArr[13 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[12] & 0xff; - const h3 = tmp; + tmp |= byteArr[12 + offset] & 0xff; + output.h3 = tmp; tmp = 0; - tmp |= byteArr[19] & 0xff; + tmp |= byteArr[19 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[18] & 0xff; + tmp |= byteArr[18 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[17] & 0xff; + tmp |= byteArr[17 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[16] & 0xff; - const h4 = tmp; + tmp |= byteArr[16 + offset] & 0xff; + output.h4 = tmp; tmp = 0; - tmp |= byteArr[23] & 0xff; + tmp |= byteArr[23 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[22] & 0xff; + tmp |= byteArr[22 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[21] & 0xff; + tmp |= byteArr[21 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[20] & 0xff; - const h5 = tmp; + tmp |= byteArr[20 + offset] & 0xff; + output.h5 = tmp; tmp = 0; - tmp |= byteArr[27] & 0xff; + tmp |= byteArr[27 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[26] & 0xff; + tmp |= byteArr[26 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[25] & 0xff; + tmp |= byteArr[25 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[24] & 0xff; - const h6 = tmp; + tmp |= byteArr[24 + offset] & 0xff; + output.h6 = tmp; tmp = 0; - tmp |= byteArr[31] & 0xff; + tmp |= byteArr[31 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[30] & 0xff; + tmp |= byteArr[30 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[29] & 0xff; + tmp |= byteArr[29 + offset] & 0xff; tmp = tmp << 8; - tmp |= byteArr[28] & 0xff; - const h7 = tmp; - - return { - h0, - h1, - h2, - h3, - h4, - h5, - h6, - h7, - }; + tmp |= byteArr[28 + offset] & 0xff; + output.h7 = tmp; } diff --git a/packages/as-sha256/src/index.ts b/packages/as-sha256/src/index.ts index 867aec60..97e7b9ca 100644 --- a/packages/as-sha256/src/index.ts +++ b/packages/as-sha256/src/index.ts @@ -1,7 +1,7 @@ import {newInstance} from "./wasm"; -import {HashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject"; +import {HashObject, byteArrayIntoHashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject"; import SHA256 from "./sha256"; -export {HashObject, byteArrayToHashObject, hashObjectToByteArray, SHA256}; +export {HashObject, byteArrayToHashObject, hashObjectToByteArray, byteArrayIntoHashObject, SHA256}; const ctx = newInstance(); const wasmInputValue = ctx.input.value; @@ -52,6 +52,24 @@ export function digest2Bytes32(bytes1: Uint8Array, bytes2: Uint8Array): Uint8Arr * @returns */ export function digest64HashObjects(obj1: HashObject, obj2: HashObject): HashObject { + const result: HashObject = { + h0: 0, + h1: 0, + h2: 0, + h3: 0, + h4: 0, + h5: 0, + h6: 0, + h7: 0, + }; + digest64HashObjectsInto(obj1, obj2, result); + return result; +} + +/** + * Same to above but this set result to the output param to save memory. + */ +export function digest64HashObjectsInto(obj1: HashObject, obj2: HashObject, output: HashObject): void { // TODO: expect obj1 and obj2 as HashObject inputUint32Array[0] = obj1.h0; inputUint32Array[1] = obj1.h1; @@ -73,7 +91,7 @@ export function digest64HashObjects(obj1: HashObject, obj2: HashObject): HashObj ctx.digest64(wasmInputValue, wasmOutputValue); // extracting numbers from Uint32Array causes more memory - return byteArrayToHashObject(outputUint8Array); + byteArrayIntoHashObject(outputUint8Array, 0, output); } /** @@ -121,6 +139,7 @@ export function batchHash4UintArray64s(inputs: Uint8Array[]): Uint8Array[] { * Inputs i0 i1 i2 i3 i4 i5 i6 i7 * \ / \ / \ / \ / * Outputs o0 o1 o2 o3 + * // TODO - batch: support equivalent method to hash into */ export function batchHash4HashObjectInputs(inputs: HashObject[]): HashObject[] { if (inputs.length !== 8) { @@ -227,14 +246,43 @@ export function batchHash4HashObjectInputs(inputs: HashObject[]): HashObject[] { ctx.batchHash4HashObjectInputs(wasmOutputValue); - const output0 = byteArrayToHashObject(outputUint8Array.subarray(0, 32)); - const output1 = byteArrayToHashObject(outputUint8Array.subarray(32, 64)); - const output2 = byteArrayToHashObject(outputUint8Array.subarray(64, 96)); - const output3 = byteArrayToHashObject(outputUint8Array.subarray(96, 128)); + const output0 = byteArrayToHashObject(outputUint8Array, 0); + const output1 = byteArrayToHashObject(outputUint8Array, 32); + const output2 = byteArrayToHashObject(outputUint8Array, 64); + const output3 = byteArrayToHashObject(outputUint8Array, 96); return [output0, output1, output2, output3]; } +/** + * Hash an input into preallocated input using batch if possible. + */ +export function hashInto(input: Uint8Array, output: Uint8Array): void { + if (input.length % 64 !== 0) { + throw new Error(`Invalid input length ${input.length}`); + } + if (input.length !== output.length * 2) { + throw new Error(`Invalid output length ${output.length}`); + } + // for every 64 x 4 = 256 bytes, do the batch hash + const endBatch = Math.floor(input.length / 256); + for (let i = 0; i < endBatch; i++) { + inputUint8Array.set(input.subarray(i * 256, (i + 1) * 256), 0); + ctx.batchHash4UintArray64s(wasmOutputValue); + output.set(outputUint8Array.subarray(0, 128), i * 128); + } + + const numHashed = endBatch * 4; + const remainingHash = Math.floor((input.length % 256) / 64); + const inputOffset = numHashed * 64; + const outputOffset = numHashed * 32; + for (let i = 0; i < remainingHash; i++) { + inputUint8Array.set(input.subarray(inputOffset + i * 64, inputOffset + (i + 1) * 64), 0); + ctx.digest64(wasmInputValue, wasmOutputValue); + output.set(outputUint8Array.subarray(0, 32), outputOffset + i * 32); + } +} + function update(data: Uint8Array): void { const INPUT_LENGTH = ctx.INPUT_LENGTH; if (data.length > INPUT_LENGTH) { diff --git a/packages/as-sha256/test/perf/index.test.ts b/packages/as-sha256/test/perf/index.test.ts index dfcd296e..c0ff38bb 100644 --- a/packages/as-sha256/test/perf/index.test.ts +++ b/packages/as-sha256/test/perf/index.test.ts @@ -16,8 +16,8 @@ describe("digestTwoHashObjects vs digest64 vs digest", () => { const input2 = "gajindergajindergajindergajinder"; const buffer1 = Buffer.from(input1, "utf-8"); const buffer2 = Buffer.from(input2, "utf-8"); - const obj1 = sha256.byteArrayToHashObject(buffer1); - const obj2 = sha256.byteArrayToHashObject(buffer2); + const obj1 = sha256.byteArrayToHashObject(buffer1, 0); + const obj2 = sha256.byteArrayToHashObject(buffer2, 0); // total number of time running hash for 200000 balances const iterations = 50023; itBench(`digestTwoHashObjects ${iterations} times`, () => { @@ -71,7 +71,7 @@ describe("hash - compare to java", () => { describe("utils", () => { const input1 = "gajindergajindergajindergajinder"; const buffer1 = Buffer.from(input1, "utf-8"); - const obj1 = sha256.byteArrayToHashObject(buffer1); + const obj1 = sha256.byteArrayToHashObject(buffer1, 0); // total number of time running hash for 200000 balances const iterations = 50023; @@ -82,6 +82,6 @@ describe("utils", () => { }); itBench(`byteArrayToHashObject ${iterations} times`, () => { - for (let j = 0; j < iterations; j++) sha256.byteArrayToHashObject(buffer1); + for (let j = 0; j < iterations; j++) sha256.byteArrayToHashObject(buffer1, 0); }); }); diff --git a/packages/as-sha256/test/perf/simd.test.ts b/packages/as-sha256/test/perf/simd.test.ts index b0fb372b..615cac33 100644 --- a/packages/as-sha256/test/perf/simd.test.ts +++ b/packages/as-sha256/test/perf/simd.test.ts @@ -5,12 +5,13 @@ import {byteArrayToHashObject} from "../../src/hashObject"; /** * This really depends on cpu, on a test ubuntu node, batch*() is 2x faster than digest64 * On Mac M1 May 2025: - * digest64 vs batchHash4UintArray64s vs batchHash4HashObjectInputs - ✓ digest64 200092 times 6.850496 ops/s 145.9748 ms/op - 66 runs 10.2 s - ✓ hash 200092 times using batchHash4UintArray64s 8.454788 ops/s 118.2762 ms/op - 82 runs 10.2 s - ✓ hash 200092 times using batchHash4HashObjectInputs 8.454464 ops/s 118.2807 ms/op - 82 runs 10.2 s + * digest64 vs batchHash4UintArray64s vs digest64HashObjects vs batchHash4HashObjectInputs + ✓ digest64 200092 times 6.648102 ops/s 150.4189 ms/op - 64 runs 10.2 s + ✓ hash 200092 times using batchHash4UintArray64s 9.120131 ops/s 109.6476 ms/op - 88 runs 10.2 s + ✓ digest64HashObjects 200092 times 7.095494 ops/s 140.9345 ms/op - 68 runs 10.2 s + ✓ hash 200092 times using batchHash4HashObjectInputs 9.211751 ops/s 108.5570 ms/op - 88 runs 10.1 s */ -describe("digest64 vs batchHash4UintArray64s vs batchHash4HashObjectInputs", function () { +describe("digest64 vs batchHash4UintArray64s vs digest64HashObjects vs batchHash4HashObjectInputs", function () { this.timeout(0); setBenchOpts({ @@ -31,7 +32,11 @@ describe("digest64 vs batchHash4UintArray64s vs batchHash4HashObjectInputs", fun } }); - const hashObject = byteArrayToHashObject(Buffer.from("gajindergajindergajindergajinder", "utf8")); + const hashObject = byteArrayToHashObject(Buffer.from("gajindergajindergajindergajinder", "utf8"), 0); + itBench(`digest64HashObjects ${iterations * 4} times`, () => { + for (let j = 0; j < iterations * 4; j++) sha256.digest64HashObjects(hashObject, hashObject); + }); + const hashInputs = Array.from({length: 8}, () => hashObject); // batchHash4HashObjectInputs do 4 sha256 in parallel itBench(`hash ${iterations * 4} times using batchHash4HashObjectInputs`, () => { diff --git a/packages/as-sha256/test/unit/index.test.ts b/packages/as-sha256/test/unit/index.test.ts index 0a08ac39..a4bf64b8 100644 --- a/packages/as-sha256/test/unit/index.test.ts +++ b/packages/as-sha256/test/unit/index.test.ts @@ -51,8 +51,8 @@ describe("sha256", function () { // digestTwoHashObjects should be the same to digest64 const buffer1 = Buffer.from(input1, "utf-8"); const buffer2 = Buffer.from(input2, "utf-8"); - const obj1 = sha256.byteArrayToHashObject(buffer1); - const obj2 = sha256.byteArrayToHashObject(buffer2); + const obj1 = sha256.byteArrayToHashObject(buffer1, 0); + const obj2 = sha256.byteArrayToHashObject(buffer2, 0); const obj = sha256.digest64HashObjects(obj1, obj2); const output2 = new Uint8Array(32); sha256.hashObjectToByteArray(obj, output2, 0); @@ -97,7 +97,7 @@ describe("sha256.hashObjectToByteArray and sha256.byteArrayToHashObject", functi ]; for (const [i, byteArr] of tcs.entries()) { it("test case " + i, function () { - const obj = sha256.byteArrayToHashObject(byteArr); + const obj = sha256.byteArrayToHashObject(byteArr, 0); const newByteArr = new Uint8Array(32); sha256.hashObjectToByteArray(obj, newByteArr, 0); expect(newByteArr).to.be.deep.equal(byteArr, "failed test case" + i); diff --git a/packages/as-sha256/test/unit/simd.test.ts b/packages/as-sha256/test/unit/simd.test.ts index 10e76b83..a17726b1 100644 --- a/packages/as-sha256/test/unit/simd.test.ts +++ b/packages/as-sha256/test/unit/simd.test.ts @@ -32,7 +32,7 @@ describe("Test SIMD implementation of as-sha256", () => { it("testHash4HashObjectInputs", () => { const input1 = "gajindergajindergajindergajinder"; - const inputHashObject = byteArrayToHashObject(Buffer.from(input1, "utf8")); + const inputHashObject = byteArrayToHashObject(Buffer.from(input1, "utf8"), 0); const outputs = sha256.batchHash4HashObjectInputs(Array.from({length: 8}, () => inputHashObject)); const expectedOutput = new Uint8Array([ 190, 57, 56, 15, 241, 208, 38, 30, 111, 55, 218, 254, 66, 120, 182, 98, 239, 97, 31, 28, 178, 247, 192, 161, @@ -44,4 +44,23 @@ describe("Test SIMD implementation of as-sha256", () => { expect(output).to.be.deep.equal(expectedOutput, "incorrect batchHash4UintArray64s result " + i); } }); + + const numHashes = [4, 5, 6, 7]; + for (const numHash of numHashes) { + it(`hashInto ${numHash} hashes`, () => { + const inputs = Array.from({length: numHash}, () => crypto.randomBytes(64)); + const input = new Uint8Array(numHash * 64); + for (let i = 0; i < numHash; i++) { + input.set(inputs[i], i * 64); + } + const output = new Uint8Array(numHash * 32); + + sha256.hashInto(input, output); + + const expectedOutputs = Array.from({length: numHash}, (_, i) => sha256.digest64(inputs[i])); + for (let i = 0; i < numHash; i++) { + expect(output.subarray(i * 32, (i + 1) * 32)).to.be.deep.equal(expectedOutputs[i]); + } + }); + } }); From 73b45664f59fa8e30680e88cea58be692c7a1132 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Sun, 14 Jul 2024 14:33:37 +0700 Subject: [PATCH 2/3] fix: uint8ArrayToHashObject compilation error --- packages/persistent-merkle-tree/src/hasher/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/persistent-merkle-tree/src/hasher/util.ts b/packages/persistent-merkle-tree/src/hasher/util.ts index b01e37bd..7f3f45ee 100644 --- a/packages/persistent-merkle-tree/src/hasher/util.ts +++ b/packages/persistent-merkle-tree/src/hasher/util.ts @@ -7,5 +7,5 @@ export function hashObjectToUint8Array(obj: HashObject): Uint8Array { } export function uint8ArrayToHashObject(byteArr: Uint8Array): HashObject { - return byteArrayToHashObject(byteArr); + return byteArrayToHashObject(byteArr, 0); } From c7dc5c23437e096e9629e9caea82f76193685c62 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Sun, 14 Jul 2024 14:38:23 +0700 Subject: [PATCH 3/3] fix: tree.test.ts compilation error --- packages/persistent-merkle-tree/test/unit/tree.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/persistent-merkle-tree/test/unit/tree.test.ts b/packages/persistent-merkle-tree/test/unit/tree.test.ts index 41d7339d..86eca941 100644 --- a/packages/persistent-merkle-tree/test/unit/tree.test.ts +++ b/packages/persistent-merkle-tree/test/unit/tree.test.ts @@ -89,7 +89,7 @@ describe("Tree.setNode vs Tree.setHashObjectFn", () => { tree.setNode(BigInt(18), LeafNode.fromRoot(Buffer.alloc(32, 2))); expect(toHex(tree.root)).to.equal("3cfd85690fdd88abcf22ca7acf45bb47835326ff3166d3c953d5a23263fea2b2"); // setHashObjectFn - const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2))); + const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2), 0)); const tree2 = new Tree(zeroNode(depth)); tree2.setNodeWithFn(BigInt(18), getNewNodeFn); expect(toHex(tree2.root)).to.equal("3cfd85690fdd88abcf22ca7acf45bb47835326ff3166d3c953d5a23263fea2b2"); @@ -103,7 +103,7 @@ describe("Tree.setNode vs Tree.setHashObjectFn", () => { tree.setNode(BigInt(60), LeafNode.fromRoot(Buffer.alloc(32, 2))); expect(toHex(tree.root)).to.equal("02607e58782c912e2f96f4ff9daf494d0d115e7c37e8c2b7ddce17213591151b"); // setHashObjectFn - const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2))); + const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2), 0)); const tree2 = new Tree(zeroNode(depth)); tree2.setNodeWithFn(BigInt(18), getNewNodeFn); tree2.setNodeWithFn(BigInt(46), getNewNodeFn);