Skip to content

Commit

Permalink
feat!: implement hashInto() api for as-sha256 (#382)
Browse files Browse the repository at this point in the history 
* feat: implement hashInto() api

* fix: uint8ArrayToHashObject compilation error

* fix: tree.test.ts compilation error
  • Loading branch information
@twoeths
twoeths authored Jul 16, 2024
1 parent 3a1c8dc commit ccadf43
Show file tree
Hide file tree
Showing 8 changed files with 156 additions and 76 deletions.
112 changes: 60 additions & 52 deletions packages/as-sha256/src/hashObject.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,95 +98,103 @@ export function hashObjectToByteArray(obj: HashObject, byteArr: Uint8Array, offs
* This function contains multiple same procedures but we intentionally
* do it step by step to improve performance a bit.
**/
export function byteArrayToHashObject(byteArr: Uint8Array): HashObject {
export function byteArrayToHashObject(byteArr: Uint8Array, offset: number): HashObject {
const result: HashObject = {
h0: 0,
h1: 0,
h2: 0,
h3: 0,
h4: 0,
h5: 0,
h6: 0,
h7: 0,
};

byteArrayIntoHashObject(byteArr, offset, result);
return result;
}

/**
* Same to above but this set result to the output param to save memory.
*/
export function byteArrayIntoHashObject(byteArr: Uint8Array, offset: number, output: HashObject): void {
let tmp = 0;
tmp |= byteArr[3] & 0xff;
tmp |= byteArr[3 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[2] & 0xff;
tmp |= byteArr[2 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[1] & 0xff;
tmp |= byteArr[1 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[0] & 0xff;
const h0 = tmp;
tmp |= byteArr[0 + offset] & 0xff;
output.h0 = tmp;

tmp = 0;
tmp |= byteArr[7] & 0xff;
tmp |= byteArr[7 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[6] & 0xff;
tmp |= byteArr[6 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[5] & 0xff;
tmp |= byteArr[5 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[4] & 0xff;
const h1 = tmp;
tmp |= byteArr[4 + offset] & 0xff;
output.h1 = tmp;

tmp = 0;
tmp |= byteArr[11] & 0xff;
tmp |= byteArr[11 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[10] & 0xff;
tmp |= byteArr[10 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[9] & 0xff;
tmp |= byteArr[9 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[8] & 0xff;
const h2 = tmp;
tmp |= byteArr[8 + offset] & 0xff;
output.h2 = tmp;

tmp = 0;
tmp |= byteArr[15] & 0xff;
tmp |= byteArr[15 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[14] & 0xff;
tmp |= byteArr[14 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[13] & 0xff;
tmp |= byteArr[13 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[12] & 0xff;
const h3 = tmp;
tmp |= byteArr[12 + offset] & 0xff;
output.h3 = tmp;

tmp = 0;
tmp |= byteArr[19] & 0xff;
tmp |= byteArr[19 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[18] & 0xff;
tmp |= byteArr[18 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[17] & 0xff;
tmp |= byteArr[17 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[16] & 0xff;
const h4 = tmp;
tmp |= byteArr[16 + offset] & 0xff;
output.h4 = tmp;

tmp = 0;
tmp |= byteArr[23] & 0xff;
tmp |= byteArr[23 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[22] & 0xff;
tmp |= byteArr[22 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[21] & 0xff;
tmp |= byteArr[21 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[20] & 0xff;
const h5 = tmp;
tmp |= byteArr[20 + offset] & 0xff;
output.h5 = tmp;

tmp = 0;
tmp |= byteArr[27] & 0xff;
tmp |= byteArr[27 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[26] & 0xff;
tmp |= byteArr[26 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[25] & 0xff;
tmp |= byteArr[25 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[24] & 0xff;
const h6 = tmp;
tmp |= byteArr[24 + offset] & 0xff;
output.h6 = tmp;

tmp = 0;
tmp |= byteArr[31] & 0xff;
tmp |= byteArr[31 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[30] & 0xff;
tmp |= byteArr[30 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[29] & 0xff;
tmp |= byteArr[29 + offset] & 0xff;
tmp = tmp << 8;
tmp |= byteArr[28] & 0xff;
const h7 = tmp;

return {
h0,
h1,
h2,
h3,
h4,
h5,
h6,
h7,
};
tmp |= byteArr[28 + offset] & 0xff;
output.h7 = tmp;
}
62 changes: 55 additions & 7 deletions packages/as-sha256/src/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {newInstance} from "./wasm";
import {HashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject";
import {HashObject, byteArrayIntoHashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject";
import SHA256 from "./sha256";
export {HashObject, byteArrayToHashObject, hashObjectToByteArray, SHA256};
export {HashObject, byteArrayToHashObject, hashObjectToByteArray, byteArrayIntoHashObject, SHA256};

const ctx = newInstance();
const wasmInputValue = ctx.input.value;
Expand Down Expand Up @@ -52,6 +52,24 @@ export function digest2Bytes32(bytes1: Uint8Array, bytes2: Uint8Array): Uint8Arr
* @returns
*/
export function digest64HashObjects(obj1: HashObject, obj2: HashObject): HashObject {
const result: HashObject = {
h0: 0,
h1: 0,
h2: 0,
h3: 0,
h4: 0,
h5: 0,
h6: 0,
h7: 0,
};
digest64HashObjectsInto(obj1, obj2, result);
return result;
}

/**
* Same to above but this set result to the output param to save memory.
*/
export function digest64HashObjectsInto(obj1: HashObject, obj2: HashObject, output: HashObject): void {
// TODO: expect obj1 and obj2 as HashObject
inputUint32Array[0] = obj1.h0;
inputUint32Array[1] = obj1.h1;
Expand All @@ -73,7 +91,7 @@ export function digest64HashObjects(obj1: HashObject, obj2: HashObject): HashObj
ctx.digest64(wasmInputValue, wasmOutputValue);

// extracting numbers from Uint32Array causes more memory
return byteArrayToHashObject(outputUint8Array);
byteArrayIntoHashObject(outputUint8Array, 0, output);
}

/**
Expand Down Expand Up @@ -121,6 +139,7 @@ export function batchHash4UintArray64s(inputs: Uint8Array[]): Uint8Array[] {
* Inputs i0 i1 i2 i3 i4 i5 i6 i7
* \ / \ / \ / \ /
* Outputs o0 o1 o2 o3
* // TODO - batch: support equivalent method to hash into
*/
export function batchHash4HashObjectInputs(inputs: HashObject[]): HashObject[] {
if (inputs.length !== 8) {
Expand Down Expand Up @@ -227,14 +246,43 @@ export function batchHash4HashObjectInputs(inputs: HashObject[]): HashObject[] {

ctx.batchHash4HashObjectInputs(wasmOutputValue);

const output0 = byteArrayToHashObject(outputUint8Array.subarray(0, 32));
const output1 = byteArrayToHashObject(outputUint8Array.subarray(32, 64));
const output2 = byteArrayToHashObject(outputUint8Array.subarray(64, 96));
const output3 = byteArrayToHashObject(outputUint8Array.subarray(96, 128));
const output0 = byteArrayToHashObject(outputUint8Array, 0);
const output1 = byteArrayToHashObject(outputUint8Array, 32);
const output2 = byteArrayToHashObject(outputUint8Array, 64);
const output3 = byteArrayToHashObject(outputUint8Array, 96);

return [output0, output1, output2, output3];
}

/**
* Hash an input into preallocated input using batch if possible.
*/
export function hashInto(input: Uint8Array, output: Uint8Array): void {
if (input.length % 64 !== 0) {
throw new Error(`Invalid input length ${input.length}`);
}
if (input.length !== output.length * 2) {
throw new Error(`Invalid output length ${output.length}`);
}
// for every 64 x 4 = 256 bytes, do the batch hash
const endBatch = Math.floor(input.length / 256);
for (let i = 0; i < endBatch; i++) {
inputUint8Array.set(input.subarray(i * 256, (i + 1) * 256), 0);
ctx.batchHash4UintArray64s(wasmOutputValue);
output.set(outputUint8Array.subarray(0, 128), i * 128);
}

const numHashed = endBatch * 4;
const remainingHash = Math.floor((input.length % 256) / 64);
const inputOffset = numHashed * 64;
const outputOffset = numHashed * 32;
for (let i = 0; i < remainingHash; i++) {
inputUint8Array.set(input.subarray(inputOffset + i * 64, inputOffset + (i + 1) * 64), 0);
ctx.digest64(wasmInputValue, wasmOutputValue);
output.set(outputUint8Array.subarray(0, 32), outputOffset + i * 32);
}
}

function update(data: Uint8Array): void {
const INPUT_LENGTH = ctx.INPUT_LENGTH;
if (data.length > INPUT_LENGTH) {
Expand Down
8 changes: 4 additions & 4 deletions packages/as-sha256/test/perf/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ describe("digestTwoHashObjects vs digest64 vs digest", () => {
const input2 = "gajindergajindergajindergajinder";
const buffer1 = Buffer.from(input1, "utf-8");
const buffer2 = Buffer.from(input2, "utf-8");
const obj1 = sha256.byteArrayToHashObject(buffer1);
const obj2 = sha256.byteArrayToHashObject(buffer2);
const obj1 = sha256.byteArrayToHashObject(buffer1, 0);
const obj2 = sha256.byteArrayToHashObject(buffer2, 0);
// total number of time running hash for 200000 balances
const iterations = 50023;
itBench(`digestTwoHashObjects ${iterations} times`, () => {
Expand Down Expand Up @@ -71,7 +71,7 @@ describe("hash - compare to java", () => {
describe("utils", () => {
const input1 = "gajindergajindergajindergajinder";
const buffer1 = Buffer.from(input1, "utf-8");
const obj1 = sha256.byteArrayToHashObject(buffer1);
const obj1 = sha256.byteArrayToHashObject(buffer1, 0);

// total number of time running hash for 200000 balances
const iterations = 50023;
Expand All @@ -82,6 +82,6 @@ describe("utils", () => {
});

itBench(`byteArrayToHashObject ${iterations} times`, () => {
for (let j = 0; j < iterations; j++) sha256.byteArrayToHashObject(buffer1);
for (let j = 0; j < iterations; j++) sha256.byteArrayToHashObject(buffer1, 0);
});
});
17 changes: 11 additions & 6 deletions packages/as-sha256/test/perf/simd.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ import {byteArrayToHashObject} from "../../src/hashObject";
/**
* This really depends on cpu, on a test ubuntu node, batch*() is 2x faster than digest64
* On Mac M1 May 2025:
* digest64 vs batchHash4UintArray64s vs batchHash4HashObjectInputs
✓ digest64 200092 times 6.850496 ops/s 145.9748 ms/op - 66 runs 10.2 s
✓ hash 200092 times using batchHash4UintArray64s 8.454788 ops/s 118.2762 ms/op - 82 runs 10.2 s
✓ hash 200092 times using batchHash4HashObjectInputs 8.454464 ops/s 118.2807 ms/op - 82 runs 10.2 s
* digest64 vs batchHash4UintArray64s vs digest64HashObjects vs batchHash4HashObjectInputs
✓ digest64 200092 times 6.648102 ops/s 150.4189 ms/op - 64 runs 10.2 s
✓ hash 200092 times using batchHash4UintArray64s 9.120131 ops/s 109.6476 ms/op - 88 runs 10.2 s
✓ digest64HashObjects 200092 times 7.095494 ops/s 140.9345 ms/op - 68 runs 10.2 s
✓ hash 200092 times using batchHash4HashObjectInputs 9.211751 ops/s 108.5570 ms/op - 88 runs 10.1 s
*/
describe("digest64 vs batchHash4UintArray64s vs batchHash4HashObjectInputs", function () {
describe("digest64 vs batchHash4UintArray64s vs digest64HashObjects vs batchHash4HashObjectInputs", function () {
this.timeout(0);

setBenchOpts({
Expand All @@ -31,7 +32,11 @@ describe("digest64 vs batchHash4UintArray64s vs batchHash4HashObjectInputs", fun
}
});

const hashObject = byteArrayToHashObject(Buffer.from("gajindergajindergajindergajinder", "utf8"));
const hashObject = byteArrayToHashObject(Buffer.from("gajindergajindergajindergajinder", "utf8"), 0);
itBench(`digest64HashObjects ${iterations * 4} times`, () => {
for (let j = 0; j < iterations * 4; j++) sha256.digest64HashObjects(hashObject, hashObject);
});

const hashInputs = Array.from({length: 8}, () => hashObject);
// batchHash4HashObjectInputs do 4 sha256 in parallel
itBench(`hash ${iterations * 4} times using batchHash4HashObjectInputs`, () => {
Expand Down
6 changes: 3 additions & 3 deletions packages/as-sha256/test/unit/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ describe("sha256", function () {
// digestTwoHashObjects should be the same to digest64
const buffer1 = Buffer.from(input1, "utf-8");
const buffer2 = Buffer.from(input2, "utf-8");
const obj1 = sha256.byteArrayToHashObject(buffer1);
const obj2 = sha256.byteArrayToHashObject(buffer2);
const obj1 = sha256.byteArrayToHashObject(buffer1, 0);
const obj2 = sha256.byteArrayToHashObject(buffer2, 0);
const obj = sha256.digest64HashObjects(obj1, obj2);
const output2 = new Uint8Array(32);
sha256.hashObjectToByteArray(obj, output2, 0);
Expand Down Expand Up @@ -97,7 +97,7 @@ describe("sha256.hashObjectToByteArray and sha256.byteArrayToHashObject", functi
];
for (const [i, byteArr] of tcs.entries()) {
it("test case " + i, function () {
const obj = sha256.byteArrayToHashObject(byteArr);
const obj = sha256.byteArrayToHashObject(byteArr, 0);
const newByteArr = new Uint8Array(32);
sha256.hashObjectToByteArray(obj, newByteArr, 0);
expect(newByteArr).to.be.deep.equal(byteArr, "failed test case" + i);
Expand Down
21 changes: 20 additions & 1 deletion packages/as-sha256/test/unit/simd.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ describe("Test SIMD implementation of as-sha256", () => {

it("testHash4HashObjectInputs", () => {
const input1 = "gajindergajindergajindergajinder";
const inputHashObject = byteArrayToHashObject(Buffer.from(input1, "utf8"));
const inputHashObject = byteArrayToHashObject(Buffer.from(input1, "utf8"), 0);
const outputs = sha256.batchHash4HashObjectInputs(Array.from({length: 8}, () => inputHashObject));
const expectedOutput = new Uint8Array([
190, 57, 56, 15, 241, 208, 38, 30, 111, 55, 218, 254, 66, 120, 182, 98, 239, 97, 31, 28, 178, 247, 192, 161,
Expand All @@ -44,4 +44,23 @@ describe("Test SIMD implementation of as-sha256", () => {
expect(output).to.be.deep.equal(expectedOutput, "incorrect batchHash4UintArray64s result " + i);
}
});

const numHashes = [4, 5, 6, 7];
for (const numHash of numHashes) {
it(`hashInto ${numHash} hashes`, () => {
const inputs = Array.from({length: numHash}, () => crypto.randomBytes(64));
const input = new Uint8Array(numHash * 64);
for (let i = 0; i < numHash; i++) {
input.set(inputs[i], i * 64);
}
const output = new Uint8Array(numHash * 32);

sha256.hashInto(input, output);

const expectedOutputs = Array.from({length: numHash}, (_, i) => sha256.digest64(inputs[i]));
for (let i = 0; i < numHash; i++) {
expect(output.subarray(i * 32, (i + 1) * 32)).to.be.deep.equal(expectedOutputs[i]);
}
});
}
});
2 changes: 1 addition & 1 deletion packages/persistent-merkle-tree/src/hasher/util.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ export function hashObjectToUint8Array(obj: HashObject): Uint8Array {
}

export function uint8ArrayToHashObject(byteArr: Uint8Array): HashObject {
return byteArrayToHashObject(byteArr);
return byteArrayToHashObject(byteArr, 0);
}
4 changes: 2 additions & 2 deletions packages/persistent-merkle-tree/test/unit/tree.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ describe("Tree.setNode vs Tree.setHashObjectFn", () => {
tree.setNode(BigInt(18), LeafNode.fromRoot(Buffer.alloc(32, 2)));
expect(toHex(tree.root)).to.equal("3cfd85690fdd88abcf22ca7acf45bb47835326ff3166d3c953d5a23263fea2b2");
// setHashObjectFn
const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2)));
const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2), 0));
const tree2 = new Tree(zeroNode(depth));
tree2.setNodeWithFn(BigInt(18), getNewNodeFn);
expect(toHex(tree2.root)).to.equal("3cfd85690fdd88abcf22ca7acf45bb47835326ff3166d3c953d5a23263fea2b2");
Expand All @@ -103,7 +103,7 @@ describe("Tree.setNode vs Tree.setHashObjectFn", () => {
tree.setNode(BigInt(60), LeafNode.fromRoot(Buffer.alloc(32, 2)));
expect(toHex(tree.root)).to.equal("02607e58782c912e2f96f4ff9daf494d0d115e7c37e8c2b7ddce17213591151b");
// setHashObjectFn
const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2)));
const getNewNodeFn = (): Node => LeafNode.fromHashObject(byteArrayToHashObject(Buffer.alloc(32, 2), 0));
const tree2 = new Tree(zeroNode(depth));
tree2.setNodeWithFn(BigInt(18), getNewNodeFn);
tree2.setNodeWithFn(BigInt(46), getNewNodeFn);
Expand Down

0 comments on commit ccadf43

Please sign in to comment.