From 17aadaf3dbe9f0fd76e4554e40d47c723b61ce2f Mon Sep 17 00:00:00 2001 From: Tommaso Allevi Date: Thu, 23 Nov 2023 11:29:02 +0100 Subject: [PATCH] Add indexable array --- src/index.ts | 75 ++++++++++++++++++++++++++--- tests/index.test.ts | 115 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 8 deletions(-) diff --git a/src/index.ts b/src/index.ts index 88157fc..2824a42 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,29 +1,35 @@ +type StrictArrayBuffer = ArrayBuffer & { buffer?: undefined }; + export interface Ser { index: number buffer: ArrayBuffer uint32Array: Uint32Array float32Array: Float32Array - getBuffer: () => ArrayBuffer + getBuffer: () => StrictArrayBuffer serializeBoolean: (b: boolean) => void serializeUInt32: (n: number) => void serializeFloat32: (n: number) => void serializeString: (str: string) => void serializeArray: (arr: T[], serialize: (ser: Ser, t: T) => void) => void serializeIterable: (iterable: Iterable, serialize: (ser: Ser, t: T) => void) => void + serializeIndexableArray: (arr: T[], serialize: (ser: Ser, t: T) => void) => void + unsafeSerializeUint32Array: (buffer: Uint32Array) => void } export interface Des { index: number - buffer: ArrayBuffer + buffer: StrictArrayBuffer uint32Array: Uint32Array float32Array: Float32Array - setBuffer: (buffer: ArrayBuffer) => void + setBuffer: (buffer: StrictArrayBuffer, byteOffset?: number, byteLength?: number) => void deserializeBoolean: () => boolean deserializeUInt32: () => number deserializeFloat32: () => number deserializeString: () => string deserializeArray: (deserialize: (des: Des) => T) => T[] deserializeIterable: (deserialize: (des: Des) => T) => Iterable + unsafeDeserializeUint32Array: () => Uint32Array + getArrayElements: (indexes: number[], deserialize: (des: Des, start: number, end: number) => T) => T[] } interface CreateSerOption { @@ -47,11 +53,13 @@ export function createSer ({ bufferSize }: CreateSerOption = {}): Ser { serializeString, serializeArray, serializeIterable, + serializeIndexableArray, + unsafeSerializeUint32Array, getBuffer: function () { return this.buffer.slice(0, this.index * 4) } } } -export function createDes (buffer: ArrayBuffer): Des { +export function createDes (buffer: StrictArrayBuffer): Des { const n32 = Math.floor(buffer.byteLength / 4) return { @@ -59,7 +67,18 @@ export function createDes (buffer: ArrayBuffer): Des { buffer, uint32Array: new Uint32Array(buffer, 0, n32), float32Array: new Float32Array(buffer, 0, n32), - setBuffer: function (buffer: ArrayBuffer) { + setBuffer: function (buffer: StrictArrayBuffer, byteOffset?: number, byteLength?: number) { + if (typeof byteOffset === 'number' && typeof byteLength === 'number') { + this.index = Math.floor(byteOffset / 4) + const n32 = this.index + Math.ceil(byteLength / 4) + + this.buffer = buffer + this.uint32Array = new Uint32Array(buffer, 0, n32) + this.float32Array = new Float32Array(buffer, 0, n32) + + return + } + const n32 = Math.floor(buffer.byteLength / 4) this.buffer = buffer this.index = 0 @@ -71,7 +90,9 @@ export function createDes (buffer: ArrayBuffer): Des { deserializeFloat32, deserializeString, deserializeArray, - deserializeIterable + deserializeIterable, + getArrayElements, + unsafeDeserializeUint32Array, } } @@ -136,7 +157,6 @@ function serializeIterable (this: Ser, iterable: Iterable, serialize: (ser } this.uint32Array[currentIndex] = n } - function deserializeIterable (this: Des, deserialize: (des: Des) => T): Iterable { const len = this.deserializeUInt32() const aGeneratorObject = (function * (des) { @@ -151,3 +171,44 @@ function deserializeIterable (this: Des, deserialize: (des: Des) => T): Itera } } } + +function unsafeSerializeUint32Array(this: Ser, arr: Uint32Array) { + const length = Math.ceil(arr.byteLength / 4) + this.uint32Array[this.index++] = length + this.uint32Array.set(arr, this.index) + this.index += length +} +function unsafeDeserializeUint32Array(this: Des): Uint32Array { + const byteLength = this.uint32Array[this.index++] + const d = new Uint32Array(this.buffer, this.index * 4, byteLength) + this.index += byteLength + return d +} + +function serializeIndexableArray(this: Ser, arr: T[], serialize: (ser: Ser, t: T) => void): void { + const l = arr.length + this.uint32Array[this.index++] = l + let indexOffsets = this.index + // Skip the length of the array twice + // to store the offset + length of the array element + this.index += l * 2 + for (let i = 0; i < l; i++) { + const offsetStart = this.index + serialize(this, arr[i]) + const offsetEnd = this.index + this.uint32Array[indexOffsets++] = offsetStart + this.uint32Array[indexOffsets++] = offsetEnd - offsetStart + } +} +function getArrayElements(this: Des, indexes: number[], deserialize: (des: Des, start: number, end: number) => T): T[] { + const currentIndex = this.index + 1 + const l = indexes.length + const arr = new Array(l) + for (let i = 0; i < l; i++) { + const indexOffset = currentIndex + indexes[i] * 2 + const start = this.uint32Array[indexOffset] + const end = this.uint32Array[indexOffset + 1] + arr[i] = deserialize(this, start * 4, end) + } + return arr +} diff --git a/tests/index.test.ts b/tests/index.test.ts index d4482b2..4aaec8a 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,6 +1,6 @@ import t from 'node:test' import assert from 'node:assert' -import { createSer, createDes } from '../src/index.js' +import { createSer, createDes, Ser, Des } from '../src/index.js' await t.test('boolean', async t => { const bools = [ @@ -261,6 +261,119 @@ await t.test('iterable', async t => { }) }) +await t.test('setBuffer with options', async t => { + await t.test('uint32', async () => { + const ser = createSer() + ser.serializeUInt32(1) + ser.serializeUInt32(2) + ser.serializeUInt32(3) + ser.serializeUInt32(4) + const buff = ser.getBuffer() + + const des = createDes(new ArrayBuffer(0)) + { + des.setBuffer(buff, 0, 4) + assert.equal(des.deserializeUInt32(), 1) + } + { + des.setBuffer(buff, 4, 4) + + assert.equal(des.deserializeUInt32(), 2) + } + { + des.setBuffer(buff, 8, 4) + assert.equal(des.deserializeUInt32(), 3) + } + { + des.setBuffer(buff, 12, 4) + assert.equal(des.deserializeUInt32(), 4) + } + { + des.setBuffer(buff, 4, 12) + assert.equal(des.deserializeUInt32(), 2) + assert.equal(des.deserializeUInt32(), 3) + assert.equal(des.deserializeUInt32(), 4) + } + }) + + await t.test('uint32 & string', async () => { + const ser = createSer() + ser.serializeUInt32(1) + ser.serializeString("v1") + const buff = ser.getBuffer() + + const des = createDes(new ArrayBuffer(0)) + { + des.setBuffer(buff, 0, 12) + assert.equal(des.deserializeUInt32(), 1) + assert.equal(des.deserializeString(), "v1") + } + { + des.setBuffer(buff, 4, 8) + assert.equal(des.deserializeString(), "v1") + } + }) +}) + +function serializeItem(ser: Ser, t: { foo: string, bar: number}) { + ser.serializeString(t.foo) + ser.serializeUInt32(t.bar) +} + +function deserializeItem(des: Des): { foo: string, bar: number} { + const foo = des.deserializeString() + const bar = des.deserializeUInt32() + return { + foo, + bar, + } +} + +await t.test('serialize + getArrayelements + serialize unsafe + deserialize with deserialize unsafe', async () => { + const arr = [ + { foo: 'v1', bar: 42 }, + { foo: 'v2', bar: 2 }, + { foo: 'v3', bar: 99 } + ] + const elementIndexes = [0, 2] + + let docsStorageBuffer: ArrayBuffer + { + const ser = createSer() + ser.serializeIndexableArray(arr, serializeItem) + docsStorageBuffer = ser.getBuffer() + } + + let foo: ArrayBuffer + { + const ser = createSer() + + const des = createDes(docsStorageBuffer) + const elements = des.getArrayElements(elementIndexes, function (_des, offset, length) { + return new Uint32Array(docsStorageBuffer, offset, length) + }) + + ser.serializeArray(elements, (ser, uint32Array) => { + ser.unsafeSerializeUint32Array(uint32Array) + }) + foo = ser.getBuffer() + } + + let found: { foo: string, bar: number }[] + { + const des = createDes(foo) + + const itemDes = createDes(new ArrayBuffer(0)) + found = des.deserializeArray((des) => { + const buff = des.unsafeDeserializeUint32Array() + itemDes.setBuffer(buff.buffer, buff.byteOffset, buff.byteLength) + return deserializeItem(itemDes) + }) + } + + assert.deepStrictEqual(found, elementIndexes.map(i => arr[i])) +}) + await t.test('with option', async t => { await t.test('bufferSize', async t => { {