From ba354756bbec60055bbeb4a6ee5bc3ab73267312 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 23 Oct 2023 15:32:05 +0300 Subject: [PATCH] feat: support jstype custom options (#117) Allow overriding 64 bit number types with strings or regular js numbers. Closes #112 --- packages/protons-benchmark/package.json | 12 +- .../src/{ => implementations}/bench.proto | 0 .../src/{ => implementations}/index.ts | 0 .../src/{ => implementations}/pbjs/bench.ts | 0 .../protobuf-ts/bench.ts | 0 .../protobufjs/bench.d.ts | 0 .../{ => implementations}/protobufjs/bench.js | 0 .../{ => implementations}/protobufjs/rpc.d.ts | 0 .../{ => implementations}/protobufjs/rpc.js | 0 .../{ => implementations}/protons/bench.ts | 0 .../src/{ => implementations}/protons/rpc.ts | 0 .../protons-benchmark/src/numbers/index.ts | 15 ++ .../protons-benchmark/src/numbers/read.ts | 42 ++++ .../protons-benchmark/src/numbers/write.ts | 39 +++ packages/protons-benchmark/tsconfig.json | 4 +- packages/protons-runtime/src/index.ts | 100 ++++++++ .../protons-runtime/src/utils/longbits.ts | 56 ++++- packages/protons-runtime/src/utils/reader.ts | 75 ++++++ packages/protons-runtime/src/utils/writer.ts | 72 ++++++ packages/protons/src/index.ts | 204 ++++++++++++++-- packages/protons/test/custom-options.spec.ts | 34 +++ .../test/fixtures/custom-option-jstype.proto | 19 ++ .../test/fixtures/custom-option-jstype.ts | 225 ++++++++++++++++++ 23 files changed, 853 insertions(+), 44 deletions(-) rename packages/protons-benchmark/src/{ => implementations}/bench.proto (100%) rename packages/protons-benchmark/src/{ => implementations}/index.ts (100%) rename packages/protons-benchmark/src/{ => implementations}/pbjs/bench.ts (100%) rename packages/protons-benchmark/src/{ => implementations}/protobuf-ts/bench.ts (100%) rename packages/protons-benchmark/src/{ => implementations}/protobufjs/bench.d.ts (100%) rename packages/protons-benchmark/src/{ => implementations}/protobufjs/bench.js (100%) rename packages/protons-benchmark/src/{ => implementations}/protobufjs/rpc.d.ts (100%) rename packages/protons-benchmark/src/{ => implementations}/protobufjs/rpc.js (100%) rename packages/protons-benchmark/src/{ => implementations}/protons/bench.ts (100%) rename packages/protons-benchmark/src/{ => implementations}/protons/rpc.ts (100%) create mode 100644 packages/protons-benchmark/src/numbers/index.ts create mode 100644 packages/protons-benchmark/src/numbers/read.ts create mode 100644 packages/protons-benchmark/src/numbers/write.ts create mode 100644 packages/protons/test/custom-options.spec.ts create mode 100644 packages/protons/test/fixtures/custom-option-jstype.proto create mode 100644 packages/protons/test/fixtures/custom-option-jstype.ts diff --git a/packages/protons-benchmark/package.json b/packages/protons-benchmark/package.json index f881ee4..f8e1354 100644 --- a/packages/protons-benchmark/package.json +++ b/packages/protons-benchmark/package.json @@ -32,19 +32,19 @@ "sourceType": "module" }, "ignorePatterns": [ - "src/pbjs/*", - "src/protobuf-ts/*", - "src/protobufjs/*" + "src/implementations/pbjs/*", + "src/implementations/protobuf-ts/*", + "src/implementations/protobufjs/*" ] }, "scripts": { "clean": "aegir clean", "lint": "aegir lint", "dep-check": "aegir dep-check --ignore @protobuf-ts/plugin pbjs protons", - "build": "aegir build --no-bundle && cp -R src/protobufjs dist/src/protobufjs", + "build": "aegir build --no-bundle && cp -R src/implementations/protobufjs dist/src/implementations/protobufjs", "prestart": "npm run build", - "start": "node dist/src/index.js", - "start:browser": "npx playwright-test dist/src/index.js --runner benchmark" + "start": "node dist/src/implementations/index.js", + "start:browser": "npx playwright-test dist/src/implementations/index.js --runner benchmark" }, "dependencies": { "@protobuf-ts/plugin": "^2.8.1", diff --git a/packages/protons-benchmark/src/bench.proto b/packages/protons-benchmark/src/implementations/bench.proto similarity index 100% rename from packages/protons-benchmark/src/bench.proto rename to packages/protons-benchmark/src/implementations/bench.proto diff --git a/packages/protons-benchmark/src/index.ts b/packages/protons-benchmark/src/implementations/index.ts similarity index 100% rename from packages/protons-benchmark/src/index.ts rename to packages/protons-benchmark/src/implementations/index.ts diff --git a/packages/protons-benchmark/src/pbjs/bench.ts b/packages/protons-benchmark/src/implementations/pbjs/bench.ts similarity index 100% rename from packages/protons-benchmark/src/pbjs/bench.ts rename to packages/protons-benchmark/src/implementations/pbjs/bench.ts diff --git a/packages/protons-benchmark/src/protobuf-ts/bench.ts b/packages/protons-benchmark/src/implementations/protobuf-ts/bench.ts similarity index 100% rename from packages/protons-benchmark/src/protobuf-ts/bench.ts rename to packages/protons-benchmark/src/implementations/protobuf-ts/bench.ts diff --git a/packages/protons-benchmark/src/protobufjs/bench.d.ts b/packages/protons-benchmark/src/implementations/protobufjs/bench.d.ts similarity index 100% rename from packages/protons-benchmark/src/protobufjs/bench.d.ts rename to packages/protons-benchmark/src/implementations/protobufjs/bench.d.ts diff --git a/packages/protons-benchmark/src/protobufjs/bench.js b/packages/protons-benchmark/src/implementations/protobufjs/bench.js similarity index 100% rename from packages/protons-benchmark/src/protobufjs/bench.js rename to packages/protons-benchmark/src/implementations/protobufjs/bench.js diff --git a/packages/protons-benchmark/src/protobufjs/rpc.d.ts b/packages/protons-benchmark/src/implementations/protobufjs/rpc.d.ts similarity index 100% rename from packages/protons-benchmark/src/protobufjs/rpc.d.ts rename to packages/protons-benchmark/src/implementations/protobufjs/rpc.d.ts diff --git a/packages/protons-benchmark/src/protobufjs/rpc.js b/packages/protons-benchmark/src/implementations/protobufjs/rpc.js similarity index 100% rename from packages/protons-benchmark/src/protobufjs/rpc.js rename to packages/protons-benchmark/src/implementations/protobufjs/rpc.js diff --git a/packages/protons-benchmark/src/protons/bench.ts b/packages/protons-benchmark/src/implementations/protons/bench.ts similarity index 100% rename from packages/protons-benchmark/src/protons/bench.ts rename to packages/protons-benchmark/src/implementations/protons/bench.ts diff --git a/packages/protons-benchmark/src/protons/rpc.ts b/packages/protons-benchmark/src/implementations/protons/rpc.ts similarity index 100% rename from packages/protons-benchmark/src/protons/rpc.ts rename to packages/protons-benchmark/src/implementations/protons/rpc.ts diff --git a/packages/protons-benchmark/src/numbers/index.ts b/packages/protons-benchmark/src/numbers/index.ts new file mode 100644 index 0000000..6af16cd --- /dev/null +++ b/packages/protons-benchmark/src/numbers/index.ts @@ -0,0 +1,15 @@ +/* eslint-disable no-console */ + +/* +$ node dist/src/numbers/index.js +$ npx playwright-test dist/src/numbers/index.js --runner benchmark +*/ + +import { readBenchmark } from './read.js' +import { writeBenchmark } from './write.js' + +console.info('-- read --') +await readBenchmark() + +console.info('-- write --') +await writeBenchmark() diff --git a/packages/protons-benchmark/src/numbers/read.ts b/packages/protons-benchmark/src/numbers/read.ts new file mode 100644 index 0000000..4f697b7 --- /dev/null +++ b/packages/protons-benchmark/src/numbers/read.ts @@ -0,0 +1,42 @@ +/* eslint-disable no-console */ + +import Benchmark from 'benchmark' +import { writer, reader } from 'protons-runtime' + +const bigint = 100n + +const w = writer() +w.uint64(bigint) + +const buf = w.finish() + +export async function readBenchmark (): Promise { + return new Promise((resolve) => { + new Benchmark.Suite() + .add('uint64 (BigInt)', () => { + const r = reader(buf) + r.uint64() + }) + .add('uint64number', () => { + const r = reader(buf) + r.uint64Number() + }) + .add('uint64string', () => { + const r = reader(buf) + r.uint64String() + }) + .on('error', (err: Error) => { + console.error(err) + }) + .on('cycle', (event: any) => { + console.info(String(event.target)) + }) + .on('complete', function () { + // @ts-expect-error types are wrong + console.info(`Fastest is ${this.filter('fastest').map('name')}`) + resolve() + }) + // run async + .run({ async: true }) + }) +} diff --git a/packages/protons-benchmark/src/numbers/write.ts b/packages/protons-benchmark/src/numbers/write.ts new file mode 100644 index 0000000..a3c87a7 --- /dev/null +++ b/packages/protons-benchmark/src/numbers/write.ts @@ -0,0 +1,39 @@ +/* eslint-disable no-console */ + +import Benchmark from 'benchmark' +import { writer } from 'protons-runtime' + +const number = 100 +const bigint = 100n +const string = '100' + +export async function writeBenchmark (): Promise { + return new Promise((resolve) => { + new Benchmark.Suite() + .add('uint64 (BigInt)', () => { + const w = writer() + w.uint64(bigint) + }) + .add('uint64number', () => { + const w = writer() + w.uint64Number(number) + }) + .add('uint64string', () => { + const w = writer() + w.uint64String(string) + }) + .on('error', (err: Error) => { + console.error(err) + }) + .on('cycle', (event: any) => { + console.info(String(event.target)) + }) + .on('complete', function () { + // @ts-expect-error types are wrong + console.info(`Fastest is ${this.filter('fastest').map('name')}`) + resolve() + }) + // run async + .run({ async: true }) + }) +} diff --git a/packages/protons-benchmark/tsconfig.json b/packages/protons-benchmark/tsconfig.json index 86b3837..9ff49b6 100644 --- a/packages/protons-benchmark/tsconfig.json +++ b/packages/protons-benchmark/tsconfig.json @@ -9,8 +9,8 @@ "test" ], "exclude": [ - "src/protobufjs/bench.js", - "src/protobufjs/rpc.js" + "src/implementations/protobufjs/bench.js", + "src/implementations/protobufjs/rpc.js" ], "references": [ { diff --git a/packages/protons-runtime/src/index.ts b/packages/protons-runtime/src/index.ts index 56a57ea..d22fa35 100644 --- a/packages/protons-runtime/src/index.ts +++ b/packages/protons-runtime/src/index.ts @@ -48,16 +48,46 @@ export interface Writer { */ uint64(value: bigint): this + /** + * Writes an unsigned 64 bit value as a varint + */ + uint64Number(value: number): this + + /** + * Writes an unsigned 64 bit value as a varint + */ + uint64String(value: string): this + /** * Writes a signed 64 bit value as a varint */ int64(value: bigint): this + /** + * Writes a signed 64 bit value as a varint + */ + int64Number(value: number): this + + /** + * Writes a signed 64 bit value as a varint + */ + int64String(value: string): this + /** * Writes a signed 64 bit value as a varint, zig-zag encoded */ sint64(value: bigint): this + /** + * Writes a signed 64 bit value as a varint, zig-zag encoded + */ + sint64Number(value: number): this + + /** + * Writes a signed 64 bit value as a varint, zig-zag encoded + */ + sint64String(value: string): this + /** * Writes a boolish value as a varint */ @@ -78,11 +108,31 @@ export interface Writer { */ fixed64(value: bigint): this + /** + * Writes an unsigned 64 bit value as fixed 64 bits + */ + fixed64Number(value: number): this + + /** + * Writes an unsigned 64 bit value as fixed 64 bits + */ + fixed64String(value: string): this + /** * Writes a signed 64 bit value as fixed 64 bits */ sfixed64(value: bigint): this + /** + * Writes a signed 64 bit value as fixed 64 bits + */ + sfixed64Number(value: number): this + + /** + * Writes a signed 64 bit value as fixed 64 bits + */ + sfixed64String(value: string): this + /** * Writes a float (32 bit) */ @@ -206,23 +256,73 @@ export interface Reader { */ int64(): bigint + /** + * Reads a varint as a signed 64 bit value + */ + int64Number(): number + + /** + * Reads a varint as a signed 64 bit value + */ + int64String(): string + /** * Reads a varint as an unsigned 64 bit value */ uint64(): bigint + /** + * Reads a varint as an unsigned 64 bit value + */ + uint64Number(): number + + /** + * Reads a varint as an unsigned 64 bit value + */ + uint64String(): string + /** * Reads a zig-zag encoded varint as a signed 64 bit value */ sint64(): bigint + /** + * Reads a zig-zag encoded varint as a signed 64 bit value + */ + sint64Number(): number + + /** + * Reads a zig-zag encoded varint as a signed 64 bit value + */ + sint64String(): string + /** * Reads fixed 64 bits */ fixed64(): bigint + /** + * Reads fixed 64 bits + */ + fixed64Number(): number + + /** + * Reads fixed 64 bits + */ + fixed64String(): string + /** * Reads zig-zag encoded fixed 64 bits */ sfixed64(): bigint + + /** + * Reads zig-zag encoded fixed 64 bits + */ + sfixed64Number(): number + + /** + * Reads zig-zag encoded fixed 64 bits + */ + sfixed64String(): string } diff --git a/packages/protons-runtime/src/utils/longbits.ts b/packages/protons-runtime/src/utils/longbits.ts index a8e41ea..12cc5d4 100644 --- a/packages/protons-runtime/src/utils/longbits.ts +++ b/packages/protons-runtime/src/utils/longbits.ts @@ -1,3 +1,7 @@ +// the largest BigInt we can safely downcast to a Number +const MAX_SAFE_NUMBER_INTEGER = BigInt(Number.MAX_SAFE_INTEGER) +const MIN_SAFE_NUMBER_INTEGER = BigInt(Number.MIN_SAFE_INTEGER) + /** * Constructs new long bits. * @@ -29,10 +33,24 @@ export class LongBits { /** * Converts this long bits to a possibly unsafe JavaScript number */ + toNumber (unsigned: boolean = false): number { + if (!unsigned && (this.hi >>> 31) > 0) { + const lo = ~this.lo + 1 >>> 0 + let hi = ~this.hi >>> 0 + if (lo === 0) { + hi = hi + 1 >>> 0 + } + return -(lo + hi * 4294967296) + } + return this.lo + this.hi * 4294967296 + } + + /** + * Converts this long bits to a bigint + */ toBigInt (unsigned: boolean = false): bigint { if (unsigned) { - const result = BigInt(this.lo >>> 0) + (BigInt(this.hi >>> 0) << 32n) - return result + return BigInt(this.lo >>> 0) + (BigInt(this.hi >>> 0) << 32n) } if ((this.hi >>> 31) !== 0) { @@ -47,6 +65,13 @@ export class LongBits { return BigInt(this.lo >>> 0) + (BigInt(this.hi >>> 0) << 32n) } + /** + * Converts this long bits to a string + */ + toString (unsigned: boolean = false): string { + return this.toBigInt(unsigned).toString() + } + /** * Zig-zag encodes this long bits */ @@ -89,25 +114,34 @@ export class LongBits { * Constructs new long bits from the specified number */ static fromBigInt (value: bigint): LongBits { - if (value === 0n) { return zero } + if (value === 0n) { + return zero + } + + if (value < MAX_SAFE_NUMBER_INTEGER && value > MIN_SAFE_NUMBER_INTEGER) { + return this.fromNumber(Number(value)) + } + + const negative = value < 0n - const negative = value < 0 if (negative) { value = -value } - let hi = Number(value >> 32n) - let lo = Number(value - (BigInt(hi) << 32n)) + + let hi = value >> 32n + let lo = value - (hi << 32n) if (negative) { - hi = ~hi >>> 0 - lo = ~lo >>> 0 + hi = ~hi | 0n + lo = ~lo | 0n + if (++lo > TWO_32) { - lo = 0 - if (++hi > TWO_32) { hi = 0 } + lo = 0n + if (++hi > TWO_32) { hi = 0n } } } - return new LongBits(lo, hi) + return new LongBits(Number(lo), Number(hi)) } /** diff --git a/packages/protons-runtime/src/utils/reader.ts b/packages/protons-runtime/src/utils/reader.ts index a70c441..a594c01 100644 --- a/packages/protons-runtime/src/utils/reader.ts +++ b/packages/protons-runtime/src/utils/reader.ts @@ -277,6 +277,21 @@ export class Uint8ArrayReader implements Reader { return this.readLongVarint().toBigInt() } + /** + * Reads a varint as a signed 64 bit value returned as a possibly unsafe + * JavaScript number + */ + int64Number (): number { + return this.readLongVarint().toNumber() + } + + /** + * Reads a varint as a signed 64 bit value returned as a string + */ + int64String (): string { + return this.readLongVarint().toString() + } + /** * Reads a varint as an unsigned 64 bit value */ @@ -284,6 +299,21 @@ export class Uint8ArrayReader implements Reader { return this.readLongVarint().toBigInt(true) } + /** + * Reads a varint as an unsigned 64 bit value returned as a possibly unsafe + * JavaScript number + */ + uint64Number (): number { + return this.readLongVarint().toNumber(true) + } + + /** + * Reads a varint as an unsigned 64 bit value returned as a string + */ + uint64String (): string { + return this.readLongVarint().toString(true) + } + /** * Reads a zig-zag encoded varint as a signed 64 bit value */ @@ -291,6 +321,22 @@ export class Uint8ArrayReader implements Reader { return this.readLongVarint().zzDecode().toBigInt() } + /** + * Reads a zig-zag encoded varint as a signed 64 bit value returned as a + * possibly unsafe JavaScript number + */ + sint64Number (): number { + return this.readLongVarint().zzDecode().toNumber() + } + + /** + * Reads a zig-zag encoded varint as a signed 64 bit value returned as a + * string + */ + sint64String (): string { + return this.readLongVarint().zzDecode().toString() + } + /** * Reads fixed 64 bits */ @@ -298,12 +344,41 @@ export class Uint8ArrayReader implements Reader { return this.readFixed64().toBigInt() } + /** + * Reads fixed 64 bits returned as a possibly unsafe JavaScript number + */ + fixed64Number (): number { + return this.readFixed64().toNumber() + } + + /** + * Reads fixed 64 bits returned as a string + */ + fixed64String (): string { + return this.readFixed64().toString() + } + /** * Reads zig-zag encoded fixed 64 bits */ sfixed64 (): bigint { return this.readFixed64().toBigInt() } + + /** + * Reads zig-zag encoded fixed 64 bits returned as a possibly unsafe + * JavaScript number + */ + sfixed64Number (): number { + return this.readFixed64().toNumber() + } + + /** + * Reads zig-zag encoded fixed 64 bits returned as a string + */ + sfixed64String (): string { + return this.readFixed64().toString() + } } export function createReader (buf: Uint8Array | Uint8ArrayList): Reader { diff --git a/packages/protons-runtime/src/utils/writer.ts b/packages/protons-runtime/src/utils/writer.ts index 0c0677e..9ebc6d5 100644 --- a/packages/protons-runtime/src/utils/writer.ts +++ b/packages/protons-runtime/src/utils/writer.ts @@ -182,6 +182,21 @@ class Uint8ArrayWriter implements Writer { return this._push(writeVarint64, bits.length(), bits) } + /** + * Writes an unsigned 64 bit value as a varint + */ + uint64Number (value: number): this { + const bits = LongBits.fromNumber(value) + return this._push(writeVarint64, bits.length(), bits) + } + + /** + * Writes an unsigned 64 bit value as a varint + */ + uint64String (value: string): this { + return this.uint64(BigInt(value)) + } + /** * Writes a signed 64 bit value as a varint */ @@ -189,6 +204,20 @@ class Uint8ArrayWriter implements Writer { return this.uint64(value) } + /** + * Writes a signed 64 bit value as a varint + */ + int64Number (value: number): this { + return this.uint64Number(value) + } + + /** + * Writes a signed 64 bit value as a varint + */ + int64String (value: string): this { + return this.uint64String(value) + } + /** * Writes a signed 64 bit value as a varint, zig-zag encoded */ @@ -197,6 +226,21 @@ class Uint8ArrayWriter implements Writer { return this._push(writeVarint64, bits.length(), bits) } + /** + * Writes a signed 64 bit value as a varint, zig-zag encoded + */ + sint64Number (value: number): this { + const bits = LongBits.fromNumber(value).zzEncode() + return this._push(writeVarint64, bits.length(), bits) + } + + /** + * Writes a signed 64 bit value as a varint, zig-zag encoded + */ + sint64String (value: string): this { + return this.sint64(BigInt(value)) + } + /** * Writes a boolish value as a varint */ @@ -223,10 +267,24 @@ class Uint8ArrayWriter implements Writer { */ fixed64 (value: bigint): this { const bits = LongBits.fromBigInt(value) + return this._push(writeFixed32, 4, bits.lo)._push(writeFixed32, 4, bits.hi) + } + /** + * Writes an unsigned 64 bit value as fixed 64 bits + */ + fixed64Number (value: number): this { + const bits = LongBits.fromNumber(value) return this._push(writeFixed32, 4, bits.lo)._push(writeFixed32, 4, bits.hi) } + /** + * Writes an unsigned 64 bit value as fixed 64 bits + */ + fixed64String (value: string): this { + return this.fixed64(BigInt(value)) + } + /** * Writes a signed 64 bit value as fixed 64 bits */ @@ -234,6 +292,20 @@ class Uint8ArrayWriter implements Writer { return this.fixed64(value) } + /** + * Writes a signed 64 bit value as fixed 64 bits + */ + sfixed64Number (value: number): this { + return this.fixed64Number(value) + } + + /** + * Writes a signed 64 bit value as fixed 64 bits + */ + sfixed64String (value: string): this { + return this.fixed64String(value) + } + /** * Writes a float (32 bit) */ diff --git a/packages/protons/src/index.ts b/packages/protons/src/index.ts index 6801293..6476832 100644 --- a/packages/protons/src/index.ts +++ b/packages/protons/src/index.ts @@ -47,40 +47,145 @@ const types: Record = { uint64: 'bigint' } -const encoderGenerators: Record string> = { +const jsTypeOverrides: Record = { + JS_NUMBER: 'number', + JS_STRING: 'string' +} + +const encoderGenerators: Record string> = { bool: (val) => `w.bool(${val})`, bytes: (val) => `w.bytes(${val})`, double: (val) => `w.double(${val})`, fixed32: (val) => `w.fixed32(${val})`, - fixed64: (val) => `w.fixed64(${val})`, + fixed64: (val, jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return `w.fixed64Number(${val})` + } + + if (jsTypeOverride === 'string') { + return `w.fixed64String(${val})` + } + + return `w.fixed64(${val})` + }, float: (val) => `w.float(${val})`, int32: (val) => `w.int32(${val})`, - int64: (val) => `w.int64(${val})`, + int64: (val, jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return `w.int64Number(${val})` + } + + if (jsTypeOverride === 'string') { + return `w.int64String(${val})` + } + + return `w.int64(${val})` + }, sfixed32: (val) => `w.sfixed32(${val})`, - sfixed64: (val) => `w.sfixed64(${val})`, + sfixed64: (val, jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return `w.sfixed64Number(${val})` + } + + if (jsTypeOverride === 'string') { + return `w.sfixed64String(${val})` + } + + return `w.sfixed64(${val})` + }, sint32: (val) => `w.sint32(${val})`, - sint64: (val) => `w.sint64(${val})`, + sint64: (val, jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return `w.sint64Number(${val})` + } + + if (jsTypeOverride === 'string') { + return `w.sint64String(${val})` + } + + return `w.sint64(${val})` + }, string: (val) => `w.string(${val})`, uint32: (val) => `w.uint32(${val})`, - uint64: (val) => `w.uint64(${val})` + uint64: (val, jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return `w.uint64Number(${val})` + } + + if (jsTypeOverride === 'string') { + return `w.uint64String(${val})` + } + + return `w.uint64(${val})` + } } -const decoderGenerators: Record string> = { +const decoderGenerators: Record string> = { bool: () => 'reader.bool()', bytes: () => 'reader.bytes()', double: () => 'reader.double()', fixed32: () => 'reader.fixed32()', - fixed64: () => 'reader.fixed64()', + fixed64: (jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return 'reader.fixed64Number()' + } + + if (jsTypeOverride === 'string') { + return 'reader.fixed64String()' + } + + return 'reader.fixed64()' + }, float: () => 'reader.float()', int32: () => 'reader.int32()', - int64: () => 'reader.int64()', + int64: (jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return 'reader.int64Number()' + } + + if (jsTypeOverride === 'string') { + return 'reader.int64String()' + } + + return 'reader.int64()' + }, sfixed32: () => 'reader.sfixed32()', - sfixed64: () => 'reader.sfixed64()', + sfixed64: (jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return 'reader.sfixed64Number()' + } + + if (jsTypeOverride === 'string') { + return 'reader.sfixed64String()' + } + + return 'reader.sfixed64()' + }, sint32: () => 'reader.sint32()', - sint64: () => 'reader.sint64()', + sint64: (jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return 'reader.sint64Number()' + } + + if (jsTypeOverride === 'string') { + return 'reader.sint64String()' + } + + return 'reader.sint64()' + }, string: () => 'reader.string()', uint32: () => 'reader.uint32()', - uint64: () => 'reader.uint64()' + uint64: (jsTypeOverride) => { + if (jsTypeOverride === 'number') { + return 'reader.uint64Number()' + } + + if (jsTypeOverride === 'string') { + return 'reader.uint64String()' + } + + return 'reader.uint64()' + } } const defaultValueGenerators: Record string> = { @@ -101,6 +206,11 @@ const defaultValueGenerators: Record string> = { uint64: () => '0n' } +const defaultValueGeneratorsJsTypeOverrides: Record string> = { + number: () => '0', + string: () => "''" +} + const defaultValueTestGenerators: Record string> = { bool: (field) => `(${field} != null && ${field} !== false)`, bytes: (field) => `(${field} != null && ${field}.byteLength > 0)`, @@ -119,7 +229,28 @@ const defaultValueTestGenerators: Record string> = { uint64: (field) => `(${field} != null && ${field} !== 0n)` } -function findTypeName (typeName: string, classDef: MessageDef, moduleDef: ModuleDef): string { +const defaultValueTestGeneratorsJsTypeOverrides: Record string> = { + number: (field) => `(${field} != null && ${field} !== 0)`, + string: (field) => `(${field} != null && ${field} !== '')` +} + +function findJsTypeOverride (defaultType: string, fieldDef: FieldDef): 'number' | 'string' | undefined { + if (fieldDef.options?.jstype != null && jsTypeOverrides[fieldDef.options?.jstype] != null) { + if (!['int64', 'uint64', 'sint64', 'fixed64', 'sfixed64'].includes(defaultType)) { + throw new Error(`jstype is only allowed on int64, uint64, sint64, fixed64 or sfixed64 fields - got "${defaultType}"`) + } + + return jsTypeOverrides[fieldDef.options?.jstype] + } +} + +function findJsTypeName (typeName: string, classDef: MessageDef, moduleDef: ModuleDef, fieldDef: FieldDef): string { + const override = findJsTypeOverride(typeName, fieldDef) + + if (override != null) { + return override + } + if (types[typeName] != null) { return types[typeName] } @@ -133,7 +264,7 @@ function findTypeName (typeName: string, classDef: MessageDef, moduleDef: Module } if (classDef.parent != null) { - return findTypeName(typeName, classDef.parent, moduleDef) + return findJsTypeName(typeName, classDef.parent, moduleDef, fieldDef) } if (moduleDef.globals[typeName] != null) { @@ -180,9 +311,16 @@ function createDefaultObject (fields: Record, messageDef: Mess const type: string = fieldDef.type let defaultValue + let defaultValueGenerator = defaultValueGenerators[type] + + if (defaultValueGenerator != null) { + const jsTypeOverride = findJsTypeOverride(type, fieldDef) + + if (jsTypeOverride != null && defaultValueGeneratorsJsTypeOverrides[jsTypeOverride] != null) { + defaultValueGenerator = defaultValueGeneratorsJsTypeOverrides[jsTypeOverride] + } - if (defaultValueGenerators[type] != null) { - defaultValue = defaultValueGenerators[type]() + defaultValue = defaultValueGenerator() } else { const def = findDef(fieldDef.type, messageDef, moduleDef) @@ -309,10 +447,10 @@ interface FieldDef { function defineFields (fields: Record, messageDef: MessageDef, moduleDef: ModuleDef): string[] { return Object.entries(fields).map(([fieldName, fieldDef]) => { if (fieldDef.map) { - return `${fieldName}: Map<${findTypeName(fieldDef.keyType ?? 'string', messageDef, moduleDef)}, ${findTypeName(fieldDef.valueType, messageDef, moduleDef)}>` + return `${fieldName}: Map<${findJsTypeName(fieldDef.keyType ?? 'string', messageDef, moduleDef, fieldDef)}, ${findJsTypeName(fieldDef.valueType, messageDef, moduleDef, fieldDef)}>` } - return `${fieldName}${fieldDef.optional ? '?' : ''}: ${findTypeName(fieldDef.type, messageDef, moduleDef)}${fieldDef.repeated ? '[]' : ''}` + return `${fieldName}${fieldDef.optional ? '?' : ''}: ${findJsTypeName(fieldDef.type, messageDef, moduleDef, fieldDef)}${fieldDef.repeated ? '[]' : ''}` }) } @@ -412,7 +550,7 @@ export interface ${messageDef.name} { type = 'message' } - typeName = findTypeName(fieldDef.type, messageDef, moduleDef) + typeName = findJsTypeName(fieldDef.type, messageDef, moduleDef, fieldDef) codec = `${typeName}.codec()` } @@ -421,9 +559,17 @@ export interface ${messageDef.name} { if (fieldDef.map) { valueTest = `obj.${name} != null && obj.${name}.size !== 0` } else if (!fieldDef.optional && !fieldDef.repeated && !fieldDef.proto2Required) { + let defaultValueTestGenerator = defaultValueTestGenerators[type] + // proto3 singular fields should only be written out if they are not the default value - if (defaultValueTestGenerators[type] != null) { - valueTest = `${defaultValueTestGenerators[type](`obj.${name}`)}` + if (defaultValueTestGenerator != null) { + const jsTypeOverride = findJsTypeOverride(type, fieldDef) + + if (jsTypeOverride != null && defaultValueTestGeneratorsJsTypeOverrides[jsTypeOverride] != null) { + defaultValueTestGenerator = defaultValueTestGeneratorsJsTypeOverrides[jsTypeOverride] + } + + valueTest = `${defaultValueTestGenerator(`obj.${name}`)}` } else if (type === 'enum') { // handle enums const def = findDef(fieldDef.type, messageDef, moduleDef) @@ -453,8 +599,13 @@ export interface ${messageDef.name} { } } - let writeField = (): string => `w.uint32(${id}) - ${encoderGenerators[type] == null ? `${codec}.encode(${valueVar}, w)` : encoderGenerators[type](valueVar)}` + let writeField = (): string => { + const encoderGenerator = encoderGenerators[type] + const jsTypeOverride = findJsTypeOverride(type, fieldDef) + + return `w.uint32(${id}) + ${encoderGenerator == null ? `${codec}.encode(${valueVar}, w)` : encoderGenerator(valueVar, jsTypeOverride)}` + } if (type === 'message') { // message fields are only written if they have values. But if a message @@ -524,11 +675,14 @@ export interface ${messageDef.name} { type = 'message' } - const typeName = findTypeName(fieldDef.type, messageDef, moduleDef) + const typeName = findJsTypeName(fieldDef.type, messageDef, moduleDef, fieldDef) codec = `${typeName}.codec()` } - const parseValue = `${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type]()}` + // override setting type on js object + const jsTypeOverride = findJsTypeOverride(fieldDef.type, fieldDef) + + const parseValue = `${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type](jsTypeOverride)}` if (fieldDef.map) { return `case ${fieldDef.id}: { diff --git a/packages/protons/test/custom-options.spec.ts b/packages/protons/test/custom-options.spec.ts new file mode 100644 index 0000000..5392060 --- /dev/null +++ b/packages/protons/test/custom-options.spec.ts @@ -0,0 +1,34 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { CustomOptionNumber, CustomOptionString } from './fixtures/custom-option-jstype.js' + +describe('custom options', () => { + it('should allow overriding 64 bit numbers with numbers', () => { + const obj: CustomOptionNumber = { + num: 5, + i64: 5, + ui64: 5, + si64: 5, + f64: 5, + sf64: 5 + } + + expect(CustomOptionNumber.decode(CustomOptionNumber.encode(obj))) + .to.deep.equal(obj) + }) + + it('should allow overriding 64 bit numbers with strings', () => { + const obj: CustomOptionString = { + num: 5, + i64: '5', + ui64: '5', + si64: '5', + f64: '5', + sf64: '5' + } + + expect(CustomOptionString.decode(CustomOptionString.encode(obj))) + .to.deep.equal(obj) + }) +}) diff --git a/packages/protons/test/fixtures/custom-option-jstype.proto b/packages/protons/test/fixtures/custom-option-jstype.proto new file mode 100644 index 0000000..777c684 --- /dev/null +++ b/packages/protons/test/fixtures/custom-option-jstype.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +message CustomOptionNumber { + int32 num = 1; + int64 i64 = 2 [jstype = JS_NUMBER]; + uint64 ui64 = 3 [jstype = JS_NUMBER]; + sint64 si64 = 4 [jstype = JS_NUMBER]; + fixed64 f64 = 5 [jstype = JS_NUMBER]; + sfixed64 sf64 = 6 [jstype = JS_NUMBER]; +} + +message CustomOptionString { + int32 num = 1; + int64 i64 = 2 [jstype = JS_STRING]; + uint64 ui64 = 3 [jstype = JS_STRING]; + sint64 si64 = 4 [jstype = JS_STRING]; + fixed64 f64 = 5 [jstype = JS_STRING]; + sfixed64 sf64 = 6 [jstype = JS_STRING]; +} diff --git a/packages/protons/test/fixtures/custom-option-jstype.ts b/packages/protons/test/fixtures/custom-option-jstype.ts new file mode 100644 index 0000000..8e352c5 --- /dev/null +++ b/packages/protons/test/fixtures/custom-option-jstype.ts @@ -0,0 +1,225 @@ +/* eslint-disable import/export */ +/* eslint-disable complexity */ +/* eslint-disable @typescript-eslint/no-namespace */ +/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ +/* eslint-disable @typescript-eslint/no-empty-interface */ + +import { encodeMessage, decodeMessage, message } from 'protons-runtime' +import type { Codec } from 'protons-runtime' +import type { Uint8ArrayList } from 'uint8arraylist' + +export interface CustomOptionNumber { + num: number + i64: number + ui64: number + si64: number + f64: number + sf64: number +} + +export namespace CustomOptionNumber { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.num != null && obj.num !== 0)) { + w.uint32(8) + w.int32(obj.num) + } + + if ((obj.i64 != null && obj.i64 !== 0)) { + w.uint32(16) + w.int64Number(obj.i64) + } + + if ((obj.ui64 != null && obj.ui64 !== 0)) { + w.uint32(24) + w.uint64Number(obj.ui64) + } + + if ((obj.si64 != null && obj.si64 !== 0)) { + w.uint32(32) + w.sint64Number(obj.si64) + } + + if ((obj.f64 != null && obj.f64 !== 0)) { + w.uint32(41) + w.fixed64Number(obj.f64) + } + + if ((obj.sf64 != null && obj.sf64 !== 0)) { + w.uint32(49) + w.sfixed64Number(obj.sf64) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + num: 0, + i64: 0, + ui64: 0, + si64: 0, + f64: 0, + sf64: 0 + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: + obj.num = reader.int32() + break + case 2: + obj.i64 = reader.int64Number() + break + case 3: + obj.ui64 = reader.uint64Number() + break + case 4: + obj.si64 = reader.sint64Number() + break + case 5: + obj.f64 = reader.fixed64Number() + break + case 6: + obj.sf64 = reader.sfixed64Number() + break + default: + reader.skipType(tag & 7) + break + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, CustomOptionNumber.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): CustomOptionNumber => { + return decodeMessage(buf, CustomOptionNumber.codec()) + } +} + +export interface CustomOptionString { + num: number + i64: string + ui64: string + si64: string + f64: string + sf64: string +} + +export namespace CustomOptionString { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.num != null && obj.num !== 0)) { + w.uint32(8) + w.int32(obj.num) + } + + if ((obj.i64 != null && obj.i64 !== '')) { + w.uint32(16) + w.int64String(obj.i64) + } + + if ((obj.ui64 != null && obj.ui64 !== '')) { + w.uint32(24) + w.uint64String(obj.ui64) + } + + if ((obj.si64 != null && obj.si64 !== '')) { + w.uint32(32) + w.sint64String(obj.si64) + } + + if ((obj.f64 != null && obj.f64 !== '')) { + w.uint32(41) + w.fixed64String(obj.f64) + } + + if ((obj.sf64 != null && obj.sf64 !== '')) { + w.uint32(49) + w.sfixed64String(obj.sf64) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + num: 0, + i64: '', + ui64: '', + si64: '', + f64: '', + sf64: '' + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: + obj.num = reader.int32() + break + case 2: + obj.i64 = reader.int64String() + break + case 3: + obj.ui64 = reader.uint64String() + break + case 4: + obj.si64 = reader.sint64String() + break + case 5: + obj.f64 = reader.fixed64String() + break + case 6: + obj.sf64 = reader.sfixed64String() + break + default: + reader.skipType(tag & 7) + break + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, CustomOptionString.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): CustomOptionString => { + return decodeMessage(buf, CustomOptionString.codec()) + } +}