Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: benchmark Protobuf-ES #89

Merged
merged 11 commits into from
Oct 23, 2023
27 changes: 13 additions & 14 deletions packages/protons-benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,17 @@

## Install

```console
$ npm i protons-benchmark
```

```console
$ git clone [email protected]:ipfs/protons.git
$ cd protons
$ npm i
$ npm run build
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Without this:

$ npm start                     

> [email protected] prestart
> npm run build


> [email protected] build
> aegir build --no-bundle && cp -R src/protobufjs dist/src/protobufjs

[22:08:47] tsc [started]
src/protons/bench.ts:6:68 - error TS2307: Cannot find module 'protons-runtime' or its corresponding type declarations.

6 import { encodeMessage, decodeMessage, message, enumeration } from 'protons-runtime'

...

Found 89 errors in 2 files.

Errors  Files
    32  src/protons/bench.ts:6
    57  src/protons/rpc.ts:6
[22:08:48] tsc [failed]
[22:08:48] → Command failed with exit code 1: tsc

...

$ cd packages/protons-benchmark
```

## Usage

Run the benchmark suite:
Run the benchmark suite in node:

```console
$ npm start
Expand All @@ -48,10 +45,11 @@ $ npm start
> [email protected] start
> node dist/src/index.js

pbjs x 11,798 ops/sec ±4.58% (88 runs sampled)
protons x 11,693 ops/sec ±2.69% (85 runs sampled)
protobuf.js x 12,419 ops/sec ±1.66% (88 runs sampled)
@protobuf-ts x 10,536 ops/sec ±3.14% (85 runs sampled)
pbjs x 19,188 ops/sec ±0.38% (98 runs sampled)
protons x 19,001 ops/sec ±0.33% (95 runs sampled)
protobuf.js x 19,558 ops/sec ±0.30% (91 runs sampled)
@protobuf-ts x 17,216 ops/sec ±0.32% (95 runs sampled)
protobuf-es x 15,673 ops/sec ±0.48% (93 runs sampled)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ℹ️ Slowest.

Fastest is protobuf.js
```

Expand All @@ -64,11 +62,12 @@ $ npm run start:browser
> npx playwright-test dist/src/index.js --runner benchmark

✔ chromium set up
pbjs x 19,027 ops/sec ±0.86% (67 runs sampled)
protons x 18,901 ops/sec ±0.65% (67 runs sampled)
protobuf.js x 18,937 ops/sec ±0.55% (65 runs sampled)
@protobuf-ts x 16,669 ops/sec ±0.49% (68 runs sampled)
Fastest is pbjs,protobuf.js
pbjs x 19,763 ops/sec ±0.35% (67 runs sampled)
protons x 19,617 ops/sec ±0.37% (68 runs sampled)
protobuf.js x 19,772 ops/sec ±0.34% (67 runs sampled)
@protobuf-ts x 17,204 ops/sec ±0.33% (69 runs sampled)
protobuf-es x 16,032 ops/sec ±0.38% (68 runs sampled)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ℹ️ Slowest.

Fastest is protobuf.js,pbjs
```

## API Docs
Expand Down
5 changes: 4 additions & 1 deletion packages/protons-benchmark/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,20 +33,23 @@
},
"ignorePatterns": [
"src/implementations/pbjs/*",
"src/implementations/protobuf-es/*",
"src/implementations/protobuf-ts/*",
"src/implementations/protobufjs/*"
]
},
"scripts": {
"clean": "aegir clean",
"lint": "aegir lint",
"dep-check": "aegir dep-check --ignore @protobuf-ts/plugin pbjs protons",
"dep-check": "aegir dep-check --ignore @protobuf-ts/plugin pbjs protons @bufbuild/protoc-gen-es",
"build": "aegir build --no-bundle && cp -R src/implementations/protobufjs dist/src/implementations/protobufjs",
"prestart": "npm run build",
"start": "node dist/src/implementations/index.js",
"start:browser": "npx playwright-test dist/src/implementations/index.js --runner benchmark"
},
"dependencies": {
"@bufbuild/protobuf": "^1.0.0",
"@bufbuild/protoc-gen-es": "^1.3.3",
"@protobuf-ts/plugin": "^2.8.1",
"@protobuf-ts/runtime": "^2.8.1",
"@types/benchmark": "^2.1.1",
Expand Down
7 changes: 7 additions & 0 deletions packages/protons-benchmark/src/implementations/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ $ npx playwright-test dist/src/index.js --runner benchmark
import { expect } from 'aegir/chai'
import Benchmark from 'benchmark'
import { encodeTest as pbjsEncodeTest, decodeTest as pbjsDecodeTest } from './pbjs/bench.js'
import { Test as ProtobufEsTest } from './protobuf-es/bench_pb.js'
import { Test as ProtobufTsTest } from './protobuf-ts/bench.js'
import { Test as ProtobufjsTest } from './protobufjs/bench.js'
import { Test as ProtonsTest } from './protons/bench.js'
Expand Down Expand Up @@ -59,6 +60,12 @@ new Benchmark.Suite()

expectDecodedCorrectly(result)
})
.add('protobuf-es', () => {
const buf = new ProtobufEsTest(message).toBinary()
const result = ProtobufEsTest.fromBinary(buf)

expectDecodedCorrectly(result)
})
.on('error', (err: Error) => {
console.error(err)
})
Expand Down
243 changes: 243 additions & 0 deletions packages/protons-benchmark/src/implementations/protobuf-es/bench_pb.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
// @generated by protoc-gen-es v1.0.0 with parameter "target=ts"
// @generated from file bench.proto (syntax proto3)
/* eslint-disable */
// @ts-nocheck

import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf";
import { Message, proto3 } from "@bufbuild/protobuf";

/**
* @generated from enum FOO
*/
export enum FOO {
/**
* @generated from enum value: NONE = 0;
*/
NONE = 0,

/**
* @generated from enum value: LOL = 1;
*/
LOL = 1,

/**
* @generated from enum value: ABE = 3;
*/
ABE = 3,
}
// Retrieve enum metadata with: proto3.getEnumType(FOO)
proto3.util.setEnumType(FOO, "FOO", [
{ no: 0, name: "NONE" },
{ no: 1, name: "LOL" },
{ no: 3, name: "ABE" },
]);

/**
* @generated from message Foo
*/
export class Foo extends Message<Foo> {
/**
* @generated from field: optional uint32 baz = 1;
*/
baz?: number;

constructor(data?: PartialMessage<Foo>) {
super();
proto3.util.initPartial(data, this);
}

static readonly runtime = proto3;
static readonly typeName = "Foo";
static readonly fields: FieldList = proto3.util.newFieldList(() => [
{ no: 1, name: "baz", kind: "scalar", T: 13 /* ScalarType.UINT32 */, opt: true },
]);

static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): Foo {
return new Foo().fromBinary(bytes, options);
}

static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): Foo {
return new Foo().fromJson(jsonValue, options);
}

static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): Foo {
return new Foo().fromJsonString(jsonString, options);
}

static equals(a: Foo | PlainMessage<Foo> | undefined, b: Foo | PlainMessage<Foo> | undefined): boolean {
return proto3.util.equals(Foo, a, b);
}
}

/**
* @generated from message Bar
*/
export class Bar extends Message<Bar> {
/**
* @generated from field: optional Foo tmp = 1;
*/
tmp?: Foo;

constructor(data?: PartialMessage<Bar>) {
super();
proto3.util.initPartial(data, this);
}

static readonly runtime = proto3;
static readonly typeName = "Bar";
static readonly fields: FieldList = proto3.util.newFieldList(() => [
{ no: 1, name: "tmp", kind: "message", T: Foo, opt: true },
]);

static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): Bar {
return new Bar().fromBinary(bytes, options);
}

static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): Bar {
return new Bar().fromJson(jsonValue, options);
}

static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): Bar {
return new Bar().fromJsonString(jsonString, options);
}

static equals(a: Bar | PlainMessage<Bar> | undefined, b: Bar | PlainMessage<Bar> | undefined): boolean {
return proto3.util.equals(Bar, a, b);
}
}

/**
* @generated from message Yo
*/
export class Yo extends Message<Yo> {
/**
* @generated from field: repeated FOO lol = 1;
*/
lol: FOO[] = [];

constructor(data?: PartialMessage<Yo>) {
super();
proto3.util.initPartial(data, this);
}

static readonly runtime = proto3;
static readonly typeName = "Yo";
static readonly fields: FieldList = proto3.util.newFieldList(() => [
{ no: 1, name: "lol", kind: "enum", T: proto3.getEnumType(FOO), repeated: true },
]);

static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): Yo {
return new Yo().fromBinary(bytes, options);
}

static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): Yo {
return new Yo().fromJson(jsonValue, options);
}

static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): Yo {
return new Yo().fromJsonString(jsonString, options);
}

static equals(a: Yo | PlainMessage<Yo> | undefined, b: Yo | PlainMessage<Yo> | undefined): boolean {
return proto3.util.equals(Yo, a, b);
}
}

/**
* @generated from message Lol
*/
export class Lol extends Message<Lol> {
/**
* @generated from field: optional string lol = 1;
*/
lol?: string;

/**
* @generated from field: Bar b = 2;
*/
b?: Bar;

constructor(data?: PartialMessage<Lol>) {
super();
proto3.util.initPartial(data, this);
}

static readonly runtime = proto3;
static readonly typeName = "Lol";
static readonly fields: FieldList = proto3.util.newFieldList(() => [
{ no: 1, name: "lol", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
{ no: 2, name: "b", kind: "message", T: Bar },
]);

static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): Lol {
return new Lol().fromBinary(bytes, options);
}

static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): Lol {
return new Lol().fromJson(jsonValue, options);
}

static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): Lol {
return new Lol().fromJsonString(jsonString, options);
}

static equals(a: Lol | PlainMessage<Lol> | undefined, b: Lol | PlainMessage<Lol> | undefined): boolean {
return proto3.util.equals(Lol, a, b);
}
}

/**
* @generated from message Test
*/
export class Test extends Message<Test> {
/**
* @generated from field: optional Lol meh = 6;
*/
meh?: Lol;

/**
* @generated from field: optional uint32 hello = 3;
*/
hello?: number;

/**
* @generated from field: optional string foo = 1;
*/
foo?: string;

/**
* @generated from field: optional bytes payload = 7;
*/
payload?: Uint8Array;

constructor(data?: PartialMessage<Test>) {
super();
proto3.util.initPartial(data, this);
}

static readonly runtime = proto3;
static readonly typeName = "Test";
static readonly fields: FieldList = proto3.util.newFieldList(() => [
{ no: 6, name: "meh", kind: "message", T: Lol, opt: true },
{ no: 3, name: "hello", kind: "scalar", T: 13 /* ScalarType.UINT32 */, opt: true },
{ no: 1, name: "foo", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
{ no: 7, name: "payload", kind: "scalar", T: 12 /* ScalarType.BYTES */, opt: true },
]);

static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): Test {
return new Test().fromBinary(bytes, options);
}

static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): Test {
return new Test().fromJson(jsonValue, options);
}

static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): Test {
return new Test().fromJsonString(jsonString, options);
}

static equals(a: Test | PlainMessage<Test> | undefined, b: Test | PlainMessage<Test> | undefined): boolean {
return proto3.util.equals(Test, a, b);
}
}