feat: compute buffer size ahead of time

The current implementation loops over message fields and each field
encoder creates a `Uint8Array` which is added to a `Uint8ArrayList`
and stitched together at the end of the process.

Other implementations compute the expected length of the serialized
message, allocate a buffer that size then have encoders write their
data into it at the correct offsets.

This PR makes protons compute the final message buffer size first in
the same way.

Thing is, it doesn't seem to make a huge amount of difference to performance.

Before:

```
Running "Encode/Decode" suite...
Progress: 100%

  pbjs:
    12 166 ops/s, ±3.92%   | 5.12% slower

  protons:
    9 755 ops/s, ±2.19%    | slowest, 23.93% slower

  protobufjs:
    12 823 ops/s, ±2.02%   | fastest

Finished 3 cases!
  Fastest: protobufjs
  Slowest: protons
```

After:

```
Running "Encode/Decode" suite...
Progress: 100%

  pbjs:
    11 866 ops/s, ±3.43%   | 2.05% slower

  protons:
    9 356 ops/s, ±2.45%    | slowest, 22.77% slower

  protobufjs:
    12 114 ops/s, ±2.16%   | fastest

Finished 3 cases!
  Fastest: protobufjs
  Slowest: protons
```

packages/protons-runtime/src/codecs/bool.ts

@@ -4,8 +4,10 @@ const encodingLength: EncodingLengthFunction<boolean> = function boolEncodingLen
   return 1
 }
 
-const encode: EncodeFunction<boolean> = function boolEncode (value) {
-  return Uint8Array.from([value ? 1 : 0])
+const encode: EncodeFunction<boolean> = function boolEncode (val, buf, offset) {
+  buf.set(offset, val ? 1 : 0)
+
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<boolean> = function boolDecode (buffer, offset) {

packages/protons-runtime/src/codecs/bytes.ts

@@ -1,5 +1,4 @@
 
-import { Uint8ArrayList } from 'uint8arraylist'
 import { unsigned } from '../utils/varint.js'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
@@ -8,12 +7,11 @@ const encodingLength: EncodingLengthFunction<Uint8Array> = function bytesEncodin
   return unsigned.encodingLength(len) + len
 }
 
-const encode: EncodeFunction<Uint8Array> = function bytesEncode (val) {
-  const prefix = new Uint8Array(unsigned.encodingLength(val.byteLength))
+const encode: EncodeFunction<Uint8Array> = function bytesEncode (val, buf, offset) {
+  offset = unsigned.encode(val.byteLength, buf, offset)
+  buf.write(val, offset)
 
-  unsigned.encode(val.byteLength, prefix)
-
-  return new Uint8ArrayList(prefix, val)
+  return offset + val.byteLength
 }
 
 const decode: DecodeFunction<Uint8Array> = function bytesDecode (buf, offset) {

packages/protons-runtime/src/codecs/codec.ts

@@ -11,7 +11,7 @@ export enum CODEC_TYPES {
 }
 
 export interface EncodeFunction<T> {
-  (value: T): Uint8Array | Uint8ArrayList
+  (value: T, buf: Uint8ArrayList, offset: number): number
 }
 
 export interface DecodeFunction<T> {

packages/protons-runtime/src/codecs/double.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<number> = function doubleEncodingLength () {
   return 8
 }
 
-const encode: EncodeFunction<number> = function doubleEncode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setFloat64(0, val, true)
+const encode: EncodeFunction<number> = function doubleEncode (val, buf, offset) {
+  buf.setFloat64(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<number> = function doubleDecode (buf, offset) {

packages/protons-runtime/src/codecs/enum.ts

@@ -10,14 +10,11 @@ export function enumeration <T> (e: T): Codec<T> {
     return unsigned.encodingLength(index)
   }
 
-  const encode: EncodeFunction<string> = function enumEncode (val) {
+  const encode: EncodeFunction<string> = function enumEncode (val, buf, offset) {
     const keys = Object.keys(e)
     const index = keys.indexOf(val)
-    const buf = new Uint8Array(unsigned.encodingLength(index))
 
-    unsigned.encode(index, buf)
-
-    return buf
+    return unsigned.encode(index, buf, offset)
   }
 
   const decode: DecodeFunction<string> = function enumDecode (buf, offset) {

packages/protons-runtime/src/codecs/fixed32.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<number> = function fixed32EncodingLength () {
   return 4
 }
 
-const encode: EncodeFunction<number> = function fixed32Encode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setInt32(0, val, true)
+const encode: EncodeFunction<number> = function fixed32Encode (val, buf, offset) {
+  buf.setInt32(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<number> = function fixed32Decode (buf, offset) {

packages/protons-runtime/src/codecs/fixed64.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<bigint> = function int64EncodingLength (val) {
   return 8
 }
 
-const encode: EncodeFunction<bigint> = function int64Encode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setBigInt64(0, val, true)
+const encode: EncodeFunction<bigint> = function int64Encode (val, buf, offset) {
+  buf.setBigInt64(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<bigint> = function int64Decode (buf, offset) {

packages/protons-runtime/src/codecs/float.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<number> = function floatEncodingLength () {
   return 4
 }
 
-const encode: EncodeFunction<number> = function floatEncode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(1)))
-  buf.setFloat32(0, val, true)
+const encode: EncodeFunction<number> = function floatEncode (val, buf, offset) {
+  buf.setFloat32(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<number> = function floatDecode (buf, offset) {

packages/protons-runtime/src/codecs/int32.ts

@@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction<number> = function int32EncodingLen
   return signed.encodingLength(val)
 }
 
-const encode: EncodeFunction<number> = function int32Encode (val) {
-  const buf = new Uint8Array(encodingLength(val))
-  signed.encode(val, buf)
-
-  return buf
+const encode: EncodeFunction<number> = function int32Encode (val, buf, offset) {
+  return signed.encode(val, buf, offset)
 }
 
 const decode: DecodeFunction<number> = function int32Decode (buf, offset) {

packages/protons-runtime/src/codecs/int64.ts

@@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction<bigint> = function int64EncodingLen
   return signed.encodingLength(val)
 }
 
-const encode: EncodeFunction<bigint> = function int64Encode (val) {
-  const buf = new Uint8Array(encodingLength(val))
-  signed.encode(val, buf)
-
-  return buf
+const encode: EncodeFunction<bigint> = function int64Encode (val, buf, offset) {
+  return signed.encode(val, buf, offset)
 }
 
 const decode: DecodeFunction<bigint> = function int64Decode (buf, offset) {

packages/protons-runtime/src/codecs/message.ts

@@ -1,64 +1,98 @@
 import { unsigned } from '../utils/varint.js'
 import type { FieldDef, FieldDefs } from '../index.js'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, Codec, CODEC_TYPES } from './codec.js'
-import { Uint8ArrayList } from 'uint8arraylist'
 
 export interface Factory<A, T> {
   new (obj: A): T
 }
 
 export function message <T> (fieldDefs: FieldDefs): Codec<T> {
   const encodingLength: EncodingLengthFunction<T> = function messageEncodingLength (val: Record<string, any>) {
+    function valueEncodingLength (value: any, fieldNumber: number, fieldDef: FieldDef): number {
+      if (value == null) {
+        if (fieldDef.optional === true) {
+          return 0
+        }
+
+        throw new Error(`Non optional field "${fieldDef.name}" was ${value === null ? 'null' : 'undefined'}`)
+      }
+
+      const key = (fieldNumber << 3) | fieldDef.codec.type
+      const prefixLength = unsigned.encodingLength(key)
+
+      return fieldDef.codec.encodingLength(value) + prefixLength
+    }
+
     let length = 0
 
-    for (const fieldDef of Object.values(fieldDefs)) {
-      length += fieldDef.codec.encodingLength(val[fieldDef.name])
+    for (const [fieldNumberStr, fieldDef] of Object.entries(fieldDefs)) {
+      const fieldNumber = parseInt(fieldNumberStr)
+      const value = val[fieldDef.name]
+
+      if (value == null && !(fieldDef.optional === true)) {
+        throw new Error(`Field ${fieldDef.name} cannot be ${value === null ? 'null' : 'undefined'}`)
+      }
+
+      if (fieldDef.repeats === true) {
+        if (!Array.isArray(value)) {
+          throw new Error(`Repeating field ${fieldDef.name} was not an array`)
+        }
+
+        for (const entry of value) {
+          length += valueEncodingLength(entry, fieldNumber, fieldDef)
+        }
+
+        continue
+      }
+
+      length += valueEncodingLength(value, fieldNumber, fieldDef)
     }
 
     return unsigned.encodingLength(length) + length
   }
 
-  const encode: EncodeFunction<Record<string, any>> = function messageEncode (val) {
-    const bytes = new Uint8ArrayList()
-
-    function encodeValue (value: any, fieldNumber: number, fieldDef: FieldDef) {
+  const encode: EncodeFunction<T> = function messageEncode (val, buf, offset): number {
+    function encodeValue (value: any, fieldNumber: number, fieldDef: FieldDef, offset: number): number {
       if (value == null) {
         if (fieldDef.optional === true) {
-          return
+          return offset
         }
 
         throw new Error(`Non optional field "${fieldDef.name}" was ${value === null ? 'null' : 'undefined'}`)
       }
 
       const key = (fieldNumber << 3) | fieldDef.codec.type
-      const prefix = new Uint8Array(unsigned.encodingLength(key))
-      unsigned.encode(key, prefix)
-      const encoded = fieldDef.codec.encode(value)
+      offset = unsigned.encode(key, buf, offset)
+      offset = fieldDef.codec.encode(value, buf, offset)
 
-      bytes.append(prefix)
-      bytes.append(encoded)
+      return offset
     }
 
+    const length = encodingLength(val)
+    offset = unsigned.encode(length - unsigned.encodingLength(length), buf, offset)
+
     for (const [fieldNumberStr, fieldDef] of Object.entries(fieldDefs)) {
       const fieldNumber = parseInt(fieldNumberStr)
 
       if (fieldDef.repeats === true) {
+        // @ts-expect-error cannot use strings to index T
         if (!Array.isArray(val[fieldDef.name])) {
           throw new Error(`Repeating field "${fieldDef.name}" was not an array`)
         }
 
+        // @ts-expect-error cannot use strings to index T
         for (const value of val[fieldDef.name]) {
-          encodeValue(value, fieldNumber, fieldDef)
+          offset = encodeValue(value, fieldNumber, fieldDef, offset)
         }
-      } else {
-        encodeValue(val[fieldDef.name], fieldNumber, fieldDef)
+
+        continue
       }
-    }
 
-    const prefix = new Uint8Array(unsigned.encodingLength(bytes.length))
-    unsigned.encode(bytes.length, prefix)
+      // @ts-expect-error cannot use strings to index T
+      offset = encodeValue(val[fieldDef.name], fieldNumber, fieldDef, offset)
+    }
 
-    return new Uint8ArrayList(prefix, bytes)
+    return offset
   }
 
   const decode: DecodeFunction<T> = function messageDecode (buffer, offset) {
@@ -78,10 +112,7 @@ export function message <T> (fieldDefs: FieldDefs): Codec<T> {
       const fieldDef = fieldDefs[fieldNumber]
       let fieldLength = 0
 
-      // console.info('fieldNumber', fieldNumber, 'wireType', wireType, 'offset', offset)
-
       if (wireType === CODEC_TYPES.VARINT) {
-        // console.info('decode varint')
         if (fieldDef != null) {
           // use the codec if it is available as this could be a bigint
           const value = fieldDef.codec.decode(buffer, offset)
@@ -91,25 +122,19 @@ export function message <T> (fieldDefs: FieldDefs): Codec<T> {
           fieldLength = unsigned.encodingLength(value)
         }
       } else if (wireType === CODEC_TYPES.BIT64) {
-        // console.info('decode 64bit')
         fieldLength = 8
       } else if (wireType === CODEC_TYPES.LENGTH_DELIMITED) {
-        // console.info('decode length delimited')
         const valueLength = unsigned.decode(buffer, offset)
         fieldLength = valueLength + unsigned.encodingLength(valueLength)
       } else if (wireType === CODEC_TYPES.BIT32) {
-        // console.info('decode 32 bit')
         fieldLength = 4
       } else if (wireType === CODEC_TYPES.START_GROUP) {
         throw new Error('Unsupported wire type START_GROUP')
       } else if (wireType === CODEC_TYPES.END_GROUP) {
         throw new Error('Unsupported wire type END_GROUP')
       }
 
-      // console.info('fieldLength', fieldLength)
-
       if (fieldDef != null) {
-        // console.info('decode', fieldDef.codec.name, fieldDef.name, 'at offset', offset)
         const value = fieldDef.codec.decode(buffer, offset)
 
         if (fieldDef.repeats === true) {
@@ -121,8 +146,6 @@ export function message <T> (fieldDefs: FieldDefs): Codec<T> {
         } else {
           fields[fieldDef.name] = value
         }
-
-        // console.info('decoded', value)
       }
 
       offset += fieldLength

packages/protons-runtime/src/codecs/sfixed32.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<number> = function sfixed32EncodingLength () {
   return 4
 }
 
-const encode: EncodeFunction<number> = function sfixed32Encode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setInt32(0, val, true)
+const encode: EncodeFunction<number> = function sfixed32Encode (val, buf, offset) {
+  buf.setInt32(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<number> = function sfixed32Decode (buf, offset) {

packages/protons-runtime/src/codecs/sfixed64.ts

@@ -1,15 +1,13 @@
-import { Uint8ArrayList } from 'uint8arraylist'
 import { DecodeFunction, EncodeFunction, createCodec, EncodingLengthFunction, CODEC_TYPES } from './codec.js'
 
 const encodingLength: EncodingLengthFunction<bigint> = function sfixed64EncodingLength () {
   return 8
 }
 
-const encode: EncodeFunction<bigint> = function sfixed64Encode (val) {
-  const buf = new Uint8ArrayList(new Uint8Array(encodingLength(val)))
-  buf.setBigInt64(0, val, true)
+const encode: EncodeFunction<bigint> = function sfixed64Encode (val, buf, offset) {
+  buf.setBigInt64(offset, val, true)
 
-  return buf
+  return offset + encodingLength(val)
 }
 
 const decode: DecodeFunction<bigint> = function sfixed64Decode (buf, offset) {

packages/protons-runtime/src/codecs/sint32.ts

@@ -5,12 +5,8 @@ const encodingLength: EncodingLengthFunction<number> = function sint32EncodingLe
   return zigzag.encodingLength(val)
 }
 
-const encode: EncodeFunction<number> = function svarintEncode (val) {
-  const buf = new Uint8Array(encodingLength(val))
-
-  zigzag.encode(val, buf)
-
-  return buf
+const encode: EncodeFunction<number> = function svarintEncode (val, buf, offset) {
+  return zigzag.encode(val, buf, offset)
 }
 
 const decode: DecodeFunction<number> = function svarintDecode (buf, offset) {

packages/protons-runtime/src/codecs/sint64.ts

@@ -5,11 +5,8 @@ const encodingLength: EncodingLengthFunction<bigint> = function int64EncodingLen
   return zigzag.encodingLength(val)
 }
 
-const encode: EncodeFunction<bigint> = function int64Encode (val) {
-  const buf = new Uint8Array(encodingLength(val))
-  zigzag.encode(val, buf)
-
-  return buf
+const encode: EncodeFunction<bigint> = function int64Encode (val, buf, offset) {
+  return zigzag.encode(val, buf, offset)
 }
 
 const decode: DecodeFunction<bigint> = function int64Decode (buf, offset) {