diff --git a/doc b/doc index bebd0480..b36b2fa1 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit bebd0480b9201978ee6c0f8eb787bdd8325e1b5e +Subproject commit b36b2fa1d38a069d04544dfad80065636ad827b0 diff --git a/etc/benchmarks/avro-serialization-implementations/scripts/decode/node-avro-io.js b/etc/benchmarks/avro-serialization-implementations/scripts/decode/node-avro-io.js index 7a4bd421..94185a96 100755 --- a/etc/benchmarks/avro-serialization-implementations/scripts/decode/node-avro-io.js +++ b/etc/benchmarks/avro-serialization-implementations/scripts/decode/node-avro-io.js @@ -3,7 +3,8 @@ 'use strict'; let io = require('node-avro-io'), - avsc = require('../../../../lib'); + avsc = require('../../../../lib'), + {isBufferLike} = require('../../../../lib/utils'); let loops = 2; @@ -30,7 +31,7 @@ avsc.createFileDecoder(process.argv[2]) }); function deserialize(buffer) { - if (!Buffer.isBuffer(buffer)) { + if (!isBufferLike(buffer)) { throw 'Buffer object expected'; } @@ -44,7 +45,7 @@ function deserialize(buffer) { this._i += len; return len == 1 ? buffer[i] : - buffer.slice(i, this._i); + buffer.subarray(i, this._i); }, skip: function(len) { if (this._i + len > buffer.length) { diff --git a/etc/benchmarks/avro-serialization-implementations/scripts/encode/node-avro-io.js b/etc/benchmarks/avro-serialization-implementations/scripts/encode/node-avro-io.js index 9e6cfb3f..aaa881bc 100755 --- a/etc/benchmarks/avro-serialization-implementations/scripts/encode/node-avro-io.js +++ b/etc/benchmarks/avro-serialization-implementations/scripts/encode/node-avro-io.js @@ -3,7 +3,8 @@ 'use strict'; let io = require('node-avro-io'), - avsc = require('../../../../lib'); + avsc = require('../../../../lib'), + {isBufferLike} = require('../../../../lib/utils'); let loops = 2; @@ -33,7 +34,7 @@ function serialize(datum) { let buffer = Buffer.from([]); let encoder = new io.IO.BinaryEncoder({ write: function(data) { - if (!Buffer.isBuffer(data)) { + if (!isBufferLike(data)) { data = Buffer.from([data]); } buffer = Buffer.concat([buffer, data]); diff --git a/etc/benchmarks/avro-serialization-implementations/scripts/encode/node-etp-avro.js b/etc/benchmarks/avro-serialization-implementations/scripts/encode/node-etp-avro.js index 7f632116..026ed515 100755 --- a/etc/benchmarks/avro-serialization-implementations/scripts/encode/node-etp-avro.js +++ b/etc/benchmarks/avro-serialization-implementations/scripts/encode/node-etp-avro.js @@ -3,6 +3,7 @@ 'use strict'; let avro = require('etp-avro'), + {Buffer} = require('buffer'), avsc = require('../../../../lib'); @@ -31,8 +32,8 @@ avsc.createFileDecoder(process.argv[2]) function loop() { let n = 0; for (let i = 0, l = records.length; i < l; i++) { - // We need to slice to force a copy otherwise the array is shared. - let buf = writer.encode(schema, records[i]).slice(); + // We need to force a copy otherwise the array is shared. + let buf = Buffer.from(writer.encode(schema, records[i])); n += buf[0] + buf.length; } return n; diff --git a/etc/benchmarks/js-serialization-libraries/index.js b/etc/benchmarks/js-serialization-libraries/index.js index d232b9cc..1fb128c9 100644 --- a/etc/benchmarks/js-serialization-libraries/index.js +++ b/etc/benchmarks/js-serialization-libraries/index.js @@ -6,6 +6,7 @@ */ let avro = require('../../../lib'), + {isBufferLike} = require('../../../../lib/utils'), Benchmark = require('benchmark'), commander = require('commander'), compactr = require('compactr'), @@ -285,7 +286,7 @@ class EncodeSuite extends Suite { let val = this.getValue(); return function () { let str = JSON.stringify(val, (key, value) => { - if (Buffer.isBuffer(value)) { + if (isBufferLike(value)) { return value.toString('binary'); } return value; diff --git a/etc/browser/avsc.js b/etc/browser/avsc.js index 367736e5..fbb29af4 100644 --- a/etc/browser/avsc.js +++ b/etc/browser/avsc.js @@ -8,10 +8,8 @@ */ let containers = require('../../lib/containers'), - utils = require('../../lib/utils'), stream = require('stream'); - /** Transform stream which lazily reads a blob's contents. */ class BlobReader extends stream.Readable { constructor (blob, opts) { @@ -39,7 +37,7 @@ class BlobReader extends stream.Readable { if (evt.error) { self.emit('error', evt.error); } else { - self.push(utils.bufferFrom(reader.result)); + self.push(reader.result); } }, false); reader.readAsArrayBuffer(blob); diff --git a/etc/browser/lib/md5.js b/etc/browser/lib/md5.js index db6ec389..3ff8d8bd 100644 --- a/etc/browser/lib/md5.js +++ b/etc/browser/lib/md5.js @@ -10,9 +10,6 @@ * */ -let buffer = require('buffer'); -let Buffer = buffer.Buffer; - function md5cycle(x, k) { let a = x[0], b = x[1], c = x[2], d = x[3]; @@ -148,9 +145,10 @@ function md5blk(s) { function md5(s) { let arr = md51(s); - let buf = Buffer.alloc(16); + let buf = new Uint8Array(16); + let dv = new DataView(buf.buffer); for (let i = 0; i < 4; i++) { - buf.writeIntLE(arr[i], i * 4, 4); + dv.setInt32(i * 4, arr[i], true); } return buf; } diff --git a/etc/schemas/Float.avsc b/etc/schemas/Float.avsc new file mode 100644 index 00000000..8116b5b9 --- /dev/null +++ b/etc/schemas/Float.avsc @@ -0,0 +1,10 @@ +{ + "name": "Float", + "type": "record", + "fields": [ + { + "name": "value", + "type": "float" + } + ] +} diff --git a/etc/scripts/infer b/etc/scripts/infer index 2db3973d..1589810d 100755 --- a/etc/scripts/infer +++ b/etc/scripts/infer @@ -33,6 +33,8 @@ switch (argv.length) { process.exit(1); } +const DECODER = new TextDecoder(); + /** * Infer a type from a stream of serialized JSON values. * @@ -42,7 +44,7 @@ function fromStdin() { let str = ''; process.stdin .on('data', (buf) => { - str += buf.toString(); + str += DECODER.decode(buf); let pos; while ((pos = utils.jsonEnd(str)) >= 0) { let val = JSON.parse(str.slice(0, pos)); diff --git a/etc/scripts/meta b/etc/scripts/meta index 3282a56b..fe16bf21 100755 --- a/etc/scripts/meta +++ b/etc/scripts/meta @@ -133,6 +133,7 @@ META_TYPE.fromBuffer = function (buf) { return avro.Type.forType(attrs, {wrapUnions: true}); }; +const DECODER = new TextDecoder(); // Example of things we can do. switch (process.argv[2]) { @@ -141,7 +142,7 @@ switch (process.argv[2]) { if (err) { throw err; } - let type = avro.Type.forSchema(buf.toString()); + let type = avro.Type.forSchema(DECODER.decode(buf)); process.stdout.write(META_TYPE.toBuffer(type)); }); break; diff --git a/etc/scripts/perf b/etc/scripts/perf index ab0fb3f4..7b185b87 100755 --- a/etc/scripts/perf +++ b/etc/scripts/perf @@ -34,6 +34,14 @@ if (~index) { // serialization speed). let NUM_VALUES = 1000; +function maybeGC() { + try { + global.gc(); + } catch (err) { + // GC not exposed + } +} + // Header formatting is done according to GitHub flavored Markdown. console.log(['fromBuffer', 'toBuffer', 'isValid ', '(ops/sec)'].join('\t| ')); console.log(['---------:', '-------:', '------: ', '---------'].join('\t| ')); @@ -58,6 +66,7 @@ paths.forEach((fpath) => { stats.push(s); }); + maybeGC(); bench.clone({fn: function () { for (let i = 0, l = NUM_VALUES; i < l; i++) { let val = type.fromBuffer(bufs[i]); @@ -67,6 +76,7 @@ paths.forEach((fpath) => { } }}).run(); + maybeGC(); bench.clone({fn: function () { for (let i = 0, l = NUM_VALUES; i < l; i++) { let buf = type.toBuffer(values[i]); @@ -76,6 +86,7 @@ paths.forEach((fpath) => { } }}).run(); + maybeGC(); bench.clone({fn: function () { for (let i = 0, l = NUM_VALUES; i < l; i++) { if (!type.isValid(values[i])) { diff --git a/lib/containers.js b/lib/containers.js index d219d2cb..2cd715c4 100644 --- a/lib/containers.js +++ b/lib/containers.js @@ -13,10 +13,11 @@ let types = require('./types'), utils = require('./utils'), - buffer = require('buffer'), stream = require('stream'); -let Buffer = buffer.Buffer; +const DECODER = new TextDecoder(); +const ENCODER = new TextEncoder(); + let OPTS = {namespace: 'org.apache.avro.file', registry: {}}; let LONG_TYPE = types.Type.forSchema('long', OPTS); @@ -44,7 +45,7 @@ let BLOCK_TYPE = types.Type.forSchema({ }, OPTS); // First 4 bytes of an Avro object container file. -let MAGIC_BYTES = utils.bufferFrom('Obj\x01'); +let MAGIC_BYTES = ENCODER.encode('Obj\x01'); // Convenience. let Tap = utils.Tap; @@ -62,7 +63,7 @@ class RawDecoder extends stream.Duplex { }); this._type = types.Type.forSchema(schema); - this._tap = new Tap(utils.newBuffer(0)); + this._tap = Tap.withCapacity(0); this._writeCb = null; this._needPush = false; this._readValue = createReader(noDecode, this._type); @@ -83,8 +84,7 @@ class RawDecoder extends stream.Duplex { this._writeCb = cb; let tap = this._tap; - tap.buf = Buffer.concat([tap.buf.slice(tap.pos), chunk]); - tap.pos = 0; + tap.forward(chunk); if (this._needPush) { this._needPush = false; this._read(); @@ -113,7 +113,6 @@ class RawDecoder extends stream.Duplex { } } - /** Duplex stream for decoding object container files. */ class BlockDecoder extends stream.Duplex { constructor (opts) { @@ -132,8 +131,8 @@ class BlockDecoder extends stream.Duplex { this._codecs = opts.codecs; this._codec = undefined; this._parseHook = opts.parseHook; - this._tap = new Tap(utils.newBuffer(0)); - this._blockTap = new Tap(utils.newBuffer(0)); + this._tap = Tap.withCapacity(0); + this._blockTap = Tap.withCapacity(0); this._syncMarker = null; this._readValue = null; this._noDecode = noDecode; @@ -164,12 +163,15 @@ class BlockDecoder extends stream.Duplex { _decodeHeader () { let tap = this._tap; - if (tap.buf.length < MAGIC_BYTES.length) { + if (tap.length < MAGIC_BYTES.length) { // Wait until more data arrives. return false; } - if (!MAGIC_BYTES.equals(tap.buf.slice(0, MAGIC_BYTES.length))) { + if (!utils.bufEqual( + MAGIC_BYTES, + tap.subarray(0, MAGIC_BYTES.length) + )) { this.emit('error', new Error('invalid magic bytes')); return false; } @@ -179,7 +181,7 @@ class BlockDecoder extends stream.Duplex { return false; } - this._codec = (header.meta['avro.codec'] || 'null').toString(); + this._codec = DECODER.decode(header.meta['avro.codec']) || 'null'; let codecs = this._codecs || BlockDecoder.getDefaultCodecs(); this._decompress = codecs[this._codec]; if (!this._decompress) { @@ -188,7 +190,7 @@ class BlockDecoder extends stream.Duplex { } try { - let schema = JSON.parse(header.meta['avro.schema'].toString()); + let schema = JSON.parse(DECODER.decode(header.meta['avro.schema'])); if (this._parseHook) { schema = this._parseHook(schema); } @@ -212,8 +214,7 @@ class BlockDecoder extends stream.Duplex { _write (chunk, encoding, cb) { let tap = this._tap; - tap.buf = Buffer.concat([tap.buf, chunk]); - tap.pos = 0; + tap.append(chunk); if (!this._decodeHeader()) { process.nextTick(cb); @@ -224,18 +225,17 @@ class BlockDecoder extends stream.Duplex { // in case we already have all the data (in which case `_write` wouldn't get // called anymore). this._write = this._writeChunk; - this._write(utils.newBuffer(0), encoding, cb); + this._write(new Uint8Array(0), encoding, cb); } _writeChunk (chunk, encoding, cb) { let tap = this._tap; - tap.buf = Buffer.concat([tap.buf.slice(tap.pos), chunk]); - tap.pos = 0; + tap.forward(chunk); let nBlocks = 1; let block; while ((block = tryReadBlock(tap))) { - if (!this._syncMarker.equals(block.sync)) { + if (!utils.bufEqual(this._syncMarker, block.sync)) { this.emit('error', new Error('invalid sync marker')); return; } @@ -293,8 +293,7 @@ class BlockDecoder extends stream.Duplex { } data.cb(); this._remaining = data.count; - tap.buf = data.buf; - tap.pos = 0; + tap.setData(data.buf); } this._remaining--; @@ -332,26 +331,25 @@ class RawEncoder extends stream.Transform { this.emit('typeError', err, val, this._type); } }; - this._tap = new Tap(utils.newBuffer(opts.batchSize || 65536)); + this._tap = Tap.withCapacity(opts.batchSize || 65536); this.on('typeError', function (err) { this.emit('error', err); }); } _transform (val, encoding, cb) { let tap = this._tap; - let buf = tap.buf; let pos = tap.pos; this._writeValue(tap, val); if (!tap.isValid()) { if (pos) { // Emit any valid data. - this.push(copyBuffer(tap.buf, 0, pos)); + this.push(tap.toBuffer()); } let len = tap.pos - pos; - if (len > buf.length) { + if (len > tap.length) { // Not enough space for last written object, need to resize. - tap.buf = utils.newBuffer(2 * len); + tap.reinitialize(2 * len); } tap.pos = 0; this._writeValue(tap, val); // Rewrite last failed write. @@ -365,7 +363,7 @@ class RawEncoder extends stream.Transform { let pos = tap.pos; if (pos) { // This should only ever be false if nothing is written to the stream. - this.push(tap.buf.slice(0, pos)); + this.push(tap.subarray(0, pos)); } cb(); } @@ -415,7 +413,7 @@ class BlockEncoder extends stream.Duplex { return true; }; this._blockSize = opts.blockSize || 65536; - this._tap = new Tap(utils.newBuffer(this._blockSize)); + this._tap = Tap.withCapacity(this._blockSize); this._codecs = opts.codecs; this._codec = opts.codec || 'null'; this._blockCount = 0; @@ -484,8 +482,8 @@ class BlockEncoder extends stream.Duplex { let meta = utils.copyOwnProperties( this._metadata, { - 'avro.schema': utils.bufferFrom(schema), - 'avro.codec': utils.bufferFrom(this._codec) + 'avro.schema': ENCODER.encode(schema), + 'avro.codec': ENCODER.encode(this._codec) }, true // Overwrite. ); @@ -515,8 +513,7 @@ class BlockEncoder extends stream.Duplex { // Not enough space for last written object, need to resize. this._blockSize = len * 2; } - tap.buf = utils.newBuffer(this._blockSize); - tap.pos = 0; + tap.reinitialize(this._blockSize); this._writeValue(tap, val); // Rewrite last failed write. } this._blockCount++; @@ -532,7 +529,10 @@ class BlockEncoder extends stream.Duplex { _flushChunk (pos, cb) { let tap = this._tap; pos = pos || tap.pos; - this._compress(tap.buf.slice(0, pos), this._createBlockCallback(pos, cb)); + this._compress( + tap.subarray(0, pos), + this._createBlockCallback(pos, cb) + ); this._blockCount = 0; } @@ -627,7 +627,7 @@ function createReader(noDecode, writerType, readerType) { return function (tap) { let pos = tap.pos; skipper(tap); - return tap.buf.slice(pos, tap.pos); + return tap.subarray(pos, tap.pos); }; })(writerType._skip); } else if (readerType) { @@ -638,13 +638,6 @@ function createReader(noDecode, writerType, readerType) { } } -/** Copy a buffer. This avoids creating a slice of the original buffer. */ -function copyBuffer(buf, pos, len) { - let copy = utils.newBuffer(len); - buf.copy(copy, 0, pos, pos + len); - return copy; -} - module.exports = { BLOCK_TYPE, // For tests. diff --git a/lib/index.js b/lib/index.js index 92b8739b..d3a9e5e1 100644 --- a/lib/index.js +++ b/lib/index.js @@ -11,11 +11,10 @@ let containers = require('./containers'), specs = require('./specs'), types = require('./types'), utils = require('./utils'), - buffer = require('buffer'), fs = require('fs'); -let Buffer = buffer.Buffer; +const DECODER = new TextDecoder(); /** Extract a container file's header synchronously. */ function extractFileHeader(path, opts) { @@ -23,13 +22,16 @@ function extractFileHeader(path, opts) { let decode = opts.decode === undefined ? true : !!opts.decode; let size = Math.max(opts.size || 4096, 4); - let buf = utils.newBuffer(size); + let buf = new Uint8Array(size); let tap = new utils.Tap(buf); let fd = fs.openSync(path, 'r'); try { let pos = fs.readSync(fd, buf, 0, size); - if (pos < 4 || !containers.MAGIC_BYTES.equals(buf.slice(0, 4))) { + if ( + pos < 4 || + !utils.bufEqual(containers.MAGIC_BYTES, buf.subarray(0, 4)) + ) { return null; } @@ -39,9 +41,9 @@ function extractFileHeader(path, opts) { } while (!isValid()); if (decode !== false) { let meta = header.meta; - meta['avro.schema'] = JSON.parse(meta['avro.schema'].toString()); + meta['avro.schema'] = JSON.parse(DECODER.decode(meta['avro.schema'])); if (meta['avro.codec'] !== undefined) { - meta['avro.codec'] = meta['avro.codec'].toString(); + meta['avro.codec'] = DECODER.decode(meta['avro.codec']); } } return header; @@ -53,11 +55,10 @@ function extractFileHeader(path, opts) { if (tap.isValid()) { return true; } - let len = 2 * tap.buf.length; - let buf = utils.newBuffer(len); + let len = 2 * tap.length; + let buf = new Uint8Array(len); len = fs.readSync(fd, buf, 0, len); - tap.buf = Buffer.concat([tap.buf, buf]); - tap.pos = 0; + tap.append(buf); return false; } } diff --git a/lib/platform.js b/lib/platform.js index bde80f00..d94ec18c 100644 --- a/lib/platform.js +++ b/lib/platform.js @@ -11,7 +11,8 @@ function getHash(str, algorithm) { algorithm = algorithm || 'md5'; let hash = crypto.createHash(algorithm); hash.end(str); - return hash.read(); + let buf = hash.read(); + return new Uint8Array(buf.buffer, buf.byteOffset, buf.length); } module.exports = { diff --git a/lib/types.js b/lib/types.js index c1aba225..1594e71d 100644 --- a/lib/types.js +++ b/lib/types.js @@ -16,7 +16,7 @@ let utils = require('./utils'), platform = require('./platform'); // Convenience imports. -let Tap = utils.Tap; +let {Tap, isBufferLike} = utils; let debug = platform.debuglog('avsc:types'); let j = utils.printJSON; @@ -28,7 +28,7 @@ let TYPES; let RANDOM = new utils.Lcg(); // Encoding tap (shared for performance). -let TAP = new Tap(Buffer.allocUnsafeSlow(1024)); +let TAP = Tap.withCapacity(1024); // Currently active logical type, used for name redirection. let LOGICAL_TYPE = null; @@ -44,7 +44,7 @@ let UNDERLYING_TYPES = []; * schemas. All type values are represented in memory similarly to their JSON * representation, except for: * - * + `bytes` and `fixed` which are represented as `Buffer`s. + * + `bytes` and `fixed` which are represented as `Uint8Array`s. * + `union`s which will be "unwrapped" unless the `wrapUnions` option is set. * * See individual subclasses for details. @@ -259,7 +259,7 @@ class Type { opts ) }, opts); - } else if (Buffer.isBuffer(val)) { + } else if (isBufferLike(val)) { return Type.forSchema('bytes', opts); } let fieldNames = Object.keys(val); @@ -439,7 +439,7 @@ class Type { static __reset (size) { debug('resetting type buffer to %d', size); - TAP.buf = Buffer.allocUnsafeSlow(size); + TAP.reinitialize(size); } get branchName () { @@ -471,7 +471,7 @@ class Type { } compareBuffers (buf1, buf2) { - return this._match(new Tap(buf1), new Tap(buf2)); + return this._match(Tap.fromBuffer(buf1), Tap.fromBuffer(buf2)); } createResolver (type, opts) { @@ -532,7 +532,7 @@ class Type { } decode (buf, pos, resolver) { - let tap = new Tap(buf, pos); + let tap = Tap.fromBuffer(buf, pos); let val = readValue(this, tap, resolver); if (!tap.isValid()) { return {value: undefined, offset: -1}; @@ -541,7 +541,7 @@ class Type { } encode (val, buf, pos) { - let tap = new Tap(buf, pos); + let tap = Tap.fromBuffer(buf, pos); this._write(tap, val); if (!tap.isValid()) { // Don't throw as there is no way to predict this. We also return the @@ -554,7 +554,7 @@ class Type { equals (type, opts) { let canon = ( // Canonical equality. Type.isType(type) && - this.fingerprint().equals(type.fingerprint()) + this._getCachedHash() === type._getCachedHash() ); if (!canon || !(opts && opts.strict)) { return canon; @@ -565,20 +565,32 @@ class Type { ); } + /** + * Get this type's schema fingerprint (lazily calculated and cached). + * Differs from {@link fingerprint} in that it returns the string + * representation of the fingerprint as it's stored internally. + * @returns {string} + */ + _getCachedHash() { + if (!this._hash.hash) { + let schemaStr = JSON.stringify(this.schema()); + // Cache the hash as a binary string to avoid overhead and also return a + // fresh copy every time + // https://stackoverflow.com/questions/45803829/memory-overhead-of-typed-arrays-vs-strings/45808835#45808835 + this._hash.hash = utils.bufferToBinaryString(utils.getHash(schemaStr)); + } + return this._hash.hash; + } + fingerprint (algorithm) { if (!algorithm) { - if (!this._hash.str) { - let schemaStr = JSON.stringify(this.schema()); - this._hash.str = utils.getHash(schemaStr).toString('binary'); - } - return utils.bufferFrom(this._hash.str, 'binary'); - } else { - return utils.getHash(JSON.stringify(this.schema()), algorithm); + return utils.binaryStringToBuffer(this._getCachedHash()); } + return utils.getHash(JSON.stringify(this.schema()), algorithm); } fromBuffer (buf, resolver, noCheck) { - let tap = new Tap(buf); + let tap = Tap.fromBuffer(buf, 0); let val = readValue(this, tap, resolver, noCheck); if (!tap.isValid()) { throw new Error('truncated buffer'); @@ -636,10 +648,10 @@ class Type { TAP.pos = 0; this._write(TAP, val); if (TAP.isValid()) { - return Uint8Array.prototype.slice.call(TAP.buf, 0, TAP.pos); + return TAP.toBuffer(); } - let buf = utils.newBuffer(TAP.pos); - this._write(new Tap(buf), val); + let buf = new Uint8Array(TAP.pos); + this._write(Tap.fromBuffer(buf), val); return buf; } @@ -1086,7 +1098,7 @@ StringType.prototype.typeName = 'string'; /** * Bytes. * - * These are represented in memory as `Buffer`s rather than binary-encoded + * These are represented in memory as `Uint8Array`s rather than binary-encoded * strings. This is more efficient (when decoding/encoding from bytes, the * common use-case), idiomatic, and convenient. * @@ -1094,7 +1106,7 @@ StringType.prototype.typeName = 'string'; */ class BytesType extends PrimitiveType { _check (val, flags, hook) { - let b = Buffer.isBuffer(val); + let b = isBufferLike(val); if (!b && hook) { hook(val, this); } @@ -1106,7 +1118,7 @@ class BytesType extends PrimitiveType { _skip (tap) { tap.skipBytes(); } _write (tap, val) { - if (!Buffer.isBuffer(val)) { + if (!isBufferLike(val)) { throwInvalidError(val, this); } tap.writeBytes(val); @@ -1129,24 +1141,24 @@ class BytesType extends PrimitiveType { switch ((opts && opts.coerce) | 0) { case 3: // Coerce buffers to strings. this._check(obj, undefined, throwInvalidError); - return obj.toString('binary'); + return utils.bufferToBinaryString(obj); case 2: // Coerce strings to buffers. if (typeof obj != 'string') { throw new Error(`cannot coerce to buffer: ${j(obj)}`); } - buf = utils.bufferFrom(obj, 'binary'); + buf = utils.binaryStringToBuffer(obj); this._check(buf, undefined, throwInvalidError); return buf; case 1: // Coerce buffer JSON representation to buffers. if (!isJsonBuffer(obj)) { throw new Error(`cannot coerce to buffer: ${j(obj)}`); } - buf = utils.bufferFrom(obj.data); + buf = new Uint8Array(obj.data); this._check(buf, undefined, throwInvalidError); return buf; default: // Copy buffer. this._check(obj, undefined, throwInvalidError); - return utils.bufferFrom(obj); + return new Uint8Array(obj); } } @@ -1155,7 +1167,7 @@ class BytesType extends PrimitiveType { } } -BytesType.prototype.compare = Buffer.compare; +BytesType.prototype.compare = utils.bufCompare; BytesType.prototype.typeName = 'bytes'; @@ -1713,7 +1725,7 @@ class EnumType extends Type { EnumType.prototype.typeName = 'enum'; -/** Avro fixed type. Represented simply as a `Buffer`. */ +/** Avro fixed type. Represented simply as a `Uint8Array`. */ class FixedType extends Type { constructor (schema, opts) { super(schema, opts); @@ -1726,7 +1738,7 @@ class FixedType extends Type { } _check (val, flags, hook) { - let b = Buffer.isBuffer(val) && val.length === this.size; + let b = isBufferLike(val) && val.length === this.size; if (!b && hook) { hook(val, this); } @@ -1742,7 +1754,7 @@ class FixedType extends Type { } _write (tap, val) { - if (!Buffer.isBuffer(val) || val.length !== this.size) { + if (!isBufferLike(val) || val.length !== this.size) { throwInvalidError(val, this); } tap.writeFixed(val, this.size); @@ -1774,7 +1786,7 @@ class FixedType extends Type { FixedType.prototype._copy = BytesType.prototype._copy; -FixedType.prototype.compare = Buffer.compare; +FixedType.prototype.compare = utils.bufCompare; FixedType.prototype.typeName = 'fixed'; @@ -2311,15 +2323,12 @@ class RecordType extends Type { if (field.defaultValue() === undefined) { body += 't' + i + '._write(t, v.' + field.name + ');\n'; } else { - let value = field.type.toBuffer(field.defaultValue()).toString('binary'); - // Convert the default value to a binary string ahead of time. We aren't - // converting it to a buffer to avoid retaining too much memory. If we - // had our own buffer pool, this could be an idea in the future. + let value = field.type.toBuffer(field.defaultValue()); args.push('d' + i); values.push(value); body += 'var v' + i + ' = v.' + field.name + ';\n'; body += 'if (v' + i + ' === undefined) {\n'; - body += ' t.writeBinary(d' + i + ', ' + value.length + ');\n'; + body += ' t.writeFixed(d' + i + ', ' + value.length + ');\n'; body += ' } else {\n t' + i + '._write(t, v' + i + ');\n }\n'; } } @@ -2704,7 +2713,7 @@ class AbstractLongType extends LongType { if (this._noUnpack) { let pos = tap.pos; tap.skipLong(); - buf = tap.buf.slice(pos, tap.pos); + buf = tap.subarray(pos, tap.pos); } else { buf = tap.unpackLongBytes(tap); } @@ -2868,7 +2877,7 @@ Resolver.prototype._peek = Type.prototype._peek; /** Mutable hash container. */ class Hash { constructor () { - this.str = undefined; + this.hash = undefined; } } @@ -3064,7 +3073,7 @@ function getValueBucket(val) { // Could be bytes, fixed, array, map, or record. if (Array.isArray(val)) { return 'array'; - } else if (Buffer.isBuffer(val)) { + } else if (isBufferLike(val)) { return 'buffer'; } } diff --git a/lib/utils.js b/lib/utils.js index 6a50ea6a..2036bc41 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -5,31 +5,13 @@ /** Various utilities used across this library. */ -let buffer = require('buffer'); let platform = require('./platform'); -let Buffer = buffer.Buffer; - // Valid (field, type, and symbol) name regex. const NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; -/** - * Create a new empty buffer. - * - * @param size {Number} The buffer's size. - */ -function newBuffer(size) { - return Buffer.alloc(size); -} - -/** - * Create a new buffer with the input contents. - * - * @param data {Array|String} The buffer's data. - * @param enc {String} Encoding, used if data is a string. - */ -function bufferFrom(data, enc) { - return Buffer.from(data, enc); +function isBufferLike(data) { + return (data instanceof Uint8Array); } /** @@ -47,6 +29,33 @@ function capitalize(s) { return s.charAt(0).toUpperCase() + s.slice(1); } */ function compare(n1, n2) { return n1 === n2 ? 0 : (n1 < n2 ? -1 : 1); } +let bufCompare, bufEqual; +if (typeof Buffer == 'function') { + bufCompare = Buffer.compare; + bufEqual = function(buf1, buf2) { + return Buffer.prototype.equals.call(buf1, buf2); + }; +} else { + bufCompare = function(buf1, buf2) { + if (buf1 === buf2) { + return 0; + } + let len = Math.min(buf1.length, buf2.length); + for (let i = 0; i < len; i++) { + if (buf1[i] !== buf2[i]) { + return Math.sign(buf1[i] - buf2[i]); + } + } + return Math.sign(buf1.length - buf2.length); + }; + bufEqual = function(buf1, buf2) { + if (buf1.length !== buf2.length) { + return false; + } + return bufCompare(buf1, buf2) === 0; + }; +} + /** * Get option or default if undefined. * @@ -257,34 +266,6 @@ function jsonEnd(str, pos) { /** "Abstract" function to help with "subclassing". */ function abstractFunction() { throw new Error('abstract'); } -/** - * Simple buffer pool to avoid allocating many small buffers. - * - * This provides significant speedups in recent versions of node (6+). - */ -class BufferPool { - constructor (len) { - this._len = len | 0; - this._pos = 0; - this._slab = newBuffer(this._len); - } - - alloc (len) { - if (len < 0) { - throw new Error('negative length'); - } - let maxLen = this._len; - if (len > maxLen) { - return newBuffer(len); - } - if (this._pos + len > maxLen) { - this._slab = newBuffer(maxLen); - this._pos = 0; - } - return this._slab.slice(this._pos, this._pos += len); - } -} - /** * Generator of random things. * @@ -350,11 +331,11 @@ class Lcg { } nextBuffer (len) { - let arr = []; + let arr = new Uint8Array(len); for (let i = 0; i < len; i++) { - arr.push(this.nextInt(256)); + arr[i] = this.nextInt(256); } - return bufferFrom(arr); + return arr; } choice (arr) { @@ -431,8 +412,108 @@ class OrderedQueue { } } -// Shared buffer pool for all taps. -const POOL = new BufferPool(4096); +let decodeSlice; +if (typeof Buffer === 'function' && typeof Buffer.prototype.utf8Slice === 'function') { + // Note that calling `Buffer.prototype.toString.call(buf, 'utf-8')` on a + // `Uint8Array` throws because Node's internal implementation expects the + // argument to be a `Buffer` specifically. + decodeSlice = Function.prototype.call.bind(Buffer.prototype.utf8Slice); +} else { + const DECODER = new TextDecoder(); + + decodeSlice = function(arr, start, end) { + return DECODER.decode(arr.subarray(start, end)); + }; +} + +const ENCODER = new TextEncoder(); +const encodeBuf = new Uint8Array(4096); +const encodeBufs = []; + +function encodeSlice(str) { + const {read, written} = ENCODER.encodeInto(str, encodeBuf); + if (read === str.length) { + // Believe it or not, `subarray` is actually quite expensive. To avoid the + // cost, we cache and reuse `subarray`s. + if (!encodeBufs[written]) { + encodeBufs[written] = encodeBuf.subarray(0, written); + } + return encodeBufs[written]; + } + + return ENCODER.encode(str); +} + +let utf8Length; +if (typeof Buffer === 'function') { + utf8Length = Buffer.byteLength; +} else { + utf8Length = function(str) { + let len = 0; + for (;;) { + // encodeInto is faster than any manual implementation (or even + // Buffer.byteLength), provided the string fits entirely within the + // buffer. Past that, it slows down but is still faster than other + // options. + const {read, written} = ENCODER.encodeInto(str, encodeBuf); + len += written; + if (read === str.length) break; + str = str.slice(read); + } + return len; + }; +} + +let bufferToBinaryString; +if (typeof Buffer === 'function' && typeof Buffer.prototype.latin1Slice === 'function') { + // Note that calling `Buffer.prototype.toString.call(buf, 'binary')` on a + // `Uint8Array` throws because Node's internal implementation expects the + // argument to be a `Buffer` specifically. + bufferToBinaryString = Function.prototype.call.bind( + Buffer.prototype.latin1Slice); +} else { + bufferToBinaryString = function(buf) { + let str = ''; + let i = 0, len = buf.length; + for (; i + 7 < len; i += 8) { + str += String.fromCharCode( + buf[i], + buf[i + 1], + buf[i + 2], + buf[i + 3], + buf[i + 4], + buf[i + 5], + buf[i + 6], + buf[i + 7] + ); + } + for (; i < len; i++) { + str += String.fromCharCode(buf[i]); + } + return str; + }; +} + +let binaryStringToBuffer; +if (typeof Buffer === 'function') { + binaryStringToBuffer = function(str) { + let buf = Buffer.from(str, 'binary'); + return new Uint8Array(buf.buffer, buf.byteOffset, buf.length); + }; +} else { + binaryStringToBuffer = function(str) { + let buf = new Uint8Array(str.length); + for (let i = 0; i < str.length; i++) { + buf[i] = str.charCodeAt(i); + } + return Buffer.from(buf); + }; +} + +// Having multiple views into the same buffer seems to massively decrease read +// performance. To read and write float and double types, copy them to and from +// this data view instead. +const FLOAT_VIEW = new DataView(new ArrayBuffer(8)); /** * A tap is a buffer which remembers what has been already read. @@ -444,13 +525,60 @@ const POOL = new BufferPool(4096); */ class Tap { constructor (buf, pos) { - this.buf = buf; + this.setData(buf, pos); + } + + setData (buf, pos) { + if (typeof Buffer === 'function' && buf instanceof Buffer) { + buf = new Uint8Array(buf.buffer, buf.byteOffset, buf.length); + } + this.arr = buf; this.pos = pos | 0; if (this.pos < 0) { throw new Error('negative offset'); } } + get length() { + return this.arr.length; + } + + reinitialize (capacity) { + this.setData(new Uint8Array(capacity)); + } + + static fromBuffer (buf, pos) { + return new Tap(buf, pos); + } + + static withCapacity (capacity) { + let buf = new Uint8Array(capacity); + return new Tap(buf); + } + + toBuffer () { + return this.arr.slice(0, this.pos); + } + + subarray (start, end) { + return this.arr.subarray(start, end); + } + + append (newBuf) { + const newArr = new Uint8Array(this.arr.length + newBuf.length); + newArr.set(this.arr, 0); + newArr.set(newBuf, this.arr.length); + this.setData(newArr, 0); + } + + forward (newBuf) { + const subArr = this.arr.subarray(this.pos); + const newArr = new Uint8Array(subArr.length + newBuf.length); + newArr.set(subArr, 0); + newArr.set(newBuf, subArr.length); + this.setData(newArr, 0); + } + /** * Check that the tap is in a valid state. * @@ -459,9 +587,9 @@ class Tap { * caller to always check that the read, skip, or write was valid by calling * this method. */ - isValid () { return this.pos <= this.buf.length; } + isValid () { return this.pos <= this.arr.length; } - _invalidate () { this.pos = this.buf.length + 1; } + _invalidate () { this.pos = this.arr.length + 1; } // Read, skip, write methods. // @@ -471,16 +599,16 @@ class Tap { // negative position offset (which will typically cause a failure in // `readFixed`). - readBoolean () { return !!this.buf[this.pos++]; } + readBoolean () { return !!this.arr[this.pos++]; } skipBoolean () { this.pos++; } - writeBoolean (b) { this.buf[this.pos++] = !!b; } + writeBoolean (b) { this.arr[this.pos++] = !!b; } readLong () { let n = 0; let k = 0; - let buf = this.buf; + let buf = this.arr; let b, h, f, fk; do { @@ -506,12 +634,12 @@ class Tap { } skipLong () { - let buf = this.buf; + let buf = this.arr; while (buf[this.pos++] & 0x80) {} } writeLong (n) { - let buf = this.buf; + let buf = this.arr; let f, m; if (n >= -1073741824 && n < 1073741824) { @@ -533,58 +661,91 @@ class Tap { } readFloat () { - let buf = this.buf; let pos = this.pos; this.pos += 4; - if (this.pos > buf.length) { + if (this.pos > this.arr.length) { return 0; } - return this.buf.readFloatLE(pos); + FLOAT_VIEW.setUint32( + 0, + this.arr[pos] | + (this.arr[pos + 1] << 8) | + (this.arr[pos + 2] << 16) | + (this.arr[pos + 3] << 24), + true); + return FLOAT_VIEW.getFloat32(0, true); } skipFloat () { this.pos += 4; } writeFloat (f) { - let buf = this.buf; let pos = this.pos; this.pos += 4; - if (this.pos > buf.length) { + if (this.pos > this.arr.length) { return; } - return this.buf.writeFloatLE(f, pos); + + FLOAT_VIEW.setFloat32(0, f, true); + const n = FLOAT_VIEW.getUint32(0, true); + this.arr[pos] = n & 0xff; + this.arr[pos + 1] = (n >> 8) & 0xff; + this.arr[pos + 2] = (n >> 16) & 0xff; + this.arr[pos + 3] = n >> 24; } readDouble () { - let buf = this.buf; let pos = this.pos; this.pos += 8; - if (this.pos > buf.length) { + if (this.pos > this.arr.length) { return 0; } - return this.buf.readDoubleLE(pos); + FLOAT_VIEW.setUint32( + 0, + this.arr[pos] | + (this.arr[pos + 1] << 8) | + (this.arr[pos + 2] << 16) | + (this.arr[pos + 3] << 24), + true + ); + FLOAT_VIEW.setUint32( + 4, + this.arr[pos + 4] | + (this.arr[pos + 5] << 8) | + (this.arr[pos + 6] << 16) | + (this.arr[pos + 7] << 24), + true + ); + return FLOAT_VIEW.getFloat64(0, true); } skipDouble () { this.pos += 8; } writeDouble (d) { - let buf = this.buf; let pos = this.pos; this.pos += 8; - if (this.pos > buf.length) { + if (this.pos > this.arr.length) { return; } - return this.buf.writeDoubleLE(d, pos); + FLOAT_VIEW.setFloat64(0, d, true); + const a = FLOAT_VIEW.getUint32(0, true); + const b = FLOAT_VIEW.getUint32(4, true); + this.arr[pos] = a & 0xff; + this.arr[pos + 1] = (a >> 8) & 0xff; + this.arr[pos + 2] = (a >> 16) & 0xff; + this.arr[pos + 3] = a >> 24; + this.arr[pos + 4] = b & 0xff; + this.arr[pos + 5] = (b >> 8) & 0xff; + this.arr[pos + 6] = (b >> 16) & 0xff; + this.arr[pos + 7] = b >> 24; } readFixed (len) { let pos = this.pos; this.pos += len; - if (this.pos > this.buf.length) { + if (this.pos > this.arr.length) { return; } - let fixed = POOL.alloc(len); - this.buf.copy(fixed, 0, pos, pos + len); - return fixed; + return this.arr.slice(pos, pos + len); } skipFixed (len) { this.pos += len; } @@ -593,10 +754,10 @@ class Tap { len = len || buf.length; let pos = this.pos; this.pos += len; - if (this.pos > this.buf.length) { + if (this.pos > this.arr.length) { return; } - buf.copy(this.buf, pos, 0, len); + this.arr.set(buf.subarray(0, len), pos); } readBytes () { @@ -632,46 +793,135 @@ class Tap { this.pos += len; } - writeString (s) { - let len = Buffer.byteLength(s); - let buf = this.buf; - this.writeLong(len); + readString () { + let len = this.readLong(); + if (len < 0) { + this._invalidate(); + return ''; + } let pos = this.pos; this.pos += len; - if (this.pos > buf.length) { + if (this.pos > this.arr.length) { return; } - if (len > 64 && typeof Buffer.prototype.utf8Write == 'function') { - // This method is roughly 50% faster than the manual implementation below - // for long strings (which is itself faster than the generic - // `Buffer#write` at least in most browsers, where `utf8Write` is not - // available). - buf.utf8Write(s, pos, len); + + let arr = this.arr; + let end = pos + len; + if (len > 24) { + return decodeSlice(arr, pos, end); + } + + let output = ''; + // Consume the string in 4-byte chunks. The performance benefit comes not + // from *reading* in chunks, but calling fromCharCode with 4 characters per + // call. + while (pos + 3 < end) { + let a = arr[pos], b = arr[pos + 1], c = arr[pos + 2], d = arr[pos + 3]; + // If the high bit of any character is set, it's a non-ASCII character. + // Fall back to TextDecoder for the remaining characters. + if ((a | b | c | d) & 0x80) { + output += decodeSlice(arr, pos, end); + return output; + } + output += String.fromCharCode(a, b, c, d); + pos += 4; + } + + // Handle the remainder of the string. + while (pos < end) { + let char = arr[pos]; + if (char & 0x80) { + output += decodeSlice(arr, pos, end); + return output; + } + output += String.fromCharCode(char); + pos++; + } + + return output; + } + + writeString (s) { + let buf = this.arr; + const stringLen = s.length; + // The maximum number that a signed varint can store in a single byte is 63. + // The maximum size of a UTF-8 representation of a UTF-16 string is 3 times + // its length, as one UTF-16 character can be represented by up to 3 bytes + // in UTF-8. Therefore, if the string is 21 characters or less, we know that + // its length can be stored in a single byte, which is why we choose 21 as + // the small-string threshold specifically. + if (stringLen > 21) { + let encodedLength, encoded; + + // If we're already over the buffer size, we don't need to encode the + // string. While encodeInto is actually faster than Buffer.byteLength, we + // could still overflow the preallocated encoding buffer and have to fall + // back to allocating, which is really really slow. + if (this.isValid()) { + encoded = encodeSlice(s); + encodedLength = encoded.length; + } else { + encodedLength = utf8Length(s); + } + this.writeLong(encodedLength); + let pos = this.pos; + this.pos += encodedLength; + + if (this.isValid() && typeof encoded != 'undefined') { + buf.set(encoded, pos); + } } else { - for (let i = 0, l = len; i < l; i++) { + // For small strings, this manual implementation is faster. + + // Set aside 1 byte to write the string length. + let pos = this.pos + 1; + let startPos = pos; + let bufLen = buf.length; + + // This is not a micro-optimization: caching the string length for the + // loop predicate really does make a difference! + for (let i = 0; i < stringLen; i++) { let c1 = s.charCodeAt(i); let c2; if (c1 < 0x80) { - buf[pos++] = c1; + if (pos < bufLen) buf[pos] = c1; + pos++; } else if (c1 < 0x800) { - buf[pos++] = c1 >> 6 | 0xc0; - buf[pos++] = c1 & 0x3f | 0x80; + if (pos + 1 < bufLen) { + buf[pos] = c1 >> 6 | 0xc0; + buf[pos + 1] = c1 & 0x3f | 0x80; + } + pos += 2; } else if ( (c1 & 0xfc00) === 0xd800 && ((c2 = s.charCodeAt(i + 1)) & 0xfc00) === 0xdc00 ) { c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff); i++; - buf[pos++] = c1 >> 18 | 0xf0; - buf[pos++] = c1 >> 12 & 0x3f | 0x80; - buf[pos++] = c1 >> 6 & 0x3f | 0x80; - buf[pos++] = c1 & 0x3f | 0x80; + if (pos + 3 < bufLen) { + buf[pos] = c1 >> 18 | 0xf0; + buf[pos + 1] = c1 >> 12 & 0x3f | 0x80; + buf[pos + 2] = c1 >> 6 & 0x3f | 0x80; + buf[pos + 3] = c1 & 0x3f | 0x80; + } + pos += 4; } else { - buf[pos++] = c1 >> 12 | 0xe0; - buf[pos++] = c1 >> 6 & 0x3f | 0x80; - buf[pos++] = c1 & 0x3f | 0x80; + if (pos + 2 < bufLen) { + buf[pos] = c1 >> 12 | 0xe0; + buf[pos + 1] = c1 >> 6 & 0x3f | 0x80; + buf[pos + 2] = c1 & 0x3f | 0x80; + } + pos += 3; } } + + // Note that we've not yet updated this.pos, so it's currently pointing to + // the place where we want to write the string length. + if (this.pos <= bufLen) { + this.writeLong(pos - startPos); + } + + this.pos = pos; } } @@ -685,7 +935,7 @@ class Tap { // valid buffers. matchBoolean (tap) { - return this.buf[this.pos++] - tap.buf[tap.pos++]; + return this.arr[this.pos++] - tap.arr[tap.pos++]; } matchLong (tap) { @@ -707,7 +957,7 @@ class Tap { } matchFixed (tap, len) { - return this.readFixed(len).compare(tap.readFixed(len)); + return bufCompare(this.readFixed(len), tap.readFixed(len)); } matchBytes (tap) { @@ -717,9 +967,9 @@ class Tap { let l2 = tap.readLong(); let p2 = tap.pos; tap.pos += l2; - let b1 = this.buf.slice(p1, this.pos); - let b2 = tap.buf.slice(p2, tap.pos); - return b1.compare(b2); + let b1 = this.arr.subarray(p1, this.pos); + let b2 = tap.arr.subarray(p2, tap.pos); + return bufCompare(b1, b2); } // Functions for supporting custom long classes. @@ -728,11 +978,11 @@ class Tap { // worry about Avro's zigzag encoding, we directly expose longs as unpacked. unpackLongBytes () { - let res = newBuffer(8); + let res = new Uint8Array(8); let n = 0; let i = 0; // Byte index in target buffer. let j = 6; // Bit offset in current target buffer byte. - let buf = this.buf; + let buf = this.arr; let b = buf[this.pos++]; let neg = b & 1; @@ -761,7 +1011,7 @@ class Tap { packLongBytes (buf) { let neg = (buf[7] & 0x80) >> 7; - let res = this.buf; + let res = this.arr; let j = 1; let k = 0; let m = 3; @@ -775,9 +1025,9 @@ class Tap { } let parts = [ - buf.readUIntLE(0, 3), - buf.readUIntLE(3, 3), - buf.readUIntLE(6, 2) + (buf[0] | (buf[1] << 8) | (buf[2] << 16)), + (buf[3] | (buf[4] << 8) | (buf[5] << 16)), + (buf[6] | (buf[7] << 8)) ]; // Not reading more than 24 bits because we need to be able to combine the // "carry" bits from the previous part and JavaScript only supports bitwise @@ -811,84 +1061,13 @@ class Tap { } } -// TODO: check if the following optimizations are really worth it or if they're -// premature micro-optimizations. - -/* istanbul ignore else */ -if (typeof Buffer.prototype.utf8Slice == 'function') { - // Use this optimized function when available. - Tap.prototype.readString = function () { - let len = this.readLong(); - if (len < 0) { - this._invalidate(); - return ''; - } - let pos = this.pos; - let buf = this.buf; - this.pos += len; - if (this.pos > buf.length) { - return; - } - return this.buf.utf8Slice(pos, pos + len); - }; -} else { - Tap.prototype.readString = function () { - let len = this.readLong(); - if (len < 0) { - this._invalidate(); - return ''; - } - let pos = this.pos; - let buf = this.buf; - this.pos += len; - if (this.pos > buf.length) { - return; - } - return this.buf.slice(pos, pos + len).toString(); - }; -} - -/* istanbul ignore else */ -if (typeof Buffer.prototype.latin1Write == 'function') { - // `binaryWrite` has been renamed to `latin1Write` in Node v6.4.0, see - // https://github.com/nodejs/node/pull/7111. Note that the `'binary'` - // encoding argument still works however. - Tap.prototype.writeBinary = function (str, len) { - let pos = this.pos; - this.pos += len; - if (this.pos > this.buf.length) { - return; - } - this.buf.latin1Write(str, pos, len); - }; -} else if (typeof Buffer.prototype.binaryWrite == 'function') { - Tap.prototype.writeBinary = function (str, len) { - let pos = this.pos; - this.pos += len; - if (this.pos > this.buf.length) { - return; - } - this.buf.binaryWrite(str, pos, len); - }; -} else { - // Slowest implementation. - Tap.prototype.writeBinary = function (s, len) { - let pos = this.pos; - this.pos += len; - if (this.pos > this.buf.length) { - return; - } - this.buf.write(s, pos, len, 'binary'); - }; -} - // Helpers. /** * Invert all bits in a buffer. * - * @param buf {Buffer} Non-empty buffer to invert. - * @param len {Number} Buffer length (must be positive). + * @param {Uint8Array} buf Non-empty buffer to invert. + * @param {number} len Buffer length (must be positive). */ function invert(buf, len) { while (len--) { @@ -920,23 +1099,25 @@ function printJSON (obj) { module.exports = { abstractFunction, - bufferFrom, + bufCompare, + bufEqual, + bufferToBinaryString, + binaryStringToBuffer, capitalize, copyOwnProperties, getHash: platform.getHash, compare, getOption, impliedNamespace, + isBufferLike, isValidName, jsonEnd, - newBuffer, objectValues, qualify, toMap, singleIndexOf, hasDuplicates, unqualify, - BufferPool, Lcg, OrderedQueue, Tap, diff --git a/package.json b/package.json index 1d5174f8..95ea28e0 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "clean": "rm -rf coverage node_modules", "cover": "nyc mocha -- --ui tdd", "lint": "eslint etc/ lib/ test/ \"etc/scripts/**\"", - "perf": "./etc/scripts/perf etc/schemas/*", + "perf": "node --expose-gc ./etc/scripts/perf etc/schemas/*", "test": "mocha --ui tdd --reporter dot" }, "devDependencies": { diff --git a/test/test_containers.js b/test/test_containers.js index 87b8f84f..5ae62431 100644 --- a/test/test_containers.js +++ b/test/test_containers.js @@ -2,7 +2,6 @@ let containers = require('../lib/containers'), types = require('../lib/types'), - utils = require('../lib/utils'), assert = require('assert'), buffer = require('buffer'), stream = require('stream'), @@ -15,11 +14,12 @@ let Block = BLOCK_TYPE.recordConstructor; let HEADER_TYPE = containers.HEADER_TYPE; let Header = HEADER_TYPE.recordConstructor; let MAGIC_BYTES = containers.MAGIC_BYTES; -let SYNC = utils.bufferFrom('atokensyncheader'); +let SYNC = Buffer.from('atokensyncheader'); let Type = types.Type; let streams = containers.streams; let builtins = types.builtins; +const DECODER = new TextDecoder(); suite('containers', () => { @@ -38,7 +38,7 @@ suite('containers', () => { buf = chunk; }) .on('end', () => { - assert.deepEqual(buf, utils.bufferFrom([2, 0, 3])); + assert.deepEqual(buf, Buffer.from([2, 0, 3])); cb(); }); encoder.write(1); @@ -55,8 +55,8 @@ suite('containers', () => { }) .on('end', () => { assert.deepEqual(bufs, [ - utils.bufferFrom([1]), - utils.bufferFrom([2]) + Buffer.from([1]), + Buffer.from([2]) ]); cb(); }); @@ -66,7 +66,7 @@ suite('containers', () => { test('resize', (cb) => { let t = Type.forSchema({type: 'fixed', name: 'A', size: 2}); - let data = utils.bufferFrom([48, 18]); + let data = Buffer.from([48, 18]); let buf; let encoder = new RawEncoder(t, {batchSize: 1}) .on('data', (chunk) => { @@ -83,7 +83,7 @@ suite('containers', () => { test('flush when full', (cb) => { let t = Type.forSchema({type: 'fixed', name: 'A', size: 2}); - let data = utils.bufferFrom([48, 18]); + let data = Buffer.from([48, 18]); let chunks = []; let encoder = new RawEncoder(t, {batchSize: 2}) .on('data', (chunk) => { chunks.push(chunk); }) @@ -139,7 +139,7 @@ suite('containers', () => { assert.deepEqual(objs, [0]); cb(); }); - decoder.end(utils.bufferFrom([0])); + decoder.end(Buffer.from([0])); }); test('no writer type', () => { @@ -155,13 +155,13 @@ suite('containers', () => { assert.deepEqual(objs, [1, 2]); cb(); }); - decoder.write(utils.bufferFrom([2])); - decoder.end(utils.bufferFrom([4])); + decoder.write(Buffer.from([2])); + decoder.end(Buffer.from([4])); }); test('no decoding', (cb) => { let t = Type.forSchema('int'); - let bufs = [utils.bufferFrom([3]), utils.bufferFrom([124])]; + let bufs = [Buffer.from([3]), Buffer.from([124])]; let objs = []; let decoder = new RawDecoder(t, {noDecode: true}) .on('data', (obj) => { objs.push(obj); }) @@ -179,12 +179,12 @@ suite('containers', () => { let decoder = new RawDecoder(t) .on('data', (obj) => { objs.push(obj); }) .on('end', () => { - assert.deepEqual(objs, [utils.bufferFrom([6])]); + assert.deepEqual(objs, [Buffer.from([6])]); cb(); }); - decoder.write(utils.bufferFrom([2])); + decoder.write(Buffer.from([2])); // Let the first read go through (and return null). - process.nextTick(() => { decoder.end(utils.bufferFrom([6])); }); + process.nextTick(() => { decoder.end(Buffer.from([6])); }); }); test('read before write', (cb) => { @@ -197,7 +197,7 @@ suite('containers', () => { cb(); }); setTimeout(() => { - decoder.end(utils.bufferFrom([2])); + decoder.end(Buffer.from([2])); }, 50); }); @@ -271,9 +271,9 @@ suite('containers', () => { }).on('data', (chunk) => { chunks.push(chunk); }) .on('end', () => { assert.deepEqual(chunks, [ - utils.bufferFrom([6]), - utils.bufferFrom([6]), - utils.bufferFrom([24, 0, 8]), + Buffer.from([6]), + Buffer.from([6]), + Buffer.from([24, 0, 8]), SYNC ]); cb(); @@ -315,14 +315,14 @@ suite('containers', () => { assert.deepEqual( chunks, [ - utils.bufferFrom([2]), - utils.bufferFrom([2]), - utils.bufferFrom([2]), + Buffer.from([2]), + Buffer.from([2]), + Buffer.from([2]), SYNC, - utils.bufferFrom([2]), - utils.bufferFrom([4]), - utils.bufferFrom([128, 1]), + Buffer.from([2]), + Buffer.from([4]), + Buffer.from([128, 1]), SYNC ] ); @@ -334,7 +334,7 @@ suite('containers', () => { test('resize', (cb) => { let t = Type.forSchema({type: 'fixed', size: 8, name: 'Eight'}); - let buf = utils.bufferFrom('abcdefgh'); + let buf = Buffer.from('abcdefgh'); let chunks = []; let encoder = new BlockEncoder(t, { omitHeader: true, @@ -342,8 +342,8 @@ suite('containers', () => { blockSize: 4 }).on('data', (chunk) => { chunks.push(chunk); }) .on('end', () => { - let b1 = utils.bufferFrom([4]); - let b2 = utils.bufferFrom([32]); + let b1 = Buffer.from([4]); + let b2 = Buffer.from([32]); assert.deepEqual(chunks, [b1, b2, Buffer.concat([buf, buf]), SYNC]); cb(); }); @@ -363,12 +363,12 @@ suite('containers', () => { test('write non-canonical schema', (cb) => { let obj = {type: 'fixed', size: 2, name: 'Id', doc: 'An id.'}; - let id = utils.bufferFrom([1, 2]); + let id = Buffer.from([1, 2]); let ids = []; let encoder = new BlockEncoder(obj); let decoder = new streams.BlockDecoder() .on('metadata', (type, codec, header) => { - let schema = JSON.parse(header.meta['avro.schema'].toString()); + let schema = JSON.parse(DECODER.decode(header.meta['avro.schema'])); assert.deepEqual(schema, obj); // Check that doc field not stripped. }) .on('data', (id) => { ids.push(id); }) @@ -390,8 +390,8 @@ suite('containers', () => { let decoder = new BlockDecoder() .on('data', () => {}) .on('error', () => { cb(); }); - decoder.write(utils.bufferFrom([0, 3, 2])); - decoder.write(utils.bufferFrom([1])); + decoder.write(Buffer.from([0, 3, 2])); + decoder.write(Buffer.from([1])); }); test('invalid sync marker', (cb) => { @@ -401,14 +401,14 @@ suite('containers', () => { let header = new Header( MAGIC_BYTES, { - 'avro.schema': utils.bufferFrom('"int"'), - 'avro.codec': utils.bufferFrom('null') + 'avro.schema': Buffer.from('"int"'), + 'avro.codec': Buffer.from('null') }, SYNC ); decoder.write(header.toBuffer()); - decoder.write(utils.bufferFrom([0, 0])); // Empty block. - decoder.end(utils.bufferFrom('alongerstringthansixteenbytes')); + decoder.write(Buffer.from([0, 0])); // Empty block. + decoder.end(Buffer.from('alongerstringthansixteenbytes')); }); test('missing codec', (cb) => { @@ -417,7 +417,7 @@ suite('containers', () => { .on('end', () => { cb(); }); let header = new Header( MAGIC_BYTES, - {'avro.schema': utils.bufferFrom('"int"')}, + {'avro.schema': Buffer.from('"int"')}, SYNC ); decoder.end(header.toBuffer()); @@ -430,8 +430,8 @@ suite('containers', () => { let header = new Header( MAGIC_BYTES, { - 'avro.schema': utils.bufferFrom('"int"'), - 'avro.codec': utils.bufferFrom('"foo"') + 'avro.schema': Buffer.from('"int"'), + 'avro.codec': Buffer.from('"foo"') }, SYNC ); @@ -445,8 +445,8 @@ suite('containers', () => { let header = new Header( MAGIC_BYTES, { - 'avro.schema': utils.bufferFrom('"int2"'), - 'avro.codec': utils.bufferFrom('null') + 'avro.schema': Buffer.from('"int2"'), + 'avro.codec': Buffer.from('null') }, SYNC ); @@ -463,12 +463,12 @@ suite('containers', () => { }); let buf = new Header( MAGIC_BYTES, - {'avro.schema': utils.bufferFrom('"int"')}, + {'avro.schema': Buffer.from('"int"')}, SYNC ).toBuffer(); - decoder.write(buf.slice(0, 5)); // Part of header. - decoder.write(buf.slice(5)); - decoder.write(utils.bufferFrom([2, 2, 4])); + decoder.write(buf.subarray(0, 5)); // Part of header. + decoder.write(buf.subarray(5)); + decoder.write(Buffer.from([2, 2, 4])); decoder.write(SYNC); decoder.end(); }); @@ -481,8 +481,8 @@ suite('containers', () => { let header = new Header( MAGIC_BYTES, { - 'avro.schema': utils.bufferFrom('"string"'), - 'avro.codec': utils.bufferFrom('null') + 'avro.schema': Buffer.from('"string"'), + 'avro.codec': Buffer.from('null') }, SYNC ); @@ -491,7 +491,7 @@ suite('containers', () => { 5, Buffer.concat([ type.toBuffer('hi'), - utils.bufferFrom([77]) // Corrupt (negative length). + Buffer.from([77]) // Corrupt (negative length). ]), SYNC ).toBuffer()); @@ -535,7 +535,7 @@ suite('containers', () => { let decoder = new streams.BlockDecoder({noDecode: true}) .on('data', (obj) => { objs.push(obj); }) .on('end', () => { - assert.deepEqual(objs, [utils.bufferFrom([96])]); + assert.deepEqual(objs, [Buffer.from([96])]); cb(); }); encoder.pipe(decoder); @@ -788,8 +788,8 @@ suite('containers', () => { decoder.write(HEADER_TYPE.toBuffer({ magic: MAGIC_BYTES, meta: { - 'avro.schema': utils.bufferFrom('"int"'), - 'avro.codec': utils.bufferFrom('null') + 'avro.schema': Buffer.from('"int"'), + 'avro.codec': Buffer.from('null') }, sync: SYNC })); @@ -797,7 +797,7 @@ suite('containers', () => { count: 1, data: t.toBuffer(1), sync: SYNC })); decoder.write(BLOCK_TYPE.toBuffer({ - count: 0, data: utils.bufferFrom([]), sync: SYNC + count: 0, data: Buffer.from([]), sync: SYNC })); decoder.write(BLOCK_TYPE.toBuffer({ count: 1, data: t.toBuffer(2), sync: SYNC diff --git a/test/test_index.js b/test/test_index.js index 08a25f7e..9c5866d5 100644 --- a/test/test_index.js +++ b/test/test_index.js @@ -7,13 +7,11 @@ if (process.browser) { let index = require('../lib'), specs = require('../lib/specs'), types = require('../lib/types'), + {isBufferLike} = require('../lib/utils'), assert = require('assert'), - buffer = require('buffer'), path = require('path'), tmp = require('tmp'); -let Buffer = buffer.Buffer; - let DPATH = path.join(__dirname, 'dat'); @@ -72,7 +70,7 @@ suite('index', () => { assert(header !== null); assert.equal(typeof header.meta['avro.schema'], 'object'); header = index.extractFileHeader(fpath, {decode: false}); - assert(Buffer.isBuffer(header.meta['avro.schema'])); + assert(isBufferLike(header.meta['avro.schema'])); header = index.extractFileHeader(fpath, {size: 2}); assert.equal(typeof header.meta['avro.schema'], 'object'); header = index.extractFileHeader(path.join(DPATH, 'person-10.avro.raw')); diff --git a/test/test_types.js b/test/test_types.js index 1f310108..888a2143 100644 --- a/test/test_types.js +++ b/test/test_types.js @@ -63,8 +63,8 @@ suite('types', () => { test('toBuffer int', () => { let type = Type.forSchema('int'); - assert.equal(type.fromBuffer(utils.bufferFrom([0x80, 0x01])), 64); - assert(utils.bufferFrom([0]).equals(type.toBuffer(0))); + assert.equal(type.fromBuffer(Buffer.from([0x80, 0x01])), 64); + assert(Buffer.from([0]).equals(type.toBuffer(0))); }); @@ -154,7 +154,7 @@ suite('types', () => { test('precision loss', () => { let type = Type.forSchema('long'); - let buf = utils.bufferFrom( + let buf = Buffer.from( [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x20] ); assert.throws(() => { type.fromBuffer(buf); }); @@ -188,7 +188,7 @@ suite('types', () => { test('fromBuffer string', () => { let type = Type.forSchema('string'); - let buf = utils.bufferFrom([0x06, 0x68, 0x69, 0x21]); + let buf = Buffer.from([0x06, 0x68, 0x69, 0x21]); let s = 'hi!'; assert.equal(type.fromBuffer(buf), s); assert(buf.equals(type.toBuffer(s))); @@ -196,7 +196,7 @@ suite('types', () => { test('toBuffer string', () => { let type = Type.forSchema('string'); - let buf = utils.bufferFrom([0x06, 0x68, 0x69, 0x21]); + let buf = Buffer.from([0x06, 0x68, 0x69, 0x21]); assert(buf.equals(type.toBuffer('hi!', 1))); }); @@ -206,7 +206,7 @@ suite('types', () => { let buf = stringT.toBuffer('\x00\x01'); assert.deepEqual( bytesT.fromBuffer(buf, bytesT.createResolver(stringT)), - utils.bufferFrom([0, 1]) + Buffer.from([0, 1]) ); }); @@ -214,10 +214,10 @@ suite('types', () => { let t = Type.forSchema('string'); let s = 'hello'; let b, pos; - b = utils.newBuffer(2); + b = Buffer.alloc(2); pos = t.encode(s, b); assert(pos < 0); - b = utils.newBuffer(b.length - pos); + b = Buffer.alloc(b.length - pos); pos = t.encode(s, b); assert(pos >= 0); assert.equal(s, t.fromBuffer(b)); // Also checks exact length match. @@ -322,7 +322,7 @@ suite('types', () => { let data = [ { - valid: [utils.newBuffer(1), utils.bufferFrom('abc')], + valid: [Buffer.alloc(1), Buffer.from('abc')], invalid: [null, 'hi', undefined, 1, 0, -3.5] } ]; @@ -332,7 +332,7 @@ suite('types', () => { test('resolve string > bytes', () => { let bytesT = Type.forSchema('bytes'); let stringT = Type.forSchema('string'); - let buf = utils.bufferFrom([4, 0, 1]); + let buf = Buffer.from([4, 0, 1]); assert.deepEqual( stringT.fromBuffer(buf, stringT.createResolver(bytesT)), '\x00\x01' @@ -342,7 +342,7 @@ suite('types', () => { test('clone', () => { let t = Type.forSchema('bytes'); let s = '\x01\x02'; - let buf = utils.bufferFrom(s); + let buf = Buffer.from(s); let clone; clone = t.clone(buf); assert.deepEqual(clone, buf); @@ -359,7 +359,7 @@ suite('types', () => { test('fromString', () => { let t = Type.forSchema('bytes'); let s = '\x01\x02'; - let buf = utils.bufferFrom(s); + let buf = Buffer.from(s); let clone; clone = t.fromString(JSON.stringify(s)); assert.deepEqual(clone, buf); @@ -369,11 +369,11 @@ suite('types', () => { test('compare', () => { let t = Type.forSchema('bytes'); - let b1 = t.toBuffer(utils.bufferFrom([0, 2])); + let b1 = t.toBuffer(Buffer.from([0, 2])); assert.equal(t.compareBuffers(b1, b1), 0); - let b2 = t.toBuffer(utils.bufferFrom([0, 2, 3])); + let b2 = t.toBuffer(Buffer.from([0, 2, 3])); assert.equal(t.compareBuffers(b1, b2), -1); - let b3 = t.toBuffer(utils.bufferFrom([1])); + let b3 = t.toBuffer(Buffer.from([1])); assert.equal(t.compareBuffers(b3, b1), 1); }); @@ -392,8 +392,8 @@ suite('types', () => { { name: 'qualified name', schema: ['null', {type: 'fixed', name: 'a.B', size: 2}], - valid: [null, utils.newBuffer(2)], - invalid: [{'a.B': utils.newBuffer(2)}], + valid: [null, Buffer.alloc(2)], + invalid: [{'a.B': Buffer.alloc(2)}], check: assert.deepEqual }, { @@ -441,7 +441,7 @@ suite('types', () => { test('invalid read', () => { let type = new builtins.UnwrappedUnionType(['null', 'int']); - assert.throws(() => { type.fromBuffer(utils.bufferFrom([4])); }); + assert.throws(() => { type.fromBuffer(Buffer.from([4])); }); }); test('missing bucket write', () => { @@ -471,8 +471,8 @@ suite('types', () => { test('non wrapped write', () => { let type = new builtins.UnwrappedUnionType(['null', 'int']); - assert.deepEqual(type.toBuffer(23), utils.bufferFrom([2, 46])); - assert.deepEqual(type.toBuffer(null), utils.bufferFrom([0])); + assert.deepEqual(type.toBuffer(23), Buffer.from([2, 46])); + assert.deepEqual(type.toBuffer(null), Buffer.from([0])); }); test('coerce buffers', () => { @@ -481,7 +481,7 @@ suite('types', () => { assert.throws(() => { type.clone(obj); }); assert.deepEqual( type.clone(obj, {coerceBuffers: true}), - utils.bufferFrom([1, 2]) + Buffer.from([1, 2]) ); assert.deepEqual(type.clone(null, {coerceBuffers: true}), null); }); @@ -509,7 +509,7 @@ suite('types', () => { let t1 = Type.forSchema('null'); let t2 = new builtins.UnwrappedUnionType(['null', 'int']); let a = t2.createResolver(t1); - assert.deepEqual(t2.fromBuffer(utils.newBuffer(0), a), null); + assert.deepEqual(t2.fromBuffer(Buffer.alloc(0), a), null); }); test('resolve [string, int] to unwrapped [float, bytes]', () => { @@ -518,7 +518,7 @@ suite('types', () => { let a = t2.createResolver(t1); let buf; buf = t1.toBuffer({string: 'hi'}); - assert.deepEqual(t2.fromBuffer(buf, a), utils.bufferFrom('hi')); + assert.deepEqual(t2.fromBuffer(buf, a), Buffer.from('hi')); buf = t1.toBuffer({'int': 1}); assert.deepEqual(t2.fromBuffer(buf, a), 1); }); @@ -563,7 +563,7 @@ suite('types', () => { {name: 'id2', type: ['null', 'an.Id']} ] }, {wrapUnions: false}); - let b = utils.bufferFrom([0]); + let b = Buffer.from([0]); let o = {id1: b, id2: b}; assert.deepEqual(t.clone(o), o); }); @@ -608,8 +608,8 @@ suite('types', () => { { name: 'qualified name', schema: ['null', {type: 'fixed', name: 'a.B', size: 2}], - valid: [null, {'a.B': utils.newBuffer(2)}], - invalid: [utils.newBuffer(2)], + valid: [null, {'a.B': Buffer.alloc(2)}], + invalid: [Buffer.alloc(2)], check: assert.deepEqual }, { @@ -661,7 +661,7 @@ suite('types', () => { test('read invalid index', () => { let type = new builtins.WrappedUnionType(['null', 'int']); - let buf = utils.bufferFrom([1, 0]); + let buf = Buffer.from([1, 0]); assert.throws(() => { type.fromBuffer(buf); }); }); @@ -689,7 +689,7 @@ suite('types', () => { let t1 = Type.forSchema('null'); let t2 = new builtins.WrappedUnionType(['null', 'int']); let a = t2.createResolver(t1); - assert.deepEqual(t2.fromBuffer(utils.newBuffer(0), a), null); + assert.deepEqual(t2.fromBuffer(Buffer.alloc(0), a), null); }); test('resolve [string, int] to [long, bytes]', () => { @@ -700,7 +700,7 @@ suite('types', () => { buf = t1.toBuffer({string: 'hi'}); assert.deepEqual( t2.fromBuffer(buf, a), - {'bytes': utils.bufferFrom('hi')} + {'bytes': Buffer.from('hi')} ); buf = t1.toBuffer({'int': 1}); assert.deepEqual(t2.fromBuffer(buf, a), {'long': 1}); @@ -714,7 +714,7 @@ suite('types', () => { buf = t1.toBuffer('hi'); assert.deepEqual( t2.fromBuffer(buf, a), - {'bytes': utils.bufferFrom('hi')} + {'bytes': Buffer.from('hi')} ); buf = t1.toBuffer(1); assert.deepEqual(t2.fromBuffer(buf, a), {'long': 1}); @@ -776,7 +776,7 @@ suite('types', () => { {name: 'id2', type: ['null', 'an.Id']} ] }, {wrapUnions: true}); - let b = utils.bufferFrom([0]); + let b = Buffer.from([0]); let o = {id1: b, id2: {Id: b}}; let c = {id1: b, id2: {'an.Id': b}}; assert.throws(() => { t.clone(o, {}); }); @@ -792,7 +792,7 @@ suite('types', () => { {name: 'id2', type: ['null', 'Id']} ] }, {wrapUnions: true}); - let b = utils.bufferFrom([0]); + let b = Buffer.from([0]); let o = {id1: b, id2: {'an.Id': b}}; assert.throws(() => { t.clone(o); }); assert.throws(() => { t.clone(o, {}); }); @@ -920,7 +920,7 @@ suite('types', () => { let type = new builtins.EnumType({ type: 'enum', symbols: ['A'], name: 'a' }); - let buf = utils.bufferFrom([2]); + let buf = Buffer.from([2]); assert.throws(() => { type.fromBuffer(buf); }); }); @@ -1020,16 +1020,16 @@ suite('types', () => { { name: 'size 1', schema: {name: 'Foo', size: 2}, - valid: [utils.bufferFrom([1, 2]), utils.bufferFrom([2, 3])], + valid: [Buffer.from([1, 2]), Buffer.from([2, 3])], invalid: [ 'HEY', null, undefined, 0, - utils.newBuffer(1), - utils.newBuffer(3) + Buffer.alloc(1), + Buffer.alloc(3) ], - check: function (a, b) { assert(a.equals(b)); } + check: function (a, b) { assert(Buffer.compare(a, b) === 0); } } ]; @@ -1090,7 +1090,7 @@ suite('types', () => { test('clone', () => { let t = new builtins.FixedType({name: 'Id', size: 2}); let s = '\x01\x02'; - let buf = utils.bufferFrom(s); + let buf = Buffer.from(s); let clone; clone = t.clone(buf); assert.deepEqual(clone, buf); @@ -1103,22 +1103,22 @@ suite('types', () => { clone = t.clone(buf.toJSON(), {coerceBuffers: true}); assert.deepEqual(clone, buf); assert.throws(() => { t.clone(1, {coerceBuffers: true}); }); - assert.throws(() => { t.clone(utils.bufferFrom([2])); }); + assert.throws(() => { t.clone(Buffer.from([2])); }); }); test('fromString', () => { let t = new builtins.FixedType({name: 'Id', size: 2}); let s = '\x01\x02'; - let buf = utils.bufferFrom(s); + let buf = Buffer.from(s); let clone = t.fromString(JSON.stringify(s)); assert.deepEqual(clone, buf); }); test('compare buffers', () => { let t = Type.forSchema({type: 'fixed', name: 'Id', size: 2}); - let b1 = utils.bufferFrom([1, 2]); + let b1 = Buffer.from([1, 2]); assert.equal(t.compareBuffers(b1, b1), 0); - let b2 = utils.bufferFrom([2, 2]); + let b2 = Buffer.from([2, 2]); assert.equal(t.compareBuffers(b1, b2), -1); }); @@ -1166,18 +1166,18 @@ suite('types', () => { test('write int', () => { let t = new builtins.MapType({type: 'map', values: 'int'}); let buf = t.toBuffer({'\x01': 3, '\x02': 4}); - assert.deepEqual(buf, utils.bufferFrom([4, 2, 1, 6, 2, 2, 8, 0])); + assert.deepEqual(buf, Buffer.from([4, 2, 1, 6, 2, 2, 8, 0])); }); test('read long', () => { let t = new builtins.MapType({type: 'map', values: 'long'}); - let buf = utils.bufferFrom([4, 2, 1, 6, 2, 2, 8, 0]); + let buf = Buffer.from([4, 2, 1, 6, 2, 2, 8, 0]); assert.deepEqual(t.fromBuffer(buf), {'\x01': 3, '\x02': 4}); }); test('read with sizes', () => { let t = new builtins.MapType({type: 'map', values: 'int'}); - let buf = utils.bufferFrom([1,6,2,97,2,0]); + let buf = Buffer.from([1,6,2,97,2,0]); assert.deepEqual(t.fromBuffer(buf), {a: 1}); }); @@ -1195,8 +1195,8 @@ suite('types', () => { type: 'record', fields: [{name: 'val', type: 'int'}] }); - let b1 = utils.bufferFrom([2,2,97,2,0,6]); // Without sizes. - let b2 = utils.bufferFrom([1,6,2,97,2,0,6]); // With sizes. + let b1 = Buffer.from([2,2,97,2,0,6]); // Without sizes. + let b2 = Buffer.from([1,6,2,97,2,0,6]); // With sizes. let resolver = v2.createResolver(v1); assert.deepEqual(v2.fromBuffer(b1, resolver), {val: 3}); assert.deepEqual(v2.fromBuffer(b2, resolver), {val: 3}); @@ -1237,7 +1237,7 @@ suite('types', () => { } }); let resolver = t2.createResolver(t1); - let obj = {one: utils.bufferFrom([1, 2])}; + let obj = {one: Buffer.from([1, 2])}; let buf = t1.toBuffer(obj); assert.deepEqual(t2.fromBuffer(buf, resolver), obj); }); @@ -1262,7 +1262,7 @@ suite('types', () => { assert.throws(() => { t.clone(o, {}); }); assert.throws(() => { t.clone(o); }); let c = t.clone(o, {coerceBuffers: true}); - assert.deepEqual(c, {one: utils.bufferFrom([1])}); + assert.deepEqual(c, {one: Buffer.from([1])}); }); test('compare buffers', () => { @@ -1319,7 +1319,7 @@ suite('types', () => { test('read with sizes', () => { let t = new builtins.ArrayType({type: 'array', items: 'int'}); - let buf = utils.bufferFrom([1,2,2,0]); + let buf = Buffer.from([1,2,2,0]); assert.deepEqual(t.fromBuffer(buf), [1]); }); @@ -1337,8 +1337,8 @@ suite('types', () => { type: 'record', fields: [{name: 'val', type: 'int'}] }); - let b1 = utils.bufferFrom([2,2,0,6]); // Without sizes. - let b2 = utils.bufferFrom([1,2,2,0,6]); // With sizes. + let b1 = Buffer.from([2,2,0,6]); // Without sizes. + let b2 = Buffer.from([1,2,2,0,6]); // With sizes. let resolver = v2.createResolver(v1); assert.deepEqual(v2.fromBuffer(b1, resolver), {val: 3}); assert.deepEqual(v2.fromBuffer(b2, resolver), {val: 3}); @@ -1352,7 +1352,7 @@ suite('types', () => { let buf = t1.toBuffer(obj); assert.deepEqual( t2.fromBuffer(buf, resolver), - [utils.bufferFrom([1, 2])] + [Buffer.from([1, 2])] ); }); @@ -1387,7 +1387,7 @@ suite('types', () => { assert.throws(() => { t.clone(o); }); assert.throws(() => { t.clone(o, {}); }); let c = t.clone(o, {coerceBuffers: true}); - assert.deepEqual(c, [utils.bufferFrom([1, 2])]); + assert.deepEqual(c, [Buffer.from([1, 2])]); }); test('compare buffers', () => { @@ -1451,7 +1451,7 @@ suite('types', () => { }); test('round-trip multi-block array', () => { - let tap = new Tap(utils.newBuffer(64)); + let tap = Tap.withCapacity(64); tap.writeLong(2); tap.writeString('hi'); tap.writeString('hey'); @@ -1460,7 +1460,7 @@ suite('types', () => { tap.writeLong(0); let t = new builtins.ArrayType({items: 'string'}); assert.deepEqual( - t.fromBuffer(tap.buf.slice(0, tap.pos)), + t.fromBuffer(tap.subarray(0, tap.pos)), ['hi', 'hey', 'hello'] ); }); @@ -1563,12 +1563,12 @@ suite('types', () => { {name: 'name', type: 'string', 'default': '\x01'} ] }); - assert.deepEqual(type.toBuffer({}), utils.bufferFrom([50, 2, 1])); + assert.deepEqual(type.toBuffer({}), Buffer.from([50, 2, 1])); }); test('fixed string default', () => { let s = '\x01\x04'; - let b = utils.bufferFrom(s); + let b = Buffer.from(s); let type = Type.forSchema({ type: 'record', name: 'Object', @@ -1581,7 +1581,7 @@ suite('types', () => { ] }); let obj = new (type.getRecordConstructor())(); - assert.deepEqual(obj.id, utils.bufferFrom([1, 4])); + assert.deepEqual(obj.id, Buffer.from([1, 4])); assert.deepEqual(type.toBuffer({}), b); }); @@ -1594,7 +1594,7 @@ suite('types', () => { { name: 'id', type: {type: 'fixed', size: 2, name: 'Id'}, - 'default': utils.bufferFrom([0]) + 'default': Buffer.from([0]) } ] }); @@ -1672,7 +1672,7 @@ suite('types', () => { fields: [{name: 'age', type: 'int'}] }); let Person = type.getRecordConstructor(); - assert.deepEqual((new Person(48)).toBuffer(), utils.bufferFrom([96])); + assert.deepEqual((new Person(48)).toBuffer(), Buffer.from([96])); assert.throws(() => { (new Person()).toBuffer(); }); }); @@ -2070,7 +2070,7 @@ suite('types', () => { name: 'Person', fields: [{name: 'pwd', type: 'bytes'}] }).getRecordConstructor(); - let r = new T(utils.bufferFrom([1, 2])); + let r = new T(Buffer.from([1, 2])); assert.equal(r.toString(), T.getType().toString(r)); }); @@ -2474,11 +2474,12 @@ suite('types', () => { let slowLongType = builtins.LongType.__with({ fromBuffer: function (buf) { + let dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); let neg = buf[7] >> 7; if (neg) { // Negative number. invert(buf); } - let n = buf.readInt32LE(0) + Math.pow(2, 32) * buf.readInt32LE(4); + let n = dv.getInt32(0, true) + Math.pow(2, 32) * dv.getInt32(4, true); if (neg) { invert(buf); n = -n - 1; @@ -2486,15 +2487,16 @@ suite('types', () => { return n; }, toBuffer: function (n) { - let buf = utils.newBuffer(8); + let buf = new Uint8Array(8); + let dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); let neg = n < 0; if (neg) { invert(buf); n = -n - 1; } - buf.writeInt32LE(n | 0); + dv.setInt32(0, n | 0, true); let h = n / Math.pow(2, 32) | 0; - buf.writeInt32LE(h ? h : (n >= 0 ? 0 : -1), 4); + dv.setInt32(4, h ? h : (n >= 0 ? 0 : -1), true); if (neg) { invert(buf); } @@ -2586,14 +2588,14 @@ suite('types', () => { let slowLongType = builtins.LongType.__with({ fromBuffer: function (buf) { - let tap = new Tap(buf); + let tap = Tap.fromBuffer(buf); return tap.readLong(); }, toBuffer: function (n) { - let buf = utils.newBuffer(10); - let tap = new Tap(buf); + let buf = Buffer.alloc(10); + let tap = Tap.fromBuffer(buf); tap.writeLong(n); - return buf.slice(0, tap.pos); + return buf.subarray(0, tap.pos); }, fromJSON: function (n) { return n; }, toJSON: function (n) { return n; }, @@ -2654,7 +2656,7 @@ suite('types', () => { compare: function () { throw new Error(); } }, true); let t = Type.forSchema(['null', 'long'], {registry: {'long': longType}}); - let v = {value: utils.bufferFrom([4])}; // Long encoding of 2. + let v = {value: Buffer.from([4])}; // Long encoding of 2. assert(t.isValid(null)); assert(t.isValid(v)); @@ -2673,7 +2675,7 @@ suite('types', () => { }); let buf = fastLongType.toBuffer(12314); assert.deepEqual( - slowLongType.decode(buf.slice(0, 1)), + slowLongType.decode(buf.subarray(0, 1)), {value: undefined, offset: -1} ); }); @@ -2951,7 +2953,7 @@ suite('types', () => { assert(t.isValid(2)); assert(!t.isValid(3)); assert(!t.isValid('abc')); - assert.equal(t.fromBuffer(utils.bufferFrom([4])), 2); + assert.equal(t.fromBuffer(Buffer.from([4])), 2); assert.equal(t.clone(4), 4); assert.equal(t.fromString('6'), 6); assert.equal(t.getSchema(), 'long'); @@ -2965,7 +2967,7 @@ suite('types', () => { assert.throws(() => { t.clone(3); }); assert.throws(() => { t.fromString('5'); }); assert.throws(() => { t.toBuffer(3); }); - assert.throws(() => { t.fromBuffer(utils.bufferFrom([2])); }); + assert.throws(() => { t.fromBuffer(Buffer.from([2])); }); }); test('inside unwrapped union', () => { @@ -3325,28 +3327,28 @@ suite('types', () => { test('fromBuffer truncated', () => { let type = Type.forSchema('int'); assert.throws(() => { - type.fromBuffer(utils.bufferFrom([128])); + type.fromBuffer(Buffer.from([128])); }); }); test('fromBuffer bad resolver', () => { let type = Type.forSchema('int'); assert.throws(() => { - type.fromBuffer(utils.bufferFrom([0]), 123, {}); + type.fromBuffer(Buffer.from([0]), 123, {}); }); }); test('fromBuffer trailing', () => { let type = Type.forSchema('int'); assert.throws(() => { - type.fromBuffer(utils.bufferFrom([0, 2])); + type.fromBuffer(Buffer.from([0, 2])); }); }); test('fromBuffer trailing with resolver', () => { let type = Type.forSchema('int'); let resolver = type.createResolver(Type.forSchema(['int'])); - assert.equal(type.fromBuffer(utils.bufferFrom([0, 2]), resolver), 1); + assert.equal(type.fromBuffer(Buffer.from([0, 2]), resolver), 1); }); test('toBuffer', () => { @@ -3357,7 +3359,7 @@ suite('types', () => { test('toBuffer and resize', () => { let type = Type.forSchema('string'); - assert.deepEqual(type.toBuffer('\x01', 1), utils.bufferFrom([2, 1])); + assert.deepEqual(type.toBuffer('\x01', 1), Buffer.from([2, 1])); }); test('type hook', () => { @@ -3426,7 +3428,7 @@ suite('types', () => { test('fingerprint', () => { let t = Type.forSchema('int'); - let buf = utils.bufferFrom('ef524ea1b91e73173d938ade36c1db32', 'hex'); + let buf = Buffer.from('ef524ea1b91e73173d938ade36c1db32', 'hex'); assert.deepEqual(t.fingerprint('md5'), buf); assert.deepEqual(t.fingerprint(), buf); }); @@ -3521,7 +3523,7 @@ suite('types', () => { type: 'record', fields: [{name: 'id1', type: {name: 'Id1', type: 'fixed', size: 2}}] }); - let o = {id1: utils.bufferFrom([0, 1])}; + let o = {id1: Buffer.from([0, 1])}; let s = '{"id1": "\\u0000\\u0001"}'; let c = t.fromString(s); assert.deepEqual(c, o); @@ -3568,7 +3570,7 @@ suite('types', () => { let resolver = t2.createResolver(t1); let buf = t1.toBuffer({'int': 12}); assert.equal(t2.fromBuffer(buf, resolver), 12); - buf = utils.bufferFrom([4, 0]); + buf = Buffer.from([4, 0]); assert.throws(() => { t2.fromBuffer(buf, resolver); }); }); @@ -3577,7 +3579,7 @@ suite('types', () => { let t2 = Type.forSchema('bytes'); let resolver = t2.createResolver(t1); let buf = t1.toBuffer('\x01\x02'); - assert.deepEqual(t2.fromBuffer(buf, resolver), utils.bufferFrom([1, 2])); + assert.deepEqual(t2.fromBuffer(buf, resolver), Buffer.from([1, 2])); }); test('union to invalid non union', () => { @@ -3750,14 +3752,14 @@ suite('types', () => { test('long valid', () => { let t = Type.forSchema('long'); - let buf = utils.bufferFrom([0, 128, 2, 0]); + let buf = Buffer.from([0, 128, 2, 0]); let res = t.decode(buf, 1); assert.deepEqual(res, {value: 128, offset: 3}); }); test('bytes invalid', () => { let t = Type.forSchema('bytes'); - let buf = utils.bufferFrom([4, 1]); + let buf = Buffer.from([4, 1]); let res = t.decode(buf, 0); assert.deepEqual(res, {value: undefined, offset: -1}); }); @@ -3768,29 +3770,29 @@ suite('types', () => { test('int valid', () => { let t = Type.forSchema('int'); - let buf = utils.newBuffer(2); + let buf = Buffer.alloc(2); buf.fill(0); let n = t.encode(5, buf, 1); assert.equal(n, 2); - assert.deepEqual(buf, utils.bufferFrom([0, 10])); + assert.deepEqual(buf, Buffer.from([0, 10])); }); test('too short', () => { let t = Type.forSchema('string'); - let buf = utils.newBuffer(1); + let buf = Buffer.alloc(1); let n = t.encode('\x01\x02', buf, 0); assert.equal(n, -2); }); test('invalid', () => { let t = Type.forSchema('float'); - let buf = utils.newBuffer(2); + let buf = Buffer.alloc(2); assert.throws(() => { t.encode('hi', buf, 0); }); }); test('invalid offset', () => { let t = Type.forSchema('string'); - let buf = utils.newBuffer(2); + let buf = Buffer.alloc(2); assert.throws(() => { t.encode('hi', buf, -1); }); }); @@ -3871,7 +3873,7 @@ suite('types', () => { types.Type.__reset(0); let t = Type.forSchema('string'); let buf = t.toBuffer('\x01'); - assert.deepEqual(buf, utils.bufferFrom([2, 1])); + assert.deepEqual(buf, Buffer.from([2, 1])); }); suite('forTypes', () => { @@ -4113,7 +4115,7 @@ suite('types', () => { }); test('record', () => { - let t = infer({b: true, n: null, s: '', f: utils.newBuffer(0)}); + let t = infer({b: true, n: null, s: '', f: Buffer.alloc(0)}); assert.deepEqual( t.getSchema(), { @@ -4225,8 +4227,8 @@ function testType(Type, data, invalidSchemas) { let items = elem.valid; if (items.length > 1) { let type = new Type(elem.schema); - let buf = utils.newBuffer(1024); - let tap = new Tap(buf); + let buf = Buffer.alloc(1024); + let tap = Tap.fromBuffer(buf); type._write(tap, items[0]); type._write(tap, items[1]); tap.pos = 0; diff --git a/test/test_utils.js b/test/test_utils.js index aea77343..70273b5c 100644 --- a/test/test_utils.js +++ b/test/test_utils.js @@ -1,7 +1,10 @@ 'use strict'; let utils = require('../lib/utils'), - assert = require('assert'); + assert = require('assert'), + buffer = require('buffer'); + +let Buffer = buffer.Buffer; suite('utils', () => { @@ -140,23 +143,6 @@ suite('utils', () => { }); - suite('Tap', () => { - - let BufferPool = utils.BufferPool; - - test('alloc negative length', () => { - let pool = new BufferPool(16); - assert.throws(() => { pool.alloc(-1); }); - }); - - test('alloc beyond pool size', () => { - let pool = new BufferPool(4); - assert.equal(pool.alloc(3).length, 3); - assert.equal(pool.alloc(2).length, 2); - }); - - }); - suite('Tap', () => { let Tap = utils.Tap; @@ -172,18 +158,18 @@ suite('utils', () => { test('write', () => { - let tap = newTap(6); + let tap = Tap.withCapacity(6); tap.writeLong(1440756011948); - let buf = utils.bufferFrom(['0xd8', '0xce', '0x80', '0xbc', '0xee', '0x53']); + let buf = Buffer.from(['0xd8', '0xce', '0x80', '0xbc', '0xee', '0x53']); assert(tap.isValid()); - assert(buf.equals(tap.buf)); + assert(buf.equals(tap.toBuffer())); }); test('read', () => { - let buf = utils.bufferFrom(['0xd8', '0xce', '0x80', '0xbc', '0xee', '0x53']); - assert.equal((new Tap(buf)).readLong(), 1440756011948); + let buf = Buffer.from(['0xd8', '0xce', '0x80', '0xbc', '0xee', '0x53']); + assert.equal((Tap.fromBuffer(buf)).readLong(), 1440756011948); }); @@ -225,7 +211,13 @@ suite('utils', () => { suite('string', () => { testWriterReader({ - elems: ['ahierw', '', 'alh hewlii! rew'], + elems: [ + 'ahierw', + '', + 'alh hewlii! rew', + 'sérialisation', + 'this string should be long enough that a different code path is exercised' + ], reader: function () { return this.readString(); }, skipper: function () { this.skipString(); }, writer: function (s) { this.writeString(s); } @@ -236,7 +228,7 @@ suite('utils', () => { suite('bytes', () => { testWriterReader({ - elems: [utils.bufferFrom('abc'), utils.newBuffer(0), utils.bufferFrom([1, 5, 255])], + elems: [Buffer.from('abc'), Buffer.alloc(0), Buffer.from([1, 5, 255])], reader: function () { return this.readBytes(); }, skipper: function () { this.skipBytes(); }, writer: function (b) { this.writeBytes(b); } @@ -247,7 +239,7 @@ suite('utils', () => { suite('fixed', () => { testWriterReader({ - elems: [utils.bufferFrom([1, 5, 255])], + elems: [Buffer.from([1, 5, 255])], reader: function () { return this.readFixed(3); }, skipper: function () { this.skipFixed(3); }, writer: function (b) { this.writeFixed(b, 3); } @@ -255,61 +247,53 @@ suite('utils', () => { }); - suite('binary', () => { - - test('write valid', () => { - let tap = newTap(3); - let s = '\x01\x02'; - tap.writeBinary(s, 2); - assert.deepEqual(tap.buf, utils.bufferFrom([1,2,0])); - }); - - test('write invalid', () => { - let tap = newTap(1); - let s = '\x01\x02'; - tap.writeBinary(s, 2); - assert.deepEqual(tap.buf, utils.bufferFrom([0])); - }); - - }); - suite('pack & unpack longs', () => { test('unpack single byte', () => { - let t = newTap(10); + let t = Tap.withCapacity(10); t.writeLong(5); t.pos = 0; assert.deepEqual( t.unpackLongBytes(), - utils.bufferFrom([5, 0, 0, 0, 0, 0, 0, 0]) + Buffer.from([5, 0, 0, 0, 0, 0, 0, 0]) ); t.pos = 0; t.writeLong(-5); t.pos = 0; assert.deepEqual( t.unpackLongBytes(), - utils.bufferFrom([-5, -1, -1, -1, -1, -1, -1, -1]) + Buffer.from([-5, -1, -1, -1, -1, -1, -1, -1]) ); t.pos = 0; }); test('unpack multiple bytes', () => { - let t = newTap(10); - let l; + let t = Tap.withCapacity(10); + let l, unpacked, dv; l = 18932; t.writeLong(l); t.pos = 0; - assert.deepEqual(t.unpackLongBytes().readInt32LE(0), l); + unpacked = t.unpackLongBytes(); + dv = new DataView( + unpacked.buffer, + unpacked.byteOffset, + unpacked.byteLength); + assert.deepEqual(dv.getInt32(0, true), l); t.pos = 0; l = -3210984; t.writeLong(l); t.pos = 0; - assert.deepEqual(t.unpackLongBytes().readInt32LE(0), l); + unpacked = t.unpackLongBytes(); + dv = new DataView( + unpacked.buffer, + unpacked.byteOffset, + unpacked.byteLength); + assert.deepEqual(dv.getInt32(0, true), l); }); test('pack single byte', () => { - let t = newTap(10); - let b = utils.newBuffer(8); + let t = Tap.withCapacity(10); + let b = Buffer.alloc(8); b.fill(0); b.writeInt32LE(12); t.packLongBytes(b); @@ -327,7 +311,7 @@ suite('utils', () => { b.writeInt32LE(-1); b.writeInt32LE(-1, 4); t.packLongBytes(b); - assert.deepEqual(t.buf.slice(0, t.pos), utils.bufferFrom([1])); + assert.deepEqual(t.subarray(0, t.pos), Buffer.from([1])); t.pos = 0; assert.deepEqual(t.readLong(), -1); }); @@ -343,8 +327,8 @@ suite('utils', () => { roundtrip(-1); function roundtrip(n) { - let t1 = newTap(10); - let t2 = newTap(10); + let t1 = Tap.withCapacity(10); + let t2 = Tap.withCapacity(10); t1.writeLong(n); t1.pos = 0; t2.packLongBytes(t1.unpackLongBytes()); @@ -353,11 +337,11 @@ suite('utils', () => { }); test('roundtrip bytes', () => { - roundtrip(utils.bufferFrom([0, 0, 0, 0, 0, 0, 0, 0])); - roundtrip(utils.bufferFrom('9007199254740995', 'hex')); + roundtrip(Buffer.from([0, 0, 0, 0, 0, 0, 0, 0])); + roundtrip(Buffer.from('9007199254740995', 'hex')); function roundtrip(b1) { - let t = newTap(10); + let t = Tap.withCapacity(10); t.packLongBytes(b1); t.pos = 0; let b2 = t.unpackLongBytes(); @@ -366,12 +350,6 @@ suite('utils', () => { }); }); - function newTap(n) { - let buf = utils.newBuffer(n); - buf.fill(0); - return new Tap(buf); - } - function testWriterReader(opts) { let size = opts.size; let elems = opts.elems; @@ -381,9 +359,9 @@ suite('utils', () => { let name = opts.name || ''; test('write read ' + name, () => { - let tap = newTap(size || 1024); + let tap = Tap.withCapacity(size || 1024); for (let i = 0, l = elems.length; i < l; i++) { - tap.buf.fill(0); + tap.arr.fill(0); tap.pos = 0; let elem = elems[i]; writeFn.call(tap, elem); @@ -393,21 +371,21 @@ suite('utils', () => { }); test('read over ' + name, () => { - let tap = new Tap(utils.newBuffer(0)); + let tap = Tap.withCapacity(0); readFn.call(tap); // Shouldn't throw. assert(!tap.isValid()); }); test('write over ' + name, () => { - let tap = new Tap(utils.newBuffer(0)); + let tap = Tap.withCapacity(0); writeFn.call(tap, elems[0]); // Shouldn't throw. assert(!tap.isValid()); }); test('skip ' + name, () => { - let tap = newTap(size || 1024); + let tap = Tap.withCapacity(size || 1024); for (let i = 0, l = elems.length; i < l; i++) { - tap.buf.fill(0); + tap.arr.fill(0); tap.pos = 0; let elem = elems[i]; writeFn.call(tap, elem);