From 53419a5175e70903abc9651553158359592576c9 Mon Sep 17 00:00:00 2001 From: Justin Grant Date: Thu, 7 Mar 2019 07:45:39 -0800 Subject: [PATCH] fix(object-id): support 4.x->1.x interop for MinKey and ObjectId * Fix 4.x->1.x interop for MinKey and ObjectId * updated browser_build/bson.js --- browser_build/bson.js | 18 +- lib/bson/parser/calculate_size.js | 2 +- lib/bson/parser/serializer.js | 9 +- test/node/4x-interop/objectid.js | 364 +++++++++++++++++++++++++++ test/node/4x-interop/parser/utils.js | 37 +++ test/node/bson_test.js | 54 ++++ 6 files changed, 469 insertions(+), 15 deletions(-) create mode 100644 test/node/4x-interop/objectid.js create mode 100644 test/node/4x-interop/parser/utils.js diff --git a/browser_build/bson.js b/browser_build/bson.js index a02bf14a..2a48840b 100644 --- a/browser_build/bson.js +++ b/browser_build/bson.js @@ -16447,11 +16447,10 @@ return /******/ (function(modules) { // webpackBootstrap var writeIEEE754 = __webpack_require__(353).writeIEEE754, Long = __webpack_require__(334).Long, - MinKey = __webpack_require__(347).MinKey, Map = __webpack_require__(333), Binary = __webpack_require__(350).Binary; - const normalizedFunctionString = __webpack_require__(354).normalizedFunctionString; + var normalizedFunctionString = __webpack_require__(354).normalizedFunctionString; // try { // var _Buffer = Uint8Array; @@ -16460,6 +16459,7 @@ return /******/ (function(modules) { // webpackBootstrap // } var regexp = /\x00/; // eslint-disable-line no-control-regex + var ignoreKeys = ['$db', '$ref', '$id', '$clusterTime']; // To ensure that 0.4 of node works correctly var isDate = function isDate(d) { @@ -16666,7 +16666,7 @@ return /******/ (function(modules) { // webpackBootstrap // Write the type of either min or max key if (value === null) { buffer[index++] = BSON.BSON_DATA_NULL; - } else if (value instanceof MinKey) { + } else if (value._bsontype === 'MinKey') { buffer[index++] = BSON.BSON_DATA_MIN_KEY; } else { buffer[index++] = BSON.BSON_DATA_MAX_KEY; @@ -17044,7 +17044,7 @@ return /******/ (function(modules) { // webpackBootstrap index = serializeNull(buffer, key, value, index, true); } else if (value === null) { index = serializeNull(buffer, key, value, index, true); - } else if (value['_bsontype'] === 'ObjectID') { + } else if (value['_bsontype'] === 'ObjectID' || value['_bsontype'] === 'ObjectId') { index = serializeObjectId(buffer, key, value, index, true); } else if (Buffer.isBuffer(value)) { index = serializeBuffer(buffer, key, value, index, true); @@ -17095,7 +17095,7 @@ return /******/ (function(modules) { // webpackBootstrap type = typeof value; // Check the key and throw error if it's illegal - if (key !== '$db' && key !== '$ref' && key !== '$id') { + if (typeof key === 'string' && ignoreKeys.indexOf(key) === -1) { if (key.match(regexp) != null) { // The BSON spec doesn't allow keys with null bytes because keys are // null-terminated. @@ -17122,7 +17122,7 @@ return /******/ (function(modules) { // webpackBootstrap // } else if (value === undefined && ignoreUndefined === true) { } else if (value === null || value === undefined && ignoreUndefined === false) { index = serializeNull(buffer, key, value, index); - } else if (value['_bsontype'] === 'ObjectID') { + } else if (value['_bsontype'] === 'ObjectID' || value['_bsontype'] === 'ObjectId') { index = serializeObjectId(buffer, key, value, index); } else if (Buffer.isBuffer(value)) { index = serializeBuffer(buffer, key, value, index); @@ -17175,7 +17175,7 @@ return /******/ (function(modules) { // webpackBootstrap type = typeof value; // Check the key and throw error if it's illegal - if (key !== '$db' && key !== '$ref' && key !== '$id') { + if (typeof key === 'string' && ignoreKeys.indexOf(key) === -1) { if (key.match(regexp) != null) { // The BSON spec doesn't allow keys with null bytes because keys are // null-terminated. @@ -17203,7 +17203,7 @@ return /******/ (function(modules) { // webpackBootstrap if (ignoreUndefined === false) index = serializeNull(buffer, key, value, index); } else if (value === null) { index = serializeNull(buffer, key, value, index); - } else if (value['_bsontype'] === 'ObjectID') { + } else if (value['_bsontype'] === 'ObjectID' || value['_bsontype'] === 'ObjectId') { index = serializeObjectId(buffer, key, value, index); } else if (Buffer.isBuffer(value)) { index = serializeBuffer(buffer, key, value, index); @@ -17667,7 +17667,7 @@ return /******/ (function(modules) { // webpackBootstrap case 'object': if (value == null || value instanceof MinKey || value instanceof MaxKey || value['_bsontype'] === 'MinKey' || value['_bsontype'] === 'MaxKey') { return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + 1; - } else if (value instanceof ObjectID || value['_bsontype'] === 'ObjectID') { + } else if (value instanceof ObjectID || value['_bsontype'] === 'ObjectID' || value['_bsontype'] === 'ObjectId') { return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (12 + 1); } else if (value instanceof Date || isDate(value)) { return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (8 + 1); diff --git a/lib/bson/parser/calculate_size.js b/lib/bson/parser/calculate_size.js index f174519d..7e0026ca 100644 --- a/lib/bson/parser/calculate_size.js +++ b/lib/bson/parser/calculate_size.js @@ -92,7 +92,7 @@ function calculateElement(name, value, serializeFunctions, isArray, ignoreUndefi value['_bsontype'] === 'MaxKey' ) { return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + 1; - } else if (value instanceof ObjectID || value['_bsontype'] === 'ObjectID') { + } else if (value instanceof ObjectID || value['_bsontype'] === 'ObjectID' || value['_bsontype'] === 'ObjectId') { return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (12 + 1); } else if (value instanceof Date || isDate(value)) { return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (8 + 1); diff --git a/lib/bson/parser/serializer.js b/lib/bson/parser/serializer.js index bd3e12a5..e4ff2bd9 100644 --- a/lib/bson/parser/serializer.js +++ b/lib/bson/parser/serializer.js @@ -2,7 +2,6 @@ var writeIEEE754 = require('../float_parser').writeIEEE754, Long = require('../long').Long, - MinKey = require('../min_key').MinKey, Map = require('../map'), Binary = require('../binary').Binary; @@ -251,7 +250,7 @@ var serializeMinMax = function(buffer, key, value, index, isArray) { // Write the type of either min or max key if (value === null) { buffer[index++] = BSON.BSON_DATA_NULL; - } else if (value instanceof MinKey) { + } else if (value._bsontype === 'MinKey') { buffer[index++] = BSON.BSON_DATA_MIN_KEY; } else { buffer[index++] = BSON.BSON_DATA_MAX_KEY; @@ -718,7 +717,7 @@ var serializeInto = function serializeInto( index = serializeNull(buffer, key, value, index, true); } else if (value === null) { index = serializeNull(buffer, key, value, index, true); - } else if (value['_bsontype'] === 'ObjectID') { + } else if (value['_bsontype'] === 'ObjectID' || value['_bsontype'] === 'ObjectId') { index = serializeObjectId(buffer, key, value, index, true); } else if (Buffer.isBuffer(value)) { index = serializeBuffer(buffer, key, value, index, true); @@ -826,7 +825,7 @@ var serializeInto = function serializeInto( // } else if (value === undefined && ignoreUndefined === true) { } else if (value === null || (value === undefined && ignoreUndefined === false)) { index = serializeNull(buffer, key, value, index); - } else if (value['_bsontype'] === 'ObjectID') { + } else if (value['_bsontype'] === 'ObjectID' || value['_bsontype'] === 'ObjectId') { index = serializeObjectId(buffer, key, value, index); } else if (Buffer.isBuffer(value)) { index = serializeBuffer(buffer, key, value, index); @@ -928,7 +927,7 @@ var serializeInto = function serializeInto( if (ignoreUndefined === false) index = serializeNull(buffer, key, value, index); } else if (value === null) { index = serializeNull(buffer, key, value, index); - } else if (value['_bsontype'] === 'ObjectID') { + } else if (value['_bsontype'] === 'ObjectID' || value['_bsontype'] === 'ObjectId') { index = serializeObjectId(buffer, key, value, index); } else if (Buffer.isBuffer(value)) { index = serializeBuffer(buffer, key, value, index); diff --git a/test/node/4x-interop/objectid.js b/test/node/4x-interop/objectid.js new file mode 100644 index 00000000..88b7d6ad --- /dev/null +++ b/test/node/4x-interop/objectid.js @@ -0,0 +1,364 @@ +'use strict'; + +var Buffer = require('buffer').Buffer; +var randomBytes = require('./parser/utils').randomBytes; +var deprecate = require('util').deprecate; + +// constants +var PROCESS_UNIQUE = randomBytes(5); + +// Regular expression that checks for hex value +var checkForHexRegExp = new RegExp('^[0-9a-fA-F]{24}$'); +var hasBufferType = false; + +// Check if buffer exists +try { + if (Buffer && Buffer.from) hasBufferType = true; +} catch (err) { + hasBufferType = false; +} + +// Precomputed hex table enables speedy hex string conversion +var hexTable = []; +for (var i = 0; i < 256; i++) { + hexTable[i] = (i <= 15 ? '0' : '') + i.toString(16); +} + +// Lookup tables +var decodeLookup = []; +var i = 0; +while (i < 10) decodeLookup[0x30 + i] = i++; +while (i < 16) decodeLookup[0x41 - 10 + i] = decodeLookup[0x61 - 10 + i] = i++; + +var _Buffer = Buffer; +function convertToHex(bytes) { + return bytes.toString('hex'); +} + +function makeObjectIdError(invalidString, index) { + var invalidCharacter = invalidString[index]; + return new TypeError( + 'ObjectId string "'+invalidString+'" contains invalid character "'+invalidCharacter+'" with character code ('+invalidString.charCodeAt(index)+'). All character codes for a non-hex string must be less than 256.' + ); +} + +/** + * @ignore + */ +function ObjectId(id) { + + // Duck-typing to support ObjectId from different npm packages + if (id instanceof ObjectId) return id; + + // The most common usecase (blank id, new objectId instance) + if (id == null || typeof id === 'number') { + // Generate a new id + this.id = ObjectId.generate(id); + // If we are caching the hex string + if (ObjectId.cacheHexString) this.__id = this.toString('hex'); + // Return the object + return; + } + + // Check if the passed in id is valid + var valid = ObjectId.isValid(id); + + // Throw an error if it's not a valid setup + if (!valid && id != null) { + throw new TypeError( + 'Argument passed in must be a single String of 12 bytes or a string of 24 hex characters' + ); + } else if (valid && typeof id === 'string' && id.length === 24 && hasBufferType) { + return new ObjectId(Buffer.from(id, 'hex')); + } else if (valid && typeof id === 'string' && id.length === 24) { + return ObjectId.createFromHexString(id); + } else if (id != null && id.length === 12) { + // assume 12 byte string + this.id = id; + } else if (id != null && id.toHexString) { + // Duck-typing to support ObjectId from different npm packages + return id; + } else { + throw new TypeError( + 'Argument passed in must be a single String of 12 bytes or a string of 24 hex characters' + ); + } + + if (ObjectId.cacheHexString) this.__id = this.toString('hex'); +} + + /** + * @ignore + */ + ObjectId.prototype.toHexString = function() { + if (ObjectId.cacheHexString && this.__id) return this.__id; + + var hexString = ''; + if (!this.id || !this.id.length) { + throw new TypeError( + 'invalid ObjectId, ObjectId.id must be either a string or a Buffer, but is [' + + JSON.stringify(this.id) + + ']' + ); + } + + if (this.id instanceof _Buffer) { + hexString = convertToHex(this.id); + if (ObjectId.cacheHexString) this.__id = hexString; + return hexString; + } + + for (var i = 0; i < this.id.length; i++) { + var hexChar = hexTable[this.id.charCodeAt(i)]; + if (typeof hexChar !== 'string') { + throw makeObjectIdError(this.id, i); + } + hexString += hexChar; + } + + if (ObjectId.cacheHexString) this.__id = hexString; + return hexString; + } + + /** + * @ignore + */ + ObjectId.prototype.toString = function(format) { + // Is the id a buffer then use the buffer toString method to return the format + if (this.id && this.id.copy) { + return this.id.toString(typeof format === 'string' ? format : 'hex'); + } + + return this.toHexString(); + } + + /** + * @ignore + */ + ObjectId.prototype.toJSON = function() { + return this.toHexString(); + } + + /** + * @ignore + */ + ObjectId.prototype.equals = function(otherId) { + if (otherId instanceof ObjectId) { + return this.toString() === otherId.toString(); + } + + if ( + typeof otherId === 'string' && + ObjectId.isValid(otherId) && + otherId.length === 12 && + this.id instanceof _Buffer + ) { + return otherId === this.id.toString('binary'); + } + + if (typeof otherId === 'string' && ObjectId.isValid(otherId) && otherId.length === 24) { + return otherId.toLowerCase() === this.toHexString(); + } + + if (typeof otherId === 'string' && ObjectId.isValid(otherId) && otherId.length === 12) { + return otherId === this.id; + } + + if (otherId != null && (otherId instanceof ObjectId || otherId.toHexString)) { + return otherId.toHexString() === this.toHexString(); + } + + return false; + } + + /** + * @ignore + */ + ObjectId.prototype.getTimestamp = function() { + var timestamp = new Date(); + var time = this.id[3] | (this.id[2] << 8) | (this.id[1] << 16) | (this.id[0] << 24); + timestamp.setTime(Math.floor(time) * 1000); + return timestamp; + } + + /** + * @ignore + */ + ObjectId.prototype.toExtendedJSON = function() { + if (this.toHexString) return { $oid: this.toHexString() }; + return { $oid: this.toString('hex') }; + } + + ObjectId.prototype.constructor = ObjectId; + + /** + * @ignore + */ + ObjectId.createPk = function() { + return new ObjectId(); + } + + /** + * @ignore + */ + ObjectId.createFromTime = function(time) { + var buffer = Buffer.from([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + // Encode time into first 4 bytes + buffer[3] = time & 0xff; + buffer[2] = (time >> 8) & 0xff; + buffer[1] = (time >> 16) & 0xff; + buffer[0] = (time >> 24) & 0xff; + // Return the new objectId + return new ObjectId(buffer); + } + + /** + * @ignore + */ + ObjectId.createFromHexString = function(string) { + // Throw an error if it's not a valid setup + if (typeof string === 'undefined' || (string != null && string.length !== 24)) { + throw new TypeError( + 'Argument passed in must be a single String of 12 bytes or a string of 24 hex characters' + ); + } + + // Use Buffer.from method if available + if (hasBufferType) return new ObjectId(Buffer.from(string, 'hex')); + + // Calculate lengths + var array = new _Buffer(12); + + var n = 0; + var i = 0; + while (i < 24) { + array[n++] = + (decodeLookup[string.charCodeAt(i++)] << 4) | decodeLookup[string.charCodeAt(i++)]; + } + + return new ObjectId(array); + } + + /** + * @ignore + */ + ObjectId.isValid = function(id) { + if (id == null) return false; + + if (typeof id === 'number') { + return true; + } + + if (typeof id === 'string') { + return id.length === 12 || (id.length === 24 && checkForHexRegExp.test(id)); + } + + if (id instanceof ObjectId) { + return true; + } + + if (id instanceof _Buffer && id.length === 12) { + return true; + } + + // Duck-Typing detection of ObjectId like objects + if (id.toHexString) { + return id.id.length === 12 || (id.id.length === 24 && checkForHexRegExp.test(id.id)); + } + + return false; + } + + /** + * @ignore + */ + ObjectId.getInc = function() { + return (ObjectId.index = (ObjectId.index + 1) % 0xffffff); + } + + /** + * @ignore + */ + ObjectId.generate = function(time) { + if ('number' !== typeof time) { + time = ~~(Date.now() / 1000); + } + + var inc = ObjectId.getInc(); + var buffer = Buffer.alloc ? Buffer.alloc(12) : new Buffer(12); + + // 4-byte timestamp + buffer[3] = time & 0xff; + buffer[2] = (time >> 8) & 0xff; + buffer[1] = (time >> 16) & 0xff; + buffer[0] = (time >> 24) & 0xff; + + // 5-byte process unique + buffer[4] = PROCESS_UNIQUE[0]; + buffer[5] = PROCESS_UNIQUE[1]; + buffer[6] = PROCESS_UNIQUE[2]; + buffer[7] = PROCESS_UNIQUE[3]; + buffer[8] = PROCESS_UNIQUE[4]; + + // 3-byte counter + buffer[11] = inc & 0xff; + buffer[10] = (inc >> 8) & 0xff; + buffer[9] = (inc >> 16) & 0xff; + + return buffer; + } + + /** + * @ignore + */ + ObjectId.fromExtendedJSON = function(doc) { + return new ObjectId(doc.$oid); + } + + + +// Deprecated methods +ObjectId.get_inc = deprecate( + function() {return ObjectId.getInc()}, + 'Please use the static `ObjectId.getInc()` instead' +); + +ObjectId.prototype.getInc = deprecate( + function() {return ObjectId.getInc()}, + 'Please use the static `ObjectId.getInc()` instead' +); + +ObjectId.prototype.generate = deprecate( + function(time) {return ObjectId.generate(time)}, + 'Please use the static `ObjectId.generate(time)` instead' +); + +Object.defineProperty(ObjectId.prototype, 'generationTime', { + enumerable: true, + get: function() { + return this.id[3] | (this.id[2] << 8) | (this.id[1] << 16) | (this.id[0] << 24); + }, + set: function(value) { + // Encode time into first 4 bytes + this.id[3] = value & 0xff; + this.id[2] = (value >> 8) & 0xff; + this.id[1] = (value >> 16) & 0xff; + this.id[0] = (value >> 24) & 0xff; + } +}); + +/** + * Converts to a string representation of this Id. + * + * @return {String} return the 24 byte hex string representation. + * @ignore + */ +ObjectId.prototype.inspect = ObjectId.prototype.toString; + +/** + * @ignore + */ +ObjectId.index = ~~(Math.random() * 0xffffff); + +Object.defineProperty(ObjectId.prototype, '_bsontype', { value: 'ObjectId' }); +module.exports = ObjectId; diff --git a/test/node/4x-interop/parser/utils.js b/test/node/4x-interop/parser/utils.js new file mode 100644 index 00000000..4fbfbdb0 --- /dev/null +++ b/test/node/4x-interop/parser/utils.js @@ -0,0 +1,37 @@ +'use strict'; + +/* global window */ + +/** + * @ignore + */ +function normalizedFunctionString(fn) { + return fn.toString().replace('function(', 'function ('); +} + +function insecureRandomBytes(size) { + var result = new Uint8Array(size); + for (var i = 0; i < size; ++i) result[i] = Math.floor(Math.random() * 256); + return result; +} + +var randomBytes = insecureRandomBytes; +if (typeof window !== 'undefined' && window.crypto && window.crypto.getRandomValues) { + randomBytes = function(size) {return window.crypto.getRandomValues(new Uint8Array(size));} +} else { + try { + randomBytes = require('crypto').randomBytes; + } catch (e) { + // keep the fallback + } + + // NOTE: in transpiled cases the above require might return null/undefined + if (randomBytes == null) { + randomBytes = insecureRandomBytes; + } +} + +module.exports = { + normalizedFunctionString: normalizedFunctionString, + randomBytes: randomBytes +}; diff --git a/test/node/bson_test.js b/test/node/bson_test.js index a74f86bb..5c948df2 100644 --- a/test/node/bson_test.js +++ b/test/node/bson_test.js @@ -2346,3 +2346,57 @@ exports['Should return boolean for ObjectID equality check'] = function(test) { test.equal(false, id.equals(undefined)); test.done(); }; + +/** + * @ignore + */ +exports['Should correctly serialize MinKey from another library version'] = function(test) { + // clone the class defn to simulate another library sending us a MinKey instance + // note that the 1.x library tests need to run on Node 0.12.48 which doesn't support + // the JS "class" keyword so the 4.x defn of MinKey was ported to pre-ES5 syntax. + function MinKey4x() { + this.toExtendedJSON = function() { + return { $minKey: 1 }; + } + }; + MinKey4x.prototype.fromExtendedJSON = function() { + return new MinKey4x(); + } + MinKey4x.prototype._bsontype = 'MinKey'; + + var doc = { + _id: new ObjectId('4e886e687ff7ef5e00000162'), + minKey: new MinKey4x() + }; + + var serialized_data = createBSON().serialize(doc); + var doc2 = createBSON().deserialize(serialized_data); + + // Ensure that MinKey can be round-tripped through the serializer (see #310) + test.ok(doc2.minKey instanceof MinKey); + test.done(); +}; + +/** + * @ignore + */ +exports['Should serialize _bsontype=ObjectID (capital D) from v4.0.0/4.0.1'] = function(test) { + // The ObjectId implementation in /4x-interop was copied from 4.0.1 to ensure that interop works + // In 4.0.0 and 4.0.1, ObjectID._bsontype was changed to 'ObjectId' (lowercase "d"). + // This broke interop with 1.x. Releases after 4.0.1 reverted back to use _bsontype==='ObjectID', + // which fixed interop with 1.x, but because we had to rev 1.x anyways to fix #310 for interop + // with MinKey, it made sense to also fix interop with 4.0.0/4.0.1 ObjectId. + // The ObjectId implementation in /4x-interop was copied from 4.0.1 source and slightly modified + // so it could run the Node 0.12 tests where class, const, let, etc. are not supported. + var ObjectId401 = require('./4x-interop/objectid'); + var id = new ObjectId401(); + var doc = { _id: id }; + var serialized_data = createBSON().serialize(doc); + + var serialized_data2 = new Buffer(createBSON().calculateObjectSize(doc)); + createBSON().serializeWithBufferAndIndex(doc, serialized_data2); + assertBuffersEqual(test, serialized_data, serialized_data2, 0); + + test.equal(doc._id.toHexString(), createBSON().deserialize(serialized_data)._id.toHexString()); + test.done(); +};