From 8eb0081815bd27031a5390c6ed560347f31db3e7 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 3 Jun 2021 14:20:58 -0400 Subject: [PATCH] fix(NODE-2944): Reintroduce bson-ext support (#2823) - Update benchmark imports - Add bson-ext test to CI --- .evergreen/config.yml | 29 +++++++++ .evergreen/config.yml.in | 11 ++++ .evergreen/generate_evergreen_tasks.js | 31 +++++++++ .evergreen/run-bson-ext-test.sh | 31 +++++++++ src/bson.ts | 62 +++++++++--------- src/index.ts | 1 + test/benchmarks/driverBench/common.js | 7 +- test/benchmarks/driverBench/index.js | 13 +++- test/functional/insert.test.js | 6 +- test/types/bson.test-d.ts | 26 ++++++++ test/unit/bson_import.test.js | 89 ++++++++++++++++++++++++++ 11 files changed, 267 insertions(+), 39 deletions(-) create mode 100755 .evergreen/run-bson-ext-test.sh create mode 100644 test/types/bson.test-d.ts create mode 100644 test/unit/bson_import.test.js diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e1f671ac10..45ffa0b470 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -516,6 +516,16 @@ functions: rm -f ./prepare_client_encryption.sh MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh + run bson-ext test: + - command: shell.exec + type: test + params: + working_dir: src + timeout_secs: 60 + script: | + ${PREPARE_SHELL} + + MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-bson-ext-test.sh upload test results: - command: attach.xunit_results params: @@ -1353,6 +1363,20 @@ tasks: VERSION: '4.4' TOPOLOGY: server - func: run custom csfle tests + - name: run-bson-ext-test + tags: + - run-bson-ext-test + commands: + - func: install dependencies + vars: + NODE_LTS_NAME: fermium + - func: bootstrap mongo-orchestration + vars: + VERSION: '4.4' + TOPOLOGY: server + - func: run bson-ext test + vars: + NODE_LTS_NAME: fermium buildvariants: - name: macos-1014-dubnium display_name: macOS 10.14 Node Dubnium @@ -1685,6 +1709,11 @@ buildvariants: run_on: ubuntu1804-test tasks: - run-custom-csfle-tests + - name: ubuntu1804-run-bson-ext-test + display_name: BSON EXT Test + run_on: ubuntu1804-test + tasks: + - run-bson-ext-test - name: mongosh_integration_tests display_name: mongosh integration tests run_on: ubuntu1804-test diff --git a/.evergreen/config.yml.in b/.evergreen/config.yml.in index b7a2cb0d36..35b10bbf1a 100644 --- a/.evergreen/config.yml.in +++ b/.evergreen/config.yml.in @@ -561,6 +561,17 @@ functions: MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh + "run bson-ext test": + - command: shell.exec + type: test + params: + working_dir: "src" + timeout_secs: 60 + script: | + ${PREPARE_SHELL} + + MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-bson-ext-test.sh + "upload test results": # Upload the xunit-format test results. - command: attach.xunit_results diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index f050234724..30c89047a3 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -533,6 +533,11 @@ BUILD_VARIANTS.push({ display_name: 'Custom FLE Version Test', run_on: 'ubuntu1804-test', tasks: ['run-custom-csfle-tests'] +},{ + name: 'ubuntu1804-run-bson-ext-test', + display_name: 'BSON EXT Test', + run_on: 'ubuntu1804-test', + tasks: ['run-bson-ext-test'] }); // singleton build variant for mongosh integration tests @@ -591,6 +596,32 @@ SINGLETON_TASKS.push({ ] }); +// special case for custom BSON-ext test +SINGLETON_TASKS.push({ + name: 'run-bson-ext-test', + tags: ['run-bson-ext-test'], + commands: [ + { + func: 'install dependencies', + vars: { + NODE_LTS_NAME: 'fermium', + }, + }, + { + func: 'bootstrap mongo-orchestration', + vars: { + VERSION: '4.4', + TOPOLOGY: 'server' + } + }, + { func: 'run bson-ext test', + vars: { + NODE_LTS_NAME: 'fermium', + } + } + ] +}); + const fileData = yaml.safeLoad(fs.readFileSync(`${__dirname}/config.yml.in`, 'utf8')); fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS).concat(TASKS).concat(SINGLETON_TASKS); fileData.buildvariants = (fileData.buildvariants || []).concat(BUILD_VARIANTS); diff --git a/.evergreen/run-bson-ext-test.sh b/.evergreen/run-bson-ext-test.sh new file mode 100755 index 0000000000..7e66583c6c --- /dev/null +++ b/.evergreen/run-bson-ext-test.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh + +set -o xtrace # Write all commands first to stderr +set -o errexit # Exit the script with error if any of the commands fail + +# Supported/used environment variables: +# SSL Set to enable SSL. Defaults to "nossl" +# MONGODB_URI Set the suggested connection MONGODB_URI (including credentials and topology info) +# TEST_NPM_SCRIPT Script to npm run. Defaults to "check:test" + +MONGODB_URI=${MONGODB_URI:-} +TEST_NPM_SCRIPT=${TEST_NPM_SCRIPT:-check:test} + +# ssl setup +SSL=${SSL:-nossl} +if [ "$SSL" != "nossl" ]; then + export SSL_KEY_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem" + export SSL_CA_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" +fi + +# run tests +echo "Running $AUTH tests over $SSL, connecting to $MONGODB_URI" + +npm install bson-ext + +export MONGODB_API_VERSION=${MONGODB_API_VERSION} +export MONGODB_URI=${MONGODB_URI} + +npm run "${TEST_NPM_SCRIPT}" diff --git a/src/bson.ts b/src/bson.ts index 1d0ce701ab..862b96de46 100644 --- a/src/bson.ts +++ b/src/bson.ts @@ -1,10 +1,21 @@ -// import type * as _BSON from 'bson'; -// let BSON: typeof _BSON = require('bson'); -// try { -// BSON = require('bson-ext'); -// } catch {} // eslint-disable-line +import type { + serialize as serializeFn, + deserialize as deserializeFn, + calculateObjectSize as calculateObjectSizeFn +} from 'bson'; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +let BSON = require('bson'); +try { + BSON = require('bson-ext'); +} catch {} // eslint-disable-line -// export = BSON; +/** @internal */ +export const deserialize = BSON.deserialize as typeof deserializeFn; +/** @internal */ +export const serialize = BSON.serialize as typeof serializeFn; +/** @internal */ +export const calculateObjectSize = BSON.calculateObjectSize as typeof calculateObjectSizeFn; export { Long, @@ -21,38 +32,27 @@ export { BSONRegExp, BSONSymbol, Map, - deserialize, - serialize, - calculateObjectSize + Document } from 'bson'; -/** @public */ -export interface Document { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - [key: string]: any; -} +import type { DeserializeOptions, SerializeOptions } from 'bson'; -import type { SerializeOptions } from 'bson'; - -// TODO: Remove me when types from BSON are updated /** * BSON Serialization options. * @public */ -export interface BSONSerializeOptions extends Omit { - /** Return document results as raw BSON buffers */ - fieldsAsRaw?: { [key: string]: boolean }; - /** Promotes BSON values to native types where possible, set to false to only receive wrapper types */ - promoteValues?: boolean; - /** Promotes Binary BSON values to native Node Buffers */ - promoteBuffers?: boolean; - /** Promotes long values to number if they fit inside the 53 bits resolution */ - promoteLongs?: boolean; - /** Serialize functions on any object */ - serializeFunctions?: boolean; - /** Specify if the BSON serializer should ignore undefined fields */ - ignoreUndefined?: boolean; - +export interface BSONSerializeOptions + extends Omit, + Omit< + DeserializeOptions, + | 'evalFunctions' + | 'cacheFunctions' + | 'cacheFunctionsCrc32' + | 'bsonRegExp' + | 'allowObjectSmallerThanBufferSize' + | 'index' + > { + /** Return BSON filled buffers from operations */ raw?: boolean; } diff --git a/src/index.ts b/src/index.ts index 9948cf242a..b1cdc23c12 100644 --- a/src/index.ts +++ b/src/index.ts @@ -358,3 +358,4 @@ export type { MetaProjectionOperators, MetaSortOperators } from './mongo_types'; +export type { serialize, deserialize } from './bson'; diff --git a/test/benchmarks/driverBench/common.js b/test/benchmarks/driverBench/common.js index 1841db4fba..f4248d02cd 100644 --- a/test/benchmarks/driverBench/common.js +++ b/test/benchmarks/driverBench/common.js @@ -1,9 +1,10 @@ +/* eslint-disable no-restricted-modules */ 'use strict'; const fs = require('fs'); const path = require('path'); -const { MongoClient } = require('../../../src/mongo_client'); -const { GridFsBucket } = require('../../../src/gridfs-stream'); +const { MongoClient } = require('../../..'); +const { GridFSBucket } = require('../../..'); const DB_NAME = 'perftest'; const COLLECTION_NAME = 'corpus'; @@ -54,7 +55,7 @@ function dropCollection() { } function initBucket() { - this.bucket = new GridFsBucket(this.db); + this.bucket = new GridFSBucket(this.db); } function dropBucket() { diff --git a/test/benchmarks/driverBench/index.js b/test/benchmarks/driverBench/index.js index 37509663dc..0a70337e74 100644 --- a/test/benchmarks/driverBench/index.js +++ b/test/benchmarks/driverBench/index.js @@ -5,7 +5,13 @@ const MongoBench = require('../mongoBench'); const Runner = MongoBench.Runner; const commonHelpers = require('./common'); -const BSON = require('bson'); +let BSON = require('bson'); +try { + BSON = require('bson-ext'); +} catch (_) { + // do not care +} + const { EJSON } = require('bson'); const makeClient = commonHelpers.makeClient; @@ -359,5 +365,8 @@ benchmarkRunner driverBench }; }) - .then(data => console.log(data)) + .then(data => { + data.bsonType = BSON.serialize.toString().includes('native code') ? 'bson-ext' : 'js-bson'; + console.log(data); + }) .catch(err => console.error(err)); diff --git a/test/functional/insert.test.js b/test/functional/insert.test.js index 71647f211e..48d59f5090 100644 --- a/test/functional/insert.test.js +++ b/test/functional/insert.test.js @@ -1577,7 +1577,7 @@ describe('Insert', function () { var collection = db.collection('bson_types_insert_1'); var document = { - symbol: new BSONSymbol('abcdefghijkl'), + string: 'abcdefghijkl', objid: new ObjectId('abcdefghijkl'), double: new Double(1), binary: new Binary(Buffer.from('hello world')), @@ -1590,9 +1590,9 @@ describe('Insert', function () { expect(err).to.not.exist; test.ok(result); - collection.findOne({ symbol: new BSONSymbol('abcdefghijkl') }, function (err, doc) { + collection.findOne({ string: 'abcdefghijkl' }, function (err, doc) { expect(err).to.not.exist; - test.equal('abcdefghijkl', doc.symbol.toString()); + test.equal('abcdefghijkl', doc.string.toString()); collection.findOne({ objid: new ObjectId('abcdefghijkl') }, function (err, doc) { expect(err).to.not.exist; diff --git a/test/types/bson.test-d.ts b/test/types/bson.test-d.ts new file mode 100644 index 0000000000..8030bd2d47 --- /dev/null +++ b/test/types/bson.test-d.ts @@ -0,0 +1,26 @@ +import { expectType } from 'tsd'; +import type { BSONSerializeOptions, Document } from '../../src/bson'; + +const options: BSONSerializeOptions = {}; + +expectType(options.checkKeys); +expectType(options.serializeFunctions); +expectType(options.ignoreUndefined); +expectType(options.promoteLongs); +expectType(options.promoteBuffers); +expectType(options.promoteValues); +expectType(options.fieldsAsRaw); + +type PermittedBSONOptionKeys = + | 'checkKeys' + | 'serializeFunctions' + | 'ignoreUndefined' + | 'promoteLongs' + | 'promoteBuffers' + | 'promoteValues' + | 'fieldsAsRaw' + | 'raw'; + +const keys = (null as unknown) as PermittedBSONOptionKeys; +// creates an explicit allow list assertion +expectType(keys); diff --git a/test/unit/bson_import.test.js b/test/unit/bson_import.test.js new file mode 100644 index 0000000000..28b3922b7d --- /dev/null +++ b/test/unit/bson_import.test.js @@ -0,0 +1,89 @@ +'use strict'; + +const { expect } = require('chai'); +const BSON = require('../../src/bson'); + +function isBSONExtInstalled() { + try { + require.resolve('bson-ext'); + return true; + } catch (_) { + return false; + } +} + +describe('When importing BSON', function () { + const types = [ + ['Long', 23], + ['ObjectId', '123456789123456789123456'], + ['Binary', Buffer.from('abc', 'ascii')], + ['Timestamp', 23], + ['Code', 'function(){}'], + ['MinKey', undefined], + ['MaxKey', undefined], + ['Decimal128', '2.34'], + ['Int32', 23], + ['Double', 2.3], + ['BSONRegExp', 'abc'] + ]; + // Omitted types since they're deprecated: + // BSONSymbol + // DBRef + + const options = { + promoteValues: false, + bsonRegExp: true + }; + + function testTypes() { + for (const [type, ctorArg] of types) { + it(`should correctly round trip ${type}`, function () { + const typeCtor = BSON[type]; + expect(typeCtor).to.be.a('function'); + const doc = { key: new typeCtor(ctorArg) }; + const outputDoc = BSON.deserialize(BSON.serialize(doc), options); + expect(outputDoc).to.have.property('key').that.is.instanceOf(typeCtor); + expect(outputDoc).to.deep.equal(doc); + }); + } + + it('should correctly round trip Map', function () { + expect(BSON.Map).to.be.a('function'); + const doc = { key: new BSON.Map([['2', 2]]) }; + const outputDoc = BSON.deserialize(BSON.serialize(doc)); + expect(outputDoc).to.have.nested.property('key.2', 2); + }); + } + + describe('bson-ext', function () { + before(function () { + if (!isBSONExtInstalled()) { + this.skip(); + } + }); + + it('should be imported if it exists', function () { + expect(BSON.deserialize.toString()).to.include('[native code]'); + expect(BSON.serialize.toString()).to.include('[native code]'); + expect(BSON.calculateObjectSize.toString()).to.include('[native code]'); + }); + + testTypes(); + }); + + describe('js-bson', function () { + before(function () { + if (isBSONExtInstalled()) { + this.skip(); + } + }); + + it('should be imported by default', function () { + expect(BSON.deserialize.toString()).to.not.include('[native code]'); + expect(BSON.serialize.toString()).to.not.include('[native code]'); + expect(BSON.calculateObjectSize.toString()).to.not.include('[native code]'); + }); + + testTypes(); + }); +});