From 0f7f3003b948d230edf1491fab775e7acc29381e Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Mon, 23 Aug 2021 16:43:06 -0400 Subject: [PATCH] fix(NODE-3528): add support for snappy 7 (#2939) --- .evergreen/config.yml | 48 ++- .evergreen/config.yml.in | 12 + .evergreen/generate_evergreen_tasks.js | 495 ++++++++++++------------- .evergreen/run-snappy-version-test.sh | 12 + package-lock.json | 26 +- package.json | 9 +- src/cmap/wire_protocol/compression.ts | 26 +- src/deps.ts | 51 ++- src/utils.ts | 9 + test/unit/snappy.test.js | 62 ++++ 10 files changed, 448 insertions(+), 302 deletions(-) create mode 100644 .evergreen/run-snappy-version-test.sh create mode 100644 test/unit/snappy.test.js diff --git a/.evergreen/config.yml b/.evergreen/config.yml index fc766a7541..a9617ced5b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -609,6 +609,17 @@ functions: rm -f ./prepare_client_encryption.sh MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh + run custom snappy tests: + - command: subprocess.exec + params: + working_dir: src + timeout_secs: 60 + env: + MONGODB_URI: '${MONGODB_URI}' + PROJECT_DIRECTORY: '${PROJECT_DIRECTORY}' + binary: bash + args: + - '${PROJECT_DIRECTORY}/.evergreen/run-snappy-version-test.sh' run bson-ext test: - command: shell.exec type: test @@ -1677,26 +1688,38 @@ tasks: - func: run mongosh integration tests - name: run-custom-csfle-tests tags: - - run-custom-csfle-tests + - run-custom-dependency-tests commands: - func: install dependencies vars: NODE_LTS_NAME: erbium - func: bootstrap mongo-orchestration vars: - VERSION: '4.4' + VERSION: '5.0' TOPOLOGY: server - func: run custom csfle tests + - name: run-custom-snappy-tests + tags: + - run-custom-dependency-tests + commands: + - func: install dependencies + vars: + NODE_LTS_NAME: erbium + - func: bootstrap mongo-orchestration + vars: + VERSION: '5.0' + TOPOLOGY: server + - func: run custom snappy tests - name: run-bson-ext-test tags: - - run-bson-ext-test + - run-custom-dependency-tests commands: - func: install dependencies vars: NODE_LTS_NAME: erbium - func: bootstrap mongo-orchestration vars: - VERSION: '4.4' + VERSION: '5.0' TOPOLOGY: server - func: run bson-ext test vars: @@ -1993,16 +2016,6 @@ buildvariants: run_on: ubuntu1804-large tasks: - run-checks - - name: ubuntu1804-custom-csfle-tests - display_name: Custom FLE Version Test - run_on: ubuntu1804-large - tasks: - - run-custom-csfle-tests - - name: ubuntu1804-run-bson-ext-test - display_name: BSON EXT Test - run_on: ubuntu1804-large - tasks: - - run-bson-ext-test - name: mongosh_integration_tests display_name: mongosh integration tests run_on: ubuntu1804-test @@ -2032,6 +2045,13 @@ buildvariants: - aws-4.4-auth-test-run-aws-auth-test-with-aws-credentials-as-environment-variables - aws-4.4-auth-test-run-aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables - aws-4.4-auth-test-run-aws-ECS-auth-test + - name: ubuntu1804-custom-dependency-tests + display_name: Custom Dependency Version Test + run_on: ubuntu1804-large + tasks: + - run-custom-csfle-tests + - run-custom-snappy-tests + - run-bson-ext-test - name: ubuntu1804-test-serverless display_name: Serverless Test run_on: ubuntu1804-test diff --git a/.evergreen/config.yml.in b/.evergreen/config.yml.in index c4b328940f..6436974faa 100644 --- a/.evergreen/config.yml.in +++ b/.evergreen/config.yml.in @@ -650,6 +650,18 @@ functions: MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh + "run custom snappy tests": + - command: subprocess.exec + params: + working_dir: "src" + timeout_secs: 60 + env: + MONGODB_URI: ${MONGODB_URI} + PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} + binary: bash + args: + - "${PROJECT_DIRECTORY}/.evergreen/run-snappy-version-test.sh" + "run bson-ext test": - command: shell.exec type: test diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 72d35a1f28..c75773a54e 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -5,7 +5,7 @@ const yaml = require('js-yaml'); const LATEST_EFFECTIVE_VERSION = '5.0'; const MONGODB_VERSIONS = ['latest', '5.0', '4.4', '4.2', '4.0', '3.6', '3.4', '3.2', '3.0', '2.6']; const NODE_VERSIONS = ['erbium', 'fermium']; -NODE_VERSIONS.sort() +NODE_VERSIONS.sort(); const LOWEST_LTS = NODE_VERSIONS[0]; const TOPOLOGIES = ['server', 'replica_set', 'sharded_cluster']; @@ -13,7 +13,7 @@ const AWS_AUTH_VERSIONS = ['latest', '5.0', '4.4']; const OCSP_VERSIONS = ['latest', '5.0', '4.4']; const TLS_VERSIONS = ['latest', '5.0', '4.4', '4.2']; -const DEFAULT_OS = 'ubuntu1804-large' +const DEFAULT_OS = 'ubuntu1804-large'; const OPERATING_SYSTEMS = [ { @@ -31,7 +31,7 @@ const OPERATING_SYSTEMS = [ display_name: 'Windows (VS2019)', run_on: 'windows-64-vs2019-large', msvsVersion: 2019, - clientEncryption: false, // TODO(NODE-3401): Unskip when Windows no longer fails to launch mongocryptd occasionally + clientEncryption: false // TODO(NODE-3401): Unskip when Windows no longer fails to launch mongocryptd occasionally } ].map(osConfig => ({ mongoVersion: '>=2.6', @@ -94,199 +94,59 @@ BASE_TASKS.push({ }); // manually added tasks -TASKS.push(...[ - { - name: 'test-atlas-connectivity', - tags: ['atlas-connect'], - commands: [{ func: 'install dependencies' }, { func: 'run atlas tests' }] - }, - { - name: 'test-atlas-data-lake', - commands: [ - { func: 'install dependencies' }, - { func: 'bootstrap mongohoused' }, - { func: 'run data lake tests' } - ] - }, - { - name: 'test-load-balancer', - tags: ['latest', 'sharded_cluster', 'load_balancer'], - commands: [ - { func: 'install dependencies' }, - { - func: 'bootstrap mongo-orchestration', - vars: { - VERSION: '5.0', - TOPOLOGY: 'sharded_cluster' - } - }, - { func: 'start-load-balancer' }, - { func: 'run-lb-tests' }, - { func: 'stop-load-balancer' } - ] - }, - { - name: 'test-auth-kerberos', - tags: ['auth', 'kerberos'], - commands: [{ func: 'install dependencies' }, { func: 'run kerberos tests' }] - }, - { - name: 'test-auth-ldap', - tags: ['auth', 'ldap'], - commands: [{ func: 'install dependencies' }, { func: 'run ldap tests' }] - }, - { - name: 'test-ocsp-valid-cert-server-staples', - tags: ['ocsp'], - commands: [ - { func: 'run-valid-ocsp-server' }, - { func: 'install dependencies' }, - { - func: 'bootstrap mongo-orchestration', - vars: { - ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple.json', - VERSION: 'latest', - TOPOLOGY: 'server' - } - }, - { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 1 } } - ] - }, - { - name: 'test-ocsp-invalid-cert-server-staples', - tags: ['ocsp'], - commands: [ - { func: 'run-revoked-ocsp-server' }, - { func: 'install dependencies' }, - { - func: 'bootstrap mongo-orchestration', - vars: { - ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple.json', - VERSION: 'latest', - TOPOLOGY: 'server' - } - }, - { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 0 } } - ] - }, - { - name: 'test-ocsp-valid-cert-server-does-not-staple', - tags: ['ocsp'], - commands: [ - { func: 'run-valid-ocsp-server' }, - { func: 'install dependencies' }, - { - func: 'bootstrap mongo-orchestration', - vars: { - ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-disableStapling.json', - VERSION: 'latest', - TOPOLOGY: 'server' - } - }, - { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 1 } } - ] - }, - { - name: 'test-ocsp-invalid-cert-server-does-not-staple', - tags: ['ocsp'], - commands: [ - { func: 'run-revoked-ocsp-server' }, - { func: 'install dependencies' }, - { - func: 'bootstrap mongo-orchestration', - vars: { - ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-disableStapling.json', - VERSION: 'latest', - TOPOLOGY: 'server' - } - }, - { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 0 } } - ] - }, - { - name: 'test-ocsp-soft-fail', - tags: ['ocsp'], - commands: [ - { func: 'install dependencies' }, - { - func: 'bootstrap mongo-orchestration', - vars: { - ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-disableStapling.json', - VERSION: 'latest', - TOPOLOGY: 'server' - } - }, - { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 1 } } - ] - }, - { - name: 'test-ocsp-malicious-invalid-cert-mustStaple-server-does-not-staple', - tags: ['ocsp'], - commands: [ - { func: 'run-revoked-ocsp-server' }, - { func: 'install dependencies' }, - { - func: 'bootstrap mongo-orchestration', - vars: { - ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple-disableStapling.json', - VERSION: 'latest', - TOPOLOGY: 'server' - } - }, - { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 0 } } - ] - }, - { - name: 'test-ocsp-malicious-no-responder-mustStaple-server-does-not-staple', - tags: ['ocsp'], - commands: [ - { func: 'install dependencies' }, - { - func: 'bootstrap mongo-orchestration', - vars: { - ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple-disableStapling.json', - VERSION: 'latest', - TOPOLOGY: 'server' - } - }, - { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 0 } } - ] - } -]); - -TLS_VERSIONS.forEach(VERSION => { - TASKS.push({ - name: `test-tls-support-${VERSION}`, - tags: ['tls-support'], - commands: [ - { func: 'install dependencies' }, - { - func: 'bootstrap mongo-orchestration', - vars: { - VERSION, - SSL: 'ssl', - TOPOLOGY: 'server' - } - }, - { func: 'run tls tests' } - ] - }); -}); - -OCSP_VERSIONS.forEach(VERSION => { - // manually added tasks - TASKS.push(...[ +TASKS.push( + ...[ + { + name: 'test-atlas-connectivity', + tags: ['atlas-connect'], + commands: [{ func: 'install dependencies' }, { func: 'run atlas tests' }] + }, + { + name: 'test-atlas-data-lake', + commands: [ + { func: 'install dependencies' }, + { func: 'bootstrap mongohoused' }, + { func: 'run data lake tests' } + ] + }, + { + name: 'test-load-balancer', + tags: ['latest', 'sharded_cluster', 'load_balancer'], + commands: [ + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + VERSION: '5.0', + TOPOLOGY: 'sharded_cluster' + } + }, + { func: 'start-load-balancer' }, + { func: 'run-lb-tests' }, + { func: 'stop-load-balancer' } + ] + }, { - name: `test-${VERSION}-ocsp-valid-cert-server-staples`, + name: 'test-auth-kerberos', + tags: ['auth', 'kerberos'], + commands: [{ func: 'install dependencies' }, { func: 'run kerberos tests' }] + }, + { + name: 'test-auth-ldap', + tags: ['auth', 'ldap'], + commands: [{ func: 'install dependencies' }, { func: 'run ldap tests' }] + }, + { + name: 'test-ocsp-valid-cert-server-staples', tags: ['ocsp'], commands: [ - { func: `run-valid-ocsp-server` }, + { func: 'run-valid-ocsp-server' }, { func: 'install dependencies' }, { func: 'bootstrap mongo-orchestration', vars: { ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple.json', - VERSION: VERSION, + VERSION: 'latest', TOPOLOGY: 'server' } }, @@ -294,7 +154,7 @@ OCSP_VERSIONS.forEach(VERSION => { ] }, { - name: `test-${VERSION}-ocsp-invalid-cert-server-staples`, + name: 'test-ocsp-invalid-cert-server-staples', tags: ['ocsp'], commands: [ { func: 'run-revoked-ocsp-server' }, @@ -303,7 +163,7 @@ OCSP_VERSIONS.forEach(VERSION => { func: 'bootstrap mongo-orchestration', vars: { ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple.json', - VERSION: VERSION, + VERSION: 'latest', TOPOLOGY: 'server' } }, @@ -311,7 +171,7 @@ OCSP_VERSIONS.forEach(VERSION => { ] }, { - name: `test-${VERSION}-ocsp-valid-cert-server-does-not-staple`, + name: 'test-ocsp-valid-cert-server-does-not-staple', tags: ['ocsp'], commands: [ { func: 'run-valid-ocsp-server' }, @@ -320,7 +180,7 @@ OCSP_VERSIONS.forEach(VERSION => { func: 'bootstrap mongo-orchestration', vars: { ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-disableStapling.json', - VERSION: VERSION, + VERSION: 'latest', TOPOLOGY: 'server' } }, @@ -328,7 +188,7 @@ OCSP_VERSIONS.forEach(VERSION => { ] }, { - name: `test-${VERSION}-ocsp-invalid-cert-server-does-not-staple`, + name: 'test-ocsp-invalid-cert-server-does-not-staple', tags: ['ocsp'], commands: [ { func: 'run-revoked-ocsp-server' }, @@ -337,7 +197,7 @@ OCSP_VERSIONS.forEach(VERSION => { func: 'bootstrap mongo-orchestration', vars: { ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-disableStapling.json', - VERSION: VERSION, + VERSION: 'latest', TOPOLOGY: 'server' } }, @@ -345,7 +205,7 @@ OCSP_VERSIONS.forEach(VERSION => { ] }, { - name: `test-${VERSION}-ocsp-soft-fail`, + name: 'test-ocsp-soft-fail', tags: ['ocsp'], commands: [ { func: 'install dependencies' }, @@ -353,7 +213,7 @@ OCSP_VERSIONS.forEach(VERSION => { func: 'bootstrap mongo-orchestration', vars: { ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-disableStapling.json', - VERSION: VERSION, + VERSION: 'latest', TOPOLOGY: 'server' } }, @@ -361,7 +221,7 @@ OCSP_VERSIONS.forEach(VERSION => { ] }, { - name: `test-${VERSION}-ocsp-malicious-invalid-cert-mustStaple-server-does-not-staple`, + name: 'test-ocsp-malicious-invalid-cert-mustStaple-server-does-not-staple', tags: ['ocsp'], commands: [ { func: 'run-revoked-ocsp-server' }, @@ -370,7 +230,7 @@ OCSP_VERSIONS.forEach(VERSION => { func: 'bootstrap mongo-orchestration', vars: { ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple-disableStapling.json', - VERSION: VERSION, + VERSION: 'latest', TOPOLOGY: 'server' } }, @@ -378,7 +238,7 @@ OCSP_VERSIONS.forEach(VERSION => { ] }, { - name: `test-${VERSION}-ocsp-malicious-no-responder-mustStaple-server-does-not-staple`, + name: 'test-ocsp-malicious-no-responder-mustStaple-server-does-not-staple', tags: ['ocsp'], commands: [ { func: 'install dependencies' }, @@ -386,14 +246,158 @@ OCSP_VERSIONS.forEach(VERSION => { func: 'bootstrap mongo-orchestration', vars: { ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple-disableStapling.json', - VERSION: VERSION, + VERSION: 'latest', TOPOLOGY: 'server' } }, { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 0 } } ] } - ]); + ] +); + +TLS_VERSIONS.forEach(VERSION => { + TASKS.push({ + name: `test-tls-support-${VERSION}`, + tags: ['tls-support'], + commands: [ + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + VERSION, + SSL: 'ssl', + TOPOLOGY: 'server' + } + }, + { func: 'run tls tests' } + ] + }); +}); + +OCSP_VERSIONS.forEach(VERSION => { + // manually added tasks + TASKS.push( + ...[ + { + name: `test-${VERSION}-ocsp-valid-cert-server-staples`, + tags: ['ocsp'], + commands: [ + { func: `run-valid-ocsp-server` }, + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple.json', + VERSION: VERSION, + TOPOLOGY: 'server' + } + }, + { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 1 } } + ] + }, + { + name: `test-${VERSION}-ocsp-invalid-cert-server-staples`, + tags: ['ocsp'], + commands: [ + { func: 'run-revoked-ocsp-server' }, + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple.json', + VERSION: VERSION, + TOPOLOGY: 'server' + } + }, + { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 0 } } + ] + }, + { + name: `test-${VERSION}-ocsp-valid-cert-server-does-not-staple`, + tags: ['ocsp'], + commands: [ + { func: 'run-valid-ocsp-server' }, + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-disableStapling.json', + VERSION: VERSION, + TOPOLOGY: 'server' + } + }, + { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 1 } } + ] + }, + { + name: `test-${VERSION}-ocsp-invalid-cert-server-does-not-staple`, + tags: ['ocsp'], + commands: [ + { func: 'run-revoked-ocsp-server' }, + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-disableStapling.json', + VERSION: VERSION, + TOPOLOGY: 'server' + } + }, + { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 0 } } + ] + }, + { + name: `test-${VERSION}-ocsp-soft-fail`, + tags: ['ocsp'], + commands: [ + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-disableStapling.json', + VERSION: VERSION, + TOPOLOGY: 'server' + } + }, + { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 1 } } + ] + }, + { + name: `test-${VERSION}-ocsp-malicious-invalid-cert-mustStaple-server-does-not-staple`, + tags: ['ocsp'], + commands: [ + { func: 'run-revoked-ocsp-server' }, + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple-disableStapling.json', + VERSION: VERSION, + TOPOLOGY: 'server' + } + }, + { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 0 } } + ] + }, + { + name: `test-${VERSION}-ocsp-malicious-no-responder-mustStaple-server-does-not-staple`, + tags: ['ocsp'], + commands: [ + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + ORCHESTRATION_FILE: 'rsa-basic-tls-ocsp-mustStaple-disableStapling.json', + VERSION: VERSION, + TOPOLOGY: 'server' + } + }, + { func: 'run-ocsp-test', vars: { OCSP_TLS_SHOULD_SUCCEED: 0 } } + ] + } + ] + ); }); const AWS_AUTH_TASKS = []; @@ -449,10 +453,9 @@ const getTaskList = (() => { // skip unsupported tasks on windows or macos if ( - task.tags && ( - (os.match(/^windows/) && task.tags.filter(tag => WINDOWS_SKIP_TAGS.has(tag)).length) || - (os.match(/^macos/) && task.tags.filter(tag => MACOS_SKIP_TAGS.has(tag)).length) - ) + task.tags && + ((os.match(/^windows/) && task.tags.filter(tag => WINDOWS_SKIP_TAGS.has(tag)).length) || + (os.match(/^macos/) && task.tags.filter(tag => MACOS_SKIP_TAGS.has(tag)).length)) ) { return false; } @@ -522,26 +525,12 @@ SINGLETON_TASKS.push({ ] }); -BUILD_VARIANTS.push( - { - name: 'lint', - display_name: 'lint', - run_on: DEFAULT_OS, - tasks: ['run-checks'] - }, - { - name: 'ubuntu1804-custom-csfle-tests', - display_name: 'Custom FLE Version Test', - run_on: DEFAULT_OS, - tasks: ['run-custom-csfle-tests'] - }, - { - name: 'ubuntu1804-run-bson-ext-test', - display_name: 'BSON EXT Test', - run_on: DEFAULT_OS, - tasks: ['run-bson-ext-test'] - } -); +BUILD_VARIANTS.push({ + name: 'lint', + display_name: 'lint', + run_on: DEFAULT_OS, + tasks: ['run-checks'] +}); // singleton build variant for mongosh integration tests SINGLETON_TASKS.push({ @@ -577,32 +566,20 @@ BUILD_VARIANTS.push({ tasks: AWS_AUTH_TASKS }); -// special case for custom CSFLE test -SINGLETON_TASKS.push({ - name: 'run-custom-csfle-tests', - tags: ['run-custom-csfle-tests'], - commands: [ - { - func: 'install dependencies', - vars: { - NODE_LTS_NAME: LOWEST_LTS - } - }, - { - func: 'bootstrap mongo-orchestration', - vars: { - VERSION: '4.4', - TOPOLOGY: 'server' - } - }, - { func: 'run custom csfle tests' } - ] -}); +const oneOffFuncs = [ + { func: 'run custom csfle tests' }, + { func: 'run custom snappy tests' }, + { + func: 'run bson-ext test', + vars: { + NODE_LTS_NAME: LOWEST_LTS + } + } +]; -// special case for custom BSON-ext test -SINGLETON_TASKS.push({ - name: 'run-bson-ext-test', - tags: ['run-bson-ext-test'], +const oneOffFuncAsTasks = oneOffFuncs.map(oneOffFunc => ({ + name: `${oneOffFunc.func.split(' ').join('-')}`, + tags: ['run-custom-dependency-tests'], commands: [ { func: 'install dependencies', @@ -613,17 +590,21 @@ SINGLETON_TASKS.push({ { func: 'bootstrap mongo-orchestration', vars: { - VERSION: '4.4', + VERSION: '5.0', TOPOLOGY: 'server' } }, - { - func: 'run bson-ext test', - vars: { - NODE_LTS_NAME: LOWEST_LTS - } - } + oneOffFunc ] +})); + +SINGLETON_TASKS.push(...oneOffFuncAsTasks); + +BUILD_VARIANTS.push({ + name: 'ubuntu1804-custom-dependency-tests', + display_name: 'Custom Dependency Version Test', + run_on: DEFAULT_OS, + tasks: oneOffFuncAsTasks.map(({ name }) => name) }); // special case for serverless testing diff --git a/.evergreen/run-snappy-version-test.sh b/.evergreen/run-snappy-version-test.sh new file mode 100644 index 0000000000..3dd8ba8792 --- /dev/null +++ b/.evergreen/run-snappy-version-test.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh" +export MONGODB_URI="${MONGODB_URI}" + +npm i --no-save snappy@6 + +npx mocha test/unit/snappy.test.js + +npm i --no-save snappy@7 + +npx mocha test/unit/snappy.test.js diff --git a/package-lock.json b/package-lock.json index d8b4f4a3eb..e19012800b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10889,8 +10889,7 @@ "@types/node": { "version": "15.14.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-15.14.2.tgz", - "integrity": "sha512-dvMUE/m2LbXPwlvVuzCyslTEtQ2ZwuuFClDrOQ6mp2CenCg971719PTILZ4I6bTP27xfFFc+o7x2TkLuun/MPw==", - "dev": true + "integrity": "sha512-dvMUE/m2LbXPwlvVuzCyslTEtQ2ZwuuFClDrOQ6mp2CenCg971719PTILZ4I6bTP27xfFFc+o7x2TkLuun/MPw==" }, "@types/normalize-package-data": { "version": "2.4.1", @@ -10913,14 +10912,12 @@ "@types/webidl-conversions": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-6.1.1.tgz", - "integrity": "sha512-XAahCdThVuCFDQLT7R7Pk/vqeObFNL3YqRyFZg+AqAP/W1/w3xHaIxuW7WszQqTbIBOPRcItYJIou3i/mppu3Q==", - "dev": true + "integrity": "sha512-XAahCdThVuCFDQLT7R7Pk/vqeObFNL3YqRyFZg+AqAP/W1/w3xHaIxuW7WszQqTbIBOPRcItYJIou3i/mppu3Q==" }, "@types/whatwg-url": { "version": "8.2.1", "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.1.tgz", "integrity": "sha512-2YubE1sjj5ifxievI5Ge1sckb9k/Er66HyR2c+3+I6VDUUg1TLPdYYTEbQ+DjRkS4nTxMJhgWfSfMRD2sl2EYQ==", - "dev": true, "requires": { "@types/node": "*", "@types/webidl-conversions": "*" @@ -14302,7 +14299,8 @@ "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true }, "lodash.camelcase": { "version": "4.3.0", @@ -14977,11 +14975,12 @@ } }, "mongodb-connection-string-url": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-1.0.1.tgz", - "integrity": "sha512-sXi8w9nwbMrErWbOK+8nofHz531rboasDbYAMS+sQ+W+2YnHN980RlMr+t5SDL6uKEU/kw/wG6jcjCTLiJltoA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.0.0.tgz", + "integrity": "sha512-M0I1vyLoq5+HQTuPSJWbt+hIXsMCfE8sS1fS5mvP9R2DOMoi2ZD32yWqgBIITyu0dFu4qtS50erxKjvUeBiyog==", "requires": { - "whatwg-url": "^8.4.0" + "@types/whatwg-url": "^8.2.1", + "whatwg-url": "^9.1.0" } }, "mongodb-mock-server": { @@ -17438,11 +17437,10 @@ "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w==" }, "whatwg-url": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.7.0.tgz", - "integrity": "sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-9.1.0.tgz", + "integrity": "sha512-CQ0UcrPHyomtlOCot1TL77WyMIm/bCwrJ2D6AOKGwEczU9EpyoqAokfqrf/MioU9kHcMsmJZcg1egXix2KYEsA==", "requires": { - "lodash": "^4.7.0", "tr46": "^2.1.0", "webidl-conversions": "^6.1.0" } diff --git a/package.json b/package.json index 1b19754be7..f126edfc32 100644 --- a/package.json +++ b/package.json @@ -31,16 +31,10 @@ "name": "The MongoDB NodeJS Team", "email": "dbx-node@mongodb.com" }, - "peerOptionalDependencies": { - "kerberos": "^1.1.0", - "mongodb-client-encryption": "^1.0.0", - "snappy": "^6.1.1", - "bson-ext": "^2.0.0" - }, "dependencies": { "bson": "^4.5.0", "denque": "^1.5.0", - "mongodb-connection-string-url": "^1.0.1" + "mongodb-connection-string-url": "^2.0.0" }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.1", @@ -82,7 +76,6 @@ "semver": "^5.5.0", "sinon": "^4.3.0", "sinon-chai": "^3.2.0", - "snappy": "^6.3.0", "source-map-support": "^0.5.19", "standard-version": "^9.3.0", "through2": "^3.0.1", diff --git a/src/cmap/wire_protocol/compression.ts b/src/cmap/wire_protocol/compression.ts index 368f944c5e..957fb45a2e 100644 --- a/src/cmap/wire_protocol/compression.ts +++ b/src/cmap/wire_protocol/compression.ts @@ -2,7 +2,7 @@ import * as zlib from 'zlib'; import type { Callback } from '../../utils'; import type { OperationDescription } from '../message_stream'; -import { Snappy } from '../../deps'; +import { PKG_VERSION, Snappy } from '../../deps'; import { MongoDecompressionError, MongoInvalidArgumentError } from '../../error'; /** @public */ @@ -39,12 +39,20 @@ export function compress( ): void { const zlibOptions = {} as zlib.ZlibOptions; switch (self.options.agreedCompressor) { - case 'snappy': + case 'snappy': { if ('kModuleError' in Snappy) { return callback(Snappy['kModuleError']); } - Snappy.compress(dataToBeCompressed, callback); + + if (Snappy[PKG_VERSION].major <= 6) { + Snappy.compress(dataToBeCompressed, callback); + } else { + Snappy.compress(dataToBeCompressed) + .then(buffer => callback(undefined, buffer)) + .catch(error => callback(error)); + } break; + } case 'zlib': // Determine zlibCompressionLevel if (self.options.zlibCompressionLevel) { @@ -72,12 +80,20 @@ export function decompress( } switch (compressorID) { - case Compressor.snappy: + case Compressor.snappy: { if ('kModuleError' in Snappy) { return callback(Snappy['kModuleError']); } - Snappy.uncompress(compressedData, { asBuffer: true }, callback as Callback); + + if (Snappy[PKG_VERSION].major <= 6) { + Snappy.uncompress(compressedData, { asBuffer: true }, callback); + } else { + Snappy.uncompress(compressedData, { asBuffer: true }) + .then(buffer => callback(undefined, buffer)) + .catch(error => callback(error)); + } break; + } case Compressor.zlib: zlib.inflate(compressedData, callback as zlib.CompressCallback); break; diff --git a/src/deps.ts b/src/deps.ts index 01edbd5b02..3fad9cd534 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,7 +1,10 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ import { MongoMissingDependencyError } from './error'; import type { MongoClient } from './mongo_client'; import type { deserialize, Document, serialize } from './bson'; -import type { Callback } from './utils'; +import { Callback, parsePackageVersion } from './utils'; + +export const PKG_VERSION = Symbol('kPkgVersion'); function makeErrorModule(error: any) { const props = error ? { kModuleError: error } : {}; @@ -41,9 +44,46 @@ export interface KerberosClient { unwrap: (challenge: string, callback?: Callback) => Promise | void; } -export let Snappy: - | typeof import('snappy') - | { kModuleError: MongoMissingDependencyError } = makeErrorModule( +type SnappyLib = { + [PKG_VERSION]: { major: number; minor: number; patch: number }; + + /** + * - Snappy 6.x takes a callback and returns void + * - Snappy 7.x returns a promise + * + * In order to support both we must check the return value of the function + * @param buf - Buffer to be compressed + * @param callback - ONLY USED IN SNAPPY 6.x + */ + compress(buf: Buffer): Promise; + compress(buf: Buffer, callback: (error?: Error, buffer?: Buffer) => void): Promise | void; + compress( + buf: Buffer, + callback?: (error?: Error, buffer?: Buffer) => void + ): Promise | void; + + /** + * - Snappy 6.x takes a callback and returns void + * - Snappy 7.x returns a promise + * + * In order to support both we must check the return value of the function + * @param buf - Buffer to be compressed + * @param callback - ONLY USED IN SNAPPY 6.x + */ + uncompress(buf: Buffer, opt: { asBuffer: true }): Promise; + uncompress( + buf: Buffer, + opt: { asBuffer: true }, + callback: (error?: Error, buffer?: Buffer) => void + ): Promise | void; + uncompress( + buf: Buffer, + opt: { asBuffer: true }, + callback?: (error?: Error, buffer?: Buffer) => void + ): Promise | void; +}; + +export let Snappy: SnappyLib | { kModuleError: MongoMissingDependencyError } = makeErrorModule( new MongoMissingDependencyError( 'Optional module `snappy` not found. Please install it to enable snappy compression' ) @@ -52,6 +92,9 @@ export let Snappy: try { // Ensure you always wrap an optional require in the try block NODE-3199 Snappy = require('snappy'); + try { + (Snappy as any)[PKG_VERSION] = parsePackageVersion(require('snappy/package.json')); + } catch {} // eslint-disable-line } catch {} // eslint-disable-line export let saslprep: diff --git a/src/utils.ts b/src/utils.ts index 8f53891e24..6fbf9ed12f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1429,3 +1429,12 @@ export function supportsRetryableWrites(server: Server): boolean { server.description.type !== ServerType.Standalone) ); } + +export function parsePackageVersion({ + version +}: { + version: string; +}): { major: number; minor: number; patch: number } { + const [major, minor, patch] = version.split('.').map((n: string) => Number.parseInt(n, 10)); + return { major, minor, patch }; +} diff --git a/test/unit/snappy.test.js b/test/unit/snappy.test.js new file mode 100644 index 0000000000..23b7226e43 --- /dev/null +++ b/test/unit/snappy.test.js @@ -0,0 +1,62 @@ +'use strict'; + +const { expect } = require('chai'); +const mock = require('../tools/mock'); +const snappy = optionalRequire('snappy'); +const snappyVersion = optionalRequire('snappy/package.json').version; + +function optionalRequire(mod) { + try { + return require(mod); + } catch { + return false; + } +} + +describe('Compression', function () { + let client; + /** @type {mock.MockServer} */ + let server; + before(async function () { + if (!snappy) this.skip(); + server = await mock.createServer(); + client = this.configuration.newClient(`mongodb://${server.uri()}`, { compressors: 'snappy' }); + }); + + after(async function () { + await mock.cleanup(); + await client.close(); + }); + + describe('Snappy', () => { + it(`should compress messages sent with snappy ${snappyVersion}`, async function () { + // the timeout is being set because the test should not take any longer than 5 seconds, + // and that if it doesn't complete, it will hang due to the callback never being called + this.timeout(5000); + + server.setMessageHandler(request => { + const doc = request.document; + if (doc.ismaster || doc.hello) { + return request.reply({ ...mock.DEFAULT_ISMASTER, compression: ['snappy'] }); + } + if (doc.insert === 'snappy') { + return request.reply({ ok: 1 }); + } + }); + // The mock server uses snappy to decode messages so + // if this passes we implicitly test snappy is working + // TODO(NODE-3560): Add more comprehensive round trip testing + await client.connect(); + await client.db().collection('snappy').insertOne({ a: 1 }); + }); + + it('should define a version number on the optional import', function () { + const { Snappy, PKG_VERSION } = require('../../src/deps'); + const [major, minor, patch] = snappyVersion.split('.').map(n => +n); + expect(Snappy).to.have.property(PKG_VERSION).that.is.an('object'); + expect(Snappy[PKG_VERSION]).to.have.property('major', major); + expect(Snappy[PKG_VERSION]).to.have.property('minor', minor); + expect(Snappy[PKG_VERSION]).to.have.property('patch', patch); + }); + }); +});