Skip to content

Commit

Permalink
test: adds tests for atlas data lake
Browse files Browse the repository at this point in the history
  • Loading branch information
Thomas Reggi authored Oct 19, 2020
1 parent 3155bcc commit 2704ce8
Show file tree
Hide file tree
Showing 23 changed files with 626 additions and 28 deletions.
29 changes: 29 additions & 0 deletions .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,18 @@ functions:
- command: expansions.update
params:
file: mo-expansion.yml
bootstrap mongohoused:
- command: shell.exec
params:
script: |
${PREPARE_SHELL}
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh
- command: shell.exec
params:
background: true
script: |
${PREPARE_SHELL}
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh
run tests:
- command: shell.exec
type: test
Expand Down Expand Up @@ -189,6 +201,15 @@ functions:
script: |
MONGODB_URI='${plain_auth_mongodb_uri}' NODE_LTS_NAME='${NODE_LTS_NAME}' \
bash ${PROJECT_DIRECTORY}/.evergreen/run-ldap-tests.sh
run data lake tests:
- command: shell.exec
type: test
params:
working_dir: src
script: |
${PREPARE_SHELL}
MONGODB_URI='mongodb://mhuser:pencil@localhost' NODE_LTS_NAME='${NODE_LTS_NAME}' \
bash ${PROJECT_DIRECTORY}/.evergreen/run-data-lake-tests.sh
run tls tests:
- command: shell.exec
type: test
Expand Down Expand Up @@ -723,6 +744,11 @@ tasks:
commands:
- func: install dependencies
- func: run atlas tests
- name: test-atlas-data-lake
commands:
- func: install dependencies
- func: bootstrap mongohoused
- func: run data lake tests
- name: test-auth-kerberos
tags:
- auth
Expand Down Expand Up @@ -1111,6 +1137,7 @@ buildvariants:
- test-2.6-replica_set
- test-2.6-sharded_cluster
- test-atlas-connectivity
- test-atlas-data-lake
- test-auth-kerberos
- test-auth-ldap
- test-tls-support
Expand Down Expand Up @@ -1178,6 +1205,7 @@ buildvariants:
- test-2.6-replica_set
- test-2.6-sharded_cluster
- test-atlas-connectivity
- test-atlas-data-lake
- test-auth-kerberos
- test-auth-ldap
- name: ubuntu-14.04-erbium
Expand Down Expand Up @@ -1215,6 +1243,7 @@ buildvariants:
- test-3.2-replica_set
- test-3.2-sharded_cluster
- test-atlas-connectivity
- test-atlas-data-lake
- test-auth-kerberos
- test-auth-ldap
- test-tls-support
Expand Down
26 changes: 23 additions & 3 deletions .evergreen/config.yml.in
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,19 @@ functions:
params:
file: mo-expansion.yml

bootstrap mongohoused:
- command: shell.exec
params:
script: |
${PREPARE_SHELL}
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh
- command: shell.exec
params:
background: true
script: |
${PREPARE_SHELL}
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh

"run tests":
- command: shell.exec
type: test
Expand Down Expand Up @@ -229,8 +242,15 @@ functions:
script: |
MONGODB_URI='${plain_auth_mongodb_uri}' NODE_LTS_NAME='${NODE_LTS_NAME}' \
bash ${PROJECT_DIRECTORY}/.evergreen/run-ldap-tests.sh

"run tls tests":
run data lake tests:
- command: shell.exec
type: test
params:
working_dir: src
script: |
MONGODB_URI='mongodb://mhuser:pencil@localhost' NODE_LTS_NAME='${NODE_LTS_NAME}' \
bash ${PROJECT_DIRECTORY}/.evergreen/run-data-lake-tests.sh
"run tls tests":
- command: shell.exec
type: test
params:
Expand Down Expand Up @@ -424,7 +444,7 @@ functions:
UNIFIED=${UNIFIED} \
CA_FILE="$DRIVERS_TOOLS/.evergreen/ocsp/rsa/ca.pem" \
OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
sh ${PROJECT_DIRECTORY}/.evergreen/run-ocsp-tests.sh
bash ${PROJECT_DIRECTORY}/.evergreen/run-ocsp-tests.sh

run-valid-ocsp-server:
- command: shell.exec
Expand Down
8 changes: 8 additions & 0 deletions .evergreen/generate_evergreen_tasks.js
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,14 @@ Array.prototype.push.apply(TASKS, [
{ func: 'run atlas tests' }
]
},
{
name: "test-atlas-data-lake",
commands: [
{ func: 'install dependencies' },
{ func: 'bootstrap mongohoused' },
{ func: 'run data lake tests' }
]
},
{
name: 'test-auth-kerberos',
tags: ['auth', 'kerberos'],
Expand Down
11 changes: 11 additions & 0 deletions .evergreen/run-data-lake-tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#!/bin/bash

set -o errexit # Exit the script with error if any of the commands fail

export PROJECT_DIRECTORY="$(pwd)"
NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"

echo $MONGODB_URI;
npm run check:adl
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@
"check:test": "mocha --recursive test/functional test/unit",
"check:ts": "tsc -v && tsc --noEmit",
"check:atlas": "mocha --config \"test/manual/mocharc.json\" test/manual/atlas_connectivity.test.js",
"check:adl": "mocha test/manual/data_lake.test.js",
"check:ocsp": "mocha --config \"test/manual/mocharc.json\" test/manual/ocsp_support.test.js",
"check:kerberos": "mocha --config \"test/manual/mocharc.json\" test/manual/kerberos.test.js",
"check:tls": "mocha --config \"test/manual/mocharc.json\" test/manual/tls_support.test.js",
Expand Down
17 changes: 13 additions & 4 deletions test/functional/spec-runner/context.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,18 @@ class Thread {
}

class TestRunnerContext {
constructor() {
this.url = null;
constructor(opts) {
const defaults = {
password: undefined,
user: undefined,
useSessions: true,
skipPrepareDatabase: false
};
opts = Object.assign({}, defaults, opts || {});
this.skipPrepareDatabase = opts.skipPrepareDatabase;
this.useSessions = opts.useSessions;
this.user = opts.user;
this.password = opts.password;
this.sharedClient = null;
this.failPointClients = [];
this.appliedFailPoints = [];
Expand All @@ -57,9 +67,8 @@ class TestRunnerContext {

setup(config) {
this.sharedClient = config.newClient(
resolveConnectionString(config, { useMultipleMongoses: true })
resolveConnectionString(config, { useMultipleMongoses: true }, this)
);

if (config.topologyType === 'Sharded') {
this.failPointClients = config.options.hosts.map(proxy =>
config.newClient(`mongodb://${proxy.host}:${proxy.port}/`)
Expand Down
43 changes: 27 additions & 16 deletions test/functional/spec-runner/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,6 @@ function generateTopologyTests(testSuites, testContext, filter) {
if (spec.failPoint) {
testPromise = testPromise.then(() => testContext.disableFailPoint(spec.failPoint));
}

return testPromise.then(() => validateOutcome(spec, testContext));
});
});
Expand All @@ -165,6 +164,9 @@ function prepareDatabaseForSuite(suite, context) {
context.collectionName = suite.collection_name || 'spec_collection';

const db = context.sharedClient.db(context.dbName);

if (context.skipPrepareDatabase) return Promise.resolve();

const setupPromise = db
.admin()
.command({ killAllSessions: [] })
Expand Down Expand Up @@ -279,15 +281,22 @@ function runTestSuiteTest(configuration, spec, context) {
)
);

const url = resolveConnectionString(configuration, spec);
const url = resolveConnectionString(configuration, spec, context);
const client = configuration.newClient(url, clientOptions);
CMAP_EVENTS.forEach(eventName => client.on(eventName, event => context.cmapEvents.push(event)));
SDAM_EVENTS.forEach(eventName => client.on(eventName, event => context.sdamEvents.push(event)));
let pingTracker = 0;
client.on('commandStarted', event => {
if (IGNORED_COMMANDS.has(event.commandName)) {
return;
}

// This gets rid of the first ping command
if (event.commandName === 'ping') {
pingTracker++;
if (pingTracker === 1) return;
}

context.commandEvents.push(event);

// very useful for debugging
Expand All @@ -305,22 +314,24 @@ function runTestSuiteTest(configuration, spec, context) {

let session0, session1;
let savedSessionData;
try {
session0 = client.startSession(
Object.assign({}, sessionOptions, parseSessionOptions(spec.sessionOptions.session0))
);
session1 = client.startSession(
Object.assign({}, sessionOptions, parseSessionOptions(spec.sessionOptions.session1))
);

savedSessionData = {
session0: JSON.parse(EJSON.stringify(session0.id)),
session1: JSON.parse(EJSON.stringify(session1.id))
};
} catch (err) {
// ignore
}
if (context.useSessions) {
try {
session0 = client.startSession(
Object.assign({}, sessionOptions, parseSessionOptions(spec.sessionOptions.session0))
);
session1 = client.startSession(
Object.assign({}, sessionOptions, parseSessionOptions(spec.sessionOptions.session1))
);

savedSessionData = {
session0: JSON.parse(EJSON.stringify(session0.id)),
session1: JSON.parse(EJSON.stringify(session1.id))
};
} catch (err) {
// ignore
}
}
// enable to see useful APM debug information at the time of actual test run
// displayCommands = true;

Expand Down
13 changes: 8 additions & 5 deletions test/functional/spec-runner/utils.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
'use strict';

function resolveConnectionString(configuration, spec) {
function resolveConnectionString(configuration, spec, context) {
const isShardedEnvironment = configuration.topologyType === 'Sharded';
const useMultipleMongoses = spec && !!spec.useMultipleMongoses;

return isShardedEnvironment && !useMultipleMongoses
? `mongodb://${configuration.host}:${configuration.port}/${configuration.db}?directConnection=false`
: configuration.url();
const user = context && context.user;
const password = context && context.password;
const connectionString =
isShardedEnvironment && !useMultipleMongoses
? `mongodb://${configuration.host}:${configuration.port}/${configuration.db}?directConnection=false`
: configuration.url(user, password);
return connectionString;
}

module.exports = { resolveConnectionString };
73 changes: 73 additions & 0 deletions test/manual/data_lake.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
'use strict';
const { expect } = require('chai');
const path = require('path');
const { TestRunnerContext } = require('../functional/spec-runner');
const { gatherTestSuites } = require('../functional/spec-runner');
const { generateTopologyTests } = require('../functional/spec-runner');
const { withClient } = require('../functional/shared');

describe('Atlas Data Lake', function () {
context('spec tests', function () {
const testContext = new TestRunnerContext({
skipPrepareDatabase: true,
useSessions: false,
user: 'mhuser',
password: 'pencil'
});

let testSuites = gatherTestSuites(path.resolve(__dirname, '../spec/atlas-data-lake-testing'));

after(() => testContext.teardown());
before(function () {
return testContext.setup(this.configuration);
});

generateTopologyTests(testSuites, testContext);
});

describe('prose Tests', function () {
it(
'should properly constructs and issues a killCursors command',
withClient('mongodb://mhuser:pencil@localhost', function (client, done) {
const db = client.db('admin');
db.command({ killCursors: 'kill_cursor_collection' }, err => {
expect(err).to.not.exist;
done();
});
})
);
it(
'should connect without authentication',
withClient('mongodb://localhost', function (client, done) {
expect(client).to.exist;
done();
})
);
it(
'should connect with auth SCRAM-SHA-1',
withClient('mongodb://mhuser:pencil@localhost?authMechanism=SCRAM-SHA-1', function (
client,
done
) {
const db = client.db('admin');
db.command({ killCursors: 'kill_cursor_collection' }, err => {
expect(err).to.not.exist;
done();
});
})
);
it(
'should connect with auth SCRAM-SHA-256',
withClient('mongodb://mhuser:pencil@localhost?authMechanism=SCRAM-SHA-256', function (
client,
done
) {
const db = client.db('admin');
db.command({ killCursors: 'kill_cursor_collection' }, err => {
expect(err).to.not.exist;
done();
});
})
);
});
});
Loading

0 comments on commit 2704ce8

Please sign in to comment.