Skip to content

Commit

Permalink
feat(ChangeStream): adds new resume functionality to ChangeStreams
Browse files Browse the repository at this point in the history 
- Adds support for the new startAfter options
- Adds the ability to parse postBatchResumeTokens off of aggregate/getMore
  responses and leverage them when resuming.
- Replaces property resumeToken with accessor resumeToken that
  will always have the most up to date resume token.

Fixes NODE-1824
Fixes NODE-1866
Fixes NODE-1951
Fixes NODE-1979
  • Loading branch information
@daprahamian
daprahamian committed Aug 13, 2019
1 parent e3c6418 commit 9ec9b8f
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 58 deletions.
138 changes: 96 additions & 42 deletions lib/change_stream.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,84 @@ const CHANGE_DOMAIN_TYPES = {
* @return {ChangeStream} a ChangeStream instance.
*/

class ResumeTokenTracker extends EventEmitter {
constructor(changeStream, options) {
super();
this.changeStream = changeStream;
this.options = options;
this._postBatchResumeToken = undefined;
}

get resumeToken() {
return this._resumeToken;
}

init() {
this._resumeToken = this.options.startAfter || this.options.resumeAfter;
this._operationTime = this.options.startAtOperationTime;
this._init = true;
}

resumeInfo() {
const resumeInfo = {};

if (this._init && this._resumeToken) {
resumeInfo.resumeAfter = this._resumeToken;
} else if (this._init && this._operationTime) {
resumeInfo.startAtOperationTime = this._operationTime;
} else {
if (this.options.startAfter) {
resumeInfo.startAfter = this.options.startAfter;
}

if (this.options.resumeAfter) {
resumeInfo.resumeAfter = this.options.resumeAfter;
}

if (this.options.startAtOperationTime) {
resumeInfo.startAtOperationTime = this.options.startAtOperationTime;
}
}

return resumeInfo;
}

onResponse(postBatchResumeToken, operationTime) {
if (this.changeStream.isClosed()) {
return;
}
const cursor = this.changeStream.cursor;
if (!postBatchResumeToken) {
if (
!(this._resumeToken || this._operationTime || cursor.bufferedCount()) &&
cursor.server &&
cursor.server.ismaster.maxWireVersion >= 7
) {
this._operationTime = operationTime;
}
return;
} else {
this._postBatchResumeToken = postBatchResumeToken;
if (cursor.cursorState.documents.length === 0) {
this._resumeToken = this._postBatchResumeToken;
}
}

this.emit('response');
}

onNext(doc) {
if (this.changeStream.isClosed()) {
return;
}
if (this._postBatchResumeToken && this.changeStream.cursor.bufferedCount() === 0) {
this._resumeToken = this._postBatchResumeToken;
} else {
this._resumeToken = doc._id;
}
}
}

class ChangeStream extends EventEmitter {
constructor(changeDomain, pipeline, options) {
super();
Expand Down Expand Up @@ -69,17 +147,13 @@ class ChangeStream extends EventEmitter {
this.options.readPreference = changeDomain.s.readPreference;
}

// We need to get the operationTime as early as possible
const isMaster = this.topology.lastIsMaster();
if (!isMaster) {
throw new MongoError('Topology does not have an ismaster yet.');
}

this.operationTime = isMaster.operationTime;
this._resumeTokenTracker = new ResumeTokenTracker(this, options);

// Create contained Change Stream cursor
this.cursor = createChangeStreamCursor(this);

this._resumeTokenTracker.init();

// Listen for any `change` listeners being added to ChangeStream
this.on('newListener', eventName => {
if (eventName === 'change' && this.cursor && this.listenerCount('change') === 0) {
Expand All @@ -97,6 +171,14 @@ class ChangeStream extends EventEmitter {
});
}

/**
* The cached resume token that will be used to resume
* after the most recently returned change.
*/
get resumeToken() {
return this._resumeTokenTracker.resumeToken;
}

/**
* Check if there is any document still available in the Change Stream
* @function ChangeStream.prototype.hasNext
Expand Down Expand Up @@ -217,10 +299,6 @@ class ChangeStream extends EventEmitter {

// Create a new change stream cursor based on self's configuration
var createChangeStreamCursor = function(self) {
if (self.resumeToken) {
self.options.resumeAfter = self.resumeToken;
}

var changeStreamCursor = buildChangeStreamAggregationCommand(self);

/**
Expand Down Expand Up @@ -277,39 +355,20 @@ var createChangeStreamCursor = function(self) {
return changeStreamCursor;
};

function getResumeToken(self) {
return self.resumeToken || self.options.resumeAfter;
}

function getStartAtOperationTime(self) {
const isMaster = self.topology.lastIsMaster() || {};
return (
isMaster.maxWireVersion && isMaster.maxWireVersion >= 7 && self.options.startAtOperationTime
);
}

var buildChangeStreamAggregationCommand = function(self) {
const topology = self.topology;
const namespace = self.namespace;
const pipeline = self.pipeline;
const options = self.options;
const resumeTokenTracker = self._resumeTokenTracker;

var changeStreamStageOptions = {
fullDocument: options.fullDocument || 'default'
};

const resumeToken = getResumeToken(self);
const startAtOperationTime = getStartAtOperationTime(self);
if (resumeToken) {
changeStreamStageOptions.resumeAfter = resumeToken;
}

if (startAtOperationTime) {
changeStreamStageOptions.startAtOperationTime = startAtOperationTime;
}
const changeStreamStageOptions = Object.assign(
{ fullDocument: options.fullDocument || 'default' },
resumeTokenTracker.resumeInfo()
);

// Map cursor options
var cursorOptions = {};
var cursorOptions = { resumeTokenTracker };
cursorOptionNames.forEach(function(optionName) {
if (options[optionName]) {
cursorOptions[optionName] = options[optionName];
Expand Down Expand Up @@ -384,11 +443,6 @@ function processNewChange(args) {
if (isResumableError(error) && !changeStream.attemptingResume) {
changeStream.attemptingResume = true;

if (!(getResumeToken(changeStream) || getStartAtOperationTime(changeStream))) {
const startAtOperationTime = changeStream.cursor.cursorState.operationTime;
changeStream.options = Object.assign({ startAtOperationTime }, changeStream.options);
}

// stop listening to all events from old cursor
['data', 'close', 'end', 'error'].forEach(event =>
changeStream.cursor.removeAllListeners(event)
Expand Down Expand Up @@ -450,7 +504,7 @@ function processNewChange(args) {
return changeStream.promiseLibrary.reject(noResumeTokenError);
}

changeStream.resumeToken = change._id;
changeStream._resumeTokenTracker.onNext(change);

// wipe the startAtOperationTime if there was one so that there won't be a conflict
// between resumeToken and startAtOperationTime if we need to reconnect the cursor
Expand Down
14 changes: 13 additions & 1 deletion lib/core/cursor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ var Cursor = function(bson, ns, cmd, options, topology, topologyOptions) {
currentLimit: 0,
// Result field name if not a cursor (contains the array of results)
transforms: options.transforms,
raw: options.raw || (cmd && cmd.raw)
raw: options.raw || (cmd && cmd.raw),
resumeTokenTracker: options.resumeTokenTracker
};

if (typeof options.session === 'object') {
Expand Down Expand Up @@ -462,6 +463,10 @@ var nextFunction = function(self, callback) {
return handleCallback(callback, err);
}

if (self.cursorState.resumeTokenTracker) {
self.cursorState.resumeTokenTracker.onResponse(doc.cursor.postBatchResumeToken);
}

if (self.cursorState.cursorId && self.cursorState.cursorId.isZero() && self._endSession) {
self._endSession();
}
Expand Down Expand Up @@ -670,6 +675,13 @@ function initializeCursor(cursor, callback) {
cursor.cursorState.documents = result.documents[0].cursor.firstBatch; //.reverse();
}

if (cursor.cursorState.resumeTokenTracker) {
cursor.cursorState.resumeTokenTracker.onResponse(
result.documents[0].cursor.postBatchResumeToken,
result.documents[0].operationTime
);
}

// Return after processing command cursor
return done(result);
}
Expand Down
43 changes: 28 additions & 15 deletions test/functional/change_stream_tests.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1639,12 +1639,11 @@ describe('Change Streams', function() {
const dbName = 'integration_tests';
const collectionName = 'resumeWithStartAtOperationTime';
const connectOptions = {
socketTimeoutMS: 500,
validateOptions: true
validateOptions: true,
monitorCommands: true
};

let getMoreCounter = 0;
let aggregateCounter = 0;
let changeStream;
let server;
let client;
Expand All @@ -1660,47 +1659,61 @@ describe('Change Streams', function() {
function primaryServerHandler(request) {
try {
const doc = request.document;

if (doc.ismaster) {
return request.reply(makeIsMaster(server));
} else if (doc.aggregate) {
if (aggregateCounter++ > 0) {
expect(doc).to.have.nested.property('pipeline[0].$changeStream.startAtOperationTime');
expect(doc.pipeline[0].$changeStream.startAtOperationTime.equals(OPERATION_TIME)).to
.be.ok;
expect(doc).to.not.have.nested.property('pipeline[0].$changeStream.resumeAfter');
} else {
expect(doc).to.not.have.nested.property(
'pipeline[0].$changeStream.startAtOperationTime'
);
expect(doc).to.not.have.nested.property('pipeline[0].$changeStream.resumeAfter');
}
return request.reply(AGGREGATE_RESPONSE);
} else if (doc.getMore) {
if (getMoreCounter++ === 0) {
request.reply({ ok: 0 });
return;
}

request.reply(GET_MORE_RESPONSE);
} else if (doc.endSessions) {
request.reply({ ok: 1 });
} else if (doc.killCursors) {
request.reply({ ok: 1 });
}
} catch (e) {
finish(e);
}
}

const started = [];

mock
.createServer()
.then(_server => (server = _server))
.then(() => server.setMessageHandler(primaryServerHandler))
.then(() => (client = configuration.newClient(`mongodb://${server.uri()}`, connectOptions)))
.then(() => client.connect())
.then(() => {
client.on('commandStarted', e => {
if (e.commandName === 'aggregate') {
started.push(e);
}
});
})
.then(() => client.db(dbName))
.then(db => db.collection(collectionName))
.then(col => col.watch(pipeline))
.then(_changeStream => (changeStream = _changeStream))
.then(() => changeStream.next())
.then(() => {
const first = started[0].command;
expect(first).to.have.nested.property('pipeline[0].$changeStream');
const firstStage = first.pipeline[0].$changeStream;
expect(firstStage).to.not.have.property('resumeAfter');
expect(firstStage).to.not.have.property('startAtOperationTime');

const second = started[1].command;
expect(second).to.have.nested.property('pipeline[0].$changeStream');
const secondStage = second.pipeline[0].$changeStream;
expect(secondStage).to.not.have.property('resumeAfter');
expect(secondStage).to.have.property('startAtOperationTime');
expect(secondStage.startAtOperationTime.equals(OPERATION_TIME)).to.be.ok;
})
.then(() => finish(), err => finish(err));
}
});
Expand Down

0 comments on commit 9ec9b8f

Please sign in to comment.