fix(NODE-3627): Enable flexible BSON validation for server error key containing invalid utf-8

package.json

@@ -32,7 +32,7 @@
     "email": "[email protected]"
   },
   "dependencies": {
-    "bson": "^4.5.4",
+    "bson": "^4.6.0",
     "denque": "^2.0.1",
     "mongodb-connection-string-url": "^2.2.0"
   },

src/bson.ts

@@ -54,6 +54,7 @@ export interface BSONSerializeOptions
     > {
   /** Return BSON filled buffers from operations */
   raw?: boolean;
+  validation?: { utf8: boolean | Record<string, true> | Record<string, false> };
 }
 
 export function pluckBSONSerializeOptions(options: BSONSerializeOptions): BSONSerializeOptions {

src/cmap/commands.ts

@@ -469,6 +469,7 @@ export interface MessageHeader {
 export interface OpResponseOptions extends BSONSerializeOptions {
   raw?: boolean;
   documentsReturnedIn?: string | null;
+  validation?: { utf8: boolean | Record<string, true> | Record<string, false> };
 }
 
 /** @internal */
@@ -837,13 +838,20 @@ export class BinMsg {
     const promoteValues = options.promoteValues ?? this.opts.promoteValues;
     const promoteBuffers = options.promoteBuffers ?? this.opts.promoteBuffers;
     const bsonRegExp = options.bsonRegExp ?? this.opts.bsonRegExp;
+    if (options.validation && Object.keys(options.validation.utf8)[0] !== 'writeErrors') {
+      throw new MongoInvalidArgumentError(
+        'Can only toggle validation settings for writeErrors key'
+      );
+    }
+    const validation = options.validation ?? { utf8: { writeErrors: false } };
 
     // Set up the options
     const _options: BSONSerializeOptions = {
       promoteLongs,
       promoteValues,
       promoteBuffers,
-      bsonRegExp
+      bsonRegExp,
+      validation
     };
 
     while (this.index < this.data.length) {

src/connection_string.ts

@@ -864,6 +864,10 @@ export const OPTIONS = {
     default: false,
     type: 'boolean'
   },
+  validation: {
+    default: { utf8: { writeErrors: false } },
+    type: 'record'
+  },
   readConcern: {
     transform({ values: [value], options }) {
       if (value instanceof ReadConcern || isRecord(value, ['level'] as const)) {

test/types/bson.test-d.ts

@@ -21,7 +21,8 @@ type PermittedBSONOptionKeys =
   | 'promoteValues'
   | 'bsonRegExp'
   | 'fieldsAsRaw'
-  | 'raw';
+  | 'raw'
+  | 'validation';
 
 const keys = null as unknown as PermittedBSONOptionKeys;
 // creates an explicit allow list assertion

test/unit/commands.test.ts

@@ -0,0 +1,151 @@
+import { expect } from 'chai';
+import { BinMsg, MessageHeader, OpResponseOptions } from '../../src/cmap/commands';
+import { BSONError } from 'bson';
+import * as BSON from '../../src/bson';
+
+const msgHeader: MessageHeader = {
+  length: 735,
+  requestId: 14704565,
+  responseTo: 4,
+  opCode: 2013
+};
+
+// when top-level key writeErrors contains an error message that has invalid utf8
+const invalidUtf8ErrorMsg =
+  '0000000000ca020000106e00000000000477726974654572726f727300a50200000330009d02000010696e646578000000000010636f646500f82a0000036b65795061747465726e000f0000001074657874000100000000036b657956616c756500610100000274657874005201000064e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e298830000026572726d736700f1000000453131303030206475706c6963617465206b6579206572726f7220636f6c6c656374696f6e3a20626967646174612e7465737420696e6465783a20746578745f3120647570206b65793a207b20746578743a202264e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e2982e2e2e22207d000000016f6b00000000000000f03f00';
+const msgBodyInvalidUtf8WriteErrors = Buffer.from(invalidUtf8ErrorMsg, 'hex');
+const invalidUtf8ErrorMsgDeserializeInput = Buffer.from(invalidUtf8ErrorMsg.substring(10), 'hex');
+const invalidUtf8InWriteErrorsJSON = {
+  n: 0,
+  writeErrors: [
+    {
+      index: 0,
+      code: 11000,
+      keyPattern: {
+        text: 1
+      },
+      keyValue: {
+        text: 'd☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃'
+      },
+      errmsg:
+        'E11000 duplicate key error collection: bigdata.test index: text_1 dup key: { text: "d☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃�..." }'
+    }
+  ],
+  ok: 1
+};
+
+// when another top-level key besides writeErrors has invalid utf8
+const nKeyWithInvalidUtf8 =
+  '0000000000cc020000026e0005000000f09f98ff000477726974654572726f727300a60200000330009e02000010696e646578000000000010636f646500f82a0000036b65795061747465726e000f0000001074657874000100000000036b657956616c756500610100000274657874005201000064e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e298830000026572726d736700f2000000453131303030206475706c6963617465206b6579206572726f7220636f6c6c656374696f6e3a20626967646174612e7465737420696e6465783a20746578745f3120647570206b65793a207b20746578743a202264e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883efbfbd2e2e2e22207d000000106f6b000100000000';
+const nKeyWithInvalidUtf8DeserializeInput = Buffer.from(nKeyWithInvalidUtf8.substring(10), 'hex');
+const msgBodyNKeyWithInvalidUtf8 = Buffer.from(nKeyWithInvalidUtf8, 'hex');
+const invalidUtf8InNKeyJSON = {
+  n: '��',
+  writeErrors: [
+    {
+      index: 0,
+      code: 11000,
+      keyPattern: {
+        text: 1
+      },
+      keyValue: {
+        text: 'd☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃'
+      },
+      errmsg:
+        'E11000 duplicate key error collection: bigdata.test index: text_1 dup key: { text: "d☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃�..." }'
+    }
+  ],
+  ok: 1
+};
+
+describe('BinMsg', function () {
+  context('when validation is disabled for writeErrors', function () {
+    const binMsgInvalidUtf8ErrorMsg = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyInvalidUtf8WriteErrors
+    );
+    const options: OpResponseOptions = { validation: { utf8: { writeErrors: false } } };
+
+    it('contains replacement characters for invalid utf8 in writeError object', function () {
+      expect(BSON.deserialize(invalidUtf8ErrorMsgDeserializeInput, options)).to.deep.equals(
+        invalidUtf8InWriteErrorsJSON
+      );
+    });
+
+    it('should not throw invalid utf8 error', function () {
+      expect(() => binMsgInvalidUtf8ErrorMsg.parse(options)).to.not.throw();
+    });
+  });
+
+  it('should throw error if trying to toggle validation settings for keys other than writeErrors', function () {
+    const binMsgInvalidUtf8ErrorMsg = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyInvalidUtf8WriteErrors
+    );
+    expect(() => binMsgInvalidUtf8ErrorMsg.parse({ validation: { utf8: { n: false } } })).to.throw(
+      Error,
+      'Can only toggle validation settings for writeErrors key'
+    );
+  });
+
+  it('should by default disable validation for writeErrors if no validation specified', function () {
+    const binMsgInvalidUtf8ErrorMsg = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyInvalidUtf8WriteErrors
+    );
+    const options = {
+      bsonRegExp: false,
+      promoteBuffers: false,
+      promoteLongs: true,
+      promoteValues: true
+    };
+    expect(() => binMsgInvalidUtf8ErrorMsg.parse(options)).to.not.throw();
+  });
+
+  context(
+    'when another key has invalid utf8 and validation is enabled for writeErrors',
+    function () {
+      const binMsgAnotherKeyWithInvalidUtf8 = new BinMsg(
+        Buffer.alloc(0),
+        msgHeader,
+        msgBodyNKeyWithInvalidUtf8
+      );
+      const option: OpResponseOptions = { validation: { utf8: { writeErrors: true } } };
+
+      it('should not throw invalid utf8 error', function () {
+        expect(() => binMsgAnotherKeyWithInvalidUtf8.parse(option)).to.not.throw();
+      });
+
+      it('contains replacement characters for invalid utf8 key', function () {
+        expect(BSON.deserialize(nKeyWithInvalidUtf8DeserializeInput, option)).to.deep.equals(
+          invalidUtf8InNKeyJSON
+        );
+      });
+    }
+  );
+
+  it('should throw invalid utf8 error when validation enabled for writeErrors', function () {
+    const binMsgInvalidUtf8ErrorMsg = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyInvalidUtf8WriteErrors
+    );
+    expect(() =>
+      binMsgInvalidUtf8ErrorMsg.parse({ validation: { utf8: { writeErrors: true } } })
+    ).to.throw(BSONError, 'Invalid UTF-8 string in BSON document');
+  });
+
+  it('should throw error when another key has invalid utf8 and writeErrors is not validated', function () {
+    const binMsgAnotherKeyWithInvalidUtf8 = new BinMsg(
+      Buffer.alloc(0),
+      msgHeader,
+      msgBodyNKeyWithInvalidUtf8
+    );
+    expect(() =>
+      binMsgAnotherKeyWithInvalidUtf8.parse({ validation: { utf8: { writeErrors: false } } })
+    ).to.throw(BSONError, 'Invalid UTF-8 string in BSON document');
+  });
+});