From 8b7f6b07147786d6c882f970163e85b0569c9741 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Tue, 30 Jun 2020 16:13:08 -0400 Subject: [PATCH 01/26] Adds v1 partitionKey hashing --- .../cosmos/src/utils/hashing/encoding/null.ts | 3 + .../src/utils/hashing/encoding/number.ts | 61 +++++++ .../src/utils/hashing/encoding/prefix.ts | 24 +++ .../src/utils/hashing/encoding/string.ts | 22 +++ .../cosmos/src/utils/hashing/murmurHashV1.ts | 153 ++++++++++++++++++ sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts | 63 ++++++++ sdk/cosmosdb/cosmos/test/tsconfig.json | 2 +- .../cosmos/test/unit/hashing/v1.spec.ts | 51 ++++++ sdk/cosmosdb/cosmos/tsconfig.json | 2 +- 9 files changed, 379 insertions(+), 2 deletions(-) create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/encoding/null.ts create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/encoding/prefix.ts create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/encoding/string.ts create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.ts create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts create mode 100644 sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/null.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/null.ts new file mode 100644 index 000000000000..4add58a62d50 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/null.ts @@ -0,0 +1,3 @@ +export function writeNullForBinaryEncoding() { + return Buffer.from("01", "hex"); +} diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts new file mode 100644 index 000000000000..a8c6f53fe2d6 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts @@ -0,0 +1,61 @@ +export function writeNumberForBinaryEncoding(hash: number) { + let payload: bigint = encodeNumberAsUInt64(hash); + let outputStream = Buffer.from("05", "hex"); + const firstChunk = BigInt.asUintN(64, payload >> 56n); + outputStream = Buffer.concat([outputStream, Buffer.from(firstChunk.toString(16), "hex")]); + payload = BigInt.asUintN(64, BigInt(payload) << 0x8n); + + let byteToWrite = 0n; + let firstIteration = false; + let shifted: bigint; + let padded: string; + + do { + if (!firstIteration) { + // we pad because javascrpt will produce "f" or similar for sufficiently small integers, + // which cannot be encoded as hex in a buffer https://github.com/nodejs/node/issues/24491 + padded = byteToWrite.toString(16).padStart(2, "0"); + if (padded !== "00") { + outputStream = Buffer.concat([outputStream, Buffer.from(padded, "hex")]); + } + } else { + firstIteration = false; + } + + shifted = BigInt.asUintN(64, payload >> 56n); + byteToWrite = BigInt.asUintN(64, shifted | 0x01n); + payload = BigInt.asUintN(64, payload << 7n); + } while (payload != 0n); + + const lastChunk = BigInt.asUintN(64, byteToWrite & 0xfen); + // we pad because javascrpt will produce "f" or similar for sufficiently small integers, + // which cannot be encoded as hex in a buffer https://github.com/nodejs/node/issues/24491 + padded = lastChunk.toString(16).padStart(2, "0"); + if (padded !== "00") { + outputStream = Buffer.concat([outputStream, Buffer.from(padded, "hex")]); + } + + return outputStream; +} + +function encodeNumberAsUInt64(value: number) { + const rawValueBits = getRawBits(value); + const mask = 0x8000000000000000n; + const returned = rawValueBits < mask ? rawValueBits ^ mask : ~BigInt(rawValueBits) + 1n; + return returned; +} + +export function doubleToByteArray(double: number) { + const output: Buffer = Buffer.alloc(8); + const lng = getRawBits(double); + for (let i = 0; i < 8; i++) { + output[i] = Number((lng >> (BigInt(i) * 8n)) & 0xffn); + } + return output; +} + +function getRawBits(value: number) { + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, value); + return view.getBigInt64(0); +} diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/prefix.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/prefix.ts new file mode 100644 index 000000000000..7b36891c0aea --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/prefix.ts @@ -0,0 +1,24 @@ +export const BytePrefix = { + Undefined: 0x0, + Null: 0x1, + False: 0x2, + True: 0x3, + MinNumber: 0x4, + Number: 0x5, + MaxNumber: 0x6, + MinString: 0x7, + String: 0x8, + MaxString: 0x9, + Int64: 0xa, + Int32: 0xb, + Int16: 0xc, + Int8: 0xd, + Uint64: 0xe, + Uint32: 0xf, + Uint16: 0x10, + Uint8: 0x11, + Binary: 0x12, + Guid: 0x13, + Float: 0x14, + Infinity: 0xff +}; diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/string.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/string.ts new file mode 100644 index 000000000000..4c25dea2b373 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/string.ts @@ -0,0 +1,22 @@ +export function writeStringForBinaryEncoding(payload: string) { + let outputStream = Buffer.from("08", "hex"); + const MAX_STRING_BYTES_TO_APPEND = 100; + const byteArray = [...Buffer.from(payload)]; + + const isShortString = payload.length <= MAX_STRING_BYTES_TO_APPEND; + + for ( + let index = 0; + index < (isShortString ? byteArray.length : MAX_STRING_BYTES_TO_APPEND + 1); + index++ + ) { + let charByte = byteArray[index]; + if (charByte < 0xff) charByte++; + outputStream = Buffer.concat([outputStream, Buffer.from(charByte.toString(16), "hex")]); + } + + if (isShortString) { + outputStream = Buffer.concat([outputStream, Buffer.from("00", "hex")]); + } + return outputStream; +} diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.ts new file mode 100644 index 000000000000..2422d324a80d --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.ts @@ -0,0 +1,153 @@ +/* +The MIT License (MIT) +Copyright (c) 2017 Microsoft Corporation +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +"use strict"; + +export const MurmurHash = { + /** + * Hashes a string, a unsigned 32-bit integer, or a Buffer into a new unsigned 32-bit integer that represents the output hash. + * @param {string, number of Buffer} key - The preimage of the hash + * @param {number} seed - Optional value used to initialize the hash generator + * @returns {} + */ + hash: function(key: string, seed: number) { + key = key || ""; + seed = seed || 0; + + MurmurHash._throwIfInvalidKey(key); + MurmurHash._throwIfInvalidSeed(seed); + + let buffer; + if (typeof key === "string") { + buffer = MurmurHash._getBufferFromString(key); + } else if (typeof key === "number") { + buffer = MurmurHash._getBufferFromNumber(key); + } else { + buffer = key; + } + + return MurmurHash._hashBytes(buffer, seed); + }, + /** @ignore */ + _throwIfInvalidKey: function(key: string | number | Buffer) { + if (key instanceof Buffer) { + return; + } + + if (typeof key === "string") { + return; + } + + if (typeof key === "number") { + return; + } + + throw new Error("Invalid argument: 'key' has to be a Buffer, string, or number."); + }, + /** @ignore */ + _throwIfInvalidSeed: function(seed: number) { + if (isNaN(seed)) { + throw new Error("Invalid argument: 'seed' is not and cannot be converted to a number."); + } + }, + /** @ignore */ + _getBufferFromString: function(key: string) { + let buffer = new Buffer(key); + return buffer; + }, + /** @ignore */ + _getBufferFromNumber: function(i: number) { + i = i >>> 0; + + let buffer = new Uint8Array([i >>> 0, i >>> 8, i >>> 16, i >>> 24]); + + return buffer; + }, + /** @ignore */ + _hashBytes: function(bytes: Buffer | Uint8Array, seed: number) { + let c1 = 0xcc9e2d51; + let c2 = 0x1b873593; + + let h1 = seed; + let reader = new Uint32Array(bytes); + { + for (var i = 0; i < bytes.length - 3; i += 4) { + let k1 = MurmurHash._readUInt32(reader, i); + + k1 = MurmurHash._multiply(k1, c1); + k1 = MurmurHash._rotateLeft(k1, 15); + k1 = MurmurHash._multiply(k1, c2); + + h1 ^= k1; + h1 = MurmurHash._rotateLeft(h1, 13); + h1 = MurmurHash._multiply(h1, 5) + 0xe6546b64; + } + } + + let k = 0; + switch (bytes.length & 3) { + case 3: + k ^= reader[i + 2] << 16; + k ^= reader[i + 1] << 8; + k ^= reader[i]; + break; + + case 2: + k ^= reader[i + 1] << 8; + k ^= reader[i]; + break; + + case 1: + k ^= reader[i]; + break; + } + + k = MurmurHash._multiply(k, c1); + k = MurmurHash._rotateLeft(k, 15); + k = MurmurHash._multiply(k, c2); + + h1 ^= k; + h1 ^= bytes.length; + h1 ^= h1 >>> 16; + h1 = MurmurHash._multiply(h1, 0x85ebca6b); + h1 ^= h1 >>> 13; + h1 = MurmurHash._multiply(h1, 0xc2b2ae35); + h1 ^= h1 >>> 16; + + return h1 >>> 0; + }, + /** @ignore */ + _rotateLeft: function(n: number, numBits: number) { + return (n << numBits) | (n >>> (32 - numBits)); + }, + /** @ignore */ + _multiply: function(m: number, n: number) { + return (m & 0xffff) * n + ((((m >>> 16) * n) & 0xffff) << 16); + }, + /** @ignore */ + _readUInt32: function(uintArray: Uint32Array, i: number) { + return ( + uintArray[i] | + (uintArray[i + 1] << 8) | + (uintArray[i + 2] << 16) | + ((uintArray[i + 3] << 24) >>> 0) + ); + } +}; diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts new file mode 100644 index 000000000000..ec8c97be4eec --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts @@ -0,0 +1,63 @@ +import { writeNumberForBinaryEncoding, doubleToByteArray } from "./encoding/number"; +import { writeStringForBinaryEncoding } from "./encoding/string"; +import { MurmurHash } from "./murmurHashV1"; + +type v1Key = string | number | null | {} | undefined; + +export function hashV1PartitionKey(partitionKey: v1Key): string { + const toHash = prefixKeyByType(partitionKey); + const hash = MurmurHash.hash(toHash, 0); + const encodedHash = writeNumberForBinaryEncoding(hash); + const encodedValue = encodeByType(partitionKey); + return Buffer.concat([encodedHash, encodedValue]) + .toString("hex") + .toUpperCase(); +} + +function prefixKeyByType(key: v1Key) { + let bytes: Buffer; + switch (typeof key) { + case "string": + const truncated = key.substr(0, 100); + bytes = Buffer.concat([ + Buffer.from("08", "hex"), + Buffer.from(truncated), + Buffer.from("00", "hex") + ]); + return bytes.toString(); + case "number": + const numberBytes = doubleToByteArray(key); + bytes = Buffer.concat([Buffer.from("05", "hex"), numberBytes]); + return bytes.toString(); + case "boolean": + const prefix = key ? "03" : "02"; + return Buffer.from(prefix, "hex").toString(); + case "object": + if (key === null) { + return Buffer.from("01", "hex").toString(); + } + return Buffer.from("00", "hex").toString(); + case "undefined": + return Buffer.from("00", "hex").toString(); + } +} + +function encodeByType(key: v1Key) { + switch (typeof key) { + case "string": + const truncated = key.substr(0, 100); + return writeStringForBinaryEncoding(truncated); + case "number": + return writeNumberForBinaryEncoding(key); + case "boolean": + const prefix = key ? "03" : "02"; + return Buffer.from(prefix, "hex"); + case "object": + if (key === null) { + return Buffer.from("01", "hex"); + } + return Buffer.from("00", "hex"); + case "undefined": + return Buffer.from("00", "hex"); + } +} diff --git a/sdk/cosmosdb/cosmos/test/tsconfig.json b/sdk/cosmosdb/cosmos/test/tsconfig.json index d515d04ab499..646ddc750eb4 100644 --- a/sdk/cosmosdb/cosmos/test/tsconfig.json +++ b/sdk/cosmosdb/cosmos/test/tsconfig.json @@ -9,7 +9,7 @@ "allowSyntheticDefaultImports": true, "preserveConstEnums": true, "removeComments": false, - "target": "es6", + "target": "esnext", "sourceMap": true, "newLine": "LF", "composite": true, diff --git a/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts b/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts new file mode 100644 index 000000000000..0e7c962a27d4 --- /dev/null +++ b/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts @@ -0,0 +1,51 @@ +import assert from "assert"; +import { hashV1PartitionKey } from "../../../src/utils/hashing/v1"; + +describe("effectivePartitionKey", function() { + describe("computes v1 key", function() { + const toMatch = [ + { + key: "partitionKey", + output: "05C1E1B3D9CD2608716273756A756A706F4C667A00" + }, + { + key: "", + output: "05C1CF33970FF80800" + }, + { + key: "aa", + output: "05C1C7B7270FE008626200" + }, + { + key: null, + output: "05C1ED45D7475601" + }, + { + key: true, + output: "05C1D7C5A903D803" + }, + { + key: false, + output: "05C1DB857D857C02" + }, + { + key: {}, + output: "05C1D529E345DC00" + }, + { + key: 5, + output: "05C1D9C1C5517C05C014" + }, + { + key: 5.5, + output: "05C1D7A771716C05C016" + } + ]; + toMatch.forEach(({ key, output }) => { + it("matches expected hash output", function() { + const hashed = hashV1PartitionKey(key); + assert.equal(hashed, output); + }); + }); + }); +}); diff --git a/sdk/cosmosdb/cosmos/tsconfig.json b/sdk/cosmosdb/cosmos/tsconfig.json index ac0425d109e9..699c8d811bf9 100644 --- a/sdk/cosmosdb/cosmos/tsconfig.json +++ b/sdk/cosmosdb/cosmos/tsconfig.json @@ -11,7 +11,7 @@ "outDir": "dist-esm", "preserveConstEnums": true, "removeComments": false, - "target": "es6", + "target": "esnext", "sourceMap": true, "inlineSources": true, "newLine": "LF", From a5e78ac5cff1f8197a8de6dbf96feff8e3d38781 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Tue, 30 Jun 2020 16:15:12 -0400 Subject: [PATCH 02/26] Fix comment --- .../cosmos/src/utils/hashing/encoding/number.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts index a8c6f53fe2d6..339878c5f039 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts @@ -12,8 +12,9 @@ export function writeNumberForBinaryEncoding(hash: number) { do { if (!firstIteration) { - // we pad because javascrpt will produce "f" or similar for sufficiently small integers, - // which cannot be encoded as hex in a buffer https://github.com/nodejs/node/issues/24491 + // we pad because after shifting we will produce characters like "f" or similar, + // which cannot be encoded as hex in a buffer because they are invalid hex + // https://github.com/nodejs/node/issues/24491 padded = byteToWrite.toString(16).padStart(2, "0"); if (padded !== "00") { outputStream = Buffer.concat([outputStream, Buffer.from(padded, "hex")]); @@ -28,8 +29,9 @@ export function writeNumberForBinaryEncoding(hash: number) { } while (payload != 0n); const lastChunk = BigInt.asUintN(64, byteToWrite & 0xfen); - // we pad because javascrpt will produce "f" or similar for sufficiently small integers, - // which cannot be encoded as hex in a buffer https://github.com/nodejs/node/issues/24491 + // we pad because after shifting we will produce characters like "f" or similar, + // which cannot be encoded as hex in a buffer because they are invalid hex + // https://github.com/nodejs/node/issues/24491 padded = lastChunk.toString(16).padStart(2, "0"); if (padded !== "00") { outputStream = Buffer.concat([outputStream, Buffer.from(padded, "hex")]); From e066237558c54fcc31ed5f895203ebe8ee6a0a83 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Tue, 30 Jun 2020 17:52:51 -0400 Subject: [PATCH 03/26] Adds murmurhash library and uses Bytes --- .../cosmos/src/utils/hashing/murmurHashV1.js | 552 ++++++++++++++++++ .../cosmos/src/utils/hashing/murmurHashV1.ts | 153 ----- sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts | 16 +- .../cosmos/test/unit/hashing/v1.spec.ts | 18 + 4 files changed, 578 insertions(+), 161 deletions(-) create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.js delete mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.ts diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.js b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.js new file mode 100644 index 000000000000..ff639c2d0d38 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.js @@ -0,0 +1,552 @@ +/* jshint -W086: true */ +// +----------------------------------------------------------------------+ +// | murmurHash3js.js v3.0.1 // https://github.com/pid/murmurHash3js +// | A javascript implementation of MurmurHash3's x86 hashing algorithms. | +// |----------------------------------------------------------------------| +// | Copyright (c) 2012-2015 Karan Lyons | +// | https://github.com/karanlyons/murmurHash3.js/blob/c1778f75792abef7bdd74bc85d2d4e1a3d25cfe9/murmurHash3.js | +// | Freely distributable under the MIT license. | +// +----------------------------------------------------------------------+ + +"use strict"; + +// PRIVATE FUNCTIONS +// ----------------- + +function _validBytes(bytes) { + // check the input is an array or a typed array + if (!Array.isArray(bytes) && !ArrayBuffer.isView(bytes)) { + return false; + } + + // check all bytes are actually bytes + for (var i = 0; i < bytes.length; i++) { + if (!Number.isInteger(bytes[i]) || bytes[i] < 0 || bytes[i] > 255) { + return false; + } + } + return true; +} + +function _x86Multiply(m, n) { + // + // Given two 32bit ints, returns the two multiplied together as a + // 32bit int. + // + + return (m & 0xffff) * n + ((((m >>> 16) * n) & 0xffff) << 16); +} + +function _x86Rotl(m, n) { + // + // Given a 32bit int and an int representing a number of bit positions, + // returns the 32bit int rotated left by that number of positions. + // + + return (m << n) | (m >>> (32 - n)); +} + +function _x86Fmix(h) { + // + // Given a block, returns murmurHash3's final x86 mix of that block. + // + + h ^= h >>> 16; + h = _x86Multiply(h, 0x85ebca6b); + h ^= h >>> 13; + h = _x86Multiply(h, 0xc2b2ae35); + h ^= h >>> 16; + + return h; +} + +function _x64Add(m, n) { + // + // Given two 64bit ints (as an array of two 32bit ints) returns the two + // added together as a 64bit int (as an array of two 32bit ints). + // + + m = [m[0] >>> 16, m[0] & 0xffff, m[1] >>> 16, m[1] & 0xffff]; + n = [n[0] >>> 16, n[0] & 0xffff, n[1] >>> 16, n[1] & 0xffff]; + var o = [0, 0, 0, 0]; + + o[3] += m[3] + n[3]; + o[2] += o[3] >>> 16; + o[3] &= 0xffff; + + o[2] += m[2] + n[2]; + o[1] += o[2] >>> 16; + o[2] &= 0xffff; + + o[1] += m[1] + n[1]; + o[0] += o[1] >>> 16; + o[1] &= 0xffff; + + o[0] += m[0] + n[0]; + o[0] &= 0xffff; + + return [(o[0] << 16) | o[1], (o[2] << 16) | o[3]]; +} + +function _x64Multiply(m, n) { + // + // Given two 64bit ints (as an array of two 32bit ints) returns the two + // multiplied together as a 64bit int (as an array of two 32bit ints). + // + + m = [m[0] >>> 16, m[0] & 0xffff, m[1] >>> 16, m[1] & 0xffff]; + n = [n[0] >>> 16, n[0] & 0xffff, n[1] >>> 16, n[1] & 0xffff]; + var o = [0, 0, 0, 0]; + + o[3] += m[3] * n[3]; + o[2] += o[3] >>> 16; + o[3] &= 0xffff; + + o[2] += m[2] * n[3]; + o[1] += o[2] >>> 16; + o[2] &= 0xffff; + + o[2] += m[3] * n[2]; + o[1] += o[2] >>> 16; + o[2] &= 0xffff; + + o[1] += m[1] * n[3]; + o[0] += o[1] >>> 16; + o[1] &= 0xffff; + + o[1] += m[2] * n[2]; + o[0] += o[1] >>> 16; + o[1] &= 0xffff; + + o[1] += m[3] * n[1]; + o[0] += o[1] >>> 16; + o[1] &= 0xffff; + + o[0] += m[0] * n[3] + m[1] * n[2] + m[2] * n[1] + m[3] * n[0]; + o[0] &= 0xffff; + + return [(o[0] << 16) | o[1], (o[2] << 16) | o[3]]; +} + +function _x64Rotl(m, n) { + // + // Given a 64bit int (as an array of two 32bit ints) and an int + // representing a number of bit positions, returns the 64bit int (as an + // array of two 32bit ints) rotated left by that number of positions. + // + + n %= 64; + + if (n === 32) { + return [m[1], m[0]]; + } else if (n < 32) { + return [(m[0] << n) | (m[1] >>> (32 - n)), (m[1] << n) | (m[0] >>> (32 - n))]; + } else { + n -= 32; + return [(m[1] << n) | (m[0] >>> (32 - n)), (m[0] << n) | (m[1] >>> (32 - n))]; + } +} + +function _x64LeftShift(m, n) { + // + // Given a 64bit int (as an array of two 32bit ints) and an int + // representing a number of bit positions, returns the 64bit int (as an + // array of two 32bit ints) shifted left by that number of positions. + // + + n %= 64; + + if (n === 0) { + return m; + } else if (n < 32) { + return [(m[0] << n) | (m[1] >>> (32 - n)), m[1] << n]; + } else { + return [m[1] << (n - 32), 0]; + } +} + +function _x64Xor(m, n) { + // + // Given two 64bit ints (as an array of two 32bit ints) returns the two + // xored together as a 64bit int (as an array of two 32bit ints). + // + + return [m[0] ^ n[0], m[1] ^ n[1]]; +} + +function _x64Fmix(h) { + // + // Given a block, returns murmurHash3's final x64 mix of that block. + // (`[0, h[0] >>> 1]` is a 33 bit unsigned right shift. This is the + // only place where we need to right shift 64bit ints.) + // + + h = _x64Xor(h, [0, h[0] >>> 1]); + h = _x64Multiply(h, [0xff51afd7, 0xed558ccd]); + h = _x64Xor(h, [0, h[0] >>> 1]); + h = _x64Multiply(h, [0xc4ceb9fe, 0x1a85ec53]); + h = _x64Xor(h, [0, h[0] >>> 1]); + + return h; +} + +// PUBLIC FUNCTIONS +// ---------------- + +function x86Hash32(bytes, seed) { + // + // Given a string and an optional seed as an int, returns a 32 bit hash + // using the x86 flavor of MurmurHash3, as an unsigned int. + // + seed = seed || 0; + + var remainder = bytes.length % 4; + var blocks = bytes.length - remainder; + + var h1 = seed; + + var k1 = 0; + + var c1 = 0xcc9e2d51; + var c2 = 0x1b873593; + + for (var i = 0; i < blocks; i = i + 4) { + k1 = bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24); + + k1 = _x86Multiply(k1, c1); + k1 = _x86Rotl(k1, 15); + k1 = _x86Multiply(k1, c2); + + h1 ^= k1; + h1 = _x86Rotl(h1, 13); + h1 = _x86Multiply(h1, 5) + 0xe6546b64; + } + + k1 = 0; + + switch (remainder) { + case 3: + k1 ^= bytes[i + 2] << 16; + + case 2: + k1 ^= bytes[i + 1] << 8; + + case 1: + k1 ^= bytes[i]; + k1 = _x86Multiply(k1, c1); + k1 = _x86Rotl(k1, 15); + k1 = _x86Multiply(k1, c2); + h1 ^= k1; + } + + h1 ^= bytes.length; + h1 = _x86Fmix(h1); + + return h1 >>> 0; +} + +function x86Hash128(bytes, seed) { + // + // Given a string and an optional seed as an int, returns a 128 bit + // hash using the x86 flavor of MurmurHash3, as an unsigned hex. + // + + seed = seed || 0; + var remainder = bytes.length % 16; + var blocks = bytes.length - remainder; + + var h1 = seed; + var h2 = seed; + var h3 = seed; + var h4 = seed; + + var k1 = 0; + var k2 = 0; + var k3 = 0; + var k4 = 0; + + var c1 = 0x239b961b; + var c2 = 0xab0e9789; + var c3 = 0x38b34ae5; + var c4 = 0xa1e38b93; + + for (var i = 0; i < blocks; i = i + 16) { + k1 = bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24); + k2 = bytes[i + 4] | (bytes[i + 5] << 8) | (bytes[i + 6] << 16) | (bytes[i + 7] << 24); + k3 = bytes[i + 8] | (bytes[i + 9] << 8) | (bytes[i + 10] << 16) | (bytes[i + 11] << 24); + k4 = bytes[i + 12] | (bytes[i + 13] << 8) | (bytes[i + 14] << 16) | (bytes[i + 15] << 24); + + k1 = _x86Multiply(k1, c1); + k1 = _x86Rotl(k1, 15); + k1 = _x86Multiply(k1, c2); + h1 ^= k1; + + h1 = _x86Rotl(h1, 19); + h1 += h2; + h1 = _x86Multiply(h1, 5) + 0x561ccd1b; + + k2 = _x86Multiply(k2, c2); + k2 = _x86Rotl(k2, 16); + k2 = _x86Multiply(k2, c3); + h2 ^= k2; + + h2 = _x86Rotl(h2, 17); + h2 += h3; + h2 = _x86Multiply(h2, 5) + 0x0bcaa747; + + k3 = _x86Multiply(k3, c3); + k3 = _x86Rotl(k3, 17); + k3 = _x86Multiply(k3, c4); + h3 ^= k3; + + h3 = _x86Rotl(h3, 15); + h3 += h4; + h3 = _x86Multiply(h3, 5) + 0x96cd1c35; + + k4 = _x86Multiply(k4, c4); + k4 = _x86Rotl(k4, 18); + k4 = _x86Multiply(k4, c1); + h4 ^= k4; + + h4 = _x86Rotl(h4, 13); + h4 += h1; + h4 = _x86Multiply(h4, 5) + 0x32ac3b17; + } + + k1 = 0; + k2 = 0; + k3 = 0; + k4 = 0; + + switch (remainder) { + case 15: + k4 ^= bytes[i + 14] << 16; + + case 14: + k4 ^= bytes[i + 13] << 8; + + case 13: + k4 ^= bytes[i + 12]; + k4 = _x86Multiply(k4, c4); + k4 = _x86Rotl(k4, 18); + k4 = _x86Multiply(k4, c1); + h4 ^= k4; + + case 12: + k3 ^= bytes[i + 11] << 24; + + case 11: + k3 ^= bytes[i + 10] << 16; + + case 10: + k3 ^= bytes[i + 9] << 8; + + case 9: + k3 ^= bytes[i + 8]; + k3 = _x86Multiply(k3, c3); + k3 = _x86Rotl(k3, 17); + k3 = _x86Multiply(k3, c4); + h3 ^= k3; + + case 8: + k2 ^= bytes[i + 7] << 24; + + case 7: + k2 ^= bytes[i + 6] << 16; + + case 6: + k2 ^= bytes[i + 5] << 8; + + case 5: + k2 ^= bytes[i + 4]; + k2 = _x86Multiply(k2, c2); + k2 = _x86Rotl(k2, 16); + k2 = _x86Multiply(k2, c3); + h2 ^= k2; + + case 4: + k1 ^= bytes[i + 3] << 24; + + case 3: + k1 ^= bytes[i + 2] << 16; + + case 2: + k1 ^= bytes[i + 1] << 8; + + case 1: + k1 ^= bytes[i]; + k1 = _x86Multiply(k1, c1); + k1 = _x86Rotl(k1, 15); + k1 = _x86Multiply(k1, c2); + h1 ^= k1; + } + + h1 ^= bytes.length; + h2 ^= bytes.length; + h3 ^= bytes.length; + h4 ^= bytes.length; + + h1 += h2; + h1 += h3; + h1 += h4; + h2 += h1; + h3 += h1; + h4 += h1; + + h1 = _x86Fmix(h1); + h2 = _x86Fmix(h2); + h3 = _x86Fmix(h3); + h4 = _x86Fmix(h4); + + h1 += h2; + h1 += h3; + h1 += h4; + h2 += h1; + h3 += h1; + h4 += h1; + + return ( + ("00000000" + (h1 >>> 0).toString(16)).slice(-8) + + ("00000000" + (h2 >>> 0).toString(16)).slice(-8) + + ("00000000" + (h3 >>> 0).toString(16)).slice(-8) + + ("00000000" + (h4 >>> 0).toString(16)).slice(-8) + ); +} + +function x64Hash128(bytes, seed) { + // + // Given a string and an optional seed as an int, returns a 128 bit + // hash using the x64 flavor of MurmurHash3, as an unsigned hex. + // + seed = seed || 0; + + var remainder = bytes.length % 16; + var blocks = bytes.length - remainder; + + var h1 = [0, seed]; + var h2 = [0, seed]; + + var k1 = [0, 0]; + var k2 = [0, 0]; + + var c1 = [0x87c37b91, 0x114253d5]; + var c2 = [0x4cf5ad43, 0x2745937f]; + + for (var i = 0; i < blocks; i = i + 16) { + k1 = [ + bytes[i + 4] | (bytes[i + 5] << 8) | (bytes[i + 6] << 16) | (bytes[i + 7] << 24), + bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24) + ]; + k2 = [ + bytes[i + 12] | (bytes[i + 13] << 8) | (bytes[i + 14] << 16) | (bytes[i + 15] << 24), + bytes[i + 8] | (bytes[i + 9] << 8) | (bytes[i + 10] << 16) | (bytes[i + 11] << 24) + ]; + + k1 = _x64Multiply(k1, c1); + k1 = _x64Rotl(k1, 31); + k1 = _x64Multiply(k1, c2); + h1 = _x64Xor(h1, k1); + + h1 = _x64Rotl(h1, 27); + h1 = _x64Add(h1, h2); + h1 = _x64Add(_x64Multiply(h1, [0, 5]), [0, 0x52dce729]); + + k2 = _x64Multiply(k2, c2); + k2 = _x64Rotl(k2, 33); + k2 = _x64Multiply(k2, c1); + h2 = _x64Xor(h2, k2); + + h2 = _x64Rotl(h2, 31); + h2 = _x64Add(h2, h1); + h2 = _x64Add(_x64Multiply(h2, [0, 5]), [0, 0x38495ab5]); + } + + k1 = [0, 0]; + k2 = [0, 0]; + + switch (remainder) { + case 15: + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 14]], 48)); + + case 14: + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 13]], 40)); + + case 13: + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 12]], 32)); + + case 12: + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 11]], 24)); + + case 11: + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 10]], 16)); + + case 10: + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 9]], 8)); + + case 9: + k2 = _x64Xor(k2, [0, bytes[i + 8]]); + k2 = _x64Multiply(k2, c2); + k2 = _x64Rotl(k2, 33); + k2 = _x64Multiply(k2, c1); + h2 = _x64Xor(h2, k2); + + case 8: + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 7]], 56)); + + case 7: + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 6]], 48)); + + case 6: + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 5]], 40)); + + case 5: + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 4]], 32)); + + case 4: + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 3]], 24)); + + case 3: + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 2]], 16)); + + case 2: + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 1]], 8)); + + case 1: + k1 = _x64Xor(k1, [0, bytes[i]]); + k1 = _x64Multiply(k1, c1); + k1 = _x64Rotl(k1, 31); + k1 = _x64Multiply(k1, c2); + h1 = _x64Xor(h1, k1); + } + + h1 = _x64Xor(h1, [0, bytes.length]); + h2 = _x64Xor(h2, [0, bytes.length]); + + h1 = _x64Add(h1, h2); + h2 = _x64Add(h2, h1); + + h1 = _x64Fmix(h1); + h2 = _x64Fmix(h2); + + h1 = _x64Add(h1, h2); + h2 = _x64Add(h2, h1); + + return ( + ("00000000" + (h1[0] >>> 0).toString(16)).slice(-8) + + ("00000000" + (h1[1] >>> 0).toString(16)).slice(-8) + + ("00000000" + (h2[0] >>> 0).toString(16)).slice(-8) + + ("00000000" + (h2[1] >>> 0).toString(16)).slice(-8) + ); +} + +export default { + version: "3.0.0", + x86: { + hash32: x86Hash32, + hash128: x86Hash128 + }, + x64: { + hash128: x64Hash128 + }, + inputValidation: true +}; diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.ts deleted file mode 100644 index 2422d324a80d..000000000000 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.ts +++ /dev/null @@ -1,153 +0,0 @@ -/* -The MIT License (MIT) -Copyright (c) 2017 Microsoft Corporation -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -"use strict"; - -export const MurmurHash = { - /** - * Hashes a string, a unsigned 32-bit integer, or a Buffer into a new unsigned 32-bit integer that represents the output hash. - * @param {string, number of Buffer} key - The preimage of the hash - * @param {number} seed - Optional value used to initialize the hash generator - * @returns {} - */ - hash: function(key: string, seed: number) { - key = key || ""; - seed = seed || 0; - - MurmurHash._throwIfInvalidKey(key); - MurmurHash._throwIfInvalidSeed(seed); - - let buffer; - if (typeof key === "string") { - buffer = MurmurHash._getBufferFromString(key); - } else if (typeof key === "number") { - buffer = MurmurHash._getBufferFromNumber(key); - } else { - buffer = key; - } - - return MurmurHash._hashBytes(buffer, seed); - }, - /** @ignore */ - _throwIfInvalidKey: function(key: string | number | Buffer) { - if (key instanceof Buffer) { - return; - } - - if (typeof key === "string") { - return; - } - - if (typeof key === "number") { - return; - } - - throw new Error("Invalid argument: 'key' has to be a Buffer, string, or number."); - }, - /** @ignore */ - _throwIfInvalidSeed: function(seed: number) { - if (isNaN(seed)) { - throw new Error("Invalid argument: 'seed' is not and cannot be converted to a number."); - } - }, - /** @ignore */ - _getBufferFromString: function(key: string) { - let buffer = new Buffer(key); - return buffer; - }, - /** @ignore */ - _getBufferFromNumber: function(i: number) { - i = i >>> 0; - - let buffer = new Uint8Array([i >>> 0, i >>> 8, i >>> 16, i >>> 24]); - - return buffer; - }, - /** @ignore */ - _hashBytes: function(bytes: Buffer | Uint8Array, seed: number) { - let c1 = 0xcc9e2d51; - let c2 = 0x1b873593; - - let h1 = seed; - let reader = new Uint32Array(bytes); - { - for (var i = 0; i < bytes.length - 3; i += 4) { - let k1 = MurmurHash._readUInt32(reader, i); - - k1 = MurmurHash._multiply(k1, c1); - k1 = MurmurHash._rotateLeft(k1, 15); - k1 = MurmurHash._multiply(k1, c2); - - h1 ^= k1; - h1 = MurmurHash._rotateLeft(h1, 13); - h1 = MurmurHash._multiply(h1, 5) + 0xe6546b64; - } - } - - let k = 0; - switch (bytes.length & 3) { - case 3: - k ^= reader[i + 2] << 16; - k ^= reader[i + 1] << 8; - k ^= reader[i]; - break; - - case 2: - k ^= reader[i + 1] << 8; - k ^= reader[i]; - break; - - case 1: - k ^= reader[i]; - break; - } - - k = MurmurHash._multiply(k, c1); - k = MurmurHash._rotateLeft(k, 15); - k = MurmurHash._multiply(k, c2); - - h1 ^= k; - h1 ^= bytes.length; - h1 ^= h1 >>> 16; - h1 = MurmurHash._multiply(h1, 0x85ebca6b); - h1 ^= h1 >>> 13; - h1 = MurmurHash._multiply(h1, 0xc2b2ae35); - h1 ^= h1 >>> 16; - - return h1 >>> 0; - }, - /** @ignore */ - _rotateLeft: function(n: number, numBits: number) { - return (n << numBits) | (n >>> (32 - numBits)); - }, - /** @ignore */ - _multiply: function(m: number, n: number) { - return (m & 0xffff) * n + ((((m >>> 16) * n) & 0xffff) << 16); - }, - /** @ignore */ - _readUInt32: function(uintArray: Uint32Array, i: number) { - return ( - uintArray[i] | - (uintArray[i + 1] << 8) | - (uintArray[i + 2] << 16) | - ((uintArray[i + 3] << 24) >>> 0) - ); - } -}; diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts index ec8c97be4eec..4d6d7274b35b 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts @@ -1,12 +1,12 @@ import { writeNumberForBinaryEncoding, doubleToByteArray } from "./encoding/number"; import { writeStringForBinaryEncoding } from "./encoding/string"; -import { MurmurHash } from "./murmurHashV1"; +const MurmurHash = require("./murmurHashV1").default; type v1Key = string | number | null | {} | undefined; export function hashV1PartitionKey(partitionKey: v1Key): string { const toHash = prefixKeyByType(partitionKey); - const hash = MurmurHash.hash(toHash, 0); + const hash = MurmurHash.x86.hash32(toHash); const encodedHash = writeNumberForBinaryEncoding(hash); const encodedValue = encodeByType(partitionKey); return Buffer.concat([encodedHash, encodedValue]) @@ -24,21 +24,21 @@ function prefixKeyByType(key: v1Key) { Buffer.from(truncated), Buffer.from("00", "hex") ]); - return bytes.toString(); + return bytes; case "number": const numberBytes = doubleToByteArray(key); bytes = Buffer.concat([Buffer.from("05", "hex"), numberBytes]); - return bytes.toString(); + return bytes; case "boolean": const prefix = key ? "03" : "02"; - return Buffer.from(prefix, "hex").toString(); + return Buffer.from(prefix, "hex"); case "object": if (key === null) { - return Buffer.from("01", "hex").toString(); + return Buffer.from("01", "hex"); } - return Buffer.from("00", "hex").toString(); + return Buffer.from("00", "hex"); case "undefined": - return Buffer.from("00", "hex").toString(); + return Buffer.from("00", "hex"); } } diff --git a/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts b/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts index 0e7c962a27d4..0cdca97d21f1 100644 --- a/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts +++ b/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts @@ -8,6 +8,16 @@ describe("effectivePartitionKey", function() { key: "partitionKey", output: "05C1E1B3D9CD2608716273756A756A706F4C667A00" }, + { + key: "redmond", + output: "05C1EFE313830C087366656E706F6500" + }, + { + key: + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + output: + "05C1EB5921F706086262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626200" + }, { key: "", output: "05C1CF33970FF80800" @@ -39,6 +49,14 @@ describe("effectivePartitionKey", function() { { key: 5.5, output: "05C1D7A771716C05C016" + }, + { + key: 12313.1221, + output: "05C1ED154D592E05C0C90723F50FC925D8" + }, + { + key: 123456789, + output: "05C1D9E1A5311C05C19DB7CD8B40" } ]; toMatch.forEach(({ key, output }) => { From b212ecf7d3204cca734e09cba576ed58e513af44 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Tue, 30 Jun 2020 18:21:28 -0400 Subject: [PATCH 04/26] Try es2020 --- sdk/cosmosdb/cosmos/test/tsconfig.json | 2 +- sdk/cosmosdb/cosmos/tsconfig.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmosdb/cosmos/test/tsconfig.json b/sdk/cosmosdb/cosmos/test/tsconfig.json index 646ddc750eb4..bb05fe793129 100644 --- a/sdk/cosmosdb/cosmos/test/tsconfig.json +++ b/sdk/cosmosdb/cosmos/test/tsconfig.json @@ -9,7 +9,7 @@ "allowSyntheticDefaultImports": true, "preserveConstEnums": true, "removeComments": false, - "target": "esnext", + "target": "es2020", "sourceMap": true, "newLine": "LF", "composite": true, diff --git a/sdk/cosmosdb/cosmos/tsconfig.json b/sdk/cosmosdb/cosmos/tsconfig.json index 699c8d811bf9..187d24054bed 100644 --- a/sdk/cosmosdb/cosmos/tsconfig.json +++ b/sdk/cosmosdb/cosmos/tsconfig.json @@ -11,7 +11,7 @@ "outDir": "dist-esm", "preserveConstEnums": true, "removeComments": false, - "target": "esnext", + "target": "es2020", "sourceMap": true, "inlineSources": true, "newLine": "LF", From 60034c2e1bb1cc4b8aa7f5d2167c9e5ded9e9498 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Wed, 1 Jul 2020 10:33:34 -0400 Subject: [PATCH 05/26] Use BigInt init everywhere --- .../src/utils/hashing/encoding/number.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts index 339878c5f039..10f09d7dbf03 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts @@ -1,18 +1,18 @@ export function writeNumberForBinaryEncoding(hash: number) { let payload: bigint = encodeNumberAsUInt64(hash); let outputStream = Buffer.from("05", "hex"); - const firstChunk = BigInt.asUintN(64, payload >> 56n); + const firstChunk = BigInt.asUintN(64, payload >> BigInt(56)); outputStream = Buffer.concat([outputStream, Buffer.from(firstChunk.toString(16), "hex")]); - payload = BigInt.asUintN(64, BigInt(payload) << 0x8n); + payload = BigInt.asUintN(64, BigInt(payload) << BigInt(0x8)); - let byteToWrite = 0n; + let byteToWrite = BigInt(0); let firstIteration = false; let shifted: bigint; let padded: string; do { if (!firstIteration) { - // we pad because after shifting we will produce characters like "f" or similar, + // we pad because after shifting because we will produce characters like "f" or similar, // which cannot be encoded as hex in a buffer because they are invalid hex // https://github.com/nodejs/node/issues/24491 padded = byteToWrite.toString(16).padStart(2, "0"); @@ -23,13 +23,13 @@ export function writeNumberForBinaryEncoding(hash: number) { firstIteration = false; } - shifted = BigInt.asUintN(64, payload >> 56n); - byteToWrite = BigInt.asUintN(64, shifted | 0x01n); - payload = BigInt.asUintN(64, payload << 7n); - } while (payload != 0n); + shifted = BigInt.asUintN(64, payload >> BigInt(56)); + byteToWrite = BigInt.asUintN(64, shifted | BigInt(0x01)); + payload = BigInt.asUintN(64, payload << BigInt(7)); + } while (payload != BigInt(0)); - const lastChunk = BigInt.asUintN(64, byteToWrite & 0xfen); - // we pad because after shifting we will produce characters like "f" or similar, + const lastChunk = BigInt.asUintN(64, byteToWrite & BigInt(0xfe)); + // we pad because after shifting because we will produce characters like "f" or similar, // which cannot be encoded as hex in a buffer because they are invalid hex // https://github.com/nodejs/node/issues/24491 padded = lastChunk.toString(16).padStart(2, "0"); @@ -42,8 +42,8 @@ export function writeNumberForBinaryEncoding(hash: number) { function encodeNumberAsUInt64(value: number) { const rawValueBits = getRawBits(value); - const mask = 0x8000000000000000n; - const returned = rawValueBits < mask ? rawValueBits ^ mask : ~BigInt(rawValueBits) + 1n; + const mask = BigInt(0x8000000000000000); + const returned = rawValueBits < mask ? rawValueBits ^ mask : ~BigInt(rawValueBits) + BigInt(1); return returned; } @@ -51,7 +51,7 @@ export function doubleToByteArray(double: number) { const output: Buffer = Buffer.alloc(8); const lng = getRawBits(double); for (let i = 0; i < 8; i++) { - output[i] = Number((lng >> (BigInt(i) * 8n)) & 0xffn); + output[i] = Number((lng >> (BigInt(i) * BigInt(8))) & BigInt(0xff)); } return output; } From ea7a53f2d83a50200c8a110a62d76ef04d369d0f Mon Sep 17 00:00:00 2001 From: Zach foster Date: Thu, 2 Jul 2020 18:42:41 -0400 Subject: [PATCH 06/26] Adds v2 hashing support --- .../{murmurHashV1.js => murmurHash.js} | 25 +++++++- sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts | 2 +- sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts | 45 +++++++++++++ .../cosmos/test/unit/hashing/v2.spec.ts | 64 +++++++++++++++++++ 4 files changed, 132 insertions(+), 4 deletions(-) rename sdk/cosmosdb/cosmos/src/utils/hashing/{murmurHashV1.js => murmurHash.js} (94%) create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts create mode 100644 sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.js b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.js similarity index 94% rename from sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.js rename to sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.js index ff639c2d0d38..86759f69bd44 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHashV1.js +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.js @@ -531,12 +531,31 @@ function x64Hash128(bytes, seed) { h1 = _x64Add(h1, h2); h2 = _x64Add(h2, h1); - return ( + // Here we reverse h1 and h2 in Cosmos + // This is an implementation detail and not part of the public spec + const h1Buff = Buffer.from( ("00000000" + (h1[0] >>> 0).toString(16)).slice(-8) + - ("00000000" + (h1[1] >>> 0).toString(16)).slice(-8) + + ("00000000" + (h1[1] >>> 0).toString(16)).slice(-8), + "hex" + ); + const h1Reversed = reverse(h1Buff).toString("hex"); + const h2Buff = Buffer.from( ("00000000" + (h2[0] >>> 0).toString(16)).slice(-8) + - ("00000000" + (h2[1] >>> 0).toString(16)).slice(-8) + ("00000000" + (h2[1] >>> 0).toString(16)).slice(-8), + "hex" ); + const h2Reversed = reverse(h2Buff).toString("hex"); + return h1Reversed + h2Reversed; +} + +export function reverse(buff) { + const buffer = Buffer.allocUnsafe(buff.length); + + for (let i = 0, j = buff.length - 1; i <= j; ++i, --j) { + buffer[i] = buff[j]; + buffer[j] = buff[i]; + } + return buffer; } export default { diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts index 4d6d7274b35b..4c1b1bf25495 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts @@ -1,6 +1,6 @@ import { writeNumberForBinaryEncoding, doubleToByteArray } from "./encoding/number"; import { writeStringForBinaryEncoding } from "./encoding/string"; -const MurmurHash = require("./murmurHashV1").default; +const MurmurHash = require("./murmurHash").default; type v1Key = string | number | null | {} | undefined; diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts new file mode 100644 index 000000000000..9e0f1fcba6d4 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts @@ -0,0 +1,45 @@ +import { doubleToByteArray } from "./encoding/number"; +const MurmurHash = require("./murmurHash").default; + +type v1Key = string | number | null | {} | undefined; + +export function hashV2PartitionKey(partitionKey: v1Key): string { + const toHash = prefixKeyByType(partitionKey); + const hash = MurmurHash.x64.hash128(toHash); + const reverseBuff: Buffer = reverse(Buffer.from(hash, "hex")); + reverseBuff[0] &= 0x3f; + return reverseBuff.toString("hex").toUpperCase(); +} + +function prefixKeyByType(key: v1Key) { + let bytes: Buffer; + switch (typeof key) { + case "string": + bytes = Buffer.concat([Buffer.from("08", "hex"), Buffer.from(key), Buffer.from("FF", "hex")]); + return bytes; + case "number": + const numberBytes = doubleToByteArray(key); + bytes = Buffer.concat([Buffer.from("05", "hex"), numberBytes]); + return bytes; + case "boolean": + const prefix = key ? "03" : "02"; + return Buffer.from(prefix, "hex"); + case "object": + if (key === null) { + return Buffer.from("01", "hex"); + } + return Buffer.from("00", "hex"); + case "undefined": + return Buffer.from("00", "hex"); + } +} + +export function reverse(buff: Buffer) { + const buffer = Buffer.allocUnsafe(buff.length); + + for (let i = 0, j = buff.length - 1; i <= j; ++i, --j) { + buffer[i] = buff[j]; + buffer[j] = buff[i]; + } + return buffer; +} diff --git a/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts b/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts new file mode 100644 index 000000000000..784768a03a29 --- /dev/null +++ b/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts @@ -0,0 +1,64 @@ +import assert from "assert"; +import { hashV2PartitionKey } from "../../../src/utils/hashing/v2"; + +describe("effectivePartitionKey", function() { + describe("computes v2 key", function() { + const toMatch = [ + { + key: "redmond", + output: "22E342F38A486A088463DFF7838A5963" + }, + { + key: + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + output: "0BA3E9CA8EE4C14538828D1612A4B652" + }, + { + key: "", + output: "32E9366E637A71B4E710384B2F4970A0" + }, + { + key: "aa", + output: "05033626483AE80D00E44FBD35362B19" + }, + { + key: null, + output: "378867E4430E67857ACE5C908374FE16" + }, + { + key: true, + output: "0E711127C5B5A8E4726AC6DD306A3E59" + }, + { + key: false, + output: "2FE1BE91E90A3439635E0E9E37361EF2" + }, + { + key: {}, + output: "11622DAA78F835834610ABE56EFF5CB5" + }, + { + key: 5, + output: "19C08621B135968252FB34B4CF66F811" + }, + { + key: 5.5, + output: "0E2EE47829D1AF775EEFB6540FD1D0ED" + }, + { + key: 12313.1221, + output: "27E7ECA8F2EE3E53424DE8D5220631C6" + }, + { + key: 123456789, + output: "1F56D2538088EBA82CCF988F36E16760" + } + ]; + toMatch.forEach(({ key, output }) => { + it("matches expected hash output", function() { + const hashed = hashV2PartitionKey(key); + assert.equal(hashed, output); + }); + }); + }); +}); From 90aae542fd02679d272673247e6445ae98203d5c Mon Sep 17 00:00:00 2001 From: Zach foster Date: Tue, 7 Jul 2020 16:50:51 -0400 Subject: [PATCH 07/26] Adds back Bulk operations for v1 and v2 containers --- sdk/cosmosdb/cosmos/src/ClientContext.ts | 50 +++++++ sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 73 +++++++++- sdk/cosmosdb/cosmos/src/common/constants.ts | 9 +- sdk/cosmosdb/cosmos/src/request/request.ts | 4 +- sdk/cosmosdb/cosmos/src/utils/batch.ts | 73 ++++++++++ .../cosmos/test/functional/item.spec.ts | 137 +++++++++++++++++- 6 files changed, 340 insertions(+), 6 deletions(-) create mode 100644 sdk/cosmosdb/cosmos/src/utils/batch.ts diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index bdeae68f4632..b91546b58c15 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -543,6 +543,56 @@ export class ClientContext { return this.globalEndpointManager.getReadEndpoint(); } + public async bulk({ + body, + path, + resourceId, + partitionKeyRange, + options = {} + }: { + body: T; + path: string; + partitionKeyRange: string; + resourceId: string; + options?: RequestOptions; + }): Promise> { + try { + const request: RequestContext = { + globalEndpointManager: this.globalEndpointManager, + requestAgent: this.cosmosClientOptions.agent, + connectionPolicy: this.connectionPolicy, + method: HTTPMethod.post, + client: this, + operationType: OperationType.Batch, + path, + body, + resourceType: ResourceType.item, + resourceId, + plugins: this.cosmosClientOptions.plugins, + options + }; + + request.headers = await this.buildHeaders(request); + request.headers[Constants.HttpHeaders.IsBatchRequest] = "True"; + request.headers[Constants.HttpHeaders.PartitionKeyRangeID] = partitionKeyRange; + request.headers[Constants.HttpHeaders.IsBatchAtomic] = false; + + this.applySessionToken(request); + + request.endpoint = await this.globalEndpointManager.resolveServiceEndpoint( + request.resourceType, + request.operationType + ); + const response = await executePlugins(request, executeRequest, PluginOn.operation); + console.log(response); + this.captureSessionToken(undefined, path, OperationType.Batch, response.headers); + return response; + } catch (err) { + this.captureSessionToken(err, path, OperationType.Upsert, (err as ErrorResponse).headers); + throw err; + } + } + private captureSessionToken( err: ErrorResponse, path: string, diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 161ba2cddaf0..53e340414404 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -9,10 +9,19 @@ import { extractPartitionKey } from "../../extractPartitionKey"; import { FetchFunctionCallback, SqlQuerySpec } from "../../queryExecutionContext"; import { QueryIterator } from "../../queryIterator"; import { FeedOptions, RequestOptions } from "../../request"; -import { Container } from "../Container"; +import { Container, PartitionKeyRange } from "../Container"; import { Item } from "./Item"; import { ItemDefinition } from "./ItemDefinition"; import { ItemResponse } from "./ItemResponse"; +import { + Batch, + isKeyInRange, + MAX_128_BIT_INTEGER, + Operation, + hasResource +} from "../../utils/batch"; +import { hashV1PartitionKey } from "../../utils/hashing/v1"; +import { hashV2PartitionKey } from "../../utils/hashing/v2"; /** * @ignore @@ -373,4 +382,66 @@ export class Items { ref ); } + + public async bulk(operations: Operation[], options?: RequestOptions) { + const { + resources: partitionKeyRanges + } = await this.container.readPartitionKeyRanges().fetchAll(); + const { resource: definition } = await this.container.getPartitionKeyDefinition(); + const batches: Batch[] = partitionKeyRanges.map((keyRange: PartitionKeyRange) => { + return { + min: keyRange.minInclusive, + max: keyRange.maxExclusive, + rangeId: keyRange.id, + operations: [] + }; + }); + operations.forEach((operation: Operation) => { + const partitionProp = definition.paths[0].replace("/", ""); + const isV2 = definition.version && definition.version === 2; + const toHashKey = hasResource(operation) + ? (operation.resourceBody as any)[partitionProp] + : operation.partitionKey + .replace("[", "") + .replace("]", "") + .replace("'", "") + .replace('"', ""); + const key = isV2 ? hashV2PartitionKey(toHashKey) : hashV1PartitionKey(toHashKey); + let batchForKey = batches.find((batch: Batch) => { + let minInt: bigint; + let maxInt: bigint; + if (batch.min === "") { + minInt = 0n; + } else { + minInt = BigInt(`0x${batch.min}`); + } + if (batch.max === "FF") { + maxInt = MAX_128_BIT_INTEGER; + } else { + maxInt = BigInt(`0x${batch.max}`); + } + return isKeyInRange(minInt, maxInt, BigInt(`0x${key}`)); + }); + if (!batchForKey) { + // this would mean our partitionKey isn't in any of the existing ranges + } + batchForKey.operations.push(operation); + }); + + const path = getPathFromLink(this.container.url, ResourceType.item); + + return Promise.all( + batches + .filter((batch: Batch) => batch.operations.length) + .map(async (batch: Batch) => { + return this.clientContext.bulk({ + body: batch.operations, + partitionKeyRange: batch.rangeId, + path, + resourceId: this.container.url, + options + }); + }) + ); + } } diff --git a/sdk/cosmosdb/cosmos/src/common/constants.ts b/sdk/cosmosdb/cosmos/src/common/constants.ts index b1c162f43a09..2edceb880448 100644 --- a/sdk/cosmosdb/cosmos/src/common/constants.ts +++ b/sdk/cosmosdb/cosmos/src/common/constants.ts @@ -142,7 +142,11 @@ export const Constants = { ScriptLogResults: "x-ms-documentdb-script-log-results", // Multi-Region Write - ALLOW_MULTIPLE_WRITES: "x-ms-cosmos-allow-tentative-writes" + ALLOW_MULTIPLE_WRITES: "x-ms-cosmos-allow-tentative-writes", + + // Bulk/Batch header + IsBatchRequest: "x-ms-cosmos-is-batch-request", + IsBatchAtomic: "x-ms-cosmos-batch-atomic" }, // GlobalDB related constants @@ -244,5 +248,6 @@ export enum OperationType { Delete = "delete", Read = "read", Query = "query", - Execute = "execute" + Execute = "execute", + Batch = "batch" } diff --git a/sdk/cosmosdb/cosmos/src/request/request.ts b/sdk/cosmosdb/cosmos/src/request/request.ts index 23c2dffa8c81..102b3dcc538f 100644 --- a/sdk/cosmosdb/cosmos/src/request/request.ts +++ b/sdk/cosmosdb/cosmos/src/request/request.ts @@ -63,8 +63,8 @@ export async function getHeaders({ partitionKey }: GetHeadersOptions): Promise { const headers: CosmosHeaders = { - [Constants.HttpHeaders.ResponseContinuationTokenLimitInKB]: 1, - [Constants.HttpHeaders.EnableCrossPartitionQuery]: true, + // [Constants.HttpHeaders.ResponseContinuationTokenLimitInKB]: 1, + // [Constants.HttpHeaders.EnableCrossPartitionQuery]: true, ...defaultHeaders }; diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts new file mode 100644 index 000000000000..863c522281e5 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -0,0 +1,73 @@ +export type Operation = + | CreateOperation + | UpsertOperation + | ReadOperation + | DeleteOperation + | ReplaceOperation; + +export interface Batch { + min: string; + max: string; + rangeId: string; + operations: Operation[]; +} + +export const MAX_128_BIT_INTEGER = BigInt( + "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" +); + +export function isKeyInRange(min: bigint, max: bigint, key: bigint) { + const isAfterMinInclusive = min <= key; + const isBeforeMax = max > key; + return isAfterMinInclusive && isBeforeMax; +} + +interface OperationBase { + partitionKey: string; + ifMatch?: string; + ifNoneMatch?: string; +} + +type OperationWithItem = OperationBase & { + resourceBody: { [key: string]: string }; +}; + +type CreateOperation = OperationWithItem & { + operationType: "Create"; +}; + +type UpsertOperation = OperationWithItem & { + operationType: "Upsert"; +}; + +type ReadOperation = OperationBase & { + operationType: "Read"; + id: string; +}; + +type DeleteOperation = OperationBase & { + operationType: "Delete"; + id: string; +}; + +type ReplaceOperation = OperationWithItem & { + operationType: "Replace"; + id: string; +}; + +export function hasResource( + operation: Operation +): operation is CreateOperation | UpsertOperation | ReplaceOperation { + return (operation as OperationWithItem).resourceBody !== undefined; +} + +// function reverse(buff: Buffer) { +// const buffer = Buffer.allocUnsafe(buff.length); + +// for (let i = 0, j = buff.length - 1; i <= j; ++i, --j) { +// buffer[i] = buff[j]; +// buffer[j] = buff[i]; +// } + +// return buffer; +// } diff --git a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts index b577d363cde2..2b90e01a15dc 100644 --- a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts +++ b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts @@ -12,8 +12,11 @@ import { createOrUpsertItem, getTestDatabase, removeAllDatabases, - replaceOrUpsertItem + replaceOrUpsertItem, + addEntropy, + getTestContainer } from "../common/TestHelpers"; +import { Operation } from "../../src/utils/batch"; /** * @ignore @@ -212,3 +215,135 @@ describe("Item CRUD", function() { await bulkDeleteItems(container, returnedDocuments, partitionKey); }); }); + +describe("bulk item operations", function() { + describe("with v1 container", function() { + let container: Container; + let readItemId: string; + let replaceItemId: string; + let deleteItemId: string; + before(async function() { + container = await getTestContainer("bulk container", undefined, { + partitionKey: { + paths: ["/key"], + version: undefined + }, + throughput: 25100 + }); + readItemId = addEntropy("item1"); + await container.items.create({ + id: readItemId, + key: "A", + class: "2010" + }); + deleteItemId = addEntropy("item2"); + await container.items.create({ + id: deleteItemId, + key: "A", + class: "2010" + }); + replaceItemId = addEntropy("item2"); + await container.items.create({ + id: replaceItemId, + key: "A", + class: "2010" + }); + }); + it("handles create, upsert, replace, delete", async function() { + const operations: Operation[] = [ + { + operationType: "Create", + partitionKey: `["A"]`, + resourceBody: { id: "doc1", name: "sample", key: "A" } + }, + { + operationType: "Upsert", + partitionKey: `["A"]`, + resourceBody: { id: "doc2", name: "other", key: "A" } + }, + { + operationType: "Read", + id: readItemId, + partitionKey: `["A"]` + }, + { + operationType: "Delete", + id: deleteItemId, + partitionKey: `["A"]` + }, + { + operationType: "Replace", + partitionKey: `["A"]`, + id: replaceItemId, + resourceBody: { id: replaceItemId, name: "nice", key: "A" } + } + ]; + const response = await container.items.bulk(operations); + assert.equal(response[0].code, 200); + }); + }); + describe("with v2 container", function() { + let container: Container; + let readItemId: string; + let replaceItemId: string; + let deleteItemId: string; + before(async function() { + container = await getTestContainer("bulk container", undefined, { + partitionKey: { + paths: ["/key"] + }, + throughput: 25100 + }); + readItemId = addEntropy("item1"); + await container.items.create({ + id: readItemId, + key: "A", + class: "2010" + }); + deleteItemId = addEntropy("item2"); + await container.items.create({ + id: deleteItemId, + key: "A", + class: "2010" + }); + replaceItemId = addEntropy("item2"); + await container.items.create({ + id: replaceItemId, + key: "A", + class: "2010" + }); + }); + it("handles create, upsert, replace, delete", async function() { + const operations: Operation[] = [ + { + operationType: "Create", + partitionKey: `["A"]`, + resourceBody: { id: "doc1", name: "sample", key: "A" } + }, + { + operationType: "Upsert", + partitionKey: `["A"]`, + resourceBody: { id: "doc2", name: "other", key: "A" } + }, + { + operationType: "Read", + id: readItemId, + partitionKey: `["A"]` + }, + { + operationType: "Delete", + id: deleteItemId, + partitionKey: `["A"]` + }, + { + operationType: "Replace", + partitionKey: `["A"]`, + id: replaceItemId, + resourceBody: { id: replaceItemId, name: "nice", key: "A" } + } + ]; + const response = await container.items.bulk(operations); + assert.equal(response[0].code, 200); + }); + }); +}); From 684772b576de791f57755227d524e17da4d5e029 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Thu, 9 Jul 2020 19:00:25 -0400 Subject: [PATCH 08/26] Replaces bigint with JSBI --- sdk/cosmosdb/cosmos/package.json | 1 + sdk/cosmosdb/cosmos/src/ClientContext.ts | 1 - sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 13 +++-- sdk/cosmosdb/cosmos/src/utils/batch.ts | 21 ++----- .../src/utils/hashing/encoding/number.ts | 58 ++++++++++++------- sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts | 16 +++-- 6 files changed, 64 insertions(+), 46 deletions(-) diff --git a/sdk/cosmosdb/cosmos/package.json b/sdk/cosmosdb/cosmos/package.json index 272871045624..c5e193c51937 100644 --- a/sdk/cosmosdb/cosmos/package.json +++ b/sdk/cosmosdb/cosmos/package.json @@ -84,6 +84,7 @@ "@types/debug": "^4.1.4", "debug": "^4.1.1", "fast-json-stable-stringify": "^2.0.0", + "jsbi": "^3.1.3", "node-abort-controller": "^1.0.4", "node-fetch": "^2.6.0", "os-name": "^3.1.0", diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index b91546b58c15..17e6555884d1 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -584,7 +584,6 @@ export class ClientContext { request.operationType ); const response = await executePlugins(request, executeRequest, PluginOn.operation); - console.log(response); this.captureSessionToken(undefined, path, OperationType.Batch, response.headers); return response; } catch (err) { diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 53e340414404..e871c5bae839 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -22,6 +22,7 @@ import { } from "../../utils/batch"; import { hashV1PartitionKey } from "../../utils/hashing/v1"; import { hashV2PartitionKey } from "../../utils/hashing/v2"; +import JSBI from "jsbi"; /** * @ignore @@ -408,19 +409,19 @@ export class Items { .replace('"', ""); const key = isV2 ? hashV2PartitionKey(toHashKey) : hashV1PartitionKey(toHashKey); let batchForKey = batches.find((batch: Batch) => { - let minInt: bigint; - let maxInt: bigint; + let minInt: JSBI; + let maxInt: JSBI; if (batch.min === "") { - minInt = 0n; + minInt = JSBI.BigInt(0); } else { - minInt = BigInt(`0x${batch.min}`); + minInt = JSBI.BigInt(`0x${batch.min}`); } if (batch.max === "FF") { maxInt = MAX_128_BIT_INTEGER; } else { - maxInt = BigInt(`0x${batch.max}`); + maxInt = JSBI.BigInt(`0x${batch.max}`); } - return isKeyInRange(minInt, maxInt, BigInt(`0x${key}`)); + return isKeyInRange(minInt, maxInt, JSBI.BigInt(`0x${key}`)); }); if (!batchForKey) { // this would mean our partitionKey isn't in any of the existing ranges diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index 863c522281e5..5eba07af2971 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -1,3 +1,5 @@ +import JSBI from "jsbi"; + export type Operation = | CreateOperation | UpsertOperation @@ -12,13 +14,13 @@ export interface Batch { operations: Operation[]; } -export const MAX_128_BIT_INTEGER = BigInt( +export const MAX_128_BIT_INTEGER = JSBI.BigInt( "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" ); -export function isKeyInRange(min: bigint, max: bigint, key: bigint) { - const isAfterMinInclusive = min <= key; - const isBeforeMax = max > key; +export function isKeyInRange(min: JSBI, max: JSBI, key: JSBI) { + const isAfterMinInclusive = JSBI.lessThanOrEqual(min, key); + const isBeforeMax = JSBI.greaterThan(max, key); return isAfterMinInclusive && isBeforeMax; } @@ -60,14 +62,3 @@ export function hasResource( ): operation is CreateOperation | UpsertOperation | ReplaceOperation { return (operation as OperationWithItem).resourceBody !== undefined; } - -// function reverse(buff: Buffer) { -// const buffer = Buffer.allocUnsafe(buff.length); - -// for (let i = 0, j = buff.length - 1; i <= j; ++i, --j) { -// buffer[i] = buff[j]; -// buffer[j] = buff[i]; -// } - -// return buffer; -// } diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts index 10f09d7dbf03..f7f7ed54588e 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts @@ -1,13 +1,16 @@ -export function writeNumberForBinaryEncoding(hash: number) { - let payload: bigint = encodeNumberAsUInt64(hash); +import JSBI from "jsbi"; + +export function writeNumberForBinaryEncodingJSBI(hash: number) { + let payload = encodeNumberAsUInt64JSBI(hash); let outputStream = Buffer.from("05", "hex"); - const firstChunk = BigInt.asUintN(64, payload >> BigInt(56)); + const firstChunk = JSBI.asUintN(64, JSBI.signedRightShift(payload, JSBI.BigInt(56))); + outputStream = Buffer.concat([outputStream, Buffer.from(firstChunk.toString(16), "hex")]); - payload = BigInt.asUintN(64, BigInt(payload) << BigInt(0x8)); + payload = JSBI.asUintN(64, JSBI.leftShift(JSBI.BigInt(payload), JSBI.BigInt(0x8))); - let byteToWrite = BigInt(0); + let byteToWrite = JSBI.BigInt(0); let firstIteration = false; - let shifted: bigint; + let shifted: JSBI; let padded: string; do { @@ -23,12 +26,12 @@ export function writeNumberForBinaryEncoding(hash: number) { firstIteration = false; } - shifted = BigInt.asUintN(64, payload >> BigInt(56)); - byteToWrite = BigInt.asUintN(64, shifted | BigInt(0x01)); - payload = BigInt.asUintN(64, payload << BigInt(7)); - } while (payload != BigInt(0)); + shifted = JSBI.asUintN(64, JSBI.signedRightShift(payload, JSBI.BigInt(56))); + byteToWrite = JSBI.asUintN(64, JSBI.bitwiseOr(shifted, JSBI.BigInt(0x01))); + payload = JSBI.asUintN(64, JSBI.leftShift(payload, JSBI.BigInt(7))); + } while (JSBI.notEqual(payload, JSBI.BigInt(0))); - const lastChunk = BigInt.asUintN(64, byteToWrite & BigInt(0xfe)); + const lastChunk = JSBI.asUintN(64, JSBI.bitwiseAnd(byteToWrite, JSBI.BigInt(0xfe))); // we pad because after shifting because we will produce characters like "f" or similar, // which cannot be encoded as hex in a buffer because they are invalid hex // https://github.com/nodejs/node/issues/24491 @@ -40,24 +43,39 @@ export function writeNumberForBinaryEncoding(hash: number) { return outputStream; } -function encodeNumberAsUInt64(value: number) { - const rawValueBits = getRawBits(value); - const mask = BigInt(0x8000000000000000); - const returned = rawValueBits < mask ? rawValueBits ^ mask : ~BigInt(rawValueBits) + BigInt(1); +function encodeNumberAsUInt64JSBI(value: number) { + const rawValueBits = getRawBitsJSBI(value); + const mask = JSBI.BigInt(0x8000000000000000); + const returned = + rawValueBits < mask + ? JSBI.bitwiseXor(rawValueBits, mask) + : JSBI.add(JSBI.bitwiseNot(rawValueBits), JSBI.BigInt(1)); return returned; } -export function doubleToByteArray(double: number) { +export function doubleToByteArrayJSBI(double: number) { const output: Buffer = Buffer.alloc(8); - const lng = getRawBits(double); + const otherOutput: Buffer = Buffer.alloc(8); + const lng = getRawBitsJSBI(double); for (let i = 0; i < 8; i++) { - output[i] = Number((lng >> (BigInt(i) * BigInt(8))) & BigInt(0xff)); + otherOutput[i] = JSBI.toNumber( + JSBI.bitwiseAnd( + JSBI.signedRightShift(lng, JSBI.multiply(JSBI.BigInt(i), JSBI.BigInt(8))), + JSBI.BigInt(0xff) + ) + ); } return output; } -function getRawBits(value: number) { +function getRawBitsJSBI(value: number) { const view = new DataView(new ArrayBuffer(8)); view.setFloat64(0, value); - return view.getBigInt64(0); + return JSBI.BigInt(`0x${buf2hex(view.buffer)}`); +} + +function buf2hex(buffer: ArrayBuffer) { + return Array.prototype.map + .call(new Uint8Array(buffer), (x: number) => ("00" + x.toString(16)).slice(-2)) + .join(""); } diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts index 4c1b1bf25495..c7da6467a8db 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts @@ -1,4 +1,8 @@ -import { writeNumberForBinaryEncoding, doubleToByteArray } from "./encoding/number"; +import { + // writeNumberForBinaryEncoding, + doubleToByteArray, + writeNumberForBinaryEncodingJSBI +} from "./encoding/number"; import { writeStringForBinaryEncoding } from "./encoding/string"; const MurmurHash = require("./murmurHash").default; @@ -7,9 +11,10 @@ type v1Key = string | number | null | {} | undefined; export function hashV1PartitionKey(partitionKey: v1Key): string { const toHash = prefixKeyByType(partitionKey); const hash = MurmurHash.x86.hash32(toHash); - const encodedHash = writeNumberForBinaryEncoding(hash); + const encodedJSBI = writeNumberForBinaryEncodingJSBI(hash); + // const encodedHash = writeNumberForBinaryEncoding(hash); const encodedValue = encodeByType(partitionKey); - return Buffer.concat([encodedHash, encodedValue]) + return Buffer.concat([encodedJSBI, encodedValue]) .toString("hex") .toUpperCase(); } @@ -48,7 +53,10 @@ function encodeByType(key: v1Key) { const truncated = key.substr(0, 100); return writeStringForBinaryEncoding(truncated); case "number": - return writeNumberForBinaryEncoding(key); + const encodedJSBI = writeNumberForBinaryEncodingJSBI(key); + // const encoded = writeNumberForBinaryEncoding(key); + return encodedJSBI; + // return encoded; case "boolean": const prefix = key ? "03" : "02"; return Buffer.from(prefix, "hex"); From 6c03482801ef12028712b461574cb3f88c7abd94 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Fri, 10 Jul 2020 19:55:06 -0400 Subject: [PATCH 09/26] Try other keys --- sdk/cosmosdb/cosmos/src/ClientContext.ts | 4 +- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 50 +++++------- sdk/cosmosdb/cosmos/src/utils/batch.ts | 14 ++-- sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts | 4 +- sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts | 4 +- .../cosmos/test/functional/item.spec.ts | 78 +++++++++---------- 6 files changed, 67 insertions(+), 87 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index 17e6555884d1..7dd70b4901b1 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -547,7 +547,7 @@ export class ClientContext { body, path, resourceId, - partitionKeyRange, + // partitionKeyRange, options = {} }: { body: T; @@ -574,7 +574,7 @@ export class ClientContext { request.headers = await this.buildHeaders(request); request.headers[Constants.HttpHeaders.IsBatchRequest] = "True"; - request.headers[Constants.HttpHeaders.PartitionKeyRangeID] = partitionKeyRange; + request.headers[Constants.HttpHeaders.PartitionKeyRangeID] = "3"; request.headers[Constants.HttpHeaders.IsBatchAtomic] = false; this.applySessionToken(request); diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index e871c5bae839..4f69720febcc 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -13,16 +13,9 @@ import { Container, PartitionKeyRange } from "../Container"; import { Item } from "./Item"; import { ItemDefinition } from "./ItemDefinition"; import { ItemResponse } from "./ItemResponse"; -import { - Batch, - isKeyInRange, - MAX_128_BIT_INTEGER, - Operation, - hasResource -} from "../../utils/batch"; +import { Batch, isKeyInRange, Operation, hasResource } from "../../utils/batch"; import { hashV1PartitionKey } from "../../utils/hashing/v1"; import { hashV2PartitionKey } from "../../utils/hashing/v2"; -import JSBI from "jsbi"; /** * @ignore @@ -406,26 +399,13 @@ export class Items { .replace("[", "") .replace("]", "") .replace("'", "") + .replace('"', "") .replace('"', ""); - const key = isV2 ? hashV2PartitionKey(toHashKey) : hashV1PartitionKey(toHashKey); + + const hashed = isV2 ? hashV2PartitionKey(toHashKey) : hashV1PartitionKey(toHashKey); let batchForKey = batches.find((batch: Batch) => { - let minInt: JSBI; - let maxInt: JSBI; - if (batch.min === "") { - minInt = JSBI.BigInt(0); - } else { - minInt = JSBI.BigInt(`0x${batch.min}`); - } - if (batch.max === "FF") { - maxInt = MAX_128_BIT_INTEGER; - } else { - maxInt = JSBI.BigInt(`0x${batch.max}`); - } - return isKeyInRange(minInt, maxInt, JSBI.BigInt(`0x${key}`)); + return isKeyInRange(batch.min, batch.max, hashed); }); - if (!batchForKey) { - // this would mean our partitionKey isn't in any of the existing ranges - } batchForKey.operations.push(operation); }); @@ -435,13 +415,19 @@ export class Items { batches .filter((batch: Batch) => batch.operations.length) .map(async (batch: Batch) => { - return this.clientContext.bulk({ - body: batch.operations, - partitionKeyRange: batch.rangeId, - path, - resourceId: this.container.url, - options - }); + console.log({ batch }); + try { + const response = await this.clientContext.bulk({ + body: batch.operations, + partitionKeyRange: batch.rangeId, + path, + resourceId: this.container.url, + options + }); + return response; + } catch (e) { + console.log({ e }); + } }) ); } diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index 5eba07af2971..a8f8946d81d6 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -1,4 +1,4 @@ -import JSBI from "jsbi"; +import { JSONObject } from "../queryExecutionContext"; export type Operation = | CreateOperation @@ -14,13 +14,9 @@ export interface Batch { operations: Operation[]; } -export const MAX_128_BIT_INTEGER = JSBI.BigInt( - "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" -); - -export function isKeyInRange(min: JSBI, max: JSBI, key: JSBI) { - const isAfterMinInclusive = JSBI.lessThanOrEqual(min, key); - const isBeforeMax = JSBI.greaterThan(max, key); +export function isKeyInRange(min: string, max: string, key: string) { + const isAfterMinInclusive = key.localeCompare(min) >= 0; + const isBeforeMax = key.localeCompare(max) < 0; return isAfterMinInclusive && isBeforeMax; } @@ -31,7 +27,7 @@ interface OperationBase { } type OperationWithItem = OperationBase & { - resourceBody: { [key: string]: string }; + resourceBody: JSONObject; }; type CreateOperation = OperationWithItem & { diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts index c7da6467a8db..b0b04e3d8ef1 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts @@ -1,6 +1,6 @@ import { // writeNumberForBinaryEncoding, - doubleToByteArray, + doubleToByteArrayJSBI, writeNumberForBinaryEncodingJSBI } from "./encoding/number"; import { writeStringForBinaryEncoding } from "./encoding/string"; @@ -31,7 +31,7 @@ function prefixKeyByType(key: v1Key) { ]); return bytes; case "number": - const numberBytes = doubleToByteArray(key); + const numberBytes = doubleToByteArrayJSBI(key); bytes = Buffer.concat([Buffer.from("05", "hex"), numberBytes]); return bytes; case "boolean": diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts index 9e0f1fcba6d4..2bbf9dfaf6b6 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts @@ -1,4 +1,4 @@ -import { doubleToByteArray } from "./encoding/number"; +import { doubleToByteArrayJSBI } from "./encoding/number"; const MurmurHash = require("./murmurHash").default; type v1Key = string | number | null | {} | undefined; @@ -18,7 +18,7 @@ function prefixKeyByType(key: v1Key) { bytes = Buffer.concat([Buffer.from("08", "hex"), Buffer.from(key), Buffer.from("FF", "hex")]); return bytes; case "number": - const numberBytes = doubleToByteArray(key); + const numberBytes = doubleToByteArrayJSBI(key); bytes = Buffer.concat([Buffer.from("05", "hex"), numberBytes]); return bytes; case "boolean": diff --git a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts index 2b90e01a15dc..8e25708e4312 100644 --- a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts +++ b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts @@ -216,7 +216,7 @@ describe("Item CRUD", function() { }); }); -describe("bulk item operations", function() { +describe.only("bulk item operations", function() { describe("with v1 container", function() { let container: Container; let readItemId: string; @@ -242,7 +242,7 @@ describe("bulk item operations", function() { key: "A", class: "2010" }); - replaceItemId = addEntropy("item2"); + replaceItemId = addEntropy("item3"); await container.items.create({ id: replaceItemId, key: "A", @@ -282,10 +282,8 @@ describe("bulk item operations", function() { assert.equal(response[0].code, 200); }); }); - describe("with v2 container", function() { + describe.only("with v2 container", function() { let container: Container; - let readItemId: string; - let replaceItemId: string; let deleteItemId: string; before(async function() { container = await getTestContainer("bulk container", undefined, { @@ -294,53 +292,53 @@ describe("bulk item operations", function() { }, throughput: 25100 }); - readItemId = addEntropy("item1"); - await container.items.create({ - id: readItemId, - key: "A", - class: "2010" - }); + // readItemId = addEntropy("item1"); + // await container.items.create({ + // id: readItemId, + // key: "A", + // class: "2010" + // }); deleteItemId = addEntropy("item2"); await container.items.create({ id: deleteItemId, - key: "A", - class: "2010" - }); - replaceItemId = addEntropy("item2"); - await container.items.create({ - id: replaceItemId, - key: "A", + key: 5, class: "2010" }); + // replaceItemId = addEntropy("item3"); + // await container.items.create({ + // id: replaceItemId, + // key: "A", + // class: "2010" + // }); }); it("handles create, upsert, replace, delete", async function() { const operations: Operation[] = [ - { - operationType: "Create", - partitionKey: `["A"]`, - resourceBody: { id: "doc1", name: "sample", key: "A" } - }, - { - operationType: "Upsert", - partitionKey: `["A"]`, - resourceBody: { id: "doc2", name: "other", key: "A" } - }, - { - operationType: "Read", - id: readItemId, - partitionKey: `["A"]` - }, + // { + // operationType: "Create", + // partitionKey: `["A"]`, + // resourceBody: { id: "doc1", name: "sample", key: "A" } + // }, + // { + // operationType: "Upsert", + // partitionKey: `["A"]`, + // resourceBody: { id: "doc2", name: "other", key: "A" } + // }, + // { + // operationType: "Read", + // id: readItemId, + // partitionKey: `["A"]` + // }, { operationType: "Delete", id: deleteItemId, - partitionKey: `["A"]` - }, - { - operationType: "Replace", - partitionKey: `["A"]`, - id: replaceItemId, - resourceBody: { id: replaceItemId, name: "nice", key: "A" } + partitionKey: `[5]` } + // { + // operationType: "Replace", + // partitionKey: `["A"]`, + // id: replaceItemId, + // resourceBody: { id: replaceItemId, name: "nice", key: "A" } + // } ]; const response = await container.items.bulk(operations); assert.equal(response[0].code, 200); From 4cdef29b85a5a2b4e98942b56e3d8f04f87b6dce Mon Sep 17 00:00:00 2001 From: Zach foster Date: Fri, 10 Jul 2020 20:08:07 -0400 Subject: [PATCH 10/26] revert comments --- sdk/cosmosdb/cosmos/src/ClientContext.ts | 7 +- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 5 +- .../cosmos/test/functional/item.spec.ts | 74 ++++++++++--------- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index 7dd70b4901b1..865260ad0433 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -547,7 +547,7 @@ export class ClientContext { body, path, resourceId, - // partitionKeyRange, + partitionKeyRange, options = {} }: { body: T; @@ -574,7 +574,7 @@ export class ClientContext { request.headers = await this.buildHeaders(request); request.headers[Constants.HttpHeaders.IsBatchRequest] = "True"; - request.headers[Constants.HttpHeaders.PartitionKeyRangeID] = "3"; + request.headers[Constants.HttpHeaders.PartitionKeyRangeID] = partitionKeyRange; request.headers[Constants.HttpHeaders.IsBatchAtomic] = false; this.applySessionToken(request); @@ -655,7 +655,8 @@ export class ClientContext { clientOptions: this.cosmosClientOptions, defaultHeaders: { ...this.cosmosClientOptions.defaultHeaders, - ...requestContext.options.initialHeaders + ...requestContext.options.initialHeaders, + PartitionKeyRangeID: "3" }, verb: requestContext.method, path: requestContext.path, diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 4f69720febcc..807e60e5d5be 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -401,7 +401,6 @@ export class Items { .replace("'", "") .replace('"', "") .replace('"', ""); - const hashed = isV2 ? hashV2PartitionKey(toHashKey) : hashV1PartitionKey(toHashKey); let batchForKey = batches.find((batch: Batch) => { return isKeyInRange(batch.min, batch.max, hashed); @@ -425,8 +424,8 @@ export class Items { options }); return response; - } catch (e) { - console.log({ e }); + } catch (error) { + console.log({ error }); } }) ); diff --git a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts index 8e25708e4312..d047f8feef08 100644 --- a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts +++ b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts @@ -282,8 +282,10 @@ describe.only("bulk item operations", function() { assert.equal(response[0].code, 200); }); }); - describe.only("with v2 container", function() { + describe("with v2 container", function() { let container: Container; + let readItemId: string; + let replaceItemId: string; let deleteItemId: string; before(async function() { container = await getTestContainer("bulk container", undefined, { @@ -292,53 +294,53 @@ describe.only("bulk item operations", function() { }, throughput: 25100 }); - // readItemId = addEntropy("item1"); - // await container.items.create({ - // id: readItemId, - // key: "A", - // class: "2010" - // }); + readItemId = addEntropy("item1"); + await container.items.create({ + id: readItemId, + key: "A", + class: "2010" + }); deleteItemId = addEntropy("item2"); await container.items.create({ id: deleteItemId, - key: 5, + key: "A", + class: "2010" + }); + replaceItemId = addEntropy("item3"); + await container.items.create({ + id: replaceItemId, + key: "A", class: "2010" }); - // replaceItemId = addEntropy("item3"); - // await container.items.create({ - // id: replaceItemId, - // key: "A", - // class: "2010" - // }); }); it("handles create, upsert, replace, delete", async function() { const operations: Operation[] = [ - // { - // operationType: "Create", - // partitionKey: `["A"]`, - // resourceBody: { id: "doc1", name: "sample", key: "A" } - // }, - // { - // operationType: "Upsert", - // partitionKey: `["A"]`, - // resourceBody: { id: "doc2", name: "other", key: "A" } - // }, - // { - // operationType: "Read", - // id: readItemId, - // partitionKey: `["A"]` - // }, + { + operationType: "Create", + partitionKey: `["A"]`, + resourceBody: { id: "doc1", name: "sample", key: "A" } + }, + { + operationType: "Upsert", + partitionKey: `["A"]`, + resourceBody: { id: "doc2", name: "other", key: "A" } + }, + { + operationType: "Read", + id: readItemId, + partitionKey: `["A"]` + }, { operationType: "Delete", id: deleteItemId, - partitionKey: `[5]` + partitionKey: `["A"]` + }, + { + operationType: "Replace", + partitionKey: `["A"]`, + id: replaceItemId, + resourceBody: { id: replaceItemId + "230984", name: "nice", key: "A" } } - // { - // operationType: "Replace", - // partitionKey: `["A"]`, - // id: replaceItemId, - // resourceBody: { id: replaceItemId, name: "nice", key: "A" } - // } ]; const response = await container.items.bulk(operations); assert.equal(response[0].code, 200); From 61d41cec7071caddb3bcaa2440fa4ca4160d9f89 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Tue, 14 Jul 2020 15:57:09 -0400 Subject: [PATCH 11/26] Pushes logs to mess with number partitioning --- sdk/cosmosdb/cosmos/src/ClientContext.ts | 4 +- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 20 +++- sdk/cosmosdb/cosmos/src/common/constants.ts | 5 +- sdk/cosmosdb/cosmos/src/utils/batch.ts | 1 + .../cosmos/test/functional/item.spec.ts | 100 +++++++++--------- 5 files changed, 74 insertions(+), 56 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index 865260ad0433..1a0e03c97762 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -139,6 +139,7 @@ export class ClientContext { request.operationType ); request.headers = await this.buildHeaders(request); + // request.headers[Constants.HttpHeaders.ForceRefresh] = "True"; if (query !== undefined) { request.headers[Constants.HttpHeaders.IsQuery] = "true"; request.headers[Constants.HttpHeaders.ContentType] = QueryJsonContentType; @@ -655,8 +656,7 @@ export class ClientContext { clientOptions: this.cosmosClientOptions, defaultHeaders: { ...this.cosmosClientOptions.defaultHeaders, - ...requestContext.options.initialHeaders, - PartitionKeyRangeID: "3" + ...requestContext.options.initialHeaders }, verb: requestContext.method, path: requestContext.path, diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 807e60e5d5be..46d6395980e0 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -382,6 +382,7 @@ export class Items { resources: partitionKeyRanges } = await this.container.readPartitionKeyRanges().fetchAll(); const { resource: definition } = await this.container.getPartitionKeyDefinition(); + console.log({ container: this.container }); const batches: Batch[] = partitionKeyRanges.map((keyRange: PartitionKeyRange) => { return { min: keyRange.minInclusive, @@ -410,12 +411,13 @@ export class Items { const path = getPathFromLink(this.container.url, ResourceType.item); - return Promise.all( + const responses = await Promise.all( batches .filter((batch: Batch) => batch.operations.length) .map(async (batch: Batch) => { - console.log({ batch }); + console.log({ batch: JSON.stringify(batch) }); try { + console.log({ rangeId: batch.rangeId }); const response = await this.clientContext.bulk({ body: batch.operations, partitionKeyRange: batch.rangeId, @@ -424,10 +426,20 @@ export class Items { options }); return response; - } catch (error) { - console.log({ error }); + } catch (err) { + console.log({ err }); + const { + resources: partitionKeyRanges + } = await this.container.readPartitionKeyRanges().fetchAll(); + console.log({ partitionKeyRanges }); } }) ); + return responses + .map((resp) => { + console.log(resp); + return resp.result; + }) + .flat(); } } diff --git a/sdk/cosmosdb/cosmos/src/common/constants.ts b/sdk/cosmosdb/cosmos/src/common/constants.ts index 2edceb880448..b29aed519a2b 100644 --- a/sdk/cosmosdb/cosmos/src/common/constants.ts +++ b/sdk/cosmosdb/cosmos/src/common/constants.ts @@ -146,7 +146,10 @@ export const Constants = { // Bulk/Batch header IsBatchRequest: "x-ms-cosmos-is-batch-request", - IsBatchAtomic: "x-ms-cosmos-batch-atomic" + IsBatchAtomic: "x-ms-cosmos-batch-atomic", + + // Cache Refresh header + ForceRefresh: "x-ms-force-refresh" }, // GlobalDB related constants diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index a8f8946d81d6..3969772aab48 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -15,6 +15,7 @@ export interface Batch { } export function isKeyInRange(min: string, max: string, key: string) { + console.log({ min, max, key }); const isAfterMinInclusive = key.localeCompare(min) >= 0; const isBeforeMax = key.localeCompare(max) < 0; return isAfterMinInclusive && isBeforeMax; diff --git a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts index d047f8feef08..cf730b90729e 100644 --- a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts +++ b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts @@ -245,7 +245,7 @@ describe.only("bulk item operations", function() { replaceItemId = addEntropy("item3"); await container.items.create({ id: replaceItemId, - key: "A", + key: 5, class: "2010" }); }); @@ -273,77 +273,79 @@ describe.only("bulk item operations", function() { }, { operationType: "Replace", - partitionKey: `["A"]`, + partitionKey: "5", id: replaceItemId, - resourceBody: { id: replaceItemId, name: "nice", key: "A" } + resourceBody: { id: replaceItemId, name: "nice", key: 5 } } ]; const response = await container.items.bulk(operations); - assert.equal(response[0].code, 200); + console.log({ response }); }); }); - describe("with v2 container", function() { - let container: Container; - let readItemId: string; + describe.only("with v2 container", function() { + let v2Container: Container; + // let readItemId: string; let replaceItemId: string; - let deleteItemId: string; + // let deleteItemId: string; before(async function() { - container = await getTestContainer("bulk container", undefined, { + v2Container = await getTestContainer("bulk container v2", undefined, { partitionKey: { - paths: ["/key"] + paths: ["/key"], + version: 2 }, throughput: 25100 }); - readItemId = addEntropy("item1"); - await container.items.create({ - id: readItemId, - key: "A", - class: "2010" - }); - deleteItemId = addEntropy("item2"); - await container.items.create({ - id: deleteItemId, - key: "A", - class: "2010" - }); - replaceItemId = addEntropy("item3"); - await container.items.create({ + // readItemId = addEntropy("item1"); + // await container.items.create({ + // id: readItemId, + // key: "A", + // class: "2010" + // }); + // deleteItemId = addEntropy("item2"); + // await container.items.create({ + // id: deleteItemId, + // key: "A", + // class: "2010" + // }); + // replaceItemId = addEntropy("item3"); + await v2Container.items.create({ id: replaceItemId, - key: "A", + key: 5, class: "2010" }); }); it("handles create, upsert, replace, delete", async function() { const operations: Operation[] = [ - { - operationType: "Create", - partitionKey: `["A"]`, - resourceBody: { id: "doc1", name: "sample", key: "A" } - }, - { - operationType: "Upsert", - partitionKey: `["A"]`, - resourceBody: { id: "doc2", name: "other", key: "A" } - }, - { - operationType: "Read", - id: readItemId, - partitionKey: `["A"]` - }, - { - operationType: "Delete", - id: deleteItemId, - partitionKey: `["A"]` - }, + // { + // operationType: "Create", + // partitionKey: `["A"]`, + // resourceBody: { id: "doc1", name: "sample", key: "A" } + // }, + // { + // operationType: "Upsert", + // partitionKey: `["A"]`, + // resourceBody: { id: "doc2", name: "other", key: "A" } + // }, + // { + // operationType: "Read", + // id: readItemId, + // partitionKey: `["A"]` + // }, + // { + // operationType: "Delete", + // id: deleteItemId, + // partitionKey: `["A"]` + // }, { operationType: "Replace", - partitionKey: `["A"]`, + partitionKey: "[5]", id: replaceItemId, - resourceBody: { id: replaceItemId + "230984", name: "nice", key: "A" } + resourceBody: { id: replaceItemId, name: "nice", key: 5 } } ]; - const response = await container.items.bulk(operations); - assert.equal(response[0].code, 200); + const response = await v2Container.items.bulk(operations); + console.log(response.map((item: any) => item.resourceBody)); + // assert.equal(response[0], 200); }); }); }); From 18864b5a874d5eed1a4ebb00338030cb12abfd65 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Thu, 16 Jul 2020 17:22:15 -0400 Subject: [PATCH 12/26] Remove pk from public API, fix test --- sdk/cosmosdb/cosmos/src/ClientContext.ts | 2 +- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 61 +++++----- sdk/cosmosdb/cosmos/src/utils/batch.ts | 24 +++- .../src/utils/hashing/encoding/number.ts | 3 +- .../cosmos/test/functional/item.spec.ts | 106 ++++++++++-------- 5 files changed, 116 insertions(+), 80 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index 1a0e03c97762..e98331263ad4 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -556,7 +556,7 @@ export class ClientContext { partitionKeyRange: string; resourceId: string; options?: RequestOptions; - }): Promise> { + }) { try { const request: RequestContext = { globalEndpointManager: this.globalEndpointManager, diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 46d6395980e0..f352bd3a2dc6 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -6,14 +6,20 @@ import { ChangeFeedOptions } from "../../ChangeFeedOptions"; import { ClientContext } from "../../ClientContext"; import { getIdFromLink, getPathFromLink, isResourceValid, ResourceType } from "../../common"; import { extractPartitionKey } from "../../extractPartitionKey"; -import { FetchFunctionCallback, SqlQuerySpec } from "../../queryExecutionContext"; +import { FetchFunctionCallback, SqlQuerySpec, JSONObject } from "../../queryExecutionContext"; import { QueryIterator } from "../../queryIterator"; import { FeedOptions, RequestOptions } from "../../request"; import { Container, PartitionKeyRange } from "../Container"; import { Item } from "./Item"; import { ItemDefinition } from "./ItemDefinition"; import { ItemResponse } from "./ItemResponse"; -import { Batch, isKeyInRange, Operation, hasResource } from "../../utils/batch"; +import { + Batch, + isKeyInRange, + Operation, + getPartitionKeyToHash, + addPKToOperation +} from "../../utils/batch"; import { hashV1PartitionKey } from "../../utils/hashing/v1"; import { hashV2PartitionKey } from "../../utils/hashing/v2"; @@ -377,12 +383,21 @@ export class Items { ); } - public async bulk(operations: Operation[], options?: RequestOptions) { + public async bulk( + operations: Operation[], + options?: RequestOptions + ): Promise< + { + statusCode: number; + requestCharge: number; + eTag?: string; + resourceBody?: JSONObject; + }[] + > { const { resources: partitionKeyRanges } = await this.container.readPartitionKeyRanges().fetchAll(); const { resource: definition } = await this.container.getPartitionKeyDefinition(); - console.log({ container: this.container }); const batches: Batch[] = partitionKeyRanges.map((keyRange: PartitionKeyRange) => { return { min: keyRange.minInclusive, @@ -391,23 +406,18 @@ export class Items { operations: [] }; }); - operations.forEach((operation: Operation) => { - const partitionProp = definition.paths[0].replace("/", ""); - const isV2 = definition.version && definition.version === 2; - const toHashKey = hasResource(operation) - ? (operation.resourceBody as any)[partitionProp] - : operation.partitionKey - .replace("[", "") - .replace("]", "") - .replace("'", "") - .replace('"', "") - .replace('"', ""); - const hashed = isV2 ? hashV2PartitionKey(toHashKey) : hashV1PartitionKey(toHashKey); - let batchForKey = batches.find((batch: Batch) => { - return isKeyInRange(batch.min, batch.max, hashed); + operations + .map((operation) => addPKToOperation(operation, definition)) + .forEach((operation: Operation) => { + const partitionProp = definition.paths[0].replace("/", ""); + const isV2 = definition.version && definition.version === 2; + const toHashKey = getPartitionKeyToHash(operation, partitionProp); + const hashed = isV2 ? hashV2PartitionKey(toHashKey) : hashV1PartitionKey(toHashKey); + let batchForKey = batches.find((batch: Batch) => { + return isKeyInRange(batch.min, batch.max, hashed); + }); + batchForKey.operations.push(operation); }); - batchForKey.operations.push(operation); - }); const path = getPathFromLink(this.container.url, ResourceType.item); @@ -415,9 +425,7 @@ export class Items { batches .filter((batch: Batch) => batch.operations.length) .map(async (batch: Batch) => { - console.log({ batch: JSON.stringify(batch) }); try { - console.log({ rangeId: batch.rangeId }); const response = await this.clientContext.bulk({ body: batch.operations, partitionKeyRange: batch.rangeId, @@ -426,18 +434,11 @@ export class Items { options }); return response; - } catch (err) { - console.log({ err }); - const { - resources: partitionKeyRanges - } = await this.container.readPartitionKeyRanges().fetchAll(); - console.log({ partitionKeyRanges }); - } + } catch (err) {} }) ); return responses .map((resp) => { - console.log(resp); return resp.result; }) .flat(); diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index 3969772aab48..0eb450ce1847 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -1,4 +1,6 @@ import { JSONObject } from "../queryExecutionContext"; +import { extractPartitionKey } from "../extractPartitionKey"; +import { PartitionKeyDefinition } from "../documents"; export type Operation = | CreateOperation @@ -15,14 +17,13 @@ export interface Batch { } export function isKeyInRange(min: string, max: string, key: string) { - console.log({ min, max, key }); const isAfterMinInclusive = key.localeCompare(min) >= 0; const isBeforeMax = key.localeCompare(max) < 0; return isAfterMinInclusive && isBeforeMax; } interface OperationBase { - partitionKey: string; + partitionKey?: string; ifMatch?: string; ifNoneMatch?: string; } @@ -59,3 +60,22 @@ export function hasResource( ): operation is CreateOperation | UpsertOperation | ReplaceOperation { return (operation as OperationWithItem).resourceBody !== undefined; } + +export function getPartitionKeyToHash(operation: Operation, partitionProperty: string) { + return hasResource(operation) + ? (operation.resourceBody as any)[partitionProperty] + : operation.partitionKey + .replace("[", "") + .replace("]", "") + .replace("'", "") + .replace('"', "") + .replace('"', ""); +} + +export function addPKToOperation(operation: Operation, definition: PartitionKeyDefinition) { + if (operation.partitionKey || !hasResource(operation)) { + return operation; + } + const pk = extractPartitionKey(operation.resourceBody, definition); + return { ...operation, partitionKey: JSON.stringify(pk) }; +} diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts index f7f7ed54588e..a167398482c0 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts @@ -55,10 +55,9 @@ function encodeNumberAsUInt64JSBI(value: number) { export function doubleToByteArrayJSBI(double: number) { const output: Buffer = Buffer.alloc(8); - const otherOutput: Buffer = Buffer.alloc(8); const lng = getRawBitsJSBI(double); for (let i = 0; i < 8; i++) { - otherOutput[i] = JSBI.toNumber( + output[i] = JSBI.toNumber( JSBI.bitwiseAnd( JSBI.signedRightShift(lng, JSBI.multiply(JSBI.BigInt(i), JSBI.BigInt(8))), JSBI.BigInt(0xff) diff --git a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts index cf730b90729e..0fbde1d75329 100644 --- a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts +++ b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts @@ -216,8 +216,8 @@ describe("Item CRUD", function() { }); }); -describe.only("bulk item operations", function() { - describe("with v1 container", function() { +describe("bulk item operations", function() { + describe.only("with v1 container", function() { let container: Container; let readItemId: string; let replaceItemId: string; @@ -253,13 +253,12 @@ describe.only("bulk item operations", function() { const operations: Operation[] = [ { operationType: "Create", - partitionKey: `["A"]`, - resourceBody: { id: "doc1", name: "sample", key: "A" } + resourceBody: { id: addEntropy("doc1"), name: "sample", key: "A" } }, { operationType: "Upsert", partitionKey: `["A"]`, - resourceBody: { id: "doc2", name: "other", key: "A" } + resourceBody: { id: addEntropy("doc2"), name: "other", key: "A" } }, { operationType: "Read", @@ -273,20 +272,29 @@ describe.only("bulk item operations", function() { }, { operationType: "Replace", - partitionKey: "5", + partitionKey: "[5]", id: replaceItemId, resourceBody: { id: replaceItemId, name: "nice", key: 5 } } ]; const response = await container.items.bulk(operations); - console.log({ response }); + // Create + assert.equal(response[0].statusCode, 200); + // Upsert + assert.equal(response[1].statusCode, 201); + // Read + assert.equal(response[2].statusCode, 201); + // Delete + assert.equal(response[3].statusCode, 200); + // Replace + assert.equal(response[4].statusCode, 204); }); }); - describe.only("with v2 container", function() { + describe("with v2 container", function() { let v2Container: Container; - // let readItemId: string; + let readItemId: string; let replaceItemId: string; - // let deleteItemId: string; + let deleteItemId: string; before(async function() { v2Container = await getTestContainer("bulk container v2", undefined, { partitionKey: { @@ -295,19 +303,19 @@ describe.only("bulk item operations", function() { }, throughput: 25100 }); - // readItemId = addEntropy("item1"); - // await container.items.create({ - // id: readItemId, - // key: "A", - // class: "2010" - // }); - // deleteItemId = addEntropy("item2"); - // await container.items.create({ - // id: deleteItemId, - // key: "A", - // class: "2010" - // }); - // replaceItemId = addEntropy("item3"); + readItemId = addEntropy("item1"); + await v2Container.items.create({ + id: readItemId, + key: "A", + class: "2010" + }); + deleteItemId = addEntropy("item2"); + await v2Container.items.create({ + id: deleteItemId, + key: "A", + class: "2010" + }); + replaceItemId = addEntropy("item3"); await v2Container.items.create({ id: replaceItemId, key: 5, @@ -316,26 +324,26 @@ describe.only("bulk item operations", function() { }); it("handles create, upsert, replace, delete", async function() { const operations: Operation[] = [ - // { - // operationType: "Create", - // partitionKey: `["A"]`, - // resourceBody: { id: "doc1", name: "sample", key: "A" } - // }, - // { - // operationType: "Upsert", - // partitionKey: `["A"]`, - // resourceBody: { id: "doc2", name: "other", key: "A" } - // }, - // { - // operationType: "Read", - // id: readItemId, - // partitionKey: `["A"]` - // }, - // { - // operationType: "Delete", - // id: deleteItemId, - // partitionKey: `["A"]` - // }, + { + operationType: "Create", + partitionKey: `["A"]`, + resourceBody: { id: addEntropy("doc1"), name: "sample", key: "A" } + }, + { + operationType: "Upsert", + partitionKey: `["A"]`, + resourceBody: { id: addEntropy("doc2"), name: "other", key: "A" } + }, + { + operationType: "Read", + id: readItemId, + partitionKey: `["A"]` + }, + { + operationType: "Delete", + id: deleteItemId, + partitionKey: `["A"]` + }, { operationType: "Replace", partitionKey: "[5]", @@ -344,8 +352,16 @@ describe.only("bulk item operations", function() { } ]; const response = await v2Container.items.bulk(operations); - console.log(response.map((item: any) => item.resourceBody)); - // assert.equal(response[0], 200); + // Create + assert.equal(response[0].statusCode, 201); + // Upsert + assert.equal(response[1].statusCode, 201); + // Read + assert.equal(response[2].statusCode, 200); + // Delete + assert.equal(response[3].statusCode, 204); + // Replace + assert.equal(response[4].statusCode, 200); }); }); }); From 68b1fe7007dcbbb1ec55087431e398b334982c1a Mon Sep 17 00:00:00 2001 From: Zach foster Date: Fri, 17 Jul 2020 16:57:27 -0400 Subject: [PATCH 13/26] Correctly orders operations --- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 31 ++++------ sdk/cosmosdb/cosmos/src/utils/batch.ts | 8 +++ .../cosmos/test/functional/item.spec.ts | 62 ++++++++++++++----- 3 files changed, 68 insertions(+), 33 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index f352bd3a2dc6..9334717f8597 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -6,7 +6,7 @@ import { ChangeFeedOptions } from "../../ChangeFeedOptions"; import { ClientContext } from "../../ClientContext"; import { getIdFromLink, getPathFromLink, isResourceValid, ResourceType } from "../../common"; import { extractPartitionKey } from "../../extractPartitionKey"; -import { FetchFunctionCallback, SqlQuerySpec, JSONObject } from "../../queryExecutionContext"; +import { FetchFunctionCallback, SqlQuerySpec } from "../../queryExecutionContext"; import { QueryIterator } from "../../queryIterator"; import { FeedOptions, RequestOptions } from "../../request"; import { Container, PartitionKeyRange } from "../Container"; @@ -18,7 +18,8 @@ import { isKeyInRange, Operation, getPartitionKeyToHash, - addPKToOperation + addPKToOperation, + OperationResponse } from "../../utils/batch"; import { hashV1PartitionKey } from "../../utils/hashing/v1"; import { hashV2PartitionKey } from "../../utils/hashing/v2"; @@ -386,14 +387,7 @@ export class Items { public async bulk( operations: Operation[], options?: RequestOptions - ): Promise< - { - statusCode: number; - requestCharge: number; - eTag?: string; - resourceBody?: JSONObject; - }[] - > { + ): Promise { const { resources: partitionKeyRanges } = await this.container.readPartitionKeyRanges().fetchAll(); @@ -403,12 +397,13 @@ export class Items { min: keyRange.minInclusive, max: keyRange.maxExclusive, rangeId: keyRange.id, + indexes: [], operations: [] }; }); operations .map((operation) => addPKToOperation(operation, definition)) - .forEach((operation: Operation) => { + .forEach((operation: Operation, index: number) => { const partitionProp = definition.paths[0].replace("/", ""); const isV2 = definition.version && definition.version === 2; const toHashKey = getPartitionKeyToHash(operation, partitionProp); @@ -417,11 +412,13 @@ export class Items { return isKeyInRange(batch.min, batch.max, hashed); }); batchForKey.operations.push(operation); + batchForKey.indexes.push(index); }); const path = getPathFromLink(this.container.url, ResourceType.item); - const responses = await Promise.all( + const orderedResponses: OperationResponse[] = []; + await Promise.all( batches .filter((batch: Batch) => batch.operations.length) .map(async (batch: Batch) => { @@ -433,14 +430,12 @@ export class Items { resourceId: this.container.url, options }); - return response; + response.result.forEach((operationResponse: OperationResponse, index: number) => { + orderedResponses[batch.indexes[index]] = operationResponse; + }); } catch (err) {} }) ); - return responses - .map((resp) => { - return resp.result; - }) - .flat(); + return orderedResponses; } } diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index 0eb450ce1847..8a2d71083791 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -13,9 +13,17 @@ export interface Batch { min: string; max: string; rangeId: string; + indexes: number[]; operations: Operation[]; } +export interface OperationResponse { + statusCode: number; + requestCharge: number; + eTag?: string; + resourceBody?: JSONObject; +} + export function isKeyInRange(min: string, max: string, key: string) { const isAfterMinInclusive = key.localeCompare(min) >= 0; const isBeforeMax = key.localeCompare(max) < 0; diff --git a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts index 0fbde1d75329..921f3abe8c41 100644 --- a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts +++ b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts @@ -216,8 +216,8 @@ describe("Item CRUD", function() { }); }); -describe("bulk item operations", function() { - describe.only("with v1 container", function() { +describe.only("bulk item operations", function() { + describe("with v1 container", function() { let container: Container; let readItemId: string; let replaceItemId: string; @@ -249,7 +249,7 @@ describe("bulk item operations", function() { class: "2010" }); }); - it("handles create, upsert, replace, delete", async function() { + it.only("handles create, upsert, replace, delete", async function() { const operations: Operation[] = [ { operationType: "Create", @@ -279,15 +279,19 @@ describe("bulk item operations", function() { ]; const response = await container.items.bulk(operations); // Create - assert.equal(response[0].statusCode, 200); + assert.equal(response[0].resourceBody.name, "sample"); + assert.equal(response[0].statusCode, 201); // Upsert + assert.equal(response[1].resourceBody.name, "other"); assert.equal(response[1].statusCode, 201); // Read - assert.equal(response[2].statusCode, 201); + assert.equal(response[2].resourceBody.class, "2010"); + assert.equal(response[2].statusCode, 200); // Delete - assert.equal(response[3].statusCode, 200); + assert.equal(response[3].statusCode, 204); // Replace - assert.equal(response[4].statusCode, 204); + assert.equal(response[4].resourceBody.name, "nice"); + assert.equal(response[4].statusCode, 200); }); }); describe("with v2 container", function() { @@ -306,20 +310,20 @@ describe("bulk item operations", function() { readItemId = addEntropy("item1"); await v2Container.items.create({ id: readItemId, - key: "A", + key: true, class: "2010" }); deleteItemId = addEntropy("item2"); await v2Container.items.create({ id: deleteItemId, - key: "A", - class: "2010" + key: {}, + class: "2011" }); replaceItemId = addEntropy("item3"); await v2Container.items.create({ id: replaceItemId, key: 5, - class: "2010" + class: "2012" }); }); it("handles create, upsert, replace, delete", async function() { @@ -331,18 +335,18 @@ describe("bulk item operations", function() { }, { operationType: "Upsert", - partitionKey: `["A"]`, - resourceBody: { id: addEntropy("doc2"), name: "other", key: "A" } + partitionKey: `["U"]`, + resourceBody: { id: addEntropy("doc2"), name: "other", key: "U" } }, { operationType: "Read", id: readItemId, - partitionKey: `["A"]` + partitionKey: `[true]` }, { operationType: "Delete", id: deleteItemId, - partitionKey: `["A"]` + partitionKey: `[{}]` }, { operationType: "Replace", @@ -353,15 +357,43 @@ describe("bulk item operations", function() { ]; const response = await v2Container.items.bulk(operations); // Create + assert.equal(response[0].resourceBody.name, "sample"); assert.equal(response[0].statusCode, 201); // Upsert + assert.equal(response[1].resourceBody.name, "other"); assert.equal(response[1].statusCode, 201); // Read + assert.equal(response[2].resourceBody.class, "2010"); assert.equal(response[2].statusCode, 200); // Delete assert.equal(response[3].statusCode, 204); // Replace + assert.equal(response[4].resourceBody.name, "nice"); assert.equal(response[4].statusCode, 200); }); + it("respects order", async function() { + readItemId = addEntropy("item1"); + await v2Container.items.create({ + id: readItemId, + key: "A", + class: "2010" + }); + const operations: Operation[] = [ + { + operationType: "Delete", + id: readItemId, + partitionKey: `["A"]` + }, + { + operationType: "Read", + id: readItemId, + partitionKey: `["A"]` + } + ]; + const response = await v2Container.items.bulk(operations); + assert.equal(response[0].statusCode, 204); + // Delete occurs first, so the read returns a 404 + assert.equal(response[1].statusCode, 404); + }); }); }); From 5ef791cd62f56607ff28e68244ab1e3fcb623f86 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Mon, 20 Jul 2020 13:35:01 -0400 Subject: [PATCH 14/26] Adds recursive bulk calls on 410 errors --- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 9334717f8597..4c821ced6502 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -433,7 +433,14 @@ export class Items { response.result.forEach((operationResponse: OperationResponse, index: number) => { orderedResponses[batch.indexes[index]] = operationResponse; }); - } catch (err) {} + } catch (err) { + console.log({ err }); + // In the case of 410 errors, we need to recompute the partition key ranges + // and redo the batch request + if (err.status === "410") { + return this.bulk(batch.operations); + } + } }) ); return orderedResponses; From c67756614d3bb8c98a86840b0294a409a24bc896 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Mon, 20 Jul 2020 18:06:19 -0400 Subject: [PATCH 15/26] Use BytePrefix everywhere --- sdk/cosmosdb/cosmos/src/ClientContext.ts | 3 +- .../cosmos/src/utils/hashing/encoding/null.ts | 3 -- .../src/utils/hashing/encoding/number.ts | 3 +- .../src/utils/hashing/encoding/prefix.ts | 44 +++++++++---------- .../src/utils/hashing/encoding/string.ts | 6 ++- sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts | 38 +++++++--------- sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts | 17 ++++--- .../cosmos/test/functional/item.spec.ts | 2 +- .../cosmos/test/unit/hashing/v2.spec.ts | 2 +- 9 files changed, 59 insertions(+), 59 deletions(-) delete mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/encoding/null.ts diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index e98331263ad4..90197df67fe6 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -139,7 +139,6 @@ export class ClientContext { request.operationType ); request.headers = await this.buildHeaders(request); - // request.headers[Constants.HttpHeaders.ForceRefresh] = "True"; if (query !== undefined) { request.headers[Constants.HttpHeaders.IsQuery] = "true"; request.headers[Constants.HttpHeaders.ContentType] = QueryJsonContentType; @@ -544,7 +543,7 @@ export class ClientContext { return this.globalEndpointManager.getReadEndpoint(); } - public async bulk({ + public async bulk({ body, path, resourceId, diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/null.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/null.ts deleted file mode 100644 index 4add58a62d50..000000000000 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/null.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function writeNullForBinaryEncoding() { - return Buffer.from("01", "hex"); -} diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts index a167398482c0..288e14030e0f 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/number.ts @@ -1,8 +1,9 @@ import JSBI from "jsbi"; +import { BytePrefix } from "./prefix"; export function writeNumberForBinaryEncodingJSBI(hash: number) { let payload = encodeNumberAsUInt64JSBI(hash); - let outputStream = Buffer.from("05", "hex"); + let outputStream = Buffer.from(BytePrefix.Number, "hex"); const firstChunk = JSBI.asUintN(64, JSBI.signedRightShift(payload, JSBI.BigInt(56))); outputStream = Buffer.concat([outputStream, Buffer.from(firstChunk.toString(16), "hex")]); diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/prefix.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/prefix.ts index 7b36891c0aea..1d07e6318234 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/prefix.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/prefix.ts @@ -1,24 +1,24 @@ export const BytePrefix = { - Undefined: 0x0, - Null: 0x1, - False: 0x2, - True: 0x3, - MinNumber: 0x4, - Number: 0x5, - MaxNumber: 0x6, - MinString: 0x7, - String: 0x8, - MaxString: 0x9, - Int64: 0xa, - Int32: 0xb, - Int16: 0xc, - Int8: 0xd, - Uint64: 0xe, - Uint32: 0xf, - Uint16: 0x10, - Uint8: 0x11, - Binary: 0x12, - Guid: 0x13, - Float: 0x14, - Infinity: 0xff + Undefined: "00", + Null: "01", + False: "02", + True: "03", + MinNumber: "04", + Number: "05", + MaxNumber: "06", + MinString: "07", + String: "08", + MaxString: "09", + Int64: "0a", + Int32: "0b", + Int16: "0c", + Int8: "0d", + Uint64: "0e", + Uint32: "0f", + Uint16: "10", + Uint8: "11", + Binary: "12", + Guid: "13", + Float: "14", + Infinity: "FF" }; diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/string.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/string.ts index 4c25dea2b373..b9e44b7d0108 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/string.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/encoding/string.ts @@ -1,5 +1,7 @@ +import { BytePrefix } from "./prefix"; + export function writeStringForBinaryEncoding(payload: string) { - let outputStream = Buffer.from("08", "hex"); + let outputStream = Buffer.from(BytePrefix.String, "hex"); const MAX_STRING_BYTES_TO_APPEND = 100; const byteArray = [...Buffer.from(payload)]; @@ -16,7 +18,7 @@ export function writeStringForBinaryEncoding(payload: string) { } if (isShortString) { - outputStream = Buffer.concat([outputStream, Buffer.from("00", "hex")]); + outputStream = Buffer.concat([outputStream, Buffer.from(BytePrefix.Undefined, "hex")]); } return outputStream; } diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts index b0b04e3d8ef1..d460106c1d6d 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts @@ -1,18 +1,16 @@ -import { - // writeNumberForBinaryEncoding, - doubleToByteArrayJSBI, - writeNumberForBinaryEncodingJSBI -} from "./encoding/number"; +import { doubleToByteArrayJSBI, writeNumberForBinaryEncodingJSBI } from "./encoding/number"; import { writeStringForBinaryEncoding } from "./encoding/string"; +import { BytePrefix } from "./encoding/prefix"; const MurmurHash = require("./murmurHash").default; +const MAX_STRING_CHARS = 100; + type v1Key = string | number | null | {} | undefined; export function hashV1PartitionKey(partitionKey: v1Key): string { const toHash = prefixKeyByType(partitionKey); const hash = MurmurHash.x86.hash32(toHash); const encodedJSBI = writeNumberForBinaryEncodingJSBI(hash); - // const encodedHash = writeNumberForBinaryEncoding(hash); const encodedValue = encodeByType(partitionKey); return Buffer.concat([encodedJSBI, encodedValue]) .toString("hex") @@ -23,49 +21,47 @@ function prefixKeyByType(key: v1Key) { let bytes: Buffer; switch (typeof key) { case "string": - const truncated = key.substr(0, 100); + const truncated = key.substr(0, MAX_STRING_CHARS); bytes = Buffer.concat([ - Buffer.from("08", "hex"), + Buffer.from(BytePrefix.String, "hex"), Buffer.from(truncated), - Buffer.from("00", "hex") + Buffer.from(BytePrefix.Undefined, "hex") ]); return bytes; case "number": const numberBytes = doubleToByteArrayJSBI(key); - bytes = Buffer.concat([Buffer.from("05", "hex"), numberBytes]); + bytes = Buffer.concat([Buffer.from(BytePrefix.Number, "hex"), numberBytes]); return bytes; case "boolean": - const prefix = key ? "03" : "02"; + const prefix = key ? BytePrefix.True : BytePrefix.False; return Buffer.from(prefix, "hex"); case "object": if (key === null) { - return Buffer.from("01", "hex"); + return Buffer.from(BytePrefix.Null, "hex"); } - return Buffer.from("00", "hex"); + return Buffer.from(BytePrefix.Undefined, "hex"); case "undefined": - return Buffer.from("00", "hex"); + return Buffer.from(BytePrefix.Undefined, "hex"); } } function encodeByType(key: v1Key) { switch (typeof key) { case "string": - const truncated = key.substr(0, 100); + const truncated = key.substr(0, MAX_STRING_CHARS); return writeStringForBinaryEncoding(truncated); case "number": const encodedJSBI = writeNumberForBinaryEncodingJSBI(key); - // const encoded = writeNumberForBinaryEncoding(key); return encodedJSBI; - // return encoded; case "boolean": - const prefix = key ? "03" : "02"; + const prefix = key ? BytePrefix.True : BytePrefix.False; return Buffer.from(prefix, "hex"); case "object": if (key === null) { - return Buffer.from("01", "hex"); + return Buffer.from(BytePrefix.Null, "hex"); } - return Buffer.from("00", "hex"); + return Buffer.from(BytePrefix.Undefined, "hex"); case "undefined": - return Buffer.from("00", "hex"); + return Buffer.from(BytePrefix.Undefined, "hex"); } } diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts index 2bbf9dfaf6b6..f688d77b9301 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts @@ -1,4 +1,5 @@ import { doubleToByteArrayJSBI } from "./encoding/number"; +import { BytePrefix } from "./encoding/prefix"; const MurmurHash = require("./murmurHash").default; type v1Key = string | number | null | {} | undefined; @@ -15,22 +16,26 @@ function prefixKeyByType(key: v1Key) { let bytes: Buffer; switch (typeof key) { case "string": - bytes = Buffer.concat([Buffer.from("08", "hex"), Buffer.from(key), Buffer.from("FF", "hex")]); + bytes = Buffer.concat([ + Buffer.from(BytePrefix.String, "hex"), + Buffer.from(key), + Buffer.from(BytePrefix.Infinity, "hex") + ]); return bytes; case "number": const numberBytes = doubleToByteArrayJSBI(key); - bytes = Buffer.concat([Buffer.from("05", "hex"), numberBytes]); + bytes = Buffer.concat([Buffer.from(BytePrefix.String, "hex"), numberBytes]); return bytes; case "boolean": - const prefix = key ? "03" : "02"; + const prefix = key ? BytePrefix.True : BytePrefix.False; return Buffer.from(prefix, "hex"); case "object": if (key === null) { - return Buffer.from("01", "hex"); + return Buffer.from(BytePrefix.Null, "hex"); } - return Buffer.from("00", "hex"); + return Buffer.from(BytePrefix.Undefined, "hex"); case "undefined": - return Buffer.from("00", "hex"); + return Buffer.from(BytePrefix.Undefined, "hex"); } } diff --git a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts index 921f3abe8c41..d85b1123467d 100644 --- a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts +++ b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts @@ -249,7 +249,7 @@ describe.only("bulk item operations", function() { class: "2010" }); }); - it.only("handles create, upsert, replace, delete", async function() { + it("handles create, upsert, replace, delete", async function() { const operations: Operation[] = [ { operationType: "Create", diff --git a/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts b/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts index 784768a03a29..7eb8ffe168ac 100644 --- a/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts +++ b/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts @@ -1,7 +1,7 @@ import assert from "assert"; import { hashV2PartitionKey } from "../../../src/utils/hashing/v2"; -describe("effectivePartitionKey", function() { +describe.only("effectivePartitionKey", function() { describe("computes v2 key", function() { const toMatch = [ { From 5e256ff1782c9a22652c2ff3f8f9cfa95293007c Mon Sep 17 00:00:00 2001 From: Zach foster Date: Mon, 20 Jul 2020 18:12:34 -0400 Subject: [PATCH 16/26] Fix hash prefix --- sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts | 8 ++++---- sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts index f688d77b9301..52e6f1540140 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts @@ -2,9 +2,9 @@ import { doubleToByteArrayJSBI } from "./encoding/number"; import { BytePrefix } from "./encoding/prefix"; const MurmurHash = require("./murmurHash").default; -type v1Key = string | number | null | {} | undefined; +type v2Key = string | number | null | {} | undefined; -export function hashV2PartitionKey(partitionKey: v1Key): string { +export function hashV2PartitionKey(partitionKey: v2Key): string { const toHash = prefixKeyByType(partitionKey); const hash = MurmurHash.x64.hash128(toHash); const reverseBuff: Buffer = reverse(Buffer.from(hash, "hex")); @@ -12,7 +12,7 @@ export function hashV2PartitionKey(partitionKey: v1Key): string { return reverseBuff.toString("hex").toUpperCase(); } -function prefixKeyByType(key: v1Key) { +function prefixKeyByType(key: v2Key) { let bytes: Buffer; switch (typeof key) { case "string": @@ -24,7 +24,7 @@ function prefixKeyByType(key: v1Key) { return bytes; case "number": const numberBytes = doubleToByteArrayJSBI(key); - bytes = Buffer.concat([Buffer.from(BytePrefix.String, "hex"), numberBytes]); + bytes = Buffer.concat([Buffer.from(BytePrefix.Number, "hex"), numberBytes]); return bytes; case "boolean": const prefix = key ? BytePrefix.True : BytePrefix.False; diff --git a/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts b/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts index 7eb8ffe168ac..784768a03a29 100644 --- a/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts +++ b/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts @@ -1,7 +1,7 @@ import assert from "assert"; import { hashV2PartitionKey } from "../../../src/utils/hashing/v2"; -describe.only("effectivePartitionKey", function() { +describe("effectivePartitionKey", function() { describe("computes v2 key", function() { const toMatch = [ { From 02478707e42c1edfda59aed975dac1fecf26f0cd Mon Sep 17 00:00:00 2001 From: Zach foster Date: Mon, 20 Jul 2020 19:00:57 -0400 Subject: [PATCH 17/26] Fixes {} case and errors on 410s --- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 10 ++++++---- sdk/cosmosdb/cosmos/src/request/request.ts | 4 ++-- sdk/cosmosdb/cosmos/src/utils/batch.ts | 15 ++++++++------- sdk/cosmosdb/cosmos/test/functional/item.spec.ts | 2 +- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 4c821ced6502..2369f6c8b9b0 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -434,11 +434,13 @@ export class Items { orderedResponses[batch.indexes[index]] = operationResponse; }); } catch (err) { - console.log({ err }); // In the case of 410 errors, we need to recompute the partition key ranges - // and redo the batch request - if (err.status === "410") { - return this.bulk(batch.operations); + // and redo the batch request, however, 410 errors occur for unsupported + // partition key types as well since we don't support them, so for now we throw + if (err.code === 410) { + throw new Error( + "Partition key error. Either the partitions have split or an operation has an unsupported partitionKey type" + ); } } }) diff --git a/sdk/cosmosdb/cosmos/src/request/request.ts b/sdk/cosmosdb/cosmos/src/request/request.ts index 102b3dcc538f..23c2dffa8c81 100644 --- a/sdk/cosmosdb/cosmos/src/request/request.ts +++ b/sdk/cosmosdb/cosmos/src/request/request.ts @@ -63,8 +63,8 @@ export async function getHeaders({ partitionKey }: GetHeadersOptions): Promise { const headers: CosmosHeaders = { - // [Constants.HttpHeaders.ResponseContinuationTokenLimitInKB]: 1, - // [Constants.HttpHeaders.EnableCrossPartitionQuery]: true, + [Constants.HttpHeaders.ResponseContinuationTokenLimitInKB]: 1, + [Constants.HttpHeaders.EnableCrossPartitionQuery]: true, ...defaultHeaders }; diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index 8a2d71083791..7dfc18a0dc89 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -70,14 +70,15 @@ export function hasResource( } export function getPartitionKeyToHash(operation: Operation, partitionProperty: string) { - return hasResource(operation) + const toHashKey = hasResource(operation) ? (operation.resourceBody as any)[partitionProperty] - : operation.partitionKey - .replace("[", "") - .replace("]", "") - .replace("'", "") - .replace('"', "") - .replace('"', ""); + : operation.partitionKey.replace(/[\[\]\"\']/g, ""); + // We check for empty object since replace will stringify the value + // The second check avoids cases where the partitionKey value is actually the string '{}' + if (toHashKey === "{}" && operation.partitionKey === "[{}]") { + return {}; + } + return toHashKey; } export function addPKToOperation(operation: Operation, definition: PartitionKeyDefinition) { diff --git a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts index d85b1123467d..6dfb5b7d8113 100644 --- a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts +++ b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts @@ -216,7 +216,7 @@ describe("Item CRUD", function() { }); }); -describe.only("bulk item operations", function() { +describe("bulk item operations", function() { describe("with v1 container", function() { let container: Container; let readItemId: string; From 0e6948f69b4d2283cf5fb5ef916e0f5ce4f458a1 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Tue, 21 Jul 2020 11:46:25 -0400 Subject: [PATCH 18/26] Rush update and es6 --- common/config/rush/pnpm-lock.yaml | 7 ++++++- sdk/cosmosdb/cosmos/test/tsconfig.json | 2 +- sdk/cosmosdb/cosmos/tsconfig.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index da348009f792..04283cf12170 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -3805,6 +3805,10 @@ packages: hasBin: true resolution: integrity: sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A== + /jsbi/3.1.3: + dev: false + resolution: + integrity: sha512-nBJqA0C6Qns+ZxurbEoIR56wyjiUszpNy70FHvxO5ervMoCbZVE3z3kxr5nKGhlxr/9MhKTSUBs7cAwwuf3g9w== /jsbn/0.1.1: dev: false resolution: @@ -8014,6 +8018,7 @@ packages: esm: 3.2.25 execa: 3.4.0 fast-json-stable-stringify: 2.1.0 + jsbi: 3.1.3 mocha: 7.2.0 mocha-junit-reporter: 1.23.3_mocha@7.2.0 node-abort-controller: 1.0.4 @@ -8040,7 +8045,7 @@ packages: dev: false name: '@rush-temp/cosmos' resolution: - integrity: sha512-AEA+nq1Up+clYIiCU7RhTM+oZZ/EaPeJ44lZCT5K+9zIQhdVK7SnXEh7R1W7ySRlBc/wvUH2v5M2gZPa2pYcPA== + integrity: sha512-56MLoy65KJ3OvenjZHEQtbr9gf/FVlnjeuRqTeEcuYo1ZHfJCNx9081w7mfFiRihlVjPovM3hQ7sLKZ7/SW44A== tarball: 'file:projects/cosmos.tgz' version: 0.0.0 'file:projects/eslint-plugin-azure-sdk.tgz': diff --git a/sdk/cosmosdb/cosmos/test/tsconfig.json b/sdk/cosmosdb/cosmos/test/tsconfig.json index bb05fe793129..d515d04ab499 100644 --- a/sdk/cosmosdb/cosmos/test/tsconfig.json +++ b/sdk/cosmosdb/cosmos/test/tsconfig.json @@ -9,7 +9,7 @@ "allowSyntheticDefaultImports": true, "preserveConstEnums": true, "removeComments": false, - "target": "es2020", + "target": "es6", "sourceMap": true, "newLine": "LF", "composite": true, diff --git a/sdk/cosmosdb/cosmos/tsconfig.json b/sdk/cosmosdb/cosmos/tsconfig.json index 187d24054bed..ac0425d109e9 100644 --- a/sdk/cosmosdb/cosmos/tsconfig.json +++ b/sdk/cosmosdb/cosmos/tsconfig.json @@ -11,7 +11,7 @@ "outDir": "dist-esm", "preserveConstEnums": true, "removeComments": false, - "target": "es2020", + "target": "es6", "sourceMap": true, "inlineSources": true, "newLine": "LF", From 8025fcfbca651ee678f72ef4cae749216ed69c36 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Wed, 22 Jul 2020 14:57:02 -0400 Subject: [PATCH 19/26] Add .d.ts murmurHash file --- sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.d.ts | 1 + 1 file changed, 1 insertion(+) create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.d.ts diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.d.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.d.ts new file mode 100644 index 000000000000..66536489d541 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.d.ts @@ -0,0 +1 @@ +declare module murmurHash {} \ No newline at end of file From ec1d194f659e5353870d6c871e4e135e34b12ae7 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Wed, 22 Jul 2020 16:55:26 -0400 Subject: [PATCH 20/26] Adds murmurHash as ts file --- .../cosmos/src/utils/hashing/murmurHash.d.ts | 1 - .../hashing/{murmurHash.js => murmurHash.ts} | 186 ++++++++---------- sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts | 2 +- sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts | 2 +- 4 files changed, 89 insertions(+), 102 deletions(-) delete mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.d.ts rename sdk/cosmosdb/cosmos/src/utils/hashing/{murmurHash.js => murmurHash.ts} (76%) diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.d.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.d.ts deleted file mode 100644 index 66536489d541..000000000000 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.d.ts +++ /dev/null @@ -1 +0,0 @@ -declare module murmurHash {} \ No newline at end of file diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.js b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.ts similarity index 76% rename from sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.js rename to sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.ts index c7663119dc63..c5a01e35347c 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.js +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.ts @@ -1,4 +1,3 @@ -/* jshint -W086: true */ // +----------------------------------------------------------------------+ // | murmurHash3js.js v3.0.1 // https://github.com/pid/murmurHash3js // | A javascript implementation of MurmurHash3's x86 hashing algorithms. | @@ -8,27 +7,10 @@ // | Freely distributable under the MIT license. | // +----------------------------------------------------------------------+ -"use strict"; - // PRIVATE FUNCTIONS // ----------------- -function _validBytes(bytes) { - // check the input is an array or a typed array - if (!Array.isArray(bytes) && !ArrayBuffer.isView(bytes)) { - return false; - } - - // check all bytes are actually bytes - for (var i = 0; i < bytes.length; i++) { - if (!Number.isInteger(bytes[i]) || bytes[i] < 0 || bytes[i] > 255) { - return false; - } - } - return true; -} - -function _x86Multiply(m, n) { +function _x86Multiply(m: number, n: number) { // // Given two 32bit ints, returns the two multiplied together as a // 32bit int. @@ -37,7 +19,7 @@ function _x86Multiply(m, n) { return (m & 0xffff) * n + ((((m >>> 16) * n) & 0xffff) << 16); } -function _x86Rotl(m, n) { +function _x86Rotl(m: number, n: number) { // // Given a 32bit int and an int representing a number of bit positions, // returns the 32bit int rotated left by that number of positions. @@ -46,7 +28,7 @@ function _x86Rotl(m, n) { return (m << n) | (m >>> (32 - n)); } -function _x86Fmix(h) { +function _x86Fmix(h: number) { // // Given a block, returns murmurHash3's final x86 mix of that block. // @@ -60,7 +42,7 @@ function _x86Fmix(h) { return h; } -function _x64Add(m, n) { +function _x64Add(m: number[], n: number[]) { // // Given two 64bit ints (as an array of two 32bit ints) returns the two // added together as a 64bit int (as an array of two 32bit ints). @@ -68,7 +50,7 @@ function _x64Add(m, n) { m = [m[0] >>> 16, m[0] & 0xffff, m[1] >>> 16, m[1] & 0xffff]; n = [n[0] >>> 16, n[0] & 0xffff, n[1] >>> 16, n[1] & 0xffff]; - var o = [0, 0, 0, 0]; + const o = [0, 0, 0, 0]; o[3] += m[3] + n[3]; o[2] += o[3] >>> 16; @@ -88,7 +70,7 @@ function _x64Add(m, n) { return [(o[0] << 16) | o[1], (o[2] << 16) | o[3]]; } -function _x64Multiply(m, n) { +function _x64Multiply(m: number[], n: number[]) { // // Given two 64bit ints (as an array of two 32bit ints) returns the two // multiplied together as a 64bit int (as an array of two 32bit ints). @@ -96,7 +78,7 @@ function _x64Multiply(m, n) { m = [m[0] >>> 16, m[0] & 0xffff, m[1] >>> 16, m[1] & 0xffff]; n = [n[0] >>> 16, n[0] & 0xffff, n[1] >>> 16, n[1] & 0xffff]; - var o = [0, 0, 0, 0]; + const o = [0, 0, 0, 0]; o[3] += m[3] * n[3]; o[2] += o[3] >>> 16; @@ -128,7 +110,7 @@ function _x64Multiply(m, n) { return [(o[0] << 16) | o[1], (o[2] << 16) | o[3]]; } -function _x64Rotl(m, n) { +function _x64Rotl(m: number[], n: number) { // // Given a 64bit int (as an array of two 32bit ints) and an int // representing a number of bit positions, returns the 64bit int (as an @@ -147,7 +129,7 @@ function _x64Rotl(m, n) { } } -function _x64LeftShift(m, n) { +function _x64LeftShift(m: number[], n: number) { // // Given a 64bit int (as an array of two 32bit ints) and an int // representing a number of bit positions, returns the 64bit int (as an @@ -165,7 +147,7 @@ function _x64LeftShift(m, n) { } } -function _x64Xor(m, n) { +function _x64Xor(m: number[], n: number[]) { // // Given two 64bit ints (as an array of two 32bit ints) returns the two // xored together as a 64bit int (as an array of two 32bit ints). @@ -174,7 +156,7 @@ function _x64Xor(m, n) { return [m[0] ^ n[0], m[1] ^ n[1]]; } -function _x64Fmix(h) { +function _x64Fmix(h: number[]) { // // Given a block, returns murmurHash3's final x64 mix of that block. // (`[0, h[0] >>> 1]` is a 33 bit unsigned right shift. This is the @@ -193,25 +175,26 @@ function _x64Fmix(h) { // PUBLIC FUNCTIONS // ---------------- -function x86Hash32(bytes, seed) { +function x86Hash32(bytes: Buffer, seed?: number) { // // Given a string and an optional seed as an int, returns a 32 bit hash // using the x86 flavor of MurmurHash3, as an unsigned int. // seed = seed || 0; - var remainder = bytes.length % 4; - var blocks = bytes.length - remainder; + const remainder = bytes.length % 4; + const blocks = bytes.length - remainder; - var h1 = seed; + let h1 = seed; - var k1 = 0; + let k1 = 0; + let i; - var c1 = 0xcc9e2d51; - var c2 = 0x1b873593; + const c1 = 0xcc9e2d51; + const c2 = 0x1b873593; - for (var i = 0; i < blocks; i = i + 4) { - k1 = bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24); + for (let j = 0; j < blocks; j = j + 4) { + k1 = bytes[j] | (bytes[j + 1] << 8) | (bytes[j + 2] << 16) | (bytes[j + 3] << 24); k1 = _x86Multiply(k1, c1); k1 = _x86Rotl(k1, 15); @@ -220,6 +203,7 @@ function x86Hash32(bytes, seed) { h1 ^= k1; h1 = _x86Rotl(h1, 13); h1 = _x86Multiply(h1, 5) + 0xe6546b64; + i = j; } k1 = 0; @@ -245,32 +229,33 @@ function x86Hash32(bytes, seed) { return h1 >>> 0; } -function x86Hash128(bytes, seed) { +function x86Hash128(bytes: Buffer, seed?: number) { // // Given a string and an optional seed as an int, returns a 128 bit // hash using the x86 flavor of MurmurHash3, as an unsigned hex. // seed = seed || 0; - var remainder = bytes.length % 16; - var blocks = bytes.length - remainder; - - var h1 = seed; - var h2 = seed; - var h3 = seed; - var h4 = seed; - - var k1 = 0; - var k2 = 0; - var k3 = 0; - var k4 = 0; - - var c1 = 0x239b961b; - var c2 = 0xab0e9789; - var c3 = 0x38b34ae5; - var c4 = 0xa1e38b93; - - for (var i = 0; i < blocks; i = i + 16) { + const remainder = bytes.length % 16; + const blocks = bytes.length - remainder; + + let h1 = seed; + let h2 = seed; + let h3 = seed; + let h4 = seed; + + let k1 = 0; + let k2 = 0; + let k3 = 0; + let k4 = 0; + + const c1 = 0x239b961b; + const c2 = 0xab0e9789; + const c3 = 0x38b34ae5; + const c4 = 0xa1e38b93; + let j; + + for (let i = 0; i < blocks; i = i + 16) { k1 = bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24); k2 = bytes[i + 4] | (bytes[i + 5] << 8) | (bytes[i + 6] << 16) | (bytes[i + 7] << 24); k3 = bytes[i + 8] | (bytes[i + 9] << 8) | (bytes[i + 10] << 16) | (bytes[i + 11] << 24); @@ -311,6 +296,7 @@ function x86Hash128(bytes, seed) { h4 = _x86Rotl(h4, 13); h4 += h1; h4 = _x86Multiply(h4, 5) + 0x32ac3b17; + j = i; } k1 = 0; @@ -320,61 +306,61 @@ function x86Hash128(bytes, seed) { switch (remainder) { case 15: - k4 ^= bytes[i + 14] << 16; + k4 ^= bytes[j + 14] << 16; case 14: - k4 ^= bytes[i + 13] << 8; + k4 ^= bytes[j + 13] << 8; case 13: - k4 ^= bytes[i + 12]; + k4 ^= bytes[j + 12]; k4 = _x86Multiply(k4, c4); k4 = _x86Rotl(k4, 18); k4 = _x86Multiply(k4, c1); h4 ^= k4; case 12: - k3 ^= bytes[i + 11] << 24; + k3 ^= bytes[j + 11] << 24; case 11: - k3 ^= bytes[i + 10] << 16; + k3 ^= bytes[j + 10] << 16; case 10: - k3 ^= bytes[i + 9] << 8; + k3 ^= bytes[j + 9] << 8; case 9: - k3 ^= bytes[i + 8]; + k3 ^= bytes[j + 8]; k3 = _x86Multiply(k3, c3); k3 = _x86Rotl(k3, 17); k3 = _x86Multiply(k3, c4); h3 ^= k3; case 8: - k2 ^= bytes[i + 7] << 24; + k2 ^= bytes[j + 7] << 24; case 7: - k2 ^= bytes[i + 6] << 16; + k2 ^= bytes[j + 6] << 16; case 6: - k2 ^= bytes[i + 5] << 8; + k2 ^= bytes[j + 5] << 8; case 5: - k2 ^= bytes[i + 4]; + k2 ^= bytes[j + 4]; k2 = _x86Multiply(k2, c2); k2 = _x86Rotl(k2, 16); k2 = _x86Multiply(k2, c3); h2 ^= k2; case 4: - k1 ^= bytes[i + 3] << 24; + k1 ^= bytes[j + 3] << 24; case 3: - k1 ^= bytes[i + 2] << 16; + k1 ^= bytes[j + 2] << 16; case 2: - k1 ^= bytes[i + 1] << 8; + k1 ^= bytes[j + 1] << 8; case 1: - k1 ^= bytes[i]; + k1 ^= bytes[j]; k1 = _x86Multiply(k1, c1); k1 = _x86Rotl(k1, 15); k1 = _x86Multiply(k1, c2); @@ -413,26 +399,27 @@ function x86Hash128(bytes, seed) { ); } -function x64Hash128(bytes, seed) { +function x64Hash128(bytes: Buffer, seed?: number) { // // Given a string and an optional seed as an int, returns a 128 bit // hash using the x64 flavor of MurmurHash3, as an unsigned hex. // seed = seed || 0; - var remainder = bytes.length % 16; - var blocks = bytes.length - remainder; + const remainder = bytes.length % 16; + const blocks = bytes.length - remainder; - var h1 = [0, seed]; - var h2 = [0, seed]; + let h1 = [0, seed]; + let h2 = [0, seed]; - var k1 = [0, 0]; - var k2 = [0, 0]; + let k1 = [0, 0]; + let k2 = [0, 0]; - var c1 = [0x87c37b91, 0x114253d5]; - var c2 = [0x4cf5ad43, 0x2745937f]; + const c1 = [0x87c37b91, 0x114253d5]; + const c2 = [0x4cf5ad43, 0x2745937f]; + let j; - for (var i = 0; i < blocks; i = i + 16) { + for (let i = 0; i < blocks; i = i + 16) { k1 = [ bytes[i + 4] | (bytes[i + 5] << 8) | (bytes[i + 6] << 16) | (bytes[i + 7] << 24), bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24), @@ -459,6 +446,7 @@ function x64Hash128(bytes, seed) { h2 = _x64Rotl(h2, 31); h2 = _x64Add(h2, h1); h2 = _x64Add(_x64Multiply(h2, [0, 5]), [0, 0x38495ab5]); + j = i; } k1 = [0, 0]; @@ -466,53 +454,53 @@ function x64Hash128(bytes, seed) { switch (remainder) { case 15: - k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 14]], 48)); + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[j + 14]], 48)); case 14: - k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 13]], 40)); + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[j + 13]], 40)); case 13: - k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 12]], 32)); + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[j + 12]], 32)); case 12: - k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 11]], 24)); + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[j + 11]], 24)); case 11: - k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 10]], 16)); + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[j + 10]], 16)); case 10: - k2 = _x64Xor(k2, _x64LeftShift([0, bytes[i + 9]], 8)); + k2 = _x64Xor(k2, _x64LeftShift([0, bytes[j + 9]], 8)); case 9: - k2 = _x64Xor(k2, [0, bytes[i + 8]]); + k2 = _x64Xor(k2, [0, bytes[j + 8]]); k2 = _x64Multiply(k2, c2); k2 = _x64Rotl(k2, 33); k2 = _x64Multiply(k2, c1); h2 = _x64Xor(h2, k2); case 8: - k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 7]], 56)); + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[j + 7]], 56)); case 7: - k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 6]], 48)); + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[j + 6]], 48)); case 6: - k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 5]], 40)); + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[j + 5]], 40)); case 5: - k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 4]], 32)); + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[j + 4]], 32)); case 4: - k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 3]], 24)); + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[j + 3]], 24)); case 3: - k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 2]], 16)); + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[j + 2]], 16)); case 2: - k1 = _x64Xor(k1, _x64LeftShift([0, bytes[i + 1]], 8)); + k1 = _x64Xor(k1, _x64LeftShift([0, bytes[j + 1]], 8)); case 1: - k1 = _x64Xor(k1, [0, bytes[i]]); + k1 = _x64Xor(k1, [0, bytes[j]]); k1 = _x64Multiply(k1, c1); k1 = _x64Rotl(k1, 31); k1 = _x64Multiply(k1, c2); @@ -548,7 +536,7 @@ function x64Hash128(bytes, seed) { return h1Reversed + h2Reversed; } -export function reverse(buff) { +export function reverse(buff: Buffer) { const buffer = Buffer.allocUnsafe(buff.length); for (let i = 0, j = buff.length - 1; i <= j; ++i, --j) { diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts index a1ee735dbd15..87141f41276c 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts @@ -2,7 +2,7 @@ import { doubleToByteArrayJSBI, writeNumberForBinaryEncodingJSBI } from "./encod import { writeStringForBinaryEncoding } from "./encoding/string"; import { BytePrefix } from "./encoding/prefix"; // tslint:disable-next-line -const MurmurHash = require("./murmurHash").default; +import MurmurHash from "./murmurHash"; const MAX_STRING_CHARS = 100; diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts index 0fa1159560c7..4ec63110d642 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts @@ -1,7 +1,7 @@ import { doubleToByteArrayJSBI } from "./encoding/number"; import { BytePrefix } from "./encoding/prefix"; // tslint:disable-next-line -const MurmurHash = require("./murmurHash").default; +import MurmurHash from "./murmurHash"; type v2Key = string | number | null | {} | undefined; From d74f1e3e7b6ad52266c32e0bb9e6ffd97746cbd3 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Wed, 22 Jul 2020 19:38:26 -0400 Subject: [PATCH 21/26] Fix converted murmurhash by removing vars --- .../cosmos/src/utils/hashing/murmurHash.ts | 22 ++++++------ .../cosmos/test/unit/hashing/v1.spec.ts | 34 +++++++++---------- .../cosmos/test/unit/hashing/v2.spec.ts | 32 ++++++++--------- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.ts index c5a01e35347c..06a19f4e29fc 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/murmurHash.ts @@ -188,13 +188,13 @@ function x86Hash32(bytes: Buffer, seed?: number) { let h1 = seed; let k1 = 0; - let i; const c1 = 0xcc9e2d51; const c2 = 0x1b873593; + let j = 0; - for (let j = 0; j < blocks; j = j + 4) { - k1 = bytes[j] | (bytes[j + 1] << 8) | (bytes[j + 2] << 16) | (bytes[j + 3] << 24); + for (let i = 0; i < blocks; i = i + 4) { + k1 = bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24); k1 = _x86Multiply(k1, c1); k1 = _x86Rotl(k1, 15); @@ -203,20 +203,20 @@ function x86Hash32(bytes: Buffer, seed?: number) { h1 ^= k1; h1 = _x86Rotl(h1, 13); h1 = _x86Multiply(h1, 5) + 0xe6546b64; - i = j; + j = i + 4; } k1 = 0; switch (remainder) { case 3: - k1 ^= bytes[i + 2] << 16; + k1 ^= bytes[j + 2] << 16; case 2: - k1 ^= bytes[i + 1] << 8; + k1 ^= bytes[j + 1] << 8; case 1: - k1 ^= bytes[i]; + k1 ^= bytes[j]; k1 = _x86Multiply(k1, c1); k1 = _x86Rotl(k1, 15); k1 = _x86Multiply(k1, c2); @@ -253,7 +253,7 @@ function x86Hash128(bytes: Buffer, seed?: number) { const c2 = 0xab0e9789; const c3 = 0x38b34ae5; const c4 = 0xa1e38b93; - let j; + let j = 0; for (let i = 0; i < blocks; i = i + 16) { k1 = bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24); @@ -296,7 +296,7 @@ function x86Hash128(bytes: Buffer, seed?: number) { h4 = _x86Rotl(h4, 13); h4 += h1; h4 = _x86Multiply(h4, 5) + 0x32ac3b17; - j = i; + j = i + 16; } k1 = 0; @@ -417,7 +417,7 @@ function x64Hash128(bytes: Buffer, seed?: number) { const c1 = [0x87c37b91, 0x114253d5]; const c2 = [0x4cf5ad43, 0x2745937f]; - let j; + let j = 0; for (let i = 0; i < blocks; i = i + 16) { k1 = [ @@ -446,7 +446,7 @@ function x64Hash128(bytes: Buffer, seed?: number) { h2 = _x64Rotl(h2, 31); h2 = _x64Add(h2, h1); h2 = _x64Add(_x64Multiply(h2, [0, 5]), [0, 0x38495ab5]); - j = i; + j = i + 16; } k1 = [0, 0]; diff --git a/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts b/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts index 0cdca97d21f1..42c15e7083e7 100644 --- a/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts +++ b/sdk/cosmosdb/cosmos/test/unit/hashing/v1.spec.ts @@ -1,66 +1,66 @@ import assert from "assert"; import { hashV1PartitionKey } from "../../../src/utils/hashing/v1"; -describe("effectivePartitionKey", function() { - describe("computes v1 key", function() { +describe("effectivePartitionKey", function () { + describe("computes v1 key", function () { const toMatch = [ { key: "partitionKey", - output: "05C1E1B3D9CD2608716273756A756A706F4C667A00" + output: "05C1E1B3D9CD2608716273756A756A706F4C667A00", }, { key: "redmond", - output: "05C1EFE313830C087366656E706F6500" + output: "05C1EFE313830C087366656E706F6500", }, { key: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", output: - "05C1EB5921F706086262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626200" + "05C1EB5921F706086262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626200", }, { key: "", - output: "05C1CF33970FF80800" + output: "05C1CF33970FF80800", }, { key: "aa", - output: "05C1C7B7270FE008626200" + output: "05C1C7B7270FE008626200", }, { key: null, - output: "05C1ED45D7475601" + output: "05C1ED45D7475601", }, { key: true, - output: "05C1D7C5A903D803" + output: "05C1D7C5A903D803", }, { key: false, - output: "05C1DB857D857C02" + output: "05C1DB857D857C02", }, { key: {}, - output: "05C1D529E345DC00" + output: "05C1D529E345DC00", }, { key: 5, - output: "05C1D9C1C5517C05C014" + output: "05C1D9C1C5517C05C014", }, { key: 5.5, - output: "05C1D7A771716C05C016" + output: "05C1D7A771716C05C016", }, { key: 12313.1221, - output: "05C1ED154D592E05C0C90723F50FC925D8" + output: "05C1ED154D592E05C0C90723F50FC925D8", }, { key: 123456789, - output: "05C1D9E1A5311C05C19DB7CD8B40" - } + output: "05C1D9E1A5311C05C19DB7CD8B40", + }, ]; toMatch.forEach(({ key, output }) => { - it("matches expected hash output", function() { + it("matches expected hash output", function () { const hashed = hashV1PartitionKey(key); assert.equal(hashed, output); }); diff --git a/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts b/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts index 784768a03a29..6bf312f3fb9f 100644 --- a/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts +++ b/sdk/cosmosdb/cosmos/test/unit/hashing/v2.spec.ts @@ -1,61 +1,61 @@ import assert from "assert"; import { hashV2PartitionKey } from "../../../src/utils/hashing/v2"; -describe("effectivePartitionKey", function() { - describe("computes v2 key", function() { +describe("effectivePartitionKey", function () { + describe("computes v2 key", function () { const toMatch = [ { key: "redmond", - output: "22E342F38A486A088463DFF7838A5963" + output: "22E342F38A486A088463DFF7838A5963", }, { key: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - output: "0BA3E9CA8EE4C14538828D1612A4B652" + output: "0BA3E9CA8EE4C14538828D1612A4B652", }, { key: "", - output: "32E9366E637A71B4E710384B2F4970A0" + output: "32E9366E637A71B4E710384B2F4970A0", }, { key: "aa", - output: "05033626483AE80D00E44FBD35362B19" + output: "05033626483AE80D00E44FBD35362B19", }, { key: null, - output: "378867E4430E67857ACE5C908374FE16" + output: "378867E4430E67857ACE5C908374FE16", }, { key: true, - output: "0E711127C5B5A8E4726AC6DD306A3E59" + output: "0E711127C5B5A8E4726AC6DD306A3E59", }, { key: false, - output: "2FE1BE91E90A3439635E0E9E37361EF2" + output: "2FE1BE91E90A3439635E0E9E37361EF2", }, { key: {}, - output: "11622DAA78F835834610ABE56EFF5CB5" + output: "11622DAA78F835834610ABE56EFF5CB5", }, { key: 5, - output: "19C08621B135968252FB34B4CF66F811" + output: "19C08621B135968252FB34B4CF66F811", }, { key: 5.5, - output: "0E2EE47829D1AF775EEFB6540FD1D0ED" + output: "0E2EE47829D1AF775EEFB6540FD1D0ED", }, { key: 12313.1221, - output: "27E7ECA8F2EE3E53424DE8D5220631C6" + output: "27E7ECA8F2EE3E53424DE8D5220631C6", }, { key: 123456789, - output: "1F56D2538088EBA82CCF988F36E16760" - } + output: "1F56D2538088EBA82CCF988F36E16760", + }, ]; toMatch.forEach(({ key, output }) => { - it("matches expected hash output", function() { + it("matches expected hash output", function () { const hashed = hashV2PartitionKey(key); assert.equal(hashed, output); }); From 00d33b48fc7498675af3aca2205f04da5f61b842 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Thu, 23 Jul 2020 19:18:21 -0400 Subject: [PATCH 22/26] Exposes types publicly, makes partitionKeyRangeId consistent --- sdk/cosmosdb/cosmos/review/cosmos.api.md | 79 +++++++++++++++++--- sdk/cosmosdb/cosmos/src/ClientContext.ts | 50 ++++++------- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 2 +- sdk/cosmosdb/cosmos/src/index.ts | 15 +++- sdk/cosmosdb/cosmos/src/utils/batch.ts | 14 ++-- 5 files changed, 113 insertions(+), 47 deletions(-) diff --git a/sdk/cosmosdb/cosmos/review/cosmos.api.md b/sdk/cosmosdb/cosmos/review/cosmos.api.md index bb267b775c66..ed13dd0b6eb1 100644 --- a/sdk/cosmosdb/cosmos/review/cosmos.api.md +++ b/sdk/cosmosdb/cosmos/review/cosmos.api.md @@ -54,17 +54,17 @@ export class ChangeFeedResponse { export class ClientContext { constructor(cosmosClientOptions: CosmosClientOptions, globalEndpointManager: GlobalEndpointManager); // (undocumented) - bulk({ body, path, resourceId, partitionKeyRange, options }: { + bulk({ body, path, resourceId, partitionKeyRangeId, options, }: { body: T; path: string; - partitionKeyRange: string; + partitionKeyRangeId: string; resourceId: string; options?: RequestOptions; }): Promise>; // (undocumented) clearSessionToken(path: string): void; // (undocumented) - create({ body, path, resourceType, resourceId, options, partitionKey }: { + create({ body, path, resourceType, resourceId, options, partitionKey, }: { body: T; path: string; resourceType: ResourceType; @@ -73,7 +73,7 @@ export class ClientContext { partitionKey?: PartitionKey; }): Promise>; // (undocumented) - delete({ path, resourceType, resourceId, options, partitionKey }: { + delete({ path, resourceType, resourceId, options, partitionKey, }: { path: string; resourceType: ResourceType; resourceId: string; @@ -81,7 +81,7 @@ export class ClientContext { partitionKey?: PartitionKey; }): Promise>; // (undocumented) - execute({ sprocLink, params, options, partitionKey }: { + execute({ sprocLink, params, options, partitionKey, }: { sprocLink: string; params?: any[]; options?: RequestOptions; @@ -99,7 +99,7 @@ export class ClientContext { [containerUrl: string]: any; }; // (undocumented) - queryFeed({ path, resourceType, resourceId, resultFn, query, options, partitionKeyRangeId, partitionKey }: { + queryFeed({ path, resourceType, resourceId, resultFn, query, options, partitionKeyRangeId, partitionKey, }: { path: string; resourceType: ResourceType; resourceId: string; @@ -114,7 +114,7 @@ export class ClientContext { // (undocumented) queryPartitionKeyRanges(collectionLink: string, query?: string | SqlQuerySpec, options?: FeedOptions): QueryIterator; // (undocumented) - read({ path, resourceType, resourceId, options, partitionKey }: { + read({ path, resourceType, resourceId, options, partitionKey, }: { path: string; resourceType: ResourceType; resourceId: string; @@ -122,7 +122,7 @@ export class ClientContext { partitionKey?: PartitionKey; }): Promise>; // (undocumented) - replace({ body, path, resourceType, resourceId, options, partitionKey }: { + replace({ body, path, resourceType, resourceId, options, partitionKey, }: { body: any; path: string; resourceType: ResourceType; @@ -131,7 +131,7 @@ export class ClientContext { partitionKey?: PartitionKey; }): Promise>; // (undocumented) - upsert({ body, path, resourceType, resourceId, options, partitionKey }: { + upsert({ body, path, resourceType, resourceId, options, partitionKey, }: { body: T; path: string; resourceType: ResourceType; @@ -492,6 +492,11 @@ export interface CosmosHeaders { [key: string]: any; } +// @public (undocumented) +export type CreateOperation = OperationWithItem & { + operationType: "Create"; +}; + // @public export class Database { constructor(client: CosmosClient, id: string, clientContext: ClientContext); @@ -576,6 +581,12 @@ export enum DataType { // @public (undocumented) export const DEFAULT_PARTITION_KEY_PATH: "/_partitionKey"; +// @public (undocumented) +export type DeleteOperation = OperationBase & { + operationType: "Delete"; + id: string; +}; + // @public (undocumented) export interface ErrorBody { // (undocumented) @@ -770,9 +781,6 @@ export class ItemResponse extends ResourceResponse; changeFeed(partitionKey: string | number | boolean, changeFeedOptions?: ChangeFeedOptions): ChangeFeedIterator; @@ -871,6 +879,31 @@ export class Offers { readAll(options?: FeedOptions): QueryIterator; } +// @public (undocumented) +export type Operation = CreateOperation | UpsertOperation | ReadOperation | DeleteOperation | ReplaceOperation; + +// @public (undocumented) +export interface OperationBase { + // (undocumented) + ifMatch?: string; + // (undocumented) + ifNoneMatch?: string; + // (undocumented) + partitionKey?: string; +} + +// @public (undocumented) +export interface OperationResponse { + // (undocumented) + eTag?: string; + // (undocumented) + requestCharge: number; + // (undocumented) + resourceBody?: JSONObject; + // (undocumented) + statusCode: number; +} + // @public (undocumented) export enum OperationType { // (undocumented) @@ -891,6 +924,11 @@ export enum OperationType { Upsert = "upsert" } +// @public (undocumented) +export type OperationWithItem = OperationBase & { + resourceBody: JSONObject; +}; + // @public (undocumented) export interface PartitionedQueryExecutionInfo { // (undocumented) @@ -1149,6 +1187,18 @@ export interface QueryRange { min: string; } +// @public (undocumented) +export type ReadOperation = OperationBase & { + operationType: "Read"; + id: string; +}; + +// @public (undocumented) +export type ReplaceOperation = OperationWithItem & { + operationType: "Replace"; + id: string; +}; + // @public (undocumented) export interface RequestContext { // (undocumented) @@ -1607,6 +1657,11 @@ export interface UniqueKeyPolicy { uniqueKeys: UniqueKey[]; } +// @public (undocumented) +export type UpsertOperation = OperationWithItem & { + operationType: "Upsert"; +}; + // @public export class User { constructor(database: Database, id: string, clientContext: ClientContext); diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index 90197df67fe6..d769e726b2df 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -51,7 +51,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey + partitionKey, }: { path: string; resourceType: ResourceType; @@ -72,7 +72,7 @@ export class ClientContext { options, resourceType, plugins: this.cosmosClientOptions.plugins, - partitionKey + partitionKey, }; request.headers = await this.buildHeaders(request); @@ -100,7 +100,7 @@ export class ClientContext { query, options, partitionKeyRangeId, - partitionKey + partitionKey, }: { path: string; resourceType: ResourceType; @@ -128,7 +128,7 @@ export class ClientContext { options, body: query, plugins: this.cosmosClientOptions.plugins, - partitionKey + partitionKey, }; const requestId = uuid(); if (query !== undefined) { @@ -180,7 +180,7 @@ export class ClientContext { resourceType, options, body: query, - plugins: this.cosmosClientOptions.plugins + plugins: this.cosmosClientOptions.plugins, }; request.endpoint = await this.globalEndpointManager.resolveServiceEndpoint( @@ -217,7 +217,7 @@ export class ClientContext { resourceId: id, resultFn: (result) => result.PartitionKeyRanges, query, - options: innerOptions + options: innerOptions, }); }; return new QueryIterator(this, query, options, cb); @@ -228,7 +228,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey + partitionKey, }: { path: string; resourceType: ResourceType; @@ -249,7 +249,7 @@ export class ClientContext { options, resourceId, plugins: this.cosmosClientOptions.plugins, - partitionKey + partitionKey, }; request.headers = await this.buildHeaders(request); @@ -278,7 +278,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey + partitionKey, }: { body: T; path: string; @@ -301,7 +301,7 @@ export class ClientContext { body, options, plugins: this.cosmosClientOptions.plugins, - partitionKey + partitionKey, }; request.headers = await this.buildHeaders(request); @@ -366,7 +366,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey + partitionKey, }: { body: any; path: string; @@ -389,7 +389,7 @@ export class ClientContext { resourceId, options, plugins: this.cosmosClientOptions.plugins, - partitionKey + partitionKey, }; request.headers = await this.buildHeaders(request); @@ -415,7 +415,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey + partitionKey, }: { body: T; path: string; @@ -438,7 +438,7 @@ export class ClientContext { resourceId, options, plugins: this.cosmosClientOptions.plugins, - partitionKey + partitionKey, }; request.headers = await this.buildHeaders(request); @@ -463,7 +463,7 @@ export class ClientContext { sprocLink, params, options = {}, - partitionKey + partitionKey, }: { sprocLink: string; params?: any[]; @@ -491,7 +491,7 @@ export class ClientContext { resourceId: id, body: params, plugins: this.cosmosClientOptions.plugins, - partitionKey + partitionKey, }; request.headers = await this.buildHeaders(request); @@ -523,7 +523,7 @@ export class ClientContext { path: "", resourceType: ResourceType.none, options, - plugins: this.cosmosClientOptions.plugins + plugins: this.cosmosClientOptions.plugins, }; request.headers = await this.buildHeaders(request); @@ -547,12 +547,12 @@ export class ClientContext { body, path, resourceId, - partitionKeyRange, - options = {} + partitionKeyRangeId, + options = {}, }: { body: T; path: string; - partitionKeyRange: string; + partitionKeyRangeId: string; resourceId: string; options?: RequestOptions; }) { @@ -569,12 +569,12 @@ export class ClientContext { resourceType: ResourceType.item, resourceId, plugins: this.cosmosClientOptions.plugins, - options + options, }; request.headers = await this.buildHeaders(request); request.headers[Constants.HttpHeaders.IsBatchRequest] = "True"; - request.headers[Constants.HttpHeaders.PartitionKeyRangeID] = partitionKeyRange; + request.headers[Constants.HttpHeaders.PartitionKeyRangeID] = partitionKeyRangeId; request.headers[Constants.HttpHeaders.IsBatchAtomic] = false; this.applySessionToken(request); @@ -629,7 +629,7 @@ export class ClientContext { resourceId, resourceAddress, resourceType, - isNameBased: true + isNameBased: true, }; } @@ -655,7 +655,7 @@ export class ClientContext { clientOptions: this.cosmosClientOptions, defaultHeaders: { ...this.cosmosClientOptions.defaultHeaders, - ...requestContext.options.initialHeaders + ...requestContext.options.initialHeaders, }, verb: requestContext.method, path: requestContext.path, @@ -664,7 +664,7 @@ export class ClientContext { options: requestContext.options, partitionKeyRangeId: requestContext.partitionKeyRangeId, useMultipleWriteLocations: this.connectionPolicy.useMultipleWriteLocations, - partitionKey: requestContext.partitionKey + partitionKey: requestContext.partitionKey, }); } } diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index a906f95671d5..edbb88485a2c 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -425,7 +425,7 @@ export class Items { try { const response = await this.clientContext.bulk({ body: batch.operations, - partitionKeyRange: batch.rangeId, + partitionKeyRangeId: batch.rangeId, path, resourceId: this.container.url, options, diff --git a/sdk/cosmosdb/cosmos/src/index.ts b/sdk/cosmosdb/cosmos/src/index.ts index 664085d1a4e0..468c39358ed5 100644 --- a/sdk/cosmosdb/cosmos/src/index.ts +++ b/sdk/cosmosdb/cosmos/src/index.ts @@ -4,6 +4,17 @@ export { DEFAULT_PARTITION_KEY_PATH } from "./common/partitionKeys"; export { StatusCodes } from "./common"; export { extractPartitionKey } from "./extractPartitionKey"; export { setAuthorizationTokenHeaderUsingMasterKey } from "./auth"; +export { + Operation, + OperationResponse, + CreateOperation, + UpsertOperation, + ReplaceOperation, + DeleteOperation, + ReadOperation, + OperationBase, + OperationWithItem, +} from "./utils/batch"; export { ConnectionMode, ConsistencyLevel, @@ -24,7 +35,7 @@ export { PermissionMode, TriggerOperation, TriggerType, - UserDefinedFunctionType + UserDefinedFunctionType, } from "./documents"; export { Constants, OperationType, ResourceType, HTTPMethod } from "./common"; @@ -37,7 +48,7 @@ export { SqlQuerySpec, JSONValue, JSONArray, - JSONObject + JSONObject, } from "./queryExecutionContext"; export { QueryIterator } from "./queryIterator"; export * from "./queryMetrics"; diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index 7dfc18a0dc89..5c02bb7fbb9f 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -30,35 +30,35 @@ export function isKeyInRange(min: string, max: string, key: string) { return isAfterMinInclusive && isBeforeMax; } -interface OperationBase { +export interface OperationBase { partitionKey?: string; ifMatch?: string; ifNoneMatch?: string; } -type OperationWithItem = OperationBase & { +export type OperationWithItem = OperationBase & { resourceBody: JSONObject; }; -type CreateOperation = OperationWithItem & { +export type CreateOperation = OperationWithItem & { operationType: "Create"; }; -type UpsertOperation = OperationWithItem & { +export type UpsertOperation = OperationWithItem & { operationType: "Upsert"; }; -type ReadOperation = OperationBase & { +export type ReadOperation = OperationBase & { operationType: "Read"; id: string; }; -type DeleteOperation = OperationBase & { +export type DeleteOperation = OperationBase & { operationType: "Delete"; id: string; }; -type ReplaceOperation = OperationWithItem & { +export type ReplaceOperation = OperationWithItem & { operationType: "Replace"; id: string; }; From 03115d156d1e257cb481c251885fa6344ccd9d61 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Fri, 24 Jul 2020 18:26:52 -0400 Subject: [PATCH 23/26] Adds 100 operation limit, removes lint issues --- sdk/cosmosdb/cosmos/CHANGELOG.md | 7 ++- sdk/cosmosdb/cosmos/src/ClientContext.ts | 46 ++++++++++---------- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 42 +++++++++++++++--- sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts | 7 +-- sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts | 3 +- 5 files changed, 66 insertions(+), 39 deletions(-) diff --git a/sdk/cosmosdb/cosmos/CHANGELOG.md b/sdk/cosmosdb/cosmos/CHANGELOG.md index 9b769ada1151..b4d61fbd0309 100644 --- a/sdk/cosmosdb/cosmos/CHANGELOG.md +++ b/sdk/cosmosdb/cosmos/CHANGELOG.md @@ -2,7 +2,6 @@ ## 3.7.5 (Unreleased) - ## 3.7.4 (2020-6-30) - BUGFIX: Properly escape ASCII "DEL" character in partition key header @@ -214,14 +213,14 @@ Constructor options have been simplified: const client = new CosmosClient({ endpoint: "https://your-database.cosmos.azure.com", auth: { - masterKey: "your-primary-key", - }, + masterKey: "your-primary-key" + } }); // v3 const client = new CosmosClient({ endpoint: "https://your-database.cosmos.azure.com", - key: "your-primary-key", + key: "your-primary-key" }); ``` diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index d769e726b2df..fb7d7890c917 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -51,7 +51,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey, + partitionKey }: { path: string; resourceType: ResourceType; @@ -72,7 +72,7 @@ export class ClientContext { options, resourceType, plugins: this.cosmosClientOptions.plugins, - partitionKey, + partitionKey }; request.headers = await this.buildHeaders(request); @@ -100,7 +100,7 @@ export class ClientContext { query, options, partitionKeyRangeId, - partitionKey, + partitionKey }: { path: string; resourceType: ResourceType; @@ -128,7 +128,7 @@ export class ClientContext { options, body: query, plugins: this.cosmosClientOptions.plugins, - partitionKey, + partitionKey }; const requestId = uuid(); if (query !== undefined) { @@ -180,7 +180,7 @@ export class ClientContext { resourceType, options, body: query, - plugins: this.cosmosClientOptions.plugins, + plugins: this.cosmosClientOptions.plugins }; request.endpoint = await this.globalEndpointManager.resolveServiceEndpoint( @@ -217,7 +217,7 @@ export class ClientContext { resourceId: id, resultFn: (result) => result.PartitionKeyRanges, query, - options: innerOptions, + options: innerOptions }); }; return new QueryIterator(this, query, options, cb); @@ -228,7 +228,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey, + partitionKey }: { path: string; resourceType: ResourceType; @@ -249,7 +249,7 @@ export class ClientContext { options, resourceId, plugins: this.cosmosClientOptions.plugins, - partitionKey, + partitionKey }; request.headers = await this.buildHeaders(request); @@ -278,7 +278,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey, + partitionKey }: { body: T; path: string; @@ -301,7 +301,7 @@ export class ClientContext { body, options, plugins: this.cosmosClientOptions.plugins, - partitionKey, + partitionKey }; request.headers = await this.buildHeaders(request); @@ -366,7 +366,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey, + partitionKey }: { body: any; path: string; @@ -389,7 +389,7 @@ export class ClientContext { resourceId, options, plugins: this.cosmosClientOptions.plugins, - partitionKey, + partitionKey }; request.headers = await this.buildHeaders(request); @@ -415,7 +415,7 @@ export class ClientContext { resourceType, resourceId, options = {}, - partitionKey, + partitionKey }: { body: T; path: string; @@ -438,7 +438,7 @@ export class ClientContext { resourceId, options, plugins: this.cosmosClientOptions.plugins, - partitionKey, + partitionKey }; request.headers = await this.buildHeaders(request); @@ -463,7 +463,7 @@ export class ClientContext { sprocLink, params, options = {}, - partitionKey, + partitionKey }: { sprocLink: string; params?: any[]; @@ -491,7 +491,7 @@ export class ClientContext { resourceId: id, body: params, plugins: this.cosmosClientOptions.plugins, - partitionKey, + partitionKey }; request.headers = await this.buildHeaders(request); @@ -523,7 +523,7 @@ export class ClientContext { path: "", resourceType: ResourceType.none, options, - plugins: this.cosmosClientOptions.plugins, + plugins: this.cosmosClientOptions.plugins }; request.headers = await this.buildHeaders(request); @@ -548,7 +548,7 @@ export class ClientContext { path, resourceId, partitionKeyRangeId, - options = {}, + options = {} }: { body: T; path: string; @@ -569,11 +569,11 @@ export class ClientContext { resourceType: ResourceType.item, resourceId, plugins: this.cosmosClientOptions.plugins, - options, + options }; request.headers = await this.buildHeaders(request); - request.headers[Constants.HttpHeaders.IsBatchRequest] = "True"; + request.headers[Constants.HttpHeaders.IsBatchRequest] = true; request.headers[Constants.HttpHeaders.PartitionKeyRangeID] = partitionKeyRangeId; request.headers[Constants.HttpHeaders.IsBatchAtomic] = false; @@ -629,7 +629,7 @@ export class ClientContext { resourceId, resourceAddress, resourceType, - isNameBased: true, + isNameBased: true }; } @@ -655,7 +655,7 @@ export class ClientContext { clientOptions: this.cosmosClientOptions, defaultHeaders: { ...this.cosmosClientOptions.defaultHeaders, - ...requestContext.options.initialHeaders, + ...requestContext.options.initialHeaders }, verb: requestContext.method, path: requestContext.path, @@ -664,7 +664,7 @@ export class ClientContext { options: requestContext.options, partitionKeyRangeId: requestContext.partitionKeyRangeId, useMultipleWriteLocations: this.connectionPolicy.useMultipleWriteLocations, - partitionKey: requestContext.partitionKey, + partitionKey: requestContext.partitionKey }); } } diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index edbb88485a2c..8b67d5c4aa05 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -19,7 +19,7 @@ import { Operation, getPartitionKeyToHash, addPKToOperation, - OperationResponse, + OperationResponse } from "../../utils/batch"; import { hashV1PartitionKey } from "../../utils/hashing/v1"; import { hashV2PartitionKey } from "../../utils/hashing/v2"; @@ -95,7 +95,7 @@ export class Items { resultFn: (result) => (result ? result.Documents : []), query, options: innerOptions, - partitionKey: options.partitionKey, + partitionKey: options.partitionKey }); }; @@ -297,7 +297,7 @@ export class Items { resourceType: ResourceType.item, resourceId: id, options, - partitionKey, + partitionKey }); const ref = new Item( @@ -366,7 +366,7 @@ export class Items { resourceType: ResourceType.item, resourceId: id, options, - partitionKey, + partitionKey }); const ref = new Item( @@ -384,12 +384,37 @@ export class Items { ); } + /** + * Execute bulk operations on items. + * + * Bulk takes an array of Operations which are typed based on what the operation does. + * The choices are: Create, Upsert, Read, Replace, and Delete + * + * Usage example: + * + * const operations: Operation[] = [ + * { + * operationType: "Create", + * resourceBody: { id: "doc1", name: "sample", key: "A" } + * }, + * { + * operationType: "Upsert", + * partitionKey: `["A"]`, + * resourceBody: { id: "doc2", name: "other", key: "A" } + * } + * ] + * + * await database.container.items.bulk(operation) + * + * @param operations. List of operations. Limit 100 + * @param options Used for modifying the request. + */ public async bulk( operations: Operation[], options?: RequestOptions ): Promise { const { - resources: partitionKeyRanges, + resources: partitionKeyRanges } = await this.container.readPartitionKeyRanges().fetchAll(); const { resource: definition } = await this.container.getPartitionKeyDefinition(); const batches: Batch[] = partitionKeyRanges.map((keyRange: PartitionKeyRange) => { @@ -398,7 +423,7 @@ export class Items { max: keyRange.maxExclusive, rangeId: keyRange.id, indexes: [], - operations: [], + operations: [] }; }); operations @@ -422,13 +447,16 @@ export class Items { batches .filter((batch: Batch) => batch.operations.length) .map(async (batch: Batch) => { + if (batch.operations.length > 100) { + throw new Error("Cannot run bulk request with more than 100 operations per partition"); + } try { const response = await this.clientContext.bulk({ body: batch.operations, partitionKeyRangeId: batch.rangeId, path, resourceId: this.container.url, - options, + options }); response.result.forEach((operationResponse: OperationResponse, index: number) => { orderedResponses[batch.indexes[index]] = operationResponse; diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts index 87141f41276c..b7d26a3173f7 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts @@ -1,7 +1,6 @@ import { doubleToByteArrayJSBI, writeNumberForBinaryEncodingJSBI } from "./encoding/number"; import { writeStringForBinaryEncoding } from "./encoding/string"; import { BytePrefix } from "./encoding/prefix"; -// tslint:disable-next-line import MurmurHash from "./murmurHash"; const MAX_STRING_CHARS = 100; @@ -13,7 +12,9 @@ export function hashV1PartitionKey(partitionKey: v1Key): string { const hash = MurmurHash.x86.hash32(toHash); const encodedJSBI = writeNumberForBinaryEncodingJSBI(hash); const encodedValue = encodeByType(partitionKey); - return Buffer.concat([encodedJSBI, encodedValue]).toString("hex").toUpperCase(); + return Buffer.concat([encodedJSBI, encodedValue]) + .toString("hex") + .toUpperCase(); } function prefixKeyByType(key: v1Key) { @@ -24,7 +25,7 @@ function prefixKeyByType(key: v1Key) { bytes = Buffer.concat([ Buffer.from(BytePrefix.String, "hex"), Buffer.from(truncated), - Buffer.from(BytePrefix.Undefined, "hex"), + Buffer.from(BytePrefix.Undefined, "hex") ]); return bytes; case "number": diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts index 4ec63110d642..3696a8767667 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts @@ -1,6 +1,5 @@ import { doubleToByteArrayJSBI } from "./encoding/number"; import { BytePrefix } from "./encoding/prefix"; -// tslint:disable-next-line import MurmurHash from "./murmurHash"; type v2Key = string | number | null | {} | undefined; @@ -20,7 +19,7 @@ function prefixKeyByType(key: v2Key) { bytes = Buffer.concat([ Buffer.from(BytePrefix.String, "hex"), Buffer.from(key), - Buffer.from(BytePrefix.Infinity, "hex"), + Buffer.from(BytePrefix.Infinity, "hex") ]); return bytes; case "number": From dd162ea23e0b684f3b1f294cc716faec94c17925 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Fri, 24 Jul 2020 19:08:35 -0400 Subject: [PATCH 24/26] Fixes api diff --- sdk/cosmosdb/cosmos/review/cosmos.api.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/sdk/cosmosdb/cosmos/review/cosmos.api.md b/sdk/cosmosdb/cosmos/review/cosmos.api.md index ed13dd0b6eb1..6d1faae3d45c 100644 --- a/sdk/cosmosdb/cosmos/review/cosmos.api.md +++ b/sdk/cosmosdb/cosmos/review/cosmos.api.md @@ -54,7 +54,7 @@ export class ChangeFeedResponse { export class ClientContext { constructor(cosmosClientOptions: CosmosClientOptions, globalEndpointManager: GlobalEndpointManager); // (undocumented) - bulk({ body, path, resourceId, partitionKeyRangeId, options, }: { + bulk({ body, path, resourceId, partitionKeyRangeId, options }: { body: T; path: string; partitionKeyRangeId: string; @@ -64,7 +64,7 @@ export class ClientContext { // (undocumented) clearSessionToken(path: string): void; // (undocumented) - create({ body, path, resourceType, resourceId, options, partitionKey, }: { + create({ body, path, resourceType, resourceId, options, partitionKey }: { body: T; path: string; resourceType: ResourceType; @@ -73,7 +73,7 @@ export class ClientContext { partitionKey?: PartitionKey; }): Promise>; // (undocumented) - delete({ path, resourceType, resourceId, options, partitionKey, }: { + delete({ path, resourceType, resourceId, options, partitionKey }: { path: string; resourceType: ResourceType; resourceId: string; @@ -81,7 +81,7 @@ export class ClientContext { partitionKey?: PartitionKey; }): Promise>; // (undocumented) - execute({ sprocLink, params, options, partitionKey, }: { + execute({ sprocLink, params, options, partitionKey }: { sprocLink: string; params?: any[]; options?: RequestOptions; @@ -99,7 +99,7 @@ export class ClientContext { [containerUrl: string]: any; }; // (undocumented) - queryFeed({ path, resourceType, resourceId, resultFn, query, options, partitionKeyRangeId, partitionKey, }: { + queryFeed({ path, resourceType, resourceId, resultFn, query, options, partitionKeyRangeId, partitionKey }: { path: string; resourceType: ResourceType; resourceId: string; @@ -114,7 +114,7 @@ export class ClientContext { // (undocumented) queryPartitionKeyRanges(collectionLink: string, query?: string | SqlQuerySpec, options?: FeedOptions): QueryIterator; // (undocumented) - read({ path, resourceType, resourceId, options, partitionKey, }: { + read({ path, resourceType, resourceId, options, partitionKey }: { path: string; resourceType: ResourceType; resourceId: string; @@ -122,7 +122,7 @@ export class ClientContext { partitionKey?: PartitionKey; }): Promise>; // (undocumented) - replace({ body, path, resourceType, resourceId, options, partitionKey, }: { + replace({ body, path, resourceType, resourceId, options, partitionKey }: { body: any; path: string; resourceType: ResourceType; @@ -131,7 +131,7 @@ export class ClientContext { partitionKey?: PartitionKey; }): Promise>; // (undocumented) - upsert({ body, path, resourceType, resourceId, options, partitionKey, }: { + upsert({ body, path, resourceType, resourceId, options, partitionKey }: { body: T; path: string; resourceType: ResourceType; @@ -781,7 +781,6 @@ export class ItemResponse extends ResourceResponse; changeFeed(partitionKey: string | number | boolean, changeFeedOptions?: ChangeFeedOptions): ChangeFeedIterator; changeFeed(changeFeedOptions?: ChangeFeedOptions): ChangeFeedIterator; From 8e08212c3ab946627c63c3588c350be9c461d41d Mon Sep 17 00:00:00 2001 From: Zachary Foster Date: Mon, 27 Jul 2020 11:09:25 -0400 Subject: [PATCH 25/26] Update sdk/cosmosdb/cosmos/CHANGELOG.md Co-authored-by: Steve Faulkner --- sdk/cosmosdb/cosmos/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmosdb/cosmos/CHANGELOG.md b/sdk/cosmosdb/cosmos/CHANGELOG.md index 45be0770b8ac..b62598e536ab 100644 --- a/sdk/cosmosdb/cosmos/CHANGELOG.md +++ b/sdk/cosmosdb/cosmos/CHANGELOG.md @@ -9,7 +9,7 @@ const operations: Operation[] = [ { operationType: "Create", - resourceBody: { id: addEntropy("doc1"), name: "sample", key: "A" } + resourceBody: { id: "doc1", name: "sample", key: "A" } }, { operationType: "Upsert", From efd15723dfb4b8e4bc15ac8a79a46f6193473950 Mon Sep 17 00:00:00 2001 From: Zach foster Date: Mon, 27 Jul 2020 17:59:35 -0400 Subject: [PATCH 26/26] Remove partitionKey as header in top level --- sdk/cosmosdb/cosmos/CHANGELOG.md | 8 ++--- sdk/cosmosdb/cosmos/review/cosmos.api.md | 14 ++++++++- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 10 ++++--- sdk/cosmosdb/cosmos/src/index.ts | 5 ++-- sdk/cosmosdb/cosmos/src/utils/batch.ts | 20 +++++++++---- .../cosmos/test/functional/item.spec.ts | 29 +++++++++---------- 6 files changed, 55 insertions(+), 31 deletions(-) diff --git a/sdk/cosmosdb/cosmos/CHANGELOG.md b/sdk/cosmosdb/cosmos/CHANGELOG.md index 45be0770b8ac..99487cf8f500 100644 --- a/sdk/cosmosdb/cosmos/CHANGELOG.md +++ b/sdk/cosmosdb/cosmos/CHANGELOG.md @@ -6,19 +6,19 @@ ```js // up to 100 operations -const operations: Operation[] = [ +const operations: OperationInput[] = [ { operationType: "Create", - resourceBody: { id: addEntropy("doc1"), name: "sample", key: "A" } + resourceBody: { id: "doc1", name: "sample", key: "A" } }, { operationType: "Upsert", - resourceBody: { id: addEntropy("doc2"), name: "other", key: "A" } + resourceBody: { id: "doc2", name: "other", key: "A" } }, { operationType: "Read", id: "readItemId", - partitionKey: `["key"]` + partitionKey: "key" } ]; diff --git a/sdk/cosmosdb/cosmos/review/cosmos.api.md b/sdk/cosmosdb/cosmos/review/cosmos.api.md index 6d1faae3d45c..3615ea096cbb 100644 --- a/sdk/cosmosdb/cosmos/review/cosmos.api.md +++ b/sdk/cosmosdb/cosmos/review/cosmos.api.md @@ -781,7 +781,7 @@ export class ItemResponse extends ResourceResponse; + bulk(operations: OperationInput[], options?: RequestOptions): Promise; changeFeed(partitionKey: string | number | boolean, changeFeedOptions?: ChangeFeedOptions): ChangeFeedIterator; changeFeed(changeFeedOptions?: ChangeFeedOptions): ChangeFeedIterator; changeFeed(partitionKey: string | number | boolean, changeFeedOptions?: ChangeFeedOptions): ChangeFeedIterator; @@ -891,6 +891,18 @@ export interface OperationBase { partitionKey?: string; } +// @public (undocumented) +export interface OperationInput { + // (undocumented) + ifMatch?: string; + // (undocumented) + ifNoneMatch?: string; + // (undocumented) + partitionKey?: string | number | null | {} | undefined; + // (undocumented) + resourceBody?: JSONObject; +} + // @public (undocumented) export interface OperationResponse { // (undocumented) diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 8b67d5c4aa05..8c74d117f97a 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -19,7 +19,8 @@ import { Operation, getPartitionKeyToHash, addPKToOperation, - OperationResponse + OperationResponse, + OperationInput } from "../../utils/batch"; import { hashV1PartitionKey } from "../../utils/hashing/v1"; import { hashV2PartitionKey } from "../../utils/hashing/v2"; @@ -392,14 +393,15 @@ export class Items { * * Usage example: * - * const operations: Operation[] = [ + * // partitionKey is optional at the top level if present in the resourceBody + * const operations: OperationInput[] = [ * { * operationType: "Create", * resourceBody: { id: "doc1", name: "sample", key: "A" } * }, * { * operationType: "Upsert", - * partitionKey: `["A"]`, + * partitionKey: 'A', * resourceBody: { id: "doc2", name: "other", key: "A" } * } * ] @@ -410,7 +412,7 @@ export class Items { * @param options Used for modifying the request. */ public async bulk( - operations: Operation[], + operations: OperationInput[], options?: RequestOptions ): Promise { const { diff --git a/sdk/cosmosdb/cosmos/src/index.ts b/sdk/cosmosdb/cosmos/src/index.ts index 468c39358ed5..1d85c8a6ad05 100644 --- a/sdk/cosmosdb/cosmos/src/index.ts +++ b/sdk/cosmosdb/cosmos/src/index.ts @@ -14,6 +14,7 @@ export { ReadOperation, OperationBase, OperationWithItem, + OperationInput } from "./utils/batch"; export { ConnectionMode, @@ -35,7 +36,7 @@ export { PermissionMode, TriggerOperation, TriggerType, - UserDefinedFunctionType, + UserDefinedFunctionType } from "./documents"; export { Constants, OperationType, ResourceType, HTTPMethod } from "./common"; @@ -48,7 +49,7 @@ export { SqlQuerySpec, JSONValue, JSONArray, - JSONObject, + JSONObject } from "./queryExecutionContext"; export { QueryIterator } from "./queryIterator"; export * from "./queryMetrics"; diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index 5c02bb7fbb9f..9bc3b2e1d203 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -36,6 +36,13 @@ export interface OperationBase { ifNoneMatch?: string; } +export interface OperationInput { + partitionKey?: string | number | null | {} | undefined; + ifMatch?: string; + ifNoneMatch?: string; + resourceBody?: JSONObject; +} + export type OperationWithItem = OperationBase & { resourceBody: JSONObject; }; @@ -81,10 +88,13 @@ export function getPartitionKeyToHash(operation: Operation, partitionProperty: s return toHashKey; } -export function addPKToOperation(operation: Operation, definition: PartitionKeyDefinition) { - if (operation.partitionKey || !hasResource(operation)) { - return operation; +export function addPKToOperation(operation: OperationInput, definition: PartitionKeyDefinition) { + if (operation.partitionKey) { + const extracted = extractPartitionKey(operation, { paths: ["/partitionKey"] }); + return { ...operation, partitionKey: JSON.stringify(extracted) }; + } else if (operation.resourceBody) { + const pk = extractPartitionKey(operation.resourceBody, definition); + return { ...operation, partitionKey: JSON.stringify(pk) }; } - const pk = extractPartitionKey(operation.resourceBody, definition); - return { ...operation, partitionKey: JSON.stringify(pk) }; + return operation; } diff --git a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts index 8a9b77624db9..4d713bce5cd2 100644 --- a/sdk/cosmosdb/cosmos/test/functional/item.spec.ts +++ b/sdk/cosmosdb/cosmos/test/functional/item.spec.ts @@ -16,7 +16,6 @@ import { addEntropy, getTestContainer } from "../common/TestHelpers"; -import { Operation } from "../../src/utils/batch"; /** * @ignore @@ -257,29 +256,29 @@ describe("bulk item operations", function() { }); }); it("handles create, upsert, replace, delete", async function() { - const operations: Operation[] = [ + const operations = [ { operationType: "Create", resourceBody: { id: addEntropy("doc1"), name: "sample", key: "A" } }, { operationType: "Upsert", - partitionKey: `["A"]`, + partitionKey: "A", resourceBody: { id: addEntropy("doc2"), name: "other", key: "A" } }, { operationType: "Read", id: readItemId, - partitionKey: `["A"]` + partitionKey: "A" }, { operationType: "Delete", id: deleteItemId, - partitionKey: `["A"]` + partitionKey: "A" }, { operationType: "Replace", - partitionKey: "[5]", + partitionKey: 5, id: replaceItemId, resourceBody: { id: replaceItemId, name: "nice", key: 5 } } @@ -334,30 +333,30 @@ describe("bulk item operations", function() { }); }); it("handles create, upsert, replace, delete", async function() { - const operations: Operation[] = [ + const operations = [ { operationType: "Create", - partitionKey: `["A"]`, + partitionKey: "A", resourceBody: { id: addEntropy("doc1"), name: "sample", key: "A" } }, { operationType: "Upsert", - partitionKey: `["U"]`, + partitionKey: "U", resourceBody: { id: addEntropy("doc2"), name: "other", key: "U" } }, { operationType: "Read", id: readItemId, - partitionKey: `[true]` + partitionKey: true }, { operationType: "Delete", id: deleteItemId, - partitionKey: `[{}]` + partitionKey: {} }, { operationType: "Replace", - partitionKey: "[5]", + partitionKey: 5, id: replaceItemId, resourceBody: { id: replaceItemId, name: "nice", key: 5 } } @@ -385,16 +384,16 @@ describe("bulk item operations", function() { key: "A", class: "2010" }); - const operations: Operation[] = [ + const operations = [ { operationType: "Delete", id: readItemId, - partitionKey: `["A"]` + partitionKey: "A" }, { operationType: "Read", id: readItemId, - partitionKey: `["A"]` + partitionKey: "A" } ]; const response = await v2Container.items.bulk(operations);