From 9f5a11c71097bf47f7ae74105df9f0064a96d249 Mon Sep 17 00:00:00 2001 From: Manik Khandelwal Date: Mon, 24 Jul 2023 13:33:48 +0530 Subject: [PATCH 1/8] update header with index metrics for cross partition queries --- .../src/queryExecutionContext/headerUtils.ts | 5 ++++ .../test/public/integration/query.spec.ts | 23 +++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/headerUtils.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/headerUtils.ts index d59a514cd523..43f8ef9ddba1 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/headerUtils.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/headerUtils.ts @@ -76,4 +76,9 @@ export function mergeHeaders(headers: CosmosHeaders, toBeMergedHeaders: CosmosHe } } } + + if (Constants.HttpHeaders.IndexUtilization in toBeMergedHeaders) { + headers[Constants.HttpHeaders.IndexUtilization] = + toBeMergedHeaders[Constants.HttpHeaders.IndexUtilization]; + } } diff --git a/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts b/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts index 1ac4b78d8d7e..d92250bbb35f 100644 --- a/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts +++ b/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts @@ -214,12 +214,25 @@ describe("Test Index metrics", function (this: Suite) { await createdContainer.items.create(doc1); await createdContainer.items.create(doc2); await createdContainer.items.create(doc3); - const query = "SELECT * from " + collectionId + " where " + collectionId + ".name = 'test2'"; - const queryOptions: FeedOptions = { populateIndexMetrics: true }; - const queryIterator = createdContainer.items.query(query, queryOptions); + const query1 = "SELECT * from " + collectionId + " where " + collectionId + ".name = 'test2'"; + const query2 = "SELECT * from " + collectionId + " order by " + collectionId + ".name"; - while (queryIterator.hasMoreResults()) { - const { resources: results, indexMetrics } = await queryIterator.fetchNext(); + const queryOptions: FeedOptions = { populateIndexMetrics: true, maxItemCount: 1 }; + const queryIterator1 = createdContainer.items.query(query1, queryOptions); + const queryIterator2 = createdContainer.items.query(query2, queryOptions); + + while (queryIterator2.hasMoreResults()) { + const { resources: results, indexMetrics } = await queryIterator2.fetchNext(); + console.log("index metrics: ", indexMetrics); + + if (results === undefined) { + break; + } + assert.notEqual(indexMetrics, undefined); + } + + while (queryIterator1.hasMoreResults()) { + const { resources: results, indexMetrics } = await queryIterator1.fetchNext(); if (results === undefined) { break; From 901f33dda70dbc13f47f8c1e739a34a057fab8e0 Mon Sep 17 00:00:00 2001 From: Manik Khandelwal Date: Thu, 27 Jul 2023 12:49:46 +0530 Subject: [PATCH 2/8] update test --- .../test/public/integration/query.spec.ts | 84 +++++++++++-------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts b/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts index d92250bbb35f..d34b57fdf4f7 100644 --- a/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts +++ b/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts @@ -2,7 +2,7 @@ // Licensed under the MIT license. import assert from "assert"; import { Suite } from "mocha"; -import { FeedOptions } from "../../../src"; +import { Container, FeedOptions } from "../../../src"; import { getTestContainer, getTestDatabase, removeAllDatabases } from "../common/TestHelpers"; const doc = { id: "myId", pk: "pk" }; @@ -191,8 +191,49 @@ describe("Test Index metrics", function (this: Suite) { await removeAllDatabases(); }); - it("validate that index metrics are correct for a single partition query", async function () { - const database = await getTestDatabase("index metrics test db"); + it("validate that index metrics are correct", async function () { + const createdContainerSinglePartition = await setupContainer( + "index metrics test db", + collectionId, + 4000 + ); + const createdContainerMultiPartition = await setupContainer( + "index metrics test db multipartioned", + collectionId, + 12000 + ); + + await validateIndexMetrics(createdContainerSinglePartition, collectionId); + await validateIndexMetrics(createdContainerMultiPartition, collectionId); + }); + + async function validateIndexMetrics(container: Container, collectionId: string) { + const doc1 = { id: "myId1", pk: "pk1", name: "test1" }; + const doc2 = { id: "myId2", pk: "pk2", name: "test2" }; + const doc3 = { id: "myId3", pk: "pk2", name: "test2" }; + await container.items.create(doc1); + await container.items.create(doc2); + await container.items.create(doc3); + //stremeable query + const query1 = "SELECT * from " + collectionId + " where " + collectionId + ".name = 'test2'"; + //aggregate query + const query2 = "SELECT * from " + collectionId + " order by " + collectionId + ".name"; + const queryList = [query1, query2]; + const queryOptions: FeedOptions = { populateIndexMetrics: true, maxItemCount: 1 }; + for (const query of queryList) { + const queryIterator = container.items.query(query, queryOptions); + while (queryIterator.hasMoreResults()) { + const { resources: results, indexMetrics } = await queryIterator.fetchNext(); + + if (results === undefined) { + break; + } + assert.notEqual(indexMetrics, undefined); + } + } + } + async function setupContainer(datbaseName: string, collectionId: string, throughput?: number) { + const database = await getTestDatabase(datbaseName); const collectionDefinition = { id: collectionId, @@ -200,44 +241,13 @@ describe("Test Index metrics", function (this: Suite) { paths: ["/pk"], }, }; - const collectionOptions = { offerThroughput: 4000 }; + const collectionOptions = { offerThroughput: throughput }; const { resource: createdCollectionDef } = await database.containers.create( collectionDefinition, collectionOptions ); const createdContainer = database.container(createdCollectionDef.id); - - const doc1 = { id: "myId1", pk: "pk1", name: "test1" }; - const doc2 = { id: "myId2", pk: "pk2", name: "test2" }; - const doc3 = { id: "myId3", pk: "pk2", name: "test2" }; - await createdContainer.items.create(doc1); - await createdContainer.items.create(doc2); - await createdContainer.items.create(doc3); - const query1 = "SELECT * from " + collectionId + " where " + collectionId + ".name = 'test2'"; - const query2 = "SELECT * from " + collectionId + " order by " + collectionId + ".name"; - - const queryOptions: FeedOptions = { populateIndexMetrics: true, maxItemCount: 1 }; - const queryIterator1 = createdContainer.items.query(query1, queryOptions); - const queryIterator2 = createdContainer.items.query(query2, queryOptions); - - while (queryIterator2.hasMoreResults()) { - const { resources: results, indexMetrics } = await queryIterator2.fetchNext(); - console.log("index metrics: ", indexMetrics); - - if (results === undefined) { - break; - } - assert.notEqual(indexMetrics, undefined); - } - - while (queryIterator1.hasMoreResults()) { - const { resources: results, indexMetrics } = await queryIterator1.fetchNext(); - - if (results === undefined) { - break; - } - assert.notEqual(indexMetrics, undefined); - } - }); + return createdContainer; + } }); From a871230a06c785b6b3cb5d1b3da15232a0cf754a Mon Sep 17 00:00:00 2001 From: Manik Khandelwal Date: Wed, 9 Aug 2023 12:43:08 +0530 Subject: [PATCH 3/8] fix lint issues --- sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts b/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts index d34b57fdf4f7..15fd413e6816 100644 --- a/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts +++ b/sdk/cosmosdb/cosmos/test/public/integration/query.spec.ts @@ -185,13 +185,13 @@ describe("aggregate query over null value", function (this: Suite) { describe("Test Index metrics", function (this: Suite) { this.timeout(process.env.MOCHA_TIMEOUT || 20000); - const collectionId = "testCollection3"; beforeEach(async function () { await removeAllDatabases(); }); it("validate that index metrics are correct", async function () { + const collectionId = "testCollection3"; const createdContainerSinglePartition = await setupContainer( "index metrics test db", collectionId, @@ -214,9 +214,9 @@ describe("Test Index metrics", function (this: Suite) { await container.items.create(doc1); await container.items.create(doc2); await container.items.create(doc3); - //stremeable query + // stremeable query const query1 = "SELECT * from " + collectionId + " where " + collectionId + ".name = 'test2'"; - //aggregate query + // aggregate query const query2 = "SELECT * from " + collectionId + " order by " + collectionId + ".name"; const queryList = [query1, query2]; const queryOptions: FeedOptions = { populateIndexMetrics: true, maxItemCount: 1 }; From 7d648fd7fc64ea3ca8d0ce5c5a84e6656242345c Mon Sep 17 00:00:00 2001 From: Manik Khandelwal Date: Tue, 5 Sep 2023 14:32:09 +0530 Subject: [PATCH 4/8] add index metrics sample --- .../v3/typescript/src/ItemManagement.ts | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts index debceaa96b8f..7cf69a403e00 100644 --- a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts +++ b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts @@ -50,7 +50,7 @@ async function run(): Promise { logStep("Read item with AccessCondition and no change to _etag"); const { resource: item2, headers } = await item.read({ - accessCondition: { type: "IfNoneMatch", condition: readDoc._etag } + accessCondition: { type: "IfNoneMatch", condition: readDoc._etag }, }); if (!item2 && headers["content-length"] === 0) { console.log( @@ -63,7 +63,7 @@ async function run(): Promise { readDoc.foo = "bar"; await item.replace(readDoc); const { resource: item3, headers: headers3 } = await item.read({ - accessCondition: { type: "IfNoneMatch", condition: readDoc._etag } + accessCondition: { type: "IfNoneMatch", condition: readDoc._etag }, }); if (!item3 && headers3["content-length"] === 0) { throw "Expected item this time. Something is wrong!"; @@ -78,9 +78,9 @@ async function run(): Promise { parameters: [ { name: "@lastName", - value: "Andersen" - } - ] + value: "Andersen", + }, + ], }; logStep("Query items in container '" + container.id + "'"); @@ -101,12 +101,18 @@ async function run(): Promise { firstName: "Newborn", gender: "unknown", fingers: 10, - toes: 10 + toes: 10, }; person.children.push(childDef); person.lastName = "Updated Family"; + logStep("Query items witb index metrics enabled"); + const { resources: resultsIndexMetrics, indexMetrics } = await container.items + .query(querySpec, { populateQueryMetrics: true }) + .fetchAll(); + console.log("IndexMetrics: ", indexMetrics); + logStep("Replace item with id '" + item.id + "'"); const { resource: updatedPerson } = await container.items.upsert(person); @@ -142,8 +148,9 @@ async function run(): Promise { const upsertSource = itemDefList[1]; logStep( - `Upserting person ${upsertSource && upsertSource.id} with id ${upsertSource && - upsertSource.id}...` + `Upserting person ${upsertSource && upsertSource.id} with id ${ + upsertSource && upsertSource.id + }...` ); // a non-identity change will cause an update on upsert @@ -171,8 +178,8 @@ async function run(): Promise { { op: "replace", path: "/lastName", - value: "Martin" - } + value: "Martin", + }, ]; if (patchSource) { const patchId = patchSource && patchSource.id; @@ -187,27 +194,27 @@ async function run(): Promise { { op: "add", path: "/aka", - value: "MeFamily" + value: "MeFamily", }, { op: "replace", path: "/lastName", - value: "Jose" + value: "Jose", }, { op: "remove", - path: "/parents" + path: "/parents", }, { op: "set", path: "/address/zip", - value: 90211 + value: 90211, }, { op: "incr", path: "/address/zip", - value: 5 - } + value: 5, + }, ]; const { resource: patchSource2 } = await container.item(patchId!).patch(multipleOperations); if (patchSource2) { @@ -219,8 +226,8 @@ async function run(): Promise { { op: "add", path: "/newImproved", - value: "it works" - } + value: "it works", + }, ]; const condition = "from c where NOT IS_DEFINED(c.newImproved)"; const { resource: patchSource3 } = await container From ac7b8c9f1aeaaa5b4cfd9d0f2fe8521e581bf764 Mon Sep 17 00:00:00 2001 From: Manik Khandelwal Date: Tue, 5 Sep 2023 14:37:29 +0530 Subject: [PATCH 5/8] update sample location --- .../samples/v3/typescript/src/ItemManagement.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts index 7cf69a403e00..5f01d33bb4bd 100644 --- a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts +++ b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts @@ -107,12 +107,6 @@ async function run(): Promise { person.children.push(childDef); person.lastName = "Updated Family"; - logStep("Query items witb index metrics enabled"); - const { resources: resultsIndexMetrics, indexMetrics } = await container.items - .query(querySpec, { populateQueryMetrics: true }) - .fetchAll(); - console.log("IndexMetrics: ", indexMetrics); - logStep("Replace item with id '" + item.id + "'"); const { resource: updatedPerson } = await container.items.upsert(person); @@ -240,6 +234,12 @@ async function run(): Promise { logStep("Delete item '" + item.id + "'"); await item.delete(); + logStep("Query items witb index metrics enabled"); + const { resources: resultsIndexMetrics, indexMetrics } = await container.items + .query(querySpec, { populateQueryMetrics: true }) + .fetchAll(); + console.log("IndexMetrics: ", indexMetrics); + await finish(); } run().catch(handleError); From 93fab04e7119bfcf9256b207c04990d20f730393 Mon Sep 17 00:00:00 2001 From: Manik Khandelwal Date: Tue, 5 Sep 2023 16:30:49 +0530 Subject: [PATCH 6/8] add sample --- .../v3/typescript/src/ItemManagement.ts | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts index 5f01d33bb4bd..36f2ce024f84 100644 --- a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts +++ b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts @@ -50,7 +50,7 @@ async function run(): Promise { logStep("Read item with AccessCondition and no change to _etag"); const { resource: item2, headers } = await item.read({ - accessCondition: { type: "IfNoneMatch", condition: readDoc._etag }, + accessCondition: { type: "IfNoneMatch", condition: readDoc._etag } }); if (!item2 && headers["content-length"] === 0) { console.log( @@ -63,7 +63,7 @@ async function run(): Promise { readDoc.foo = "bar"; await item.replace(readDoc); const { resource: item3, headers: headers3 } = await item.read({ - accessCondition: { type: "IfNoneMatch", condition: readDoc._etag }, + accessCondition: { type: "IfNoneMatch", condition: readDoc._etag } }); if (!item3 && headers3["content-length"] === 0) { throw "Expected item this time. Something is wrong!"; @@ -78,9 +78,9 @@ async function run(): Promise { parameters: [ { name: "@lastName", - value: "Andersen", - }, - ], + value: "Andersen" + } + ] }; logStep("Query items in container '" + container.id + "'"); @@ -101,7 +101,7 @@ async function run(): Promise { firstName: "Newborn", gender: "unknown", fingers: 10, - toes: 10, + toes: 10 }; person.children.push(childDef); @@ -142,9 +142,8 @@ async function run(): Promise { const upsertSource = itemDefList[1]; logStep( - `Upserting person ${upsertSource && upsertSource.id} with id ${ - upsertSource && upsertSource.id - }...` + `Upserting person ${upsertSource && upsertSource.id} with id ${upsertSource && + upsertSource.id}...` ); // a non-identity change will cause an update on upsert @@ -172,8 +171,8 @@ async function run(): Promise { { op: "replace", path: "/lastName", - value: "Martin", - }, + value: "Martin" + } ]; if (patchSource) { const patchId = patchSource && patchSource.id; @@ -188,27 +187,27 @@ async function run(): Promise { { op: "add", path: "/aka", - value: "MeFamily", + value: "MeFamily" }, { op: "replace", path: "/lastName", - value: "Jose", + value: "Jose" }, { op: "remove", - path: "/parents", + path: "/parents" }, { op: "set", path: "/address/zip", - value: 90211, + value: 90211 }, { op: "incr", path: "/address/zip", - value: 5, - }, + value: 5 + } ]; const { resource: patchSource2 } = await container.item(patchId!).patch(multipleOperations); if (patchSource2) { @@ -220,8 +219,8 @@ async function run(): Promise { { op: "add", path: "/newImproved", - value: "it works", - }, + value: "it works" + } ]; const condition = "from c where NOT IS_DEFINED(c.newImproved)"; const { resource: patchSource3 } = await container @@ -231,15 +230,16 @@ async function run(): Promise { console.log(`Patched ${JSON.stringify(patchSource)} to new ${JSON.stringify(patchSource3)}.`); } } - logStep("Delete item '" + item.id + "'"); - await item.delete(); - + logStep("Query items witb index metrics enabled"); const { resources: resultsIndexMetrics, indexMetrics } = await container.items .query(querySpec, { populateQueryMetrics: true }) .fetchAll(); console.log("IndexMetrics: ", indexMetrics); + logStep("Delete item '" + item.id + "'"); + await item.delete(); + await finish(); } run().catch(handleError); From 0ec86dc67b470feb23e589b0fb1c778f73a1ba3a Mon Sep 17 00:00:00 2001 From: Manik Khandelwal Date: Wed, 6 Sep 2023 11:35:47 +0530 Subject: [PATCH 7/8] update sample --- .../cosmos/samples/v3/typescript/src/ItemManagement.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts index 36f2ce024f84..7a11014f74b6 100644 --- a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts +++ b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts @@ -230,12 +230,13 @@ async function run(): Promise { console.log(`Patched ${JSON.stringify(patchSource)} to new ${JSON.stringify(patchSource3)}.`); } } - + logStep("Query items witb index metrics enabled"); const { resources: resultsIndexMetrics, indexMetrics } = await container.items - .query(querySpec, { populateQueryMetrics: true }) + .query(querySpec, { populateIndexMetrics: true }) .fetchAll(); console.log("IndexMetrics: ", indexMetrics); + console.log("Query results: ", resultsIndexMetrics); logStep("Delete item '" + item.id + "'"); await item.delete(); From 85a64d3fd4325dcc34f3d8355fc0bbae6d295aed Mon Sep 17 00:00:00 2001 From: Manik Khandelwal <113669638+topshot99@users.noreply.github.com> Date: Thu, 7 Sep 2023 15:52:08 +0530 Subject: [PATCH 8/8] Update spelling --- sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts index 7a11014f74b6..a9146a05f22a 100644 --- a/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts +++ b/sdk/cosmosdb/cosmos/samples/v3/typescript/src/ItemManagement.ts @@ -231,7 +231,7 @@ async function run(): Promise { } } - logStep("Query items witb index metrics enabled"); + logStep("Query items with index metrics enabled"); const { resources: resultsIndexMetrics, indexMetrics } = await container.items .query(querySpec, { populateIndexMetrics: true }) .fetchAll();