From 3cbc2b55e3ff1eb7ddd9c682fd34c87e18c86cbb Mon Sep 17 00:00:00 2001 From: Florian Maunier Date: Tue, 1 Oct 2024 11:07:18 +0200 Subject: [PATCH 1/3] feat(elasticsearch): add flag to reindex collection after an update --- .../controllers/collection/update/index.md | 8 +++--- lib/service/storage/7/elasticsearch.ts | 26 ++++++++++++------- lib/service/storage/8/elasticsearch.ts | 21 +++++++++------ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/doc/2/api/controllers/collection/update/index.md b/doc/2/api/controllers/collection/update/index.md index f180cfa235..09affd1c26 100644 --- a/doc/2/api/controllers/collection/update/index.md +++ b/doc/2/api/controllers/collection/update/index.md @@ -52,6 +52,7 @@ Body: } } }, + "reindexCollection": true|false, "settings": { "analysis" : { "analyzer":{ @@ -92,6 +93,7 @@ Body: } } }, + "reindexCollection": true|false, "settings": { "analysis" : { "analyzer":{ @@ -117,8 +119,9 @@ Body: ## Body properties -* `settings`: Elasticsearch index [settings](https://www.elastic.co/guide/en/elasticsearch/reference/7.5/index-modules.html#index-modules-settings) -* `mappings`: [collection mappings](/core/2/guides/main-concepts/data-storage#mappings-properties) +- `settings`: Elasticsearch index [settings](https://www.elastic.co/guide/en/elasticsearch/reference/7.5/index-modules.html#index-modules-settings) +- `mappings`: [collection mappings](/core/2/guides/main-concepts/data-storage#mappings-properties) +- `reindexCollection`: boolean, if `true`, the collection will be reindexed after the update --- @@ -143,4 +146,3 @@ Body: - [Common errors](/core/2/api/errors/types#common-errors) - [NotFoundError](/core/2/api/errors/types#notfounderror) - diff --git a/lib/service/storage/7/elasticsearch.ts b/lib/service/storage/7/elasticsearch.ts index 2c015927ae..971fbc0b39 100644 --- a/lib/service/storage/7/elasticsearch.ts +++ b/lib/service/storage/7/elasticsearch.ts @@ -21,7 +21,7 @@ import _ from "lodash"; -import { ApiResponse, RequestParams, Client } from "sdk-es7"; +import { ApiResponse, Client, RequestParams } from "sdk-es7"; import { Index, IndicesCreate } from "sdk-es7/api/requestParams"; import { TypeMapping } from "sdk-es7/api/types"; import { @@ -39,16 +39,16 @@ import ms from "ms"; import semver from "semver"; import debug from "../../../util/debug"; -import ESWrapper from "./esWrapper"; -import { QueryTranslator } from "../commons/queryTranslator"; -import didYouMean from "../../../util/didYouMean"; +import { storeScopeEnum } from "../../../core/storage/storeScopeEnum"; import * as kerror from "../../../kerror"; -import { assertIsObject } from "../../../util/requestAssertions"; -import { isPlainObject } from "../../../util/safeObject"; +import didYouMean from "../../../util/didYouMean"; import extractFields from "../../../util/extractFields"; import { Mutex } from "../../../util/mutex"; import { randomNumber } from "../../../util/name-generator"; -import { storeScopeEnum } from "../../../core/storage/storeScopeEnum"; +import { assertIsObject } from "../../../util/requestAssertions"; +import { isPlainObject } from "../../../util/safeObject"; +import { QueryTranslator } from "../commons/queryTranslator"; +import ESWrapper from "./esWrapper"; debug("kuzzle:services:elasticsearch"); @@ -1659,8 +1659,13 @@ export class ES7 { collection: string, { mappings = {}, + reindexCollection = false, settings = {}, - }: { mappings?: TypeMapping; settings?: Record } = {}, + }: { + mappings?: TypeMapping; + reindexCollection?: boolean; + settings?: Record; + } = {}, ) { const esRequest = { index: await this._getIndice(index, collection), @@ -1690,7 +1695,10 @@ export class ES7 { await this.updateMapping(index, collection, mappings); - if (this._dynamicChanges(previousMappings, mappings)) { + if ( + reindexCollection || + this._dynamicChanges(previousMappings, mappings) + ) { await this.updateSearchIndex(index, collection); } } diff --git a/lib/service/storage/8/elasticsearch.ts b/lib/service/storage/8/elasticsearch.ts index b520c03ac7..cd0a6acebd 100644 --- a/lib/service/storage/8/elasticsearch.ts +++ b/lib/service/storage/8/elasticsearch.ts @@ -36,21 +36,21 @@ import { import assert from "assert"; -import ms from "ms"; import Bluebird from "bluebird"; +import ms from "ms"; import semver from "semver"; +import { storeScopeEnum } from "../../../core/storage/storeScopeEnum"; +import * as kerror from "../../../kerror"; import debug from "../../../util/debug"; -import ESWrapper from "./esWrapper"; -import { QueryTranslator } from "../commons/queryTranslator"; import didYouMean from "../../../util/didYouMean"; -import * as kerror from "../../../kerror"; -import { assertIsObject } from "../../../util/requestAssertions"; -import { isPlainObject } from "../../../util/safeObject"; -import { storeScopeEnum } from "../../../core/storage/storeScopeEnum"; import extractFields from "../../../util/extractFields"; import { Mutex } from "../../../util/mutex"; import { randomNumber } from "../../../util/name-generator"; +import { assertIsObject } from "../../../util/requestAssertions"; +import { isPlainObject } from "../../../util/safeObject"; +import { QueryTranslator } from "../commons/queryTranslator"; +import ESWrapper from "./esWrapper"; debug("kuzzle:services:elasticsearch"); @@ -1666,9 +1666,11 @@ export class ES8 { collection: string, { mappings = {}, + reindexCollection = false, settings = {}, }: { mappings?: estypes.MappingTypeMapping; + reindexCollection?: boolean; settings?: Record; } = {}, ) { @@ -1700,7 +1702,10 @@ export class ES8 { await this.updateMapping(index, collection, mappings); - if (this._dynamicChanges(previousMappings, mappings)) { + if ( + reindexCollection || + this._dynamicChanges(previousMappings, mappings) + ) { await this.updateSearchIndex(index, collection); } } From ef744c09596ed0dc6ececf13f8eed8650160bb9a Mon Sep 17 00:00:00 2001 From: Florian Maunier Date: Tue, 1 Oct 2024 11:45:09 +0200 Subject: [PATCH 2/3] test(collectioncontroller): add reindexCollection test for collection:update --- jest/api/controller/collection/update.test.ts | 74 +++++++++++++++++++ package-lock.json | 9 ++- package.json | 2 +- 3 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 jest/api/controller/collection/update.test.ts diff --git a/jest/api/controller/collection/update.test.ts b/jest/api/controller/collection/update.test.ts new file mode 100644 index 0000000000..ca5a93ec6f --- /dev/null +++ b/jest/api/controller/collection/update.test.ts @@ -0,0 +1,74 @@ +import { Kuzzle, WebSocket } from "kuzzle-sdk"; + +const kuzzle = new Kuzzle(new WebSocket("localhost")); +const index = "nyc-open-data"; +const collection = "green-taxi"; +const mappings = { + dynamic: "false" as const, + properties: { + name: { + type: "keyword", + }, + }, +}; + +beforeAll(async () => { + await kuzzle.connect(); + if (await kuzzle.index.exists(index)) { + await kuzzle.index.delete(index); + } + + await kuzzle.index.create(index); + await kuzzle.collection.create(index, collection, { + mappings, + }); +}); + +afterAll(async () => { + await kuzzle.index.delete(index); + kuzzle.disconnect(); +}); + +describe("collection:update", () => { + it("should reindex the collection if asked to", async () => { + await kuzzle.document.create( + index, + collection, + { age: 42, name: "Bob" }, + "document-1", + { refresh: "wait_for" }, + ); + + let result = await kuzzle.document.search(index, collection, { + query: { + range: { + age: { + gte: 40, + }, + }, + }, + }); + + expect(result.hits.length).toEqual(0); + + await kuzzle.collection.update(index, collection, { + mappings: { properties: { age: { type: "long" } } }, + reindexCollection: true, + }); + + // Wait for the reindexing to complete + await new Promise((r) => setTimeout(r, 2000)); + + result = await kuzzle.document.search(index, collection, { + query: { + range: { + age: { + gte: 40, + }, + }, + }, + }); + + expect(result.hits.length).toEqual(1); + }); +}); diff --git a/package-lock.json b/package-lock.json index 93bba18138..279c9cc78b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,7 @@ "koncorde": "4.3.0", "kuzzle-plugin-auth-passport-local": "6.4.1", "kuzzle-plugin-logger": "3.0.3", - "kuzzle-sdk": "^7.11.3", + "kuzzle-sdk": "^7.13.0", "kuzzle-vault": "2.0.4", "lodash": "4.17.21", "long": "5.2.3", @@ -10282,9 +10282,10 @@ } }, "node_modules/kuzzle-sdk": { - "version": "7.11.3", - "resolved": "https://registry.npmjs.org/kuzzle-sdk/-/kuzzle-sdk-7.11.3.tgz", - "integrity": "sha512-YHPSE99GRg1GT9UD5hOuJnt5vc/Gqmp705id4sCjyRJNRaq9uf3j3OzM6bvD+4YPZB7LTim53z66CFQP7IV7Qg==", + "version": "7.13.0", + "resolved": "https://registry.npmjs.org/kuzzle-sdk/-/kuzzle-sdk-7.13.0.tgz", + "integrity": "sha512-NootJz88hCkkuwXSRRnM9ir02NQvNn7AZbQcORDF/dOARxJLZD3ojZttt9jaeF9YmwcGXnsoSFV9pYO+2qOX+w==", + "license": "Apache-2.0", "dependencies": { "min-req-promise": "^1.0.1", "ws": "^8.13.0" diff --git a/package.json b/package.json index de7a3edf45..a470c06518 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "koncorde": "4.3.0", "kuzzle-plugin-auth-passport-local": "6.4.1", "kuzzle-plugin-logger": "3.0.3", - "kuzzle-sdk": "^7.11.3", + "kuzzle-sdk": "^7.13.0", "kuzzle-vault": "2.0.4", "lodash": "4.17.21", "long": "5.2.3", From 569bf2c680e70c47cb5b8f81a326f4039e8a5214 Mon Sep 17 00:00:00 2001 From: Juiced66 Date: Thu, 3 Oct 2024 07:26:48 +0200 Subject: [PATCH 3/3] fix: improve typing to avoid typescript build errors --- lib/kuzzle/kuzzle.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/kuzzle/kuzzle.ts b/lib/kuzzle/kuzzle.ts index 218b1b6e36..683c1701ae 100644 --- a/lib/kuzzle/kuzzle.ts +++ b/lib/kuzzle/kuzzle.ts @@ -103,7 +103,7 @@ type ImportStatus = { class Kuzzle extends KuzzleEventEmitter { public config: KuzzleConfiguration; - private _state: kuzzleStateEnum = kuzzleStateEnum.STARTING; + private _state: typeof kuzzleStateEnum = kuzzleStateEnum.STARTING; public log: Logger; private rootPath: string; /** @@ -138,7 +138,7 @@ class Kuzzle extends KuzzleEventEmitter { /** * Validation core component */ - public validation: Validation; + public validation: typeof Validation; /** * Dump generator @@ -148,7 +148,7 @@ class Kuzzle extends KuzzleEventEmitter { /** * Vault component (will be initialized after bootstrap) */ - public vault: vault; + public vault: typeof vault; /** * AsyncLocalStorage wrapper @@ -833,11 +833,11 @@ class Kuzzle extends KuzzleEventEmitter { ); } - get state() { + get state(): typeof kuzzleStateEnum { return this._state; } - set state(value) { + set state(value: typeof kuzzleStateEnum) { this._state = value; this.emit("kuzzle:state:change", value); }