From 6a410230f46318e89128457202cc0a562faf9c09 Mon Sep 17 00:00:00 2001 From: Joe Portner <5295965+jportner@users.noreply.github.com> Date: Fri, 20 Nov 2020 13:10:15 -0500 Subject: [PATCH] Add "reference" transforms for object types These are applied to all types using the `convertToMultiNamespaceTypeVersion` field when defining any object type. Any time the Kibana version is changed, all reference transforms for applicable versions are applied, and all documents' `referencesMigrationVersion` fields are updated to match the Kibana version. --- .../build_active_mappings.test.ts.snap | 8 + .../migrations/core/build_active_mappings.ts | 3 + .../migrations/core/document_migrator.test.ts | 264 +++++++++++++++++- .../migrations/core/document_migrator.ts | 173 +++++++++--- .../migrations/core/elastic_index.test.ts | 21 +- .../migrations/core/elastic_index.ts | 32 ++- .../migrations/core/index_migrator.test.ts | 7 + .../migrations/core/index_migrator.ts | 5 +- .../migrations/core/migration_context.ts | 3 + .../kibana_migrator.test.ts.snap | 4 + .../migrations/kibana/kibana_migrator.ts | 3 + .../serialization/serializer.test.ts | 41 +++ .../saved_objects/serialization/serializer.ts | 17 +- .../saved_objects/serialization/types.ts | 1 + .../service/lib/repository.test.js | 2 + .../saved_objects/service/lib/repository.ts | 2 +- .../apis/saved_objects/migrations.ts | 25 +- 17 files changed, 536 insertions(+), 75 deletions(-) diff --git a/src/core/server/saved_objects/migrations/core/__snapshots__/build_active_mappings.test.ts.snap b/src/core/server/saved_objects/migrations/core/__snapshots__/build_active_mappings.test.ts.snap index f8ef47cae894420..32c5f326f026575 100644 --- a/src/core/server/saved_objects/migrations/core/__snapshots__/build_active_mappings.test.ts.snap +++ b/src/core/server/saved_objects/migrations/core/__snapshots__/build_active_mappings.test.ts.snap @@ -11,6 +11,7 @@ Object { "namespaces": "2f4316de49999235636386fe51dc06c1", "originId": "2f4316de49999235636386fe51dc06c1", "references": "7997cf5a56cc02bdc9c93361bde732b0", + "referencesMigrationVersion": "2f4316de49999235636386fe51dc06c1", "type": "2f4316de49999235636386fe51dc06c1", "updated_at": "00da57df13e94e9d98437d13ace4bfe0", }, @@ -50,6 +51,9 @@ Object { }, "type": "nested", }, + "referencesMigrationVersion": Object { + "type": "keyword", + }, "type": Object { "type": "keyword", }, @@ -70,6 +74,7 @@ Object { "namespaces": "2f4316de49999235636386fe51dc06c1", "originId": "2f4316de49999235636386fe51dc06c1", "references": "7997cf5a56cc02bdc9c93361bde732b0", + "referencesMigrationVersion": "2f4316de49999235636386fe51dc06c1", "secondType": "72d57924f415fbadb3ee293b67d233ab", "thirdType": "510f1f0adb69830cf8a1c5ce2923ed82", "type": "2f4316de49999235636386fe51dc06c1", @@ -113,6 +118,9 @@ Object { }, "type": "nested", }, + "referencesMigrationVersion": Object { + "type": "keyword", + }, "secondType": Object { "dynamic": false, "properties": Object { diff --git a/src/core/server/saved_objects/migrations/core/build_active_mappings.ts b/src/core/server/saved_objects/migrations/core/build_active_mappings.ts index 2f4427b27b6bfd2..511eac17ed358b1 100644 --- a/src/core/server/saved_objects/migrations/core/build_active_mappings.ts +++ b/src/core/server/saved_objects/migrations/core/build_active_mappings.ts @@ -164,6 +164,9 @@ function defaultMapping(): IndexMapping { }, }, }, + referencesMigrationVersion: { + type: 'keyword', + }, }, }; } diff --git a/src/core/server/saved_objects/migrations/core/document_migrator.test.ts b/src/core/server/saved_objects/migrations/core/document_migrator.test.ts index 0dadb86e208ca69..c3ded4a41273034 100644 --- a/src/core/server/saved_objects/migrations/core/document_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/core/document_migrator.test.ts @@ -28,6 +28,7 @@ import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; const mockLoggerFactory = loggingSystemMock.create(); const mockLogger = mockLoggerFactory.get('mock logger'); +const kibanaVersion = '25.2.3'; const createRegistry = (...types: Array>) => { const registry = new SavedObjectTypeRegistry(); @@ -51,7 +52,7 @@ beforeEach(() => { describe('DocumentMigrator', () => { function testOpts() { return { - kibanaVersion: '25.2.3', + kibanaVersion, typeRegistry: createRegistry(), log: mockLogger, }; @@ -202,6 +203,7 @@ describe('DocumentMigrator', () => { type: 'user', attributes: { name: 'Chris' }, migrationVersion: { user: '1.2.3' }, + referencesMigrationVersion: kibanaVersion, }); }); @@ -252,6 +254,7 @@ describe('DocumentMigrator', () => { attributes: { name: 'Tyler' }, migrationVersion: { acl: '2.3.5' }, acl: 'admins-only, sucka!', + referencesMigrationVersion: kibanaVersion, }); }); @@ -289,10 +292,11 @@ describe('DocumentMigrator', () => { id: 'me', type: 'user', attributes: { name: 'Tyler' }, + referencesMigrationVersion: kibanaVersion, }); }); - it('assumes documents w/ undefined migrationVersion are up to date', () => { + it('assumes documents w/ undefined migrationVersion and correct referencesMigrationVersion are up to date', () => { const migrator = new DocumentMigrator({ ...testOpts(), typeRegistry: createRegistry( @@ -321,6 +325,7 @@ describe('DocumentMigrator', () => { type: 'user', attributes: { name: 'Tyler' }, bbb: 'Shazm', + referencesMigrationVersion: kibanaVersion, } as SavedObjectUnsanitizedDoc); expect(actual).toEqual({ id: 'me', @@ -331,6 +336,7 @@ describe('DocumentMigrator', () => { user: '1.0.0', bbb: '2.3.4', }, + referencesMigrationVersion: kibanaVersion, }); }); @@ -357,6 +363,7 @@ describe('DocumentMigrator', () => { type: 'dog', attributes: { name: 'Callie', b: 'B', c: 'C' }, migrationVersion: { dog: '2.0.1' }, + referencesMigrationVersion: kibanaVersion, }); }); @@ -423,6 +430,7 @@ describe('DocumentMigrator', () => { type: 'dog', attributes: { name: 'Callie', a: 1, b: 2, c: 3 }, migrationVersion: { dog: '10.0.1' }, + referencesMigrationVersion: kibanaVersion, }); }); @@ -456,6 +464,7 @@ describe('DocumentMigrator', () => { attributes: { name: 'Callie' }, animal: 'Animal: Doggie', migrationVersion: { animal: '1.0.0', dog: '2.2.4' }, + referencesMigrationVersion: kibanaVersion, }); }); @@ -482,6 +491,7 @@ describe('DocumentMigrator', () => { type: 'dog', attributes: { title: 'Title: Name: Callie' }, migrationVersion: { dog: '1.0.2' }, + referencesMigrationVersion: kibanaVersion, }); }); @@ -514,6 +524,7 @@ describe('DocumentMigrator', () => { type: 'cat', attributes: { name: 'Kitty Callie' }, migrationVersion: { dog: '2.2.4', cat: '1.0.0' }, + referencesMigrationVersion: kibanaVersion, }); }); @@ -590,6 +601,7 @@ describe('DocumentMigrator', () => { type: 'cat', attributes: { name: 'Shiny' }, migrationVersion: { cat: '3.0.0' }, + referencesMigrationVersion: kibanaVersion, }); }); @@ -614,6 +626,7 @@ describe('DocumentMigrator', () => { type: 'cat', attributes: { name: 'Boo' }, migrationVersion: { cat: '1.0.0', foo: '5.6.7' }, + referencesMigrationVersion: kibanaVersion, }); }); @@ -693,6 +706,14 @@ describe('DocumentMigrator', () => { '3.2.3': (doc: SavedObjectUnsanitizedDoc) => doc, '2.0.0': (doc: SavedObjectUnsanitizedDoc) => doc, }, + }, + { + name: 'ccc', + namespaceType: 'multiple', + migrations: { + '9.0.0': (doc: SavedObjectUnsanitizedDoc) => doc, + }, + convertToMultiNamespaceTypeVersion: '11.0.0', // this results in reference transforms getting added to other types, but does not increase the migrationVersion of those types } ), }); @@ -700,11 +721,12 @@ describe('DocumentMigrator', () => { expect(migrationVersion).toEqual({ aaa: '10.4.0', bbb: '3.2.3', + ccc: '11.0.0', }); }); describe('conversion to multi-namespace type', () => { - it('assumes documents w/ undefined migrationVersion are up to date', () => { + it('assumes documents w/ undefined migrationVersion and correct referencesMigrationVersion are up to date', () => { const migrator = new DocumentMigrator({ ...testOpts(), typeRegistry: createRegistry( @@ -716,6 +738,7 @@ describe('DocumentMigrator', () => { id: 'mischievous', type: 'dog', attributes: { name: 'Ann' }, + referencesMigrationVersion: kibanaVersion, } as SavedObjectUnsanitizedDoc; const actual = migrator.migrate(obj, { convertTypes: true }); expect(actual).toEqual({ @@ -723,33 +746,39 @@ describe('DocumentMigrator', () => { type: 'dog', attributes: { name: 'Ann' }, migrationVersion: { dog: '1.0.0' }, + referencesMigrationVersion: kibanaVersion, // there is no 'namespaces' field because no transforms were applied; this scenario is contrived for a clean test case but is not indicative of a real-world scenario }); }); - it('skips conversion transforms when the "convertTypes" option is falsy', () => { + it('skips reference transforms and conversion transforms when the "convertTypes" option is falsy', () => { const migrator = new DocumentMigrator({ ...testOpts(), - typeRegistry: createRegistry({ - name: 'dog', - namespaceType: 'multiple', - convertToMultiNamespaceTypeVersion: '1.0.0', - }), + typeRegistry: createRegistry( + { name: 'dog', namespaceType: 'multiple', convertToMultiNamespaceTypeVersion: '1.0.0' }, + { name: 'toy', namespaceType: 'multiple', convertToMultiNamespaceTypeVersion: '1.0.0' } + ), }); const obj = { id: 'cowardly', type: 'dog', attributes: { name: 'Leslie' }, migrationVersion: {}, + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], + namespace: 'foo-namespace', }; const test = (convertTypes?: boolean) => { const actual = migrator.migrate(obj, { convertTypes }); + expect(mockUuidv5).not.toHaveBeenCalled(); expect(actual).toEqual({ id: 'cowardly', type: 'dog', attributes: { name: 'Leslie' }, migrationVersion: { dog: '1.0.0' }, - // there is no 'namespaces' field because no transforms were applied; this scenario is contrived for a clean test case but is not indicative of a real-world scenario + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], + referencesMigrationVersion: kibanaVersion, + namespace: 'foo-namespace', + // there is no 'namespaces' field because no conversion transform was applied; this scenario is contrived for a clean test case but is not indicative of a real-world scenario }); }; @@ -757,6 +786,52 @@ describe('DocumentMigrator', () => { test(false); }); + describe('correctly applies reference transforms', () => { + const migrator = new DocumentMigrator({ + ...testOpts(), + typeRegistry: createRegistry( + { name: 'dog', namespaceType: 'single' }, + { name: 'toy', namespaceType: 'multiple', convertToMultiNamespaceTypeVersion: '1.0.0' } + ), + }); + const obj = { + id: 'bad', + type: 'dog', + attributes: { name: 'Sweet Peach' }, + migrationVersion: {}, + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], + }; + + it('in the default space', () => { + const actual = migrator.migrate(obj, { convertTypes: true }); + expect(mockUuidv5).not.toHaveBeenCalled(); + expect(actual).toEqual({ + id: 'bad', + type: 'dog', + attributes: { name: 'Sweet Peach' }, + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], // no change + referencesMigrationVersion: kibanaVersion, + }); + }); + + it('in a non-default space', () => { + const actual = migrator.migrate( + { ...obj, namespace: 'foo-namespace' }, + { convertTypes: true } + ); + expect(mockUuidv5).toHaveBeenCalledTimes(1); + expect(mockUuidv5).toHaveBeenCalledWith('foo-namespace:toy:favorite', 'DNSUUID'); + expect(actual).toEqual({ + id: 'bad', + type: 'dog', + attributes: { name: 'Sweet Peach' }, + references: [{ id: 'uuidv5', type: 'toy', name: 'BALL!' }], // changed + referencesMigrationVersion: kibanaVersion, + namespace: 'foo-namespace', + }); + }); + }); + describe('correctly applies conversion transforms', () => { const migrator = new DocumentMigrator({ ...testOpts(), @@ -781,6 +856,7 @@ describe('DocumentMigrator', () => { type: 'dog', attributes: { name: 'Wally' }, migrationVersion: { dog: '1.0.0' }, + referencesMigrationVersion: kibanaVersion, namespaces: ['default'], }); }); @@ -797,12 +873,119 @@ describe('DocumentMigrator', () => { type: 'dog', attributes: { name: 'Wally' }, migrationVersion: { dog: '1.0.0' }, + referencesMigrationVersion: kibanaVersion, namespaces: ['foo-namespace'], originId: 'loud', }); }); }); + describe('correctly applies reference and conversion transforms', () => { + const migrator = new DocumentMigrator({ + ...testOpts(), + typeRegistry: createRegistry( + { name: 'dog', namespaceType: 'multiple', convertToMultiNamespaceTypeVersion: '1.0.0' }, + { name: 'toy', namespaceType: 'multiple', convertToMultiNamespaceTypeVersion: '1.0.0' } + ), + }); + const obj = { + id: 'cute', + type: 'dog', + attributes: { name: 'Too' }, + migrationVersion: {}, + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], + }; + + it('in the default space', () => { + const actual = migrator.migrate(obj, { convertTypes: true }); + expect(mockUuidv5).not.toHaveBeenCalled(); + expect(actual).toEqual({ + id: 'cute', + type: 'dog', + attributes: { name: 'Too' }, + migrationVersion: { dog: '1.0.0' }, + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], // no change + referencesMigrationVersion: kibanaVersion, + namespaces: ['default'], + }); + }); + + it('in a non-default space', () => { + const actual = migrator.migrate( + { ...obj, namespace: 'foo-namespace' }, + { convertTypes: true } + ); + expect(mockUuidv5).toHaveBeenCalledTimes(2); + expect(mockUuidv5).toHaveBeenNthCalledWith(1, 'foo-namespace:toy:favorite', 'DNSUUID'); + expect(mockUuidv5).toHaveBeenNthCalledWith(2, 'foo-namespace:dog:cute', 'DNSUUID'); + expect(actual).toEqual({ + id: 'uuidv5', + type: 'dog', + attributes: { name: 'Too' }, + migrationVersion: { dog: '1.0.0' }, + references: [{ id: 'uuidv5', type: 'toy', name: 'BALL!' }], // changed + referencesMigrationVersion: kibanaVersion, + namespaces: ['foo-namespace'], + originId: 'cute', + }); + }); + }); + + describe('correctly applies reference and migration transforms', () => { + const migrator = new DocumentMigrator({ + ...testOpts(), + typeRegistry: createRegistry( + { + name: 'dog', + namespaceType: 'single', + migrations: { + '1.0.0': setAttr('migrationVersion.dog', '2.0.0'), + '2.0.0': (doc) => doc, // noop + }, + }, + { name: 'toy', namespaceType: 'multiple', convertToMultiNamespaceTypeVersion: '1.0.0' } + ), + }); + const obj = { + id: 'sleepy', + type: 'dog', + attributes: { name: 'Patches' }, + migrationVersion: {}, + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], + }; + + it('in the default space', () => { + const actual = migrator.migrate(obj, { convertTypes: true }); + expect(mockUuidv5).not.toHaveBeenCalled(); + expect(actual).toEqual({ + id: 'sleepy', + type: 'dog', + attributes: { name: 'Patches' }, + migrationVersion: { dog: '2.0.0' }, + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], // no change + referencesMigrationVersion: kibanaVersion, + }); + }); + + it('in a non-default space', () => { + const actual = migrator.migrate( + { ...obj, namespace: 'foo-namespace' }, + { convertTypes: true } + ); + expect(mockUuidv5).toHaveBeenCalledTimes(1); + expect(mockUuidv5).toHaveBeenCalledWith('foo-namespace:toy:favorite', 'DNSUUID'); + expect(actual).toEqual({ + id: 'sleepy', + type: 'dog', + attributes: { name: 'Patches' }, + migrationVersion: { dog: '2.0.0' }, + references: [{ id: 'uuidv5', type: 'toy', name: 'BALL!' }], // changed + referencesMigrationVersion: kibanaVersion, + namespace: 'foo-namespace', + }); + }); + }); + describe('correctly applies conversion and migration transforms', () => { const migrator = new DocumentMigrator({ ...testOpts(), @@ -831,6 +1014,7 @@ describe('DocumentMigrator', () => { type: 'dog', attributes: { name: 'Remy' }, migrationVersion: { dog: '2.0.0' }, + referencesMigrationVersion: kibanaVersion, namespaces: ['default'], }); }); @@ -847,11 +1031,71 @@ describe('DocumentMigrator', () => { type: 'dog', attributes: { name: 'Remy' }, migrationVersion: { dog: '2.0.0' }, + referencesMigrationVersion: kibanaVersion, namespaces: ['foo-namespace'], originId: 'hungry', }); }); }); + + describe('correctly applies reference, conversion, and migration transforms', () => { + const migrator = new DocumentMigrator({ + ...testOpts(), + typeRegistry: createRegistry( + { + name: 'dog', + namespaceType: 'multiple', + migrations: { + '1.0.0': setAttr('migrationVersion.dog', '2.0.0'), + '2.0.0': (doc) => doc, // noop + }, + convertToMultiNamespaceTypeVersion: '1.0.0', + }, + { name: 'toy', namespaceType: 'multiple', convertToMultiNamespaceTypeVersion: '1.0.0' } + ), + }); + const obj = { + id: 'pretty', + type: 'dog', + attributes: { name: 'Sasha' }, + migrationVersion: {}, + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], + }; + + it('in the default space', () => { + const actual = migrator.migrate(obj, { convertTypes: true }); + expect(mockUuidv5).not.toHaveBeenCalled(); + expect(actual).toEqual({ + id: 'pretty', + type: 'dog', + attributes: { name: 'Sasha' }, + migrationVersion: { dog: '2.0.0' }, + references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], // no change + referencesMigrationVersion: kibanaVersion, + namespaces: ['default'], + }); + }); + + it('in a non-default space', () => { + const actual = migrator.migrate( + { ...obj, namespace: 'foo-namespace' }, + { convertTypes: true } + ); + expect(mockUuidv5).toHaveBeenCalledTimes(2); + expect(mockUuidv5).toHaveBeenNthCalledWith(1, 'foo-namespace:toy:favorite', 'DNSUUID'); + expect(mockUuidv5).toHaveBeenNthCalledWith(2, 'foo-namespace:dog:pretty', 'DNSUUID'); + expect(actual).toEqual({ + id: 'uuidv5', + type: 'dog', + attributes: { name: 'Sasha' }, + migrationVersion: { dog: '2.0.0' }, + references: [{ id: 'uuidv5', type: 'toy', name: 'BALL!' }], // changed + referencesMigrationVersion: kibanaVersion, + namespaces: ['foo-namespace'], + originId: 'pretty', + }); + }); + }); }); }); }); diff --git a/src/core/server/saved_objects/migrations/core/document_migrator.ts b/src/core/server/saved_objects/migrations/core/document_migrator.ts index c098cbed12d3d1c..07a60f62f803a85 100644 --- a/src/core/server/saved_objects/migrations/core/document_migrator.ts +++ b/src/core/server/saved_objects/migrations/core/document_migrator.ts @@ -90,7 +90,8 @@ interface DocumentMigratorOptions { interface ActiveMigrations { [type: string]: { - latestVersion: string; + latestVersion?: string; + latestReferenceVersion?: string; transforms: Transform[]; }; } @@ -98,7 +99,7 @@ interface ActiveMigrations { interface Transform { version: string; transform: (doc: SavedObjectUnsanitizedDoc) => SavedObjectUnsanitizedDoc; - transformType: 'migrate' | 'convert'; + transformType: 'migrate' | 'convert' | 'reference'; } /** @@ -143,7 +144,13 @@ export class DocumentMigrator implements VersionedTransformer { * @memberof DocumentMigrator */ public get migrationVersion(): SavedObjectsMigrationVersion { - return _.mapValues(this.migrations, ({ latestVersion }) => latestVersion); + return Object.entries(this.migrations).reduce((acc, [prop, { latestVersion }]) => { + // some migration objects won't have a latestVersion (they only contain reference transforms that are applied from other types) + if (latestVersion) { + return { ...acc, [prop]: latestVersion }; + } + return acc; + }, {}); } /** @@ -253,39 +260,43 @@ function buildActiveMigrations( typeRegistry: ISavedObjectTypeRegistry, log: Logger ): ActiveMigrations { - return typeRegistry - .getAllTypes() - .filter( - (type) => - (type.migrations && Object.keys(type.migrations).length > 0) || - type.convertToMultiNamespaceTypeVersion - ) - .reduce((migrations, type) => { - const migrationTransforms = Object.entries(type.migrations!).map( - ([version, transform]) => ({ - version, - transform: wrapWithTry(version, type.name, transform, log), - transformType: 'migrate', - }) - ); - const conversionTransforms = getConversionTransforms(type); - const transforms = [...migrationTransforms, ...conversionTransforms].sort( - transformComparator - ); - return { - ...migrations, - [type.name]: { - latestVersion: _.last(transforms)!.version, - transforms, - }, - }; - }, {} as ActiveMigrations); + const referenceTransforms = getReferenceTransforms(typeRegistry); + + return typeRegistry.getAllTypes().reduce((migrations, type) => { + const migrationTransforms = Object.entries(type.migrations ?? {}).map( + ([version, transform]) => ({ + version, + transform: wrapWithTry(version, type.name, transform, log), + transformType: 'migrate', + }) + ); + const conversionTransforms = getConversionTransforms(type); + const transforms = [ + ...referenceTransforms, + ...conversionTransforms, + ...migrationTransforms, + ].sort(transformComparator); + + if (!transforms.length) { + return migrations; + } + return { + ...migrations, + [type.name]: { + latestVersion: _.last(transforms.filter((x) => x.transformType !== 'reference'))?.version, + latestReferenceVersion: _.last(transforms.filter((x) => x.transformType === 'reference')) + ?.version, + transforms, + }, + }; + }, {} as ActiveMigrations); } /** * Creates a function which migrates and validates any document that is passed to it. */ function buildDocumentTransform({ + kibanaVersion, migrations, }: { kibanaVersion: string; @@ -297,8 +308,8 @@ function buildDocumentTransform({ ) { const { convertTypes = false } = options; const result = doc.migrationVersion - ? applyMigrations(doc, migrations, convertTypes) - : markAsUpToDate(doc, migrations); + ? applyMigrations(doc, migrations, kibanaVersion, convertTypes) + : markAsUpToDate(doc, migrations, kibanaVersion); // In order to keep tests a bit more stable, we won't // tack on an empy migrationVersion to docs that have @@ -314,12 +325,15 @@ function buildDocumentTransform({ function applyMigrations( doc: SavedObjectUnsanitizedDoc, migrations: ActiveMigrations, + kibanaVersion: string, convertTypes: boolean ) { while (true) { const prop = nextUnmigratedProp(doc, migrations); if (!prop) { - return doc; + // regardless of whether or not any reference transform was applied, update the referencesMigrationVersion + // this is needed to ensure that newly created documents have an up-to-date referencesMigrationVersion field + return { ...doc, referencesMigrationVersion: kibanaVersion }; } doc = migrateProp(doc, prop, migrations, convertTypes); } @@ -345,13 +359,18 @@ function propVersion(doc: SavedObjectUnsanitizedDoc | ActiveMigrations, prop: st /** * Sets the doc's migrationVersion to be the most recent version */ -function markAsUpToDate(doc: SavedObjectUnsanitizedDoc, migrations: ActiveMigrations) { +function markAsUpToDate( + doc: SavedObjectUnsanitizedDoc, + migrations: ActiveMigrations, + kibanaVersion: string +) { return { ...doc, migrationVersion: props(doc).reduce((acc, prop) => { const version = propVersion(migrations, prop); return version ? set(acc, prop, version) : acc; }, {}), + referencesMigrationVersion: kibanaVersion, }; } @@ -394,9 +413,43 @@ function getConversionTransforms(type: SavedObjectsType): Transform[] { } /** - * Transforms are sorted in ascending order by version. One version may contain multiple transforms; 'convert' transforms always run first, - * and 'migrate' transforms always run last. This is because 'migrate' transforms are defined by the consumer, and may change the object - * type or migrationVersion which resets the migration loop and could cause any remaining transforms for this version to be skipped. + * Returns all applicable reference transforms for all object types. + */ +function getReferenceTransforms(typeRegistry: ISavedObjectTypeRegistry): Transform[] { + const transformMap = typeRegistry + .getAllTypes() + .filter((type) => type.convertToMultiNamespaceTypeVersion) + .reduce((acc, { convertToMultiNamespaceTypeVersion: key, name }) => { + const val = acc.get(key!) ?? new Set(); + return acc.set(key!, val.add(name)); + }, new Map>()); + + return Array.from(transformMap, ([key, val]) => ({ + version: key, + transform: (doc) => { + const { namespace, references } = doc; + if (namespace && references?.length) { + return { + ...doc, + references: references.map(({ type, id, ...attrs }) => ({ + ...attrs, + type, + id: val.has(type) ? uuidv5(`${namespace}:${type}:${id}`, uuidv5.DNS) : id, + })), + }; + } + return doc; + }, + transformType: 'reference', + })); +} + +/** + * Transforms are sorted in ascending order by version. One version may contain multiple transforms; 'reference' transforms always run + * first, 'convert' transforms always run second, and 'migrate' transforms always run last. This is because: + * 1. 'convert' transforms get rid of the `namespace` field, which must be present for 'reference' transforms to function correctly. + * 2. 'migrate' transforms are defined by the consumer, and may change the object type or migrationVersion which resets the migration loop + * and could cause any remaining transforms for this version to be skipped. */ function transformComparator(a: Transform, b: Transform) { const semver = Semver.compare(a.version, b.version); @@ -407,6 +460,10 @@ function transformComparator(a: Transform, b: Transform) { return 1; } else if (b.transformType === 'migrate') { return -1; + } else if (a.transformType === 'convert') { + return 1; + } else if (b.transformType === 'convert') { + return -1; } } return 0; @@ -445,6 +502,23 @@ function wrapWithTry( }; } +function getHasPendingReferenceTransform( + doc: SavedObjectUnsanitizedDoc, + migrations: ActiveMigrations, + prop: string +) { + if (!migrations[prop]) { + return false; + } + + const { latestReferenceVersion } = migrations[prop]; + const { referencesMigrationVersion } = doc; + return ( + latestReferenceVersion && + (!referencesMigrationVersion || Semver.gt(latestReferenceVersion, referencesMigrationVersion)) + ); +} + /** * Finds the first unmigrated property in the specified document. */ @@ -452,10 +526,7 @@ function nextUnmigratedProp(doc: SavedObjectUnsanitizedDoc, migrations: ActiveMi return props(doc).find((p) => { const latestVersion = propVersion(migrations, p); const docVersion = propVersion(doc, p); - - if (latestVersion === docVersion) { - return false; - } + const hasPendingReferenceTransform = getHasPendingReferenceTransform(doc, migrations, p); // We verify that the version is not greater than the version supported by Kibana. // If we didn't, this would cause an infinite loop, as we'd be unable to migrate the property @@ -470,7 +541,7 @@ function nextUnmigratedProp(doc: SavedObjectUnsanitizedDoc, migrations: ActiveMi ); } - return true; + return (latestVersion && latestVersion !== docVersion) || hasPendingReferenceTransform; }); } @@ -494,11 +565,16 @@ function migrateProp( } if (transformType === 'migrate' || convertTypes) { - // migrate transforms are always applied, but conversion transforms are only applied when Kibana is upgraded + // migrate transforms are always applied, but conversion transforms and reference transforms are only applied when Kibana is upgraded doc = transform(doc); } - migrationVersion = updateMigrationVersion(doc, migrationVersion, prop, version); - doc.migrationVersion = _.clone(migrationVersion); + if (transformType === 'reference') { + // regardless of whether or not the reference transform was applied, increment the version + doc.referencesMigrationVersion = version; + } else { + migrationVersion = updateMigrationVersion(doc, migrationVersion, prop, version); + doc.migrationVersion = _.clone(migrationVersion); + } if (doc.type !== originalType) { // the transform function changed the object's type; break out of the loop @@ -518,9 +594,14 @@ function applicableTransforms( prop: string ) { const minVersion = propVersion(doc, prop); + const minReferenceVersion = doc.referencesMigrationVersion || '0.0.0'; const { transforms } = migrations[prop]; return minVersion - ? transforms.filter(({ version }) => Semver.gt(version, minVersion)) + ? transforms.filter(({ version, transformType }) => + transformType === 'reference' + ? Semver.gt(version, minReferenceVersion) + : Semver.gt(version, minVersion) + ) : transforms; } diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts index 0b3ad1b6e3cc890..dbbbec347b69c2d 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts @@ -568,6 +568,7 @@ describe('ElasticIndex', () => { mappings, count, migrations, + kibanaVersion, }: any) { client.indices.get = jest.fn().mockReturnValueOnce( elasticsearchClientMock.createSuccessTransportRequestPromise({ @@ -581,7 +582,12 @@ describe('ElasticIndex', () => { }) ); - const hasMigrations = await Index.migrationsUpToDate(client, index, migrations); + const hasMigrations = await Index.migrationsUpToDate( + client, + index, + migrations, + kibanaVersion + ); return { hasMigrations }; } @@ -595,6 +601,7 @@ describe('ElasticIndex', () => { }, count: 0, migrations: { dashy: '2.3.4' }, + kibanaVersion: '7.10.0', }); expect(hasMigrations).toBeFalsy(); @@ -622,6 +629,7 @@ describe('ElasticIndex', () => { }, count: 2, migrations: {}, + kibanaVersion: '7.10.0', }); expect(hasMigrations).toBeTruthy(); @@ -663,6 +671,7 @@ describe('ElasticIndex', () => { }, count: 3, migrations: { dashy: '23.2.5' }, + kibanaVersion: '7.10.0', }); expect(hasMigrations).toBeFalsy(); @@ -688,6 +697,7 @@ describe('ElasticIndex', () => { bashy: '99.9.3', flashy: '3.4.5', }, + kibanaVersion: '7.10.0', }); function shouldClause(type: string, version: string) { @@ -713,6 +723,15 @@ describe('ElasticIndex', () => { shouldClause('dashy', '23.2.5'), shouldClause('bashy', '99.9.3'), shouldClause('flashy', '3.4.5'), + { + bool: { + must_not: { + term: { + referencesMigrationVersion: '7.10.0', + }, + }, + }, + }, ], }, }, diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts index d5093bfd8dc4211..ead2a635a5a0ecd 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts @@ -158,6 +158,7 @@ export async function migrationsUpToDate( client: MigrationEsClient, index: string, migrationVersion: SavedObjectsMigrationVersion, + kibanaVersion: string, retryCount: number = 10 ): Promise { try { @@ -176,18 +177,29 @@ export async function migrationsUpToDate( body: { query: { bool: { - should: Object.entries(migrationVersion).map(([type, latestVersion]) => ({ - bool: { - must: [ - { exists: { field: type } }, - { - bool: { - must_not: { term: { [`migrationVersion.${type}`]: latestVersion } }, + should: [ + ...Object.entries(migrationVersion).map(([type, latestVersion]) => ({ + bool: { + must: [ + { exists: { field: type } }, + { + bool: { + must_not: { term: { [`migrationVersion.${type}`]: latestVersion } }, + }, + }, + ], + }, + })), + { + bool: { + must_not: { + term: { + referencesMigrationVersion: kibanaVersion, }, }, - ], + }, }, - })), + ], }, }, }, @@ -205,7 +217,7 @@ export async function migrationsUpToDate( await new Promise((r) => setTimeout(r, 1000)); - return await migrationsUpToDate(client, index, migrationVersion, retryCount - 1); + return await migrationsUpToDate(client, index, migrationVersion, kibanaVersion, retryCount - 1); } } diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts index d2d0f313151544c..ba01baf5a4a8b74 100644 --- a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts @@ -35,6 +35,7 @@ describe('IndexMigrator', () => { batchSize: 10, client: elasticsearchClientMock.createElasticsearchClient(), index: '.kibana', + kibanaVersion: '7.10.0', log: loggingSystemMock.create().get(), mappingProperties: {}, pollInterval: 1, @@ -68,6 +69,7 @@ describe('IndexMigrator', () => { namespaces: '2f4316de49999235636386fe51dc06c1', originId: '2f4316de49999235636386fe51dc06c1', references: '7997cf5a56cc02bdc9c93361bde732b0', + referencesMigrationVersion: '2f4316de49999235636386fe51dc06c1', type: '2f4316de49999235636386fe51dc06c1', updated_at: '00da57df13e94e9d98437d13ace4bfe0', }, @@ -88,6 +90,7 @@ describe('IndexMigrator', () => { id: { type: 'keyword' }, }, }, + referencesMigrationVersion: { type: 'keyword' }, }, }, settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, @@ -189,6 +192,7 @@ describe('IndexMigrator', () => { namespaces: '2f4316de49999235636386fe51dc06c1', originId: '2f4316de49999235636386fe51dc06c1', references: '7997cf5a56cc02bdc9c93361bde732b0', + referencesMigrationVersion: '2f4316de49999235636386fe51dc06c1', type: '2f4316de49999235636386fe51dc06c1', updated_at: '00da57df13e94e9d98437d13ace4bfe0', }, @@ -210,6 +214,7 @@ describe('IndexMigrator', () => { id: { type: 'keyword' }, }, }, + referencesMigrationVersion: { type: 'keyword' }, }, }, settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, @@ -250,6 +255,7 @@ describe('IndexMigrator', () => { namespaces: '2f4316de49999235636386fe51dc06c1', originId: '2f4316de49999235636386fe51dc06c1', references: '7997cf5a56cc02bdc9c93361bde732b0', + referencesMigrationVersion: '2f4316de49999235636386fe51dc06c1', type: '2f4316de49999235636386fe51dc06c1', updated_at: '00da57df13e94e9d98437d13ace4bfe0', }, @@ -271,6 +277,7 @@ describe('IndexMigrator', () => { id: { type: 'keyword' }, }, }, + referencesMigrationVersion: { type: 'keyword' }, }, }, settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.ts b/src/core/server/saved_objects/migrations/core/index_migrator.ts index ceca27fa87723f2..967352f1a5b3918 100644 --- a/src/core/server/saved_objects/migrations/core/index_migrator.ts +++ b/src/core/server/saved_objects/migrations/core/index_migrator.ts @@ -70,13 +70,14 @@ export class IndexMigrator { * Determines what action the migration system needs to take (none, patch, migrate). */ async function requiresMigration(context: Context): Promise { - const { client, alias, documentMigrator, dest, log } = context; + const { client, alias, documentMigrator, dest, kibanaVersion, log } = context; // Have all of our known migrations been run against the index? const hasMigrations = await Index.migrationsUpToDate( client, alias, - documentMigrator.migrationVersion + documentMigrator.migrationVersion, + kibanaVersion ); if (!hasMigrations) { diff --git a/src/core/server/saved_objects/migrations/core/migration_context.ts b/src/core/server/saved_objects/migrations/core/migration_context.ts index 0ea362d65623e2f..6ee8dfeecf91f2d 100644 --- a/src/core/server/saved_objects/migrations/core/migration_context.ts +++ b/src/core/server/saved_objects/migrations/core/migration_context.ts @@ -43,6 +43,7 @@ export interface MigrationOpts { scrollDuration: string; client: MigrationEsClient; index: string; + kibanaVersion: string; log: Logger; mappingProperties: SavedObjectsTypeMappingDefinitions; documentMigrator: VersionedTransformer; @@ -65,6 +66,7 @@ export interface Context { source: Index.FullIndexInfo; dest: Index.FullIndexInfo; documentMigrator: VersionedTransformer; + kibanaVersion: string; log: SavedObjectsMigrationLogger; batchSize: number; pollInterval: number; @@ -89,6 +91,7 @@ export async function migrationContext(opts: MigrationOpts): Promise { alias, source, dest, + kibanaVersion: opts.kibanaVersion, log: new MigrationLogger(log), batchSize: opts.batchSize, documentMigrator: opts.documentMigrator, diff --git a/src/core/server/saved_objects/migrations/kibana/__snapshots__/kibana_migrator.test.ts.snap b/src/core/server/saved_objects/migrations/kibana/__snapshots__/kibana_migrator.test.ts.snap index 9311292a6a0edee..29e2c4970f98248 100644 --- a/src/core/server/saved_objects/migrations/kibana/__snapshots__/kibana_migrator.test.ts.snap +++ b/src/core/server/saved_objects/migrations/kibana/__snapshots__/kibana_migrator.test.ts.snap @@ -11,6 +11,7 @@ Object { "namespaces": "2f4316de49999235636386fe51dc06c1", "originId": "2f4316de49999235636386fe51dc06c1", "references": "7997cf5a56cc02bdc9c93361bde732b0", + "referencesMigrationVersion": "2f4316de49999235636386fe51dc06c1", "type": "2f4316de49999235636386fe51dc06c1", "updated_at": "00da57df13e94e9d98437d13ace4bfe0", }, @@ -58,6 +59,9 @@ Object { }, "type": "nested", }, + "referencesMigrationVersion": Object { + "type": "keyword", + }, "type": Object { "type": "keyword", }, diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index 18a385c6994b879..50ade55e692e133 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -60,6 +60,7 @@ export class KibanaMigrator { private readonly savedObjectsConfig: SavedObjectsMigrationConfigType; private readonly documentMigrator: VersionedTransformer; private readonly kibanaConfig: KibanaConfigType; + private readonly kibanaVersion: string; private readonly log: Logger; private readonly mappingProperties: SavedObjectsTypeMappingDefinitions; private readonly typeRegistry: ISavedObjectTypeRegistry; @@ -83,6 +84,7 @@ export class KibanaMigrator { }: KibanaMigratorOptions) { this.client = client; this.kibanaConfig = kibanaConfig; + this.kibanaVersion = kibanaVersion; this.savedObjectsConfig = savedObjectsConfig; this.typeRegistry = typeRegistry; this.serializer = new SavedObjectsSerializer(this.typeRegistry); @@ -156,6 +158,7 @@ export class KibanaMigrator { client: this.client, documentMigrator: this.documentMigrator, index, + kibanaVersion: this.kibanaVersion, log: this.log, mappingProperties: indexMap[index].typeMappings, pollInterval: this.savedObjectsConfig.pollInterval, diff --git a/src/core/server/saved_objects/serialization/serializer.test.ts b/src/core/server/saved_objects/serialization/serializer.test.ts index c5f060268297618..1107ad78a778277 100644 --- a/src/core/server/saved_objects/serialization/serializer.test.ts +++ b/src/core/server/saved_objects/serialization/serializer.test.ts @@ -142,6 +142,27 @@ describe('#rawToSavedObject', () => { expect(expected).toEqual(actual); }); + test('if specified it copies the _source.referencesMigrationVersion property to referencesMigrationVersion', () => { + const actual = singleNamespaceSerializer.rawToSavedObject({ + _id: 'foo:bar', + _source: { + type: 'foo', + referencesMigrationVersion: '1.2.3', + }, + }); + expect(actual).toHaveProperty('referencesMigrationVersion', '1.2.3'); + }); + + test(`if _source.referencesMigrationVersion is unspecified it doesn't set referencesMigrationVersion`, () => { + const actual = singleNamespaceSerializer.rawToSavedObject({ + _id: 'foo:bar', + _source: { + type: 'foo', + }, + }); + expect(actual).not.toHaveProperty('referencesMigrationVersion'); + }); + test(`if version is unspecified it doesn't set version`, () => { const actual = singleNamespaceSerializer.rawToSavedObject({ _id: 'foo:bar', @@ -299,6 +320,7 @@ describe('#rawToSavedObject', () => { foo: '1.2.3', bar: '9.8.7', }, + referencesMigrationVersion: '4.5.6', namespace: 'foo-namespace', updated_at: String(new Date()), references: [], @@ -533,6 +555,25 @@ describe('#savedObjectToRaw', () => { expect(actual._source).not.toHaveProperty('migrationVersion'); }); + test('it copies the referencesMigrationVersion property to _source.referencesMigrationVersion', () => { + const actual = singleNamespaceSerializer.savedObjectToRaw({ + type: '', + attributes: {}, + referencesMigrationVersion: '1.2.3', + } as any); + + expect(actual._source).toHaveProperty('referencesMigrationVersion', '1.2.3'); + }); + + test(`if unspecified it doesn't add referencesMigrationVersion property to _source`, () => { + const actual = singleNamespaceSerializer.savedObjectToRaw({ + type: '', + attributes: {}, + } as any); + + expect(actual._source).not.toHaveProperty('referencesMigrationVersion'); + }); + test('it decodes the version property to _seq_no and _primary_term', () => { const actual = singleNamespaceSerializer.savedObjectToRaw({ type: '', diff --git a/src/core/server/saved_objects/serialization/serializer.ts b/src/core/server/saved_objects/serialization/serializer.ts index e809c89165d83e7..ba370686fd09c6a 100644 --- a/src/core/server/saved_objects/serialization/serializer.ts +++ b/src/core/server/saved_objects/serialization/serializer.ts @@ -68,7 +68,15 @@ export class SavedObjectsSerializer { ): SavedObjectSanitizedDoc { const { flexibleRawId = false } = options; const { _id, _source, _seq_no, _primary_term } = doc; - const { type, namespace, namespaces, originId } = _source; + const { + type, + namespace, + namespaces, + originId, + migrationVersion, + references, + referencesMigrationVersion, + } = _source; const version = _seq_no != null || _primary_term != null @@ -82,8 +90,9 @@ export class SavedObjectsSerializer { ...(namespaces && this.registry.isMultiNamespace(type) && { namespaces }), ...(originId && { originId }), attributes: _source[type], - references: _source.references || [], - ...(_source.migrationVersion && { migrationVersion: _source.migrationVersion }), + references: references || [], + ...(migrationVersion && { migrationVersion }), + ...(referencesMigrationVersion && { referencesMigrationVersion }), ...(_source.updated_at && { updated_at: _source.updated_at }), ...(version && { version }), }; @@ -107,6 +116,7 @@ export class SavedObjectsSerializer { updated_at, version, references, + referencesMigrationVersion, } = savedObj; const source = { [type]: attributes, @@ -116,6 +126,7 @@ export class SavedObjectsSerializer { ...(namespaces && this.registry.isMultiNamespace(type) && { namespaces }), ...(originId && { originId }), ...(migrationVersion && { migrationVersion }), + ...(referencesMigrationVersion && { referencesMigrationVersion }), ...(updated_at && { updated_at }), }; diff --git a/src/core/server/saved_objects/serialization/types.ts b/src/core/server/saved_objects/serialization/types.ts index 1609454af2cfd2f..a3a25da7c0ba62b 100644 --- a/src/core/server/saved_objects/serialization/types.ts +++ b/src/core/server/saved_objects/serialization/types.ts @@ -55,6 +55,7 @@ interface SavedObjectDoc { namespace?: string; namespaces?: string[]; migrationVersion?: SavedObjectsMigrationVersion; + referencesMigrationVersion?: string; version?: string; updated_at?: string; originId?: string; diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index 6f885f17fd82b22..9c5d756a4f2530e 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -964,6 +964,7 @@ describe('SavedObjectsRepository', () => { ...response.items[0].create, _source: { ...response.items[0].create._source, + referencesMigrationVersion: '2.0.0', // the document migrator adds this to all objects before creation namespaces: response.items[0].create._source.namespaces, }, _id: expect.stringMatching(/^myspace:config:[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/), @@ -972,6 +973,7 @@ describe('SavedObjectsRepository', () => { ...response.items[1].create, _source: { ...response.items[1].create._source, + referencesMigrationVersion: '2.0.0', // the document migrator adds this to all objects before creation namespaces: response.items[1].create._source.namespaces, }, }); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index d362c02de491530..2a21bb95b2f0696 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -1637,7 +1637,7 @@ export class SavedObjectsRepository { if (this._registry.isSingleNamespace(type)) { savedObject.namespaces = [SavedObjectsUtils.namespaceIdToString(namespace)]; } - return omit(savedObject, 'namespace') as SavedObject; + return omit(savedObject, ['namespace', 'referencesMigrationVersion']) as SavedObject; } /** diff --git a/test/api_integration/apis/saved_objects/migrations.ts b/test/api_integration/apis/saved_objects/migrations.ts index 99a58620b17f569..e44db1908a2fff2 100644 --- a/test/api_integration/apis/saved_objects/migrations.ts +++ b/test/api_integration/apis/saved_objects/migrations.ts @@ -39,6 +39,7 @@ import { } from '../../../../src/core/server/saved_objects'; import { FtrProviderContext } from '../../ftr_provider_context'; +const KIBANA_VERSION = '99.9.9'; function getLogMock() { return { debug() {}, @@ -156,6 +157,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { bar: '1.9.0' }, bar: { mynum: 68 }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, { id: 'bar:o', @@ -163,14 +165,22 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { bar: '1.9.0' }, bar: { mynum: 6 }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here + }, + { + id: 'baz:u', + type: 'baz', + baz: { title: 'Terrific!' }, + references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, - { id: 'baz:u', type: 'baz', baz: { title: 'Terrific!' }, references: [] }, { id: 'foo:a', type: 'foo', migrationVersion: { foo: '1.0.0' }, foo: { name: 'FOO A' }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, { id: 'foo:e', @@ -178,6 +188,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { foo: '1.0.0' }, foo: { name: 'FOOEY' }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, ]); }); @@ -227,6 +238,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { bar: '1.9.0' }, bar: { mynum: 68 }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, { id: 'bar:o', @@ -234,6 +246,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { bar: '1.9.0' }, bar: { mynum: 6 }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, { id: 'foo:a', @@ -241,6 +254,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { foo: '1.0.0' }, foo: { name: 'FOO A' }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, { id: 'foo:e', @@ -248,6 +262,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { foo: '1.0.0' }, foo: { name: 'FOOEY' }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, ]); @@ -259,6 +274,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { bar: '2.3.4' }, bar: { mynum: 68, name: 'NAME i' }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, { id: 'bar:o', @@ -266,6 +282,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { bar: '2.3.4' }, bar: { mynum: 6, name: 'NAME o' }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, { id: 'foo:a', @@ -273,6 +290,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { foo: '2.0.1' }, foo: { name: 'FOO Av2' }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, { id: 'foo:e', @@ -280,6 +298,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { foo: '2.0.1' }, foo: { name: 'FOOEYv2' }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, ]); }); @@ -335,6 +354,7 @@ export default ({ getService }: FtrProviderContext) => { migrationVersion: { foo: '1.0.0' }, foo: { name: 'LOTR' }, references: [], + referencesMigrationVersion: KIBANA_VERSION, // this field is omitted from saved objects, but we fetched the raw document here }, ]); }); @@ -392,7 +412,7 @@ async function migrateIndex({ types.forEach((type) => typeRegistry.registerType(type)); const documentMigrator = new DocumentMigrator({ - kibanaVersion: '99.9.9', + kibanaVersion: KIBANA_VERSION, typeRegistry, log: getLogMock(), }); @@ -401,6 +421,7 @@ async function migrateIndex({ client: createMigrationEsClient(esClient, getLogMock()), documentMigrator, index, + kibanaVersion: KIBANA_VERSION, obsoleteIndexTemplatePattern, mappingProperties, batchSize: 10,