From 97d67f680e8bd0597363de2059bc95fbfcc6a1de Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 17 Feb 2021 18:27:45 -0800 Subject: [PATCH 01/14] Set all keyword and text fields for `index.query.default_field` setting --- .../epm/elasticsearch/template/install.ts | 4 +- .../epm/elasticsearch/template/template.ts | 43 ++++++++++++++++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 70afa78e723bcf..f66c8f67faa473 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -297,7 +297,8 @@ export async function installTemplate({ packageVersion: string; packageName: string; }): Promise { - const mappings = generateMappings(processFields(fields)); + const validFields = processFields(fields); + const mappings = generateMappings(validFields); const templateName = generateTemplateName(dataStream); const templateIndexPattern = generateTemplateIndexPattern(dataStream); const templatePriority = getTemplatePriority(dataStream); @@ -359,6 +360,7 @@ export async function installTemplate({ const template = getTemplate({ type: dataStream.type, templateIndexPattern, + fields: validFields, mappings, pipelineName, packageName, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index b86c989f8c24c8..e9849c8c5ccf40 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -37,6 +37,8 @@ const DEFAULT_IGNORE_ABOVE = 1024; const DEFAULT_TEMPLATE_PRIORITY = 200; const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; +const DEFAULT_FIELD_TYPES = ['keyword', 'text']; + /** * getTemplate retrieves the default template but overwrites the index pattern with the given value. * @@ -45,6 +47,7 @@ const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; export function getTemplate({ type, templateIndexPattern, + fields, mappings, pipelineName, packageName, @@ -55,6 +58,7 @@ export function getTemplate({ }: { type: string; templateIndexPattern: string; + fields: Fields; mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; @@ -66,6 +70,7 @@ export function getTemplate({ const template = getBaseTemplate( type, templateIndexPattern, + fields, mappings, packageName, composedOfTemplates, @@ -296,9 +301,28 @@ export function generateESIndexPatterns( return patterns; } +const flattenFieldsToNameAndType = ( + fields: Fields, + path: string = '' +): Array> => { + let newFields: Array> = []; + fields.forEach((field) => { + const fieldName = path ? `${path}.${field.name}` : field.name; + newFields.push({ + name: fieldName, + type: field.type, + }); + if (field.fields && field.fields.length) { + newFields = newFields.concat(flattenFieldsToNameAndType(field.fields, fieldName)); + } + }); + return newFields; +}; + function getBaseTemplate( type: string, templateIndexPattern: string, + fields: Fields, mappings: IndexTemplateMappings, packageName: string, composedOfTemplates: string[], @@ -315,6 +339,10 @@ function getBaseTemplate( managed: true, }; + const defaultFields = flattenFieldsToNameAndType(fields) + .filter((field) => field.type && DEFAULT_FIELD_TYPES.includes(field.type)) + .map((field) => field.name); + return { priority: templatePriority, // To be completed with the correct index patterns @@ -338,13 +366,18 @@ function getBaseTemplate( refresh_interval: '5s', // Default in the stack now, still good to have it in number_of_shards: '1', - // All the default fields which should be queried have to be added here. - // So far we add all keyword and text fields here. - query: { - default_field: ['message'], - }, // We are setting 30 because it can be devided by several numbers. Useful when shrinking. number_of_routing_shards: '30', + // All the default fields which should be queried have to be added here. + // So far we add all keyword and text fields here if there are any, otherwise + // this setting is skipped. + ...(defaultFields.length + ? { + query: { + default_field: defaultFields, + }, + } + : {}), }, }, mappings: { From ff920186924367211ab8edc62e28a8a31ba494b2 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 17 Feb 2021 18:28:24 -0800 Subject: [PATCH 02/14] Update tests and snapshots --- .../__snapshots__/template.test.ts.snap | 56 ++++++++++++++++--- .../elasticsearch/template/template.test.ts | 8 +++ 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index 2d2478843c4541..e57d5f6c58aa6c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -20,12 +20,12 @@ exports[`tests loading base.yml: base.yml 1`] = ` }, "refresh_interval": "5s", "number_of_shards": "1", + "number_of_routing_shards": "30", "query": { "default_field": [ - "message" + "long.nested.foo" ] - }, - "number_of_routing_shards": "30" + } } }, "mappings": { @@ -129,12 +129,17 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` }, "refresh_interval": "5s", "number_of_shards": "1", + "number_of_routing_shards": "30", "query": { "default_field": [ - "message" + "coredns.id", + "coredns.query.class", + "coredns.query.name", + "coredns.query.type", + "coredns.response.code", + "coredns.response.flags" ] - }, - "number_of_routing_shards": "30" + } } }, "mappings": { @@ -238,12 +243,45 @@ exports[`tests loading system.yml: system.yml 1`] = ` }, "refresh_interval": "5s", "number_of_shards": "1", + "number_of_routing_shards": "30", "query": { "default_field": [ - "message" + "system.diskio.name", + "system.diskio.serial_number", + "system.filesystem.device_name", + "system.filesystem.type", + "system.filesystem.mount_point", + "system.network.name", + "system.process.state", + "system.process.cmdline", + "system.process.cgroup.id", + "system.process.cgroup.path", + "system.process.cgroup.cpu.id", + "system.process.cgroup.cpu.path", + "system.process.cgroup.cpuacct.id", + "system.process.cgroup.cpuacct.path", + "system.process.cgroup.memory.id", + "system.process.cgroup.memory.path", + "system.process.cgroup.blkio.id", + "system.process.cgroup.blkio.path", + "system.raid.name", + "system.raid.status", + "system.raid.level", + "system.raid.sync_action", + "system.socket.remote.host", + "system.socket.remote.etld_plus_one", + "system.socket.remote.host_error", + "system.socket.process.cmdline", + "system.users.id", + "system.users.seat", + "system.users.path", + "system.users.type", + "system.users.service", + "system.users.state", + "system.users.scope", + "system.users.remote_host" ] - }, - "number_of_routing_shards": "30" + } } }, "mappings": { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index a176805307845c..bb5a42fa43c2c7 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -35,6 +35,7 @@ test('get template', () => { type: 'logs', templateIndexPattern, packageName: 'nginx', + fields: [], mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, @@ -49,6 +50,7 @@ test('adds composed_of correctly', () => { type: 'logs', templateIndexPattern: 'name-*', packageName: 'nginx', + fields: [], mappings: { properties: {} }, composedOfTemplates, templatePriority: 200, @@ -63,6 +65,7 @@ test('adds empty composed_of correctly', () => { type: 'logs', templateIndexPattern: 'name-*', packageName: 'nginx', + fields: [], mappings: { properties: {} }, composedOfTemplates, templatePriority: 200, @@ -77,6 +80,7 @@ test('adds hidden field correctly', () => { type: 'logs', templateIndexPattern, packageName: 'nginx', + fields: [], mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, @@ -88,6 +92,7 @@ test('adds hidden field correctly', () => { type: 'logs', templateIndexPattern, packageName: 'nginx', + fields: [], mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, @@ -106,6 +111,7 @@ test('tests loading base.yml', () => { type: 'logs', templateIndexPattern: 'foo-*', packageName: 'nginx', + fields: processedFields, mappings, composedOfTemplates: [], templatePriority: 200, @@ -125,6 +131,7 @@ test('tests loading coredns.logs.yml', () => { type: 'logs', templateIndexPattern: 'foo-*', packageName: 'coredns', + fields: processedFields, mappings, composedOfTemplates: [], templatePriority: 200, @@ -144,6 +151,7 @@ test('tests loading system.yml', () => { type: 'metrics', templateIndexPattern: 'whatsthis-*', packageName: 'system', + fields: processedFields, mappings, composedOfTemplates: [], templatePriority: 200, From 4345c0bd0dd2a6745ac62ce79b820f69074ddd30 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 17 Feb 2021 20:20:06 -0800 Subject: [PATCH 03/14] Fix test --- x-pack/test/fleet_api_integration/apis/epm/template.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index d79452ca0eb6f2..c0138ee06d9574 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -20,12 +20,20 @@ export default function ({ getService }: FtrProviderContext) { }, }, }; + const fields = [ + { + name: 'foo', + type: 'keyword', + }, + ]; + // This test was inspired by https://github.com/elastic/kibana/blob/master/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js describe('EPM - template', async () => { it('can be loaded', async () => { const template = getTemplate({ type: 'logs', templateIndexPattern, + fields, mappings, packageName: 'system', composedOfTemplates: [], From 42a81dae872efcd1b897d7cc715b7b5b89a51703 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 1 Mar 2021 12:58:50 -0800 Subject: [PATCH 04/14] Add default field limit safeguard --- .../epm/elasticsearch/template/template.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index e9849c8c5ccf40..1fa633f72a4612 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -38,6 +38,7 @@ const DEFAULT_TEMPLATE_PRIORITY = 200; const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; const DEFAULT_FIELD_TYPES = ['keyword', 'text']; +const DEFAULT_FIELD_LIMIT = 1024; /** * getTemplate retrieves the default template but overwrites the index pattern with the given value. @@ -339,9 +340,15 @@ function getBaseTemplate( managed: true, }; - const defaultFields = flattenFieldsToNameAndType(fields) - .filter((field) => field.type && DEFAULT_FIELD_TYPES.includes(field.type)) - .map((field) => field.name); + // Find all field names to set `index.query.default_field` to, which will be + // the first 1024 keyword or text fields + const defaultFields = flattenFieldsToNameAndType(fields).filter( + (field) => field.type && DEFAULT_FIELD_TYPES.includes(field.type) + ); + const defaultFieldNames = (defaultFields.length > DEFAULT_FIELD_LIMIT + ? defaultFields.slice(0, DEFAULT_FIELD_LIMIT) + : defaultFields + ).map((field) => field.name); return { priority: templatePriority, @@ -371,10 +378,10 @@ function getBaseTemplate( // All the default fields which should be queried have to be added here. // So far we add all keyword and text fields here if there are any, otherwise // this setting is skipped. - ...(defaultFields.length + ...(defaultFieldNames.length ? { query: { - default_field: defaultFields, + default_field: defaultFieldNames, }, } : {}), From 98dafb65994411309447e58d203837500efb7f4a Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 1 Mar 2021 13:08:10 -0800 Subject: [PATCH 05/14] Add logging when beyond limit --- .../services/epm/elasticsearch/template/template.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 1fa633f72a4612..41b7bf4022fab2 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -13,6 +13,7 @@ import { IndexTemplate, IndexTemplateMappings, } from '../../../../types'; +import { appContextService } from '../../../'; import { getRegistryDataStreamAssetBaseName } from '../index'; interface Properties { @@ -331,6 +332,8 @@ function getBaseTemplate( ilmPolicy?: string | undefined, hidden?: boolean ): IndexTemplate { + const logger = appContextService.getLogger(); + // Meta information to identify Ingest Manager's managed templates and indices const _meta = { package: { @@ -345,6 +348,11 @@ function getBaseTemplate( const defaultFields = flattenFieldsToNameAndType(fields).filter( (field) => field.type && DEFAULT_FIELD_TYPES.includes(field.type) ); + if (defaultFields.length > DEFAULT_FIELD_LIMIT) { + logger.warn( + `large amount of default fields detected for index template ${templateIndexPattern} in package ${packageName}, applying the first ${DEFAULT_FIELD_LIMIT} fields` + ); + } const defaultFieldNames = (defaultFields.length > DEFAULT_FIELD_LIMIT ? defaultFields.slice(0, DEFAULT_FIELD_LIMIT) : defaultFields From 6585ecb5e7d42441ad0b3d7c3f917cbca276dd6a Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 1 Mar 2021 14:33:35 -0800 Subject: [PATCH 06/14] Update tests to mock app context (because I added logger usage) --- .../__snapshots__/template.test.ts.snap | 6 +- .../elasticsearch/template/install.test.ts | 351 ++++---- .../elasticsearch/template/template.test.ts | 846 +++++++++--------- 3 files changed, 611 insertions(+), 592 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index e57d5f6c58aa6c..65eec939d58502 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`tests loading base.yml: base.yml 1`] = ` +exports[`EPM template tests loading base.yml: base.yml 1`] = ` { "priority": 200, "index_patterns": [ @@ -109,7 +109,7 @@ exports[`tests loading base.yml: base.yml 1`] = ` } `; -exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` +exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` { "priority": 200, "index_patterns": [ @@ -223,7 +223,7 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` } `; -exports[`tests loading system.yml: system.yml 1`] = ` +exports[`EPM template tests loading system.yml: system.yml 1`] = ` { "priority": 200, "index_patterns": [ diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index cdcd3972fd189a..22586f2ad8dbd3 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -6,191 +6,202 @@ */ import { RegistryDataStream } from '../../../../types'; +import { createAppContextStartContractMock } from '../../../../mocks'; +import { appContextService } from '../../../../services'; import { Field } from '../../fields/field'; - import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { installTemplate } from './install'; -test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } +describe('EPM install', () => { + beforeEach(async () => { + appContextService.start(createAppContextStartContractMock()); }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixUnset, - packageVersion: pkg.version, - packageName: pkg.name, - }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); -}); + test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { index_templates: [] }; + } + }); -test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } + const fields: Field[] = []; + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixUnset, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[1][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixFalse = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: false, - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixFalse = 200; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixFalse, - packageVersion: pkg.version, - packageName: pkg.name, - }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); -}); + test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { index_templates: [] }; + } + }); -test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } + const fields: Field[] = []; + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixFalse, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[1][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixTrue = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: true, - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; - const templatePriorityDatasetIsPrefixTrue = 150; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixTrue, - packageVersion: pkg.version, - packageName: pkg.name, + test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { index_templates: [] }; + } + }); + + const fields: Field[] = []; + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 150; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixTrue, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[1][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); -}); -test('tests installPackage remove the aliases property if the property existed', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { - index_templates: [ - { - name: 'metrics-package.dataset', - index_template: { - index_patterns: ['metrics-package.dataset-*'], - template: { aliases: {} }, + test('tests installPackage remove the aliases property if the property existed', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { + index_templates: [ + { + name: 'metrics-package.dataset', + index_template: { + index_patterns: ['metrics-package.dataset-*'], + template: { aliases: {} }, + }, }, - }, - ], - }; - } - }); + ], + }; + } + }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixUnset, - packageVersion: pkg.version, - packageName: pkg.name, - }); + const fields: Field[] = []; + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixUnset, + packageVersion: pkg.version, + packageName: pkg.name, + }); - // @ts-ignore - const removeAliases = callCluster.mock.calls[1][1].body; - expect(removeAliases.template.aliases).not.toBeDefined(); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[2][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); + // @ts-ignore + const removeAliases = callCluster.mock.calls[1][1].body; + expect(removeAliases.template.aliases).not.toBeDefined(); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[2][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index bb5a42fa43c2c7..42962711d67880 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -9,6 +9,8 @@ import { readFileSync } from 'fs'; import { safeLoad } from 'js-yaml'; import path from 'path'; import { RegistryDataStream } from '../../../../types'; +import { createAppContextStartContractMock } from '../../../../mocks'; +import { appContextService } from '../../../../services'; import { Field, processFields } from '../../fields/field'; import { generateMappings, @@ -28,140 +30,145 @@ expect.addSnapshotSerializer({ }, }); -test('get template', () => { - const templateIndexPattern = 'logs-nginx.access-abcd-*'; - - const template = getTemplate({ - type: 'logs', - templateIndexPattern, - packageName: 'nginx', - fields: [], - mappings: { properties: {} }, - composedOfTemplates: [], - templatePriority: 200, +describe('EPM template', () => { + beforeEach(async () => { + appContextService.start(createAppContextStartContractMock()); }); - expect(template.index_patterns).toStrictEqual([templateIndexPattern]); -}); -test('adds composed_of correctly', () => { - const composedOfTemplates = ['component1', 'component2']; - - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'name-*', - packageName: 'nginx', - fields: [], - mappings: { properties: {} }, - composedOfTemplates, - templatePriority: 200, + test('get template', () => { + const templateIndexPattern = 'logs-nginx.access-abcd-*'; + + const template = getTemplate({ + type: 'logs', + templateIndexPattern, + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates: [], + templatePriority: 200, + }); + expect(template.index_patterns).toStrictEqual([templateIndexPattern]); }); - expect(template.composed_of).toStrictEqual(composedOfTemplates); -}); -test('adds empty composed_of correctly', () => { - const composedOfTemplates: string[] = []; - - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'name-*', - packageName: 'nginx', - fields: [], - mappings: { properties: {} }, - composedOfTemplates, - templatePriority: 200, + test('adds composed_of correctly', () => { + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'name-*', + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates, + templatePriority: 200, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); }); - expect(template.composed_of).toStrictEqual(composedOfTemplates); -}); -test('adds hidden field correctly', () => { - const templateIndexPattern = 'logs-nginx.access-abcd-*'; - - const templateWithHidden = getTemplate({ - type: 'logs', - templateIndexPattern, - packageName: 'nginx', - fields: [], - mappings: { properties: {} }, - composedOfTemplates: [], - templatePriority: 200, - hidden: true, - }); - expect(templateWithHidden.data_stream.hidden).toEqual(true); - - const templateWithoutHidden = getTemplate({ - type: 'logs', - templateIndexPattern, - packageName: 'nginx', - fields: [], - mappings: { properties: {} }, - composedOfTemplates: [], - templatePriority: 200, + test('adds empty composed_of correctly', () => { + const composedOfTemplates: string[] = []; + + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'name-*', + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates, + templatePriority: 200, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); }); - expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); -}); -test('tests loading base.yml', () => { - const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); - const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = safeLoad(fieldsYML); - - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'nginx', - fields: processedFields, - mappings, - composedOfTemplates: [], - templatePriority: 200, + test('adds hidden field correctly', () => { + const templateIndexPattern = 'logs-nginx.access-abcd-*'; + + const templateWithHidden = getTemplate({ + type: 'logs', + templateIndexPattern, + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates: [], + templatePriority: 200, + hidden: true, + }); + expect(templateWithHidden.data_stream.hidden).toEqual(true); + + const templateWithoutHidden = getTemplate({ + type: 'logs', + templateIndexPattern, + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates: [], + templatePriority: 200, + }); + expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); -}); - -test('tests loading coredns.logs.yml', () => { - const ymlPath = path.join(__dirname, '../../fields/tests/coredns.logs.yml'); - const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = safeLoad(fieldsYML); - - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'coredns', - fields: processedFields, - mappings, - composedOfTemplates: [], - templatePriority: 200, + test('tests loading base.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'foo-*', + packageName: 'nginx', + fields: processedFields, + mappings, + composedOfTemplates: [], + templatePriority: 200, + }); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); -}); - -test('tests loading system.yml', () => { - const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); - const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = safeLoad(fieldsYML); - - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'metrics', - templateIndexPattern: 'whatsthis-*', - packageName: 'system', - fields: processedFields, - mappings, - composedOfTemplates: [], - templatePriority: 200, + test('tests loading coredns.logs.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/coredns.logs.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'foo-*', + packageName: 'coredns', + fields: processedFields, + mappings, + composedOfTemplates: [], + templatePriority: 200, + }); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); -}); + test('tests loading system.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate({ + type: 'metrics', + templateIndexPattern: 'whatsthis-*', + packageName: 'system', + fields: processedFields, + mappings, + composedOfTemplates: [], + templatePriority: 200, + }); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); + }); -test('tests processing text field with multi fields', () => { - const textWithMultiFieldsLiteralYml = ` + test('tests processing text field with multi fields', () => { + const textWithMultiFieldsLiteralYml = ` - name: textWithMultiFields type: text multi_fields: @@ -170,30 +177,30 @@ test('tests processing text field with multi fields', () => { - name: indexed type: text `; - const textWithMultiFieldsMapping = { - properties: { - textWithMultiFields: { - type: 'text', - fields: { - raw: { - ignore_above: 1024, - type: 'keyword', - }, - indexed: { - type: 'text', + const textWithMultiFieldsMapping = { + properties: { + textWithMultiFields: { + type: 'text', + fields: { + raw: { + ignore_above: 1024, + type: 'keyword', + }, + indexed: { + type: 'text', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(textWithMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(textWithMultiFieldsMapping); + }); -test('tests processing keyword field with multi fields', () => { - const keywordWithMultiFieldsLiteralYml = ` + test('tests processing keyword field with multi fields', () => { + const keywordWithMultiFieldsLiteralYml = ` - name: keywordWithMultiFields type: keyword multi_fields: @@ -203,31 +210,31 @@ test('tests processing keyword field with multi fields', () => { type: text `; - const keywordWithMultiFieldsMapping = { - properties: { - keywordWithMultiFields: { - ignore_above: 1024, - type: 'keyword', - fields: { - raw: { - ignore_above: 1024, - type: 'keyword', - }, - indexed: { - type: 'text', + const keywordWithMultiFieldsMapping = { + properties: { + keywordWithMultiFields: { + ignore_above: 1024, + type: 'keyword', + fields: { + raw: { + ignore_above: 1024, + type: 'keyword', + }, + indexed: { + type: 'text', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(keywordWithMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(keywordWithMultiFieldsMapping); + }); -test('tests processing keyword field with multi fields with analyzed text field', () => { - const keywordWithAnalyzedMultiFieldsLiteralYml = ` + test('tests processing keyword field with multi fields with analyzed text field', () => { + const keywordWithAnalyzedMultiFieldsLiteralYml = ` - name: keywordWithAnalyzedMultiField type: keyword multi_fields: @@ -237,29 +244,29 @@ test('tests processing keyword field with multi fields with analyzed text field' search_analyzer: standard `; - const keywordWithAnalyzedMultiFieldsMapping = { - properties: { - keywordWithAnalyzedMultiField: { - ignore_above: 1024, - type: 'keyword', - fields: { - analyzed: { - analyzer: 'autocomplete', - search_analyzer: 'standard', - type: 'text', + const keywordWithAnalyzedMultiFieldsMapping = { + properties: { + keywordWithAnalyzedMultiField: { + ignore_above: 1024, + type: 'keyword', + fields: { + analyzed: { + analyzer: 'autocomplete', + search_analyzer: 'standard', + type: 'text', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping); + }); -test('tests processing keyword field with multi fields with normalized keyword field', () => { - const keywordWithNormalizedMultiFieldsLiteralYml = ` + test('tests processing keyword field with multi fields with normalized keyword field', () => { + const keywordWithNormalizedMultiFieldsLiteralYml = ` - name: keywordWithNormalizedMultiField type: keyword multi_fields: @@ -268,235 +275,235 @@ test('tests processing keyword field with multi fields with normalized keyword f normalizer: lowercase `; - const keywordWithNormalizedMultiFieldsMapping = { - properties: { - keywordWithNormalizedMultiField: { - ignore_above: 1024, - type: 'keyword', - fields: { - normalized: { - type: 'keyword', - ignore_above: 1024, - normalizer: 'lowercase', + const keywordWithNormalizedMultiFieldsMapping = { + properties: { + keywordWithNormalizedMultiField: { + ignore_above: 1024, + type: 'keyword', + fields: { + normalized: { + type: 'keyword', + ignore_above: 1024, + normalizer: 'lowercase', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(keywordWithNormalizedMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(keywordWithNormalizedMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping); + }); -test('tests processing object field with no other attributes', () => { - const objectFieldLiteralYml = ` + test('tests processing object field with no other attributes', () => { + const objectFieldLiteralYml = ` - name: objectField type: object `; - const objectFieldMapping = { - properties: { - objectField: { - type: 'object', + const objectFieldMapping = { + properties: { + objectField: { + type: 'object', + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldMapping); + }); -test('tests processing object field with enabled set to false', () => { - const objectFieldEnabledFalseLiteralYml = ` + test('tests processing object field with enabled set to false', () => { + const objectFieldEnabledFalseLiteralYml = ` - name: objectField type: object enabled: false `; - const objectFieldEnabledFalseMapping = { - properties: { - objectField: { - type: 'object', - enabled: false, + const objectFieldEnabledFalseMapping = { + properties: { + objectField: { + type: 'object', + enabled: false, + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldEnabledFalseMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldEnabledFalseMapping); + }); -test('tests processing object field with dynamic set to false', () => { - const objectFieldDynamicFalseLiteralYml = ` + test('tests processing object field with dynamic set to false', () => { + const objectFieldDynamicFalseLiteralYml = ` - name: objectField type: object dynamic: false `; - const objectFieldDynamicFalseMapping = { - properties: { - objectField: { - type: 'object', - dynamic: false, + const objectFieldDynamicFalseMapping = { + properties: { + objectField: { + type: 'object', + dynamic: false, + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldDynamicFalseMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldDynamicFalseMapping); + }); -test('tests processing object field with dynamic set to true', () => { - const objectFieldDynamicTrueLiteralYml = ` + test('tests processing object field with dynamic set to true', () => { + const objectFieldDynamicTrueLiteralYml = ` - name: objectField type: object dynamic: true `; - const objectFieldDynamicTrueMapping = { - properties: { - objectField: { - type: 'object', - dynamic: true, + const objectFieldDynamicTrueMapping = { + properties: { + objectField: { + type: 'object', + dynamic: true, + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldDynamicTrueMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldDynamicTrueMapping); + }); -test('tests processing object field with dynamic set to strict', () => { - const objectFieldDynamicStrictLiteralYml = ` + test('tests processing object field with dynamic set to strict', () => { + const objectFieldDynamicStrictLiteralYml = ` - name: objectField type: object dynamic: strict `; - const objectFieldDynamicStrictMapping = { - properties: { - objectField: { - type: 'object', - dynamic: 'strict', + const objectFieldDynamicStrictMapping = { + properties: { + objectField: { + type: 'object', + dynamic: 'strict', + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldDynamicStrictMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldDynamicStrictMapping); + }); -test('tests processing object field with property', () => { - const objectFieldWithPropertyLiteralYml = ` + test('tests processing object field with property', () => { + const objectFieldWithPropertyLiteralYml = ` - name: a type: object - name: a.b type: keyword `; - const objectFieldWithPropertyMapping = { - properties: { - a: { - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const objectFieldWithPropertyMapping = { + properties: { + a: { + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldWithPropertyMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldWithPropertyMapping); + }); -test('tests processing object field with property, reverse order', () => { - const objectFieldWithPropertyReversedLiteralYml = ` + test('tests processing object field with property, reverse order', () => { + const objectFieldWithPropertyReversedLiteralYml = ` - name: a.b type: keyword - name: a type: object dynamic: false `; - const objectFieldWithPropertyReversedMapping = { - properties: { - a: { - dynamic: false, - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const objectFieldWithPropertyReversedMapping = { + properties: { + a: { + dynamic: false, + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); + }); -test('tests processing nested field with property', () => { - const nestedYaml = ` + test('tests processing nested field with property', () => { + const nestedYaml = ` - name: a.b type: keyword - name: a type: nested dynamic: false `; - const expectedMapping = { - properties: { - a: { - dynamic: false, - type: 'nested', - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const expectedMapping = { + properties: { + a: { + dynamic: false, + type: 'nested', + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(nestedYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(expectedMapping); -}); + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); -test('tests processing nested field with property, nested field first', () => { - const nestedYaml = ` + test('tests processing nested field with property, nested field first', () => { + const nestedYaml = ` - name: a type: nested include_in_parent: true - name: a.b type: keyword `; - const expectedMapping = { - properties: { - a: { - include_in_parent: true, - type: 'nested', - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const expectedMapping = { + properties: { + a: { + include_in_parent: true, + type: 'nested', + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(nestedYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(expectedMapping); -}); + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); -test('tests processing nested leaf field with properties', () => { - const nestedYaml = ` + test('tests processing nested leaf field with properties', () => { + const nestedYaml = ` - name: a type: object dynamic: false @@ -504,98 +511,99 @@ test('tests processing nested leaf field with properties', () => { type: nested enabled: false `; - const expectedMapping = { - properties: { - a: { - dynamic: false, - properties: { - b: { - enabled: false, - type: 'nested', + const expectedMapping = { + properties: { + a: { + dynamic: false, + properties: { + b: { + enabled: false, + type: 'nested', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(nestedYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(expectedMapping); -}); + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); -test('tests constant_keyword field type handling', () => { - const constantKeywordLiteralYaml = ` + test('tests constant_keyword field type handling', () => { + const constantKeywordLiteralYaml = ` - name: constantKeyword type: constant_keyword `; - const constantKeywordMapping = { - properties: { - constantKeyword: { - type: 'constant_keyword', + const constantKeywordMapping = { + properties: { + constantKeyword: { + type: 'constant_keyword', + }, }, - }, - }; - const fields: Field[] = safeLoad(constantKeywordLiteralYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); -}); + }; + const fields: Field[] = safeLoad(constantKeywordLiteralYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); + }); -test('tests priority and index pattern for data stream without dataset_is_prefix', () => { - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset); - const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset); - - expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset); - expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset); -}); + test('tests priority and index pattern for data stream without dataset_is_prefix', () => { + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset); + }); -test('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { - const dataStreamDatasetIsPrefixFalse = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: false, - } as RegistryDataStream; - const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixFalse = 200; - const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse); - const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse); - - expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse); - expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse); -}); + test('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse); + }); -test('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { - const dataStreamDatasetIsPrefixTrue = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: true, - } as RegistryDataStream; - const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; - const templatePriorityDatasetIsPrefixTrue = 150; - const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue); - const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue); - - expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue); - expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue); + test('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 150; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue); + }); }); From bde924ac79e43dfb08efbcaa81dc234290c1f138 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 1 Mar 2021 17:23:03 -0800 Subject: [PATCH 07/14] Update api integration test --- x-pack/test/fleet_api_integration/apis/epm/template.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index c0138ee06d9574..990955b311751b 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -7,6 +7,8 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { createAppContextStartContractMock } from '../../../../plugins/fleet/server/mocks'; +import { appContextService } from '../../../../plugins/fleet/server/services'; import { getTemplate } from '../../../../plugins/fleet/server/services/epm/elasticsearch/template/template'; export default function ({ getService }: FtrProviderContext) { @@ -29,6 +31,10 @@ export default function ({ getService }: FtrProviderContext) { // This test was inspired by https://github.com/elastic/kibana/blob/master/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js describe('EPM - template', async () => { + beforeEach(async () => { + appContextService.start(createAppContextStartContractMock()); + }); + it('can be loaded', async () => { const template = getTemplate({ type: 'logs', From ae35298eb77faf6f75d8491bd5ca6048c69bc4b1 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 2 Mar 2021 12:01:50 -0800 Subject: [PATCH 08/14] Rename consts --- .../epm/elasticsearch/template/template.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 41b7bf4022fab2..24cddedb817bc9 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -38,8 +38,8 @@ const DEFAULT_IGNORE_ABOVE = 1024; const DEFAULT_TEMPLATE_PRIORITY = 200; const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; -const DEFAULT_FIELD_TYPES = ['keyword', 'text']; -const DEFAULT_FIELD_LIMIT = 1024; +const QUERY_DEFAULT_FIELD_TYPES = ['keyword', 'text']; +const QUERY_DEFAULT_FIELD_LIMIT = 1024; /** * getTemplate retrieves the default template but overwrites the index pattern with the given value. @@ -346,15 +346,15 @@ function getBaseTemplate( // Find all field names to set `index.query.default_field` to, which will be // the first 1024 keyword or text fields const defaultFields = flattenFieldsToNameAndType(fields).filter( - (field) => field.type && DEFAULT_FIELD_TYPES.includes(field.type) + (field) => field.type && QUERY_DEFAULT_FIELD_TYPES.includes(field.type) ); - if (defaultFields.length > DEFAULT_FIELD_LIMIT) { + if (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT) { logger.warn( - `large amount of default fields detected for index template ${templateIndexPattern} in package ${packageName}, applying the first ${DEFAULT_FIELD_LIMIT} fields` + `large amount of default fields detected for index template ${templateIndexPattern} in package ${packageName}, applying the first ${QUERY_DEFAULT_FIELD_LIMIT} fields` ); } - const defaultFieldNames = (defaultFields.length > DEFAULT_FIELD_LIMIT - ? defaultFields.slice(0, DEFAULT_FIELD_LIMIT) + const defaultFieldNames = (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT + ? defaultFields.slice(0, QUERY_DEFAULT_FIELD_LIMIT) : defaultFields ).map((field) => field.name); From 227380f00668ff552d1f516c4ceb9b06ea3e2ba1 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 2 Mar 2021 12:28:09 -0800 Subject: [PATCH 09/14] Fix imports --- x-pack/plugins/fleet/server/mocks.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/mocks.ts b/x-pack/plugins/fleet/server/mocks.ts index 3a7f9ebcc9e539..35e1aaba102cb2 100644 --- a/x-pack/plugins/fleet/server/mocks.ts +++ b/x-pack/plugins/fleet/server/mocks.ts @@ -9,8 +9,8 @@ import { elasticsearchServiceMock, loggingSystemMock, savedObjectsServiceMock, -} from 'src/core/server/mocks'; -import { coreMock } from '../../../../src/core/server/mocks'; + coreMock, +} from '../../../../src/core/server/mocks'; import { licensingMock } from '../../../plugins/licensing/server/mocks'; import type { FleetAppContext } from './plugin'; import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks'; From a6527c949e0d8a02cdf1f413c9b998296897be77 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 3 Mar 2021 10:40:23 -0800 Subject: [PATCH 10/14] Rename test() to it() for consistency --- .../elasticsearch/template/install.test.ts | 8 +-- .../elasticsearch/template/template.test.ts | 50 +++++++++---------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index e18c783845976e..791d71af080e22 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -16,7 +16,7 @@ describe('EPM install', () => { appContextService.start(createAppContextStartContractMock()); }); - test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { + it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() .callAsCurrentUser; callCluster.mockImplementation(async (_, params) => { @@ -59,7 +59,7 @@ describe('EPM install', () => { expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); }); - test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { + it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() .callAsCurrentUser; callCluster.mockImplementation(async (_, params) => { @@ -103,7 +103,7 @@ describe('EPM install', () => { expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); }); - test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { + it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() .callAsCurrentUser; callCluster.mockImplementation(async (_, params) => { @@ -147,7 +147,7 @@ describe('EPM install', () => { expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); }); - test('tests installPackage remove the aliases property if the property existed', async () => { + it('tests installPackage remove the aliases property if the property existed', async () => { const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() .callAsCurrentUser; callCluster.mockImplementation(async (_, params) => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index f655099cdf2aca..8533c14fd2b174 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -36,7 +36,7 @@ describe('EPM template', () => { appContextService.start(createAppContextStartContractMock()); }); - test('get template', () => { + it('get template', () => { const templateIndexPattern = 'logs-nginx.access-abcd-*'; const template = getTemplate({ @@ -51,7 +51,7 @@ describe('EPM template', () => { expect(template.index_patterns).toStrictEqual([templateIndexPattern]); }); - test('adds composed_of correctly', () => { + it('adds composed_of correctly', () => { const composedOfTemplates = ['component1', 'component2']; const template = getTemplate({ @@ -66,7 +66,7 @@ describe('EPM template', () => { expect(template.composed_of).toStrictEqual(composedOfTemplates); }); - test('adds empty composed_of correctly', () => { + it('adds empty composed_of correctly', () => { const composedOfTemplates: string[] = []; const template = getTemplate({ @@ -81,7 +81,7 @@ describe('EPM template', () => { expect(template.composed_of).toStrictEqual(composedOfTemplates); }); - test('adds hidden field correctly', () => { + it('adds hidden field correctly', () => { const templateIndexPattern = 'logs-nginx.access-abcd-*'; const templateWithHidden = getTemplate({ @@ -108,7 +108,7 @@ describe('EPM template', () => { expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); - test('tests loading base.yml', () => { + it('tests loading base.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); const fields: Field[] = safeLoad(fieldsYML); @@ -128,7 +128,7 @@ describe('EPM template', () => { expect(template).toMatchSnapshot(path.basename(ymlPath)); }); - test('tests loading coredns.logs.yml', () => { + it('tests loading coredns.logs.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/coredns.logs.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); const fields: Field[] = safeLoad(fieldsYML); @@ -148,7 +148,7 @@ describe('EPM template', () => { expect(template).toMatchSnapshot(path.basename(ymlPath)); }); - test('tests loading system.yml', () => { + it('tests loading system.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); const fields: Field[] = safeLoad(fieldsYML); @@ -168,7 +168,7 @@ describe('EPM template', () => { expect(template).toMatchSnapshot(path.basename(ymlPath)); }); - test('tests processing text field with multi fields', () => { + it('tests processing text field with multi fields', () => { const textWithMultiFieldsLiteralYml = ` - name: textWithMultiFields type: text @@ -200,7 +200,7 @@ describe('EPM template', () => { expect(mappings).toEqual(textWithMultiFieldsMapping); }); - test('tests processing keyword field with multi fields', () => { + it('tests processing keyword field with multi fields', () => { const keywordWithMultiFieldsLiteralYml = ` - name: keywordWithMultiFields type: keyword @@ -234,7 +234,7 @@ describe('EPM template', () => { expect(mappings).toEqual(keywordWithMultiFieldsMapping); }); - test('tests processing keyword field with multi fields with analyzed text field', () => { + it('tests processing keyword field with multi fields with analyzed text field', () => { const keywordWithAnalyzedMultiFieldsLiteralYml = ` - name: keywordWithAnalyzedMultiField type: keyword @@ -266,7 +266,7 @@ describe('EPM template', () => { expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping); }); - test('tests processing keyword field with multi fields with normalized keyword field', () => { + it('tests processing keyword field with multi fields with normalized keyword field', () => { const keywordWithNormalizedMultiFieldsLiteralYml = ` - name: keywordWithNormalizedMultiField type: keyword @@ -297,7 +297,7 @@ describe('EPM template', () => { expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping); }); - test('tests processing object field with no other attributes', () => { + it('tests processing object field with no other attributes', () => { const objectFieldLiteralYml = ` - name: objectField type: object @@ -315,7 +315,7 @@ describe('EPM template', () => { expect(mappings).toEqual(objectFieldMapping); }); - test('tests processing object field with enabled set to false', () => { + it('tests processing object field with enabled set to false', () => { const objectFieldEnabledFalseLiteralYml = ` - name: objectField type: object @@ -335,7 +335,7 @@ describe('EPM template', () => { expect(mappings).toEqual(objectFieldEnabledFalseMapping); }); - test('tests processing object field with dynamic set to false', () => { + it('tests processing object field with dynamic set to false', () => { const objectFieldDynamicFalseLiteralYml = ` - name: objectField type: object @@ -355,7 +355,7 @@ describe('EPM template', () => { expect(mappings).toEqual(objectFieldDynamicFalseMapping); }); - test('tests processing object field with dynamic set to true', () => { + it('tests processing object field with dynamic set to true', () => { const objectFieldDynamicTrueLiteralYml = ` - name: objectField type: object @@ -375,7 +375,7 @@ describe('EPM template', () => { expect(mappings).toEqual(objectFieldDynamicTrueMapping); }); - test('tests processing object field with dynamic set to strict', () => { + it('tests processing object field with dynamic set to strict', () => { const objectFieldDynamicStrictLiteralYml = ` - name: objectField type: object @@ -395,7 +395,7 @@ describe('EPM template', () => { expect(mappings).toEqual(objectFieldDynamicStrictMapping); }); - test('tests processing object field with property', () => { + it('tests processing object field with property', () => { const objectFieldWithPropertyLiteralYml = ` - name: a type: object @@ -420,7 +420,7 @@ describe('EPM template', () => { expect(mappings).toEqual(objectFieldWithPropertyMapping); }); - test('tests processing object field with property, reverse order', () => { + it('tests processing object field with property, reverse order', () => { const objectFieldWithPropertyReversedLiteralYml = ` - name: a.b type: keyword @@ -447,7 +447,7 @@ describe('EPM template', () => { expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); }); - test('tests processing nested field with property', () => { + it('tests processing nested field with property', () => { const nestedYaml = ` - name: a.b type: keyword @@ -475,7 +475,7 @@ describe('EPM template', () => { expect(mappings).toEqual(expectedMapping); }); - test('tests processing nested field with property, nested field first', () => { + it('tests processing nested field with property, nested field first', () => { const nestedYaml = ` - name: a type: nested @@ -503,7 +503,7 @@ describe('EPM template', () => { expect(mappings).toEqual(expectedMapping); }); - test('tests processing nested leaf field with properties', () => { + it('tests processing nested leaf field with properties', () => { const nestedYaml = ` - name: a type: object @@ -531,7 +531,7 @@ describe('EPM template', () => { expect(mappings).toEqual(expectedMapping); }); - test('tests constant_keyword field type handling', () => { + it('tests constant_keyword field type handling', () => { const constantKeywordLiteralYaml = ` - name: constantKeyword type: constant_keyword @@ -549,7 +549,7 @@ describe('EPM template', () => { expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); }); - test('tests priority and index pattern for data stream without dataset_is_prefix', () => { + it('tests priority and index pattern for data stream without dataset_is_prefix', () => { const dataStreamDatasetIsPrefixUnset = { type: 'metrics', dataset: 'package.dataset', @@ -568,7 +568,7 @@ describe('EPM template', () => { expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset); }); - test('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { + it('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { const dataStreamDatasetIsPrefixFalse = { type: 'metrics', dataset: 'package.dataset', @@ -588,7 +588,7 @@ describe('EPM template', () => { expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse); }); - test('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { + it('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { const dataStreamDatasetIsPrefixTrue = { type: 'metrics', dataset: 'package.dataset', From 1cbe0cde38259e748443267beb58c6864d61ec6c Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 3 Mar 2021 11:12:39 -0800 Subject: [PATCH 11/14] Decouple jest from createAppContextStartContractMock --- .../fleet/server/mocks/app_context_mock.ts | 28 ++++++++++++++++ .../fleet/server/{mocks.ts => mocks/index.ts} | 32 ++++--------------- .../apis/epm/template.ts | 2 +- 3 files changed, 36 insertions(+), 26 deletions(-) create mode 100644 x-pack/plugins/fleet/server/mocks/app_context_mock.ts rename x-pack/plugins/fleet/server/{mocks.ts => mocks/index.ts} (61%) diff --git a/x-pack/plugins/fleet/server/mocks/app_context_mock.ts b/x-pack/plugins/fleet/server/mocks/app_context_mock.ts new file mode 100644 index 00000000000000..01c552327abc6f --- /dev/null +++ b/x-pack/plugins/fleet/server/mocks/app_context_mock.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + elasticsearchServiceMock, + loggingSystemMock, + savedObjectsServiceMock, +} from '../../../../../src/core/server/mocks'; +import { licensingMock } from '../../../../plugins/licensing/server/mocks'; +import type { FleetAppContext } from '../plugin'; +import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; +import { securityMock } from '../../../security/server/mocks'; + +export const createAppContextStartContractMock = (): FleetAppContext => { + return { + elasticsearch: elasticsearchServiceMock.createStart(), + encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(), + savedObjects: savedObjectsServiceMock.createStartContract(), + security: securityMock.createStart(), + logger: loggingSystemMock.create().get(), + isProductionMode: true, + kibanaVersion: '8.0.0', + kibanaBranch: 'master', + }; +}; diff --git a/x-pack/plugins/fleet/server/mocks.ts b/x-pack/plugins/fleet/server/mocks/index.ts similarity index 61% rename from x-pack/plugins/fleet/server/mocks.ts rename to x-pack/plugins/fleet/server/mocks/index.ts index 35e1aaba102cb2..e8f63bd7dbb875 100644 --- a/x-pack/plugins/fleet/server/mocks.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -4,32 +4,14 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { coreMock } from '../../../../../src/core/server/mocks'; +import { licensingMock } from '../../../../plugins/licensing/server/mocks'; +import type { PackagePolicyServiceInterface } from '../services/package_policy'; +import type { AgentPolicyServiceInterface, AgentService } from '../services'; -import { - elasticsearchServiceMock, - loggingSystemMock, - savedObjectsServiceMock, - coreMock, -} from '../../../../src/core/server/mocks'; -import { licensingMock } from '../../../plugins/licensing/server/mocks'; -import type { FleetAppContext } from './plugin'; -import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks'; -import { securityMock } from '../../security/server/mocks'; -import type { PackagePolicyServiceInterface } from './services/package_policy'; -import type { AgentPolicyServiceInterface, AgentService } from './services'; - -export const createAppContextStartContractMock = (): FleetAppContext => { - return { - elasticsearch: elasticsearchServiceMock.createStart(), - encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(), - savedObjects: savedObjectsServiceMock.createStartContract(), - security: securityMock.createStart(), - logger: loggingSystemMock.create().get(), - isProductionMode: true, - kibanaVersion: '8.0.0', - kibanaBranch: 'master', - }; -}; +// This mock is used in an api integration test, it lives in a separate file +// so that the jest dependency is decoupled +export { createAppContextStartContractMock } from './app_context_mock'; function createCoreRequestHandlerContextMock() { return { diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index 990955b311751b..3b53a35c7c9ba7 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -7,7 +7,7 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; -import { createAppContextStartContractMock } from '../../../../plugins/fleet/server/mocks'; +import { createAppContextStartContractMock } from '../../../../plugins/fleet/server/mocks/app_context_mock'; import { appContextService } from '../../../../plugins/fleet/server/services'; import { getTemplate } from '../../../../plugins/fleet/server/services/epm/elasticsearch/template/template'; From db9b28e2c3f4379ea59d03464228c91f47d7cd37 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 3 Mar 2021 11:55:40 -0800 Subject: [PATCH 12/14] Fix import --- x-pack/plugins/fleet/server/mocks/app_context_mock.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/mocks/app_context_mock.ts b/x-pack/plugins/fleet/server/mocks/app_context_mock.ts index 01c552327abc6f..2171f7c1db0d5e 100644 --- a/x-pack/plugins/fleet/server/mocks/app_context_mock.ts +++ b/x-pack/plugins/fleet/server/mocks/app_context_mock.ts @@ -9,7 +9,6 @@ import { loggingSystemMock, savedObjectsServiceMock, } from '../../../../../src/core/server/mocks'; -import { licensingMock } from '../../../../plugins/licensing/server/mocks'; import type { FleetAppContext } from '../plugin'; import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; import { securityMock } from '../../../security/server/mocks'; From 7c275b59afb284154a4959424509de6b19e6e4df Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 3 Mar 2021 13:04:35 -0800 Subject: [PATCH 13/14] Revert decoupling, mock directly in api integration test --- .../fleet/server/mocks/app_context_mock.ts | 27 ------------------- x-pack/plugins/fleet/server/mocks/index.ts | 26 ++++++++++++++---- .../apis/epm/template.ts | 19 +++++++++++-- 3 files changed, 38 insertions(+), 34 deletions(-) delete mode 100644 x-pack/plugins/fleet/server/mocks/app_context_mock.ts diff --git a/x-pack/plugins/fleet/server/mocks/app_context_mock.ts b/x-pack/plugins/fleet/server/mocks/app_context_mock.ts deleted file mode 100644 index 2171f7c1db0d5e..00000000000000 --- a/x-pack/plugins/fleet/server/mocks/app_context_mock.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { - elasticsearchServiceMock, - loggingSystemMock, - savedObjectsServiceMock, -} from '../../../../../src/core/server/mocks'; -import type { FleetAppContext } from '../plugin'; -import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; -import { securityMock } from '../../../security/server/mocks'; - -export const createAppContextStartContractMock = (): FleetAppContext => { - return { - elasticsearch: elasticsearchServiceMock.createStart(), - encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(), - savedObjects: savedObjectsServiceMock.createStartContract(), - security: securityMock.createStart(), - logger: loggingSystemMock.create().get(), - isProductionMode: true, - kibanaVersion: '8.0.0', - kibanaBranch: 'master', - }; -}; diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index a1eac6356e8c08..cff80f533d5e39 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -4,15 +4,31 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { coreMock } from '../../../../../src/core/server/mocks'; +import { + elasticsearchServiceMock, + loggingSystemMock, + savedObjectsServiceMock, + coreMock, +} from '../../../../../src/core/server/mocks'; import { licensingMock } from '../../../../plugins/licensing/server/mocks'; - +import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; +import { securityMock } from '../../../security/server/mocks'; import type { PackagePolicyServiceInterface } from '../services/package_policy'; import type { AgentPolicyServiceInterface, AgentService } from '../services'; +import type { FleetAppContext } from '../plugin'; -// This mock is used in an api integration test, it lives in a separate file -// so that the jest dependency is decoupled -export { createAppContextStartContractMock } from './app_context_mock'; +export const createAppContextStartContractMock = (): FleetAppContext => { + return { + elasticsearch: elasticsearchServiceMock.createStart(), + encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(), + savedObjects: savedObjectsServiceMock.createStartContract(), + security: securityMock.createStart(), + logger: loggingSystemMock.create().get(), + isProductionMode: true, + kibanaVersion: '8.0.0', + kibanaBranch: 'master', + }; +}; function createCoreRequestHandlerContextMock() { return { diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index 3b53a35c7c9ba7..9fd79c4d24bc0d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -7,7 +7,6 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; -import { createAppContextStartContractMock } from '../../../../plugins/fleet/server/mocks/app_context_mock'; import { appContextService } from '../../../../plugins/fleet/server/services'; import { getTemplate } from '../../../../plugins/fleet/server/services/epm/elasticsearch/template/template'; @@ -32,7 +31,23 @@ export default function ({ getService }: FtrProviderContext) { // This test was inspired by https://github.com/elastic/kibana/blob/master/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js describe('EPM - template', async () => { beforeEach(async () => { - appContextService.start(createAppContextStartContractMock()); + appContextService.start({ + // @ts-ignore + elasticsearch: {}, + // @ts-ignore + encryptedSavedObjectsStart: {}, + // @ts-ignore + savedObjects: {}, + // @ts-ignore + security: {}, + // @ts-ignore + logger: { + warn: () => {}, + }, + isProductionMode: true, + kibanaVersion: '8.0.0', + kibanaBranch: 'master', + }); }); it('can be loaded', async () => { From c420c0f94a77c841a677af1f67a9828b57294dda Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 4 Mar 2021 11:28:34 -0800 Subject: [PATCH 14/14] Adjust context mock --- .../test/fleet_api_integration/apis/epm/template.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index 9fd79c4d24bc0d..517d2c77d430de 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -33,20 +33,11 @@ export default function ({ getService }: FtrProviderContext) { beforeEach(async () => { appContextService.start({ // @ts-ignore - elasticsearch: {}, - // @ts-ignore - encryptedSavedObjectsStart: {}, - // @ts-ignore - savedObjects: {}, - // @ts-ignore - security: {}, + elasticsearch: { client: {} }, // @ts-ignore logger: { warn: () => {}, }, - isProductionMode: true, - kibanaVersion: '8.0.0', - kibanaBranch: 'master', }); });