From 9fb8dfae14eef4d1897f2afd4cba83e906133d66 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Mon, 1 Feb 2021 15:33:33 +0100 Subject: [PATCH 1/9] Explicitly generate ES index pattern name. --- .../plugins/fleet/common/types/models/epm.ts | 1 + .../epm/elasticsearch/template/install.ts | 9 ++++++++- .../epm/elasticsearch/template/template.ts | 18 ++++++++++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 0f59befc2e4673..e7e5a931b7429f 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -221,6 +221,7 @@ export interface RegistryDataStream { path: string; ingest_pipeline: string; elasticsearch?: RegistryElasticsearch; + dataset_is_prefix?: boolean; } export interface RegistryElasticsearch { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 10e94d93bbc8e2..32723986e4d519 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -17,7 +17,12 @@ import { import { CallESAsCurrentUser } from '../../../../types'; import { Field, loadFieldsFromYaml, processFields } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; -import { generateMappings, generateTemplateName, getTemplate } from './template'; +import { + generateIndexPatternName, + generateMappings, + generateTemplateName, + getTemplate, +} from './template'; import { getAsset, getPathParts } from '../../archive'; import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install'; @@ -293,6 +298,7 @@ export async function installTemplate({ }): Promise { const mappings = generateMappings(processFields(fields)); const templateName = generateTemplateName(dataStream); + const indexPatternName = generateIndexPatternName(dataStream); let pipelineName; if (dataStream.ingest_pipeline) { pipelineName = getPipelineNameForInstallation({ @@ -311,6 +317,7 @@ export async function installTemplate({ const template = getTemplate({ type: dataStream.type, templateName, + indexPatternName, mappings, pipelineName, packageName, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index ea0bb5dc53a1e4..e1c9763214ea69 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -41,6 +41,7 @@ const DEFAULT_IGNORE_ABOVE = 1024; export function getTemplate({ type, templateName, + indexPatternName, mappings, pipelineName, packageName, @@ -50,6 +51,7 @@ export function getTemplate({ }: { type: string; templateName: string; + indexPatternName: string; mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; @@ -60,6 +62,7 @@ export function getTemplate({ const template = getBaseTemplate( type, templateName, + indexPatternName, mappings, packageName, composedOfTemplates, @@ -242,6 +245,16 @@ export function generateTemplateName(dataStream: RegistryDataStream): string { return getRegistryDataStreamAssetBaseName(dataStream); } +export function generateIndexPatternName(dataStream: RegistryDataStream): string { + // undefined or explicitly set to false + // See also https://github.com/elastic/package-spec/pull/102 + if (!dataStream.dataset_is_prefix) { + return getRegistryDataStreamAssetBaseName(dataStream) + '-*'; + } else { + return getRegistryDataStreamAssetBaseName(dataStream) + '.*-*'; + } +} + /** * Returns a map of the data stream path fields to elasticsearch index pattern. * @param dataStreams an array of RegistryDataStream objects @@ -255,7 +268,7 @@ export function generateESIndexPatterns( const patterns: Record = {}; for (const dataStream of dataStreams) { - patterns[dataStream.path] = generateTemplateName(dataStream) + '-*'; + patterns[dataStream.path] = generateIndexPatternName(dataStream); } return patterns; } @@ -263,6 +276,7 @@ export function generateESIndexPatterns( function getBaseTemplate( type: string, templateName: string, + indexPatternName: string, mappings: IndexTemplateMappings, packageName: string, composedOfTemplates: string[], @@ -285,7 +299,7 @@ function getBaseTemplate( // default and the one the ingest manager uses. priority: 200, // To be completed with the correct index patterns - index_patterns: [`${templateName}-*`], + index_patterns: [indexPatternName], template: { settings: { index: { From 0d45453fe6dc64786e7fe1eef9cb7346186bfb0d Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Mon, 1 Feb 2021 16:37:15 +0100 Subject: [PATCH 2/9] Adjust tests. --- .../epm/elasticsearch/template/template.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 80386a2a0dd56c..308d7dcca127ba 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -24,15 +24,17 @@ expect.addSnapshotSerializer({ test('get template', () => { const templateName = 'logs-nginx-access-abcd'; + const indexPatternName = 'logs-nginx-access-abcd-*'; const template = getTemplate({ type: 'logs', templateName, + indexPatternName, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], }); - expect(template.index_patterns).toStrictEqual([`${templateName}-*`]); + expect(template.index_patterns).toStrictEqual([indexPatternName]); }); test('adds composed_of correctly', () => { @@ -41,6 +43,7 @@ test('adds composed_of correctly', () => { const template = getTemplate({ type: 'logs', templateName: 'name', + indexPatternName: 'name-*', packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates, @@ -54,6 +57,7 @@ test('adds empty composed_of correctly', () => { const template = getTemplate({ type: 'logs', templateName: 'name', + indexPatternName: 'name-*', packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates, @@ -63,10 +67,12 @@ test('adds empty composed_of correctly', () => { test('adds hidden field correctly', () => { const templateWithHiddenName = 'logs-nginx-access-abcd'; + const indexPatternName = 'logs-nginx-access-abcd-*'; const templateWithHidden = getTemplate({ type: 'logs', templateName: templateWithHiddenName, + indexPatternName, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], @@ -79,6 +85,7 @@ test('adds hidden field correctly', () => { const templateWithoutHidden = getTemplate({ type: 'logs', templateName: templateWithoutHiddenName, + indexPatternName, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], @@ -96,6 +103,7 @@ test('tests loading base.yml', () => { const template = getTemplate({ type: 'logs', templateName: 'foo', + indexPatternName: 'foo-*', packageName: 'nginx', mappings, composedOfTemplates: [], @@ -114,6 +122,7 @@ test('tests loading coredns.logs.yml', () => { const template = getTemplate({ type: 'logs', templateName: 'foo', + indexPatternName: 'foo-*', packageName: 'coredns', mappings, composedOfTemplates: [], @@ -132,6 +141,7 @@ test('tests loading system.yml', () => { const template = getTemplate({ type: 'metrics', templateName: 'whatsthis', + indexPatternName: 'whatsthis-*', packageName: 'system', mappings, composedOfTemplates: [], From f5b4dcc8bc4f16544daab5daa00d340ee4c58929 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Tue, 2 Feb 2021 17:08:49 +0100 Subject: [PATCH 3/9] Adjust and reenable tests. --- .../fleet_api_integration/apis/epm/index.js | 2 +- .../fleet_api_integration/apis/epm/template.ts | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/index.js b/x-pack/test/fleet_api_integration/apis/epm/index.js index 23b7464a317e90..0020e6bdf1bb01 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/index.js +++ b/x-pack/test/fleet_api_integration/apis/epm/index.js @@ -11,7 +11,7 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./setup')); loadTestFile(require.resolve('./get')); loadTestFile(require.resolve('./file')); - //loadTestFile(require.resolve('./template')); + loadTestFile(require.resolve('./template')); loadTestFile(require.resolve('./ilm')); loadTestFile(require.resolve('./install_by_upload')); loadTestFile(require.resolve('./install_overrides')); diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index c7e9e211552578..c6833ed0f1c06b 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -10,8 +10,8 @@ import { FtrProviderContext } from '../../../api_integration/ftr_provider_contex import { getTemplate } from '../../../../plugins/fleet/server/services/epm/elasticsearch/template/template'; export default function ({ getService }: FtrProviderContext) { - const indexPattern = 'foo'; const templateName = 'bar'; + const indexPatternName = 'bar-*'; const es = getService('es'); const mappings = { properties: { @@ -26,6 +26,7 @@ export default function ({ getService }: FtrProviderContext) { const template = getTemplate({ type: 'logs', templateName, + indexPatternName, mappings, packageName: 'system', composedOfTemplates: [], @@ -34,18 +35,24 @@ export default function ({ getService }: FtrProviderContext) { // This test is not an API integration test with Kibana // We want to test here if the template is valid and for this we need a running ES instance. // If the ES instance takes the template, we assume it is a valid template. - const { body: response1 } = await es.indices.putTemplate({ - name: templateName, + const { body: response1 } = await es.transport.request({ + method: 'PUT', + path: `/_index_template/${templateName}`, body: template, }); + // Checks if template loading worked as expected expect(response1).to.eql({ acknowledged: true }); - const { body: response2 } = await es.indices.getTemplate({ name: templateName }); + const { body: response2 } = await es.transport.request({ + method: 'GET', + path: `/_index_template/${templateName}`, + }); + // Checks if the content of the template that was loaded is as expected // We already know based on the above test that the template was valid // but we check here also if we wrote the index pattern inside the template as expected - expect(response2[templateName].index_patterns).to.eql([`${indexPattern}-*`]); + expect(response2.index_templates[0].index_template.index_patterns).to.eql([indexPatternName]); }); }); } From 0642673b3a8acb4da9997e9a7d308e63af61d109 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Wed, 3 Feb 2021 14:46:06 +0100 Subject: [PATCH 4/9] Set template priority based on dataset_is_prefix --- .../epm/elasticsearch/template/install.ts | 5 ++- .../elasticsearch/template/template.test.ts | 20 ++++------ .../epm/elasticsearch/template/template.ts | 37 ++++++++++++++----- .../apis/epm/template.ts | 2 +- 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 32723986e4d519..ccddd8f8ad9759 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -22,6 +22,7 @@ import { generateMappings, generateTemplateName, getTemplate, + getTemplatePriority, } from './template'; import { getAsset, getPathParts } from '../../archive'; import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install'; @@ -299,6 +300,8 @@ export async function installTemplate({ const mappings = generateMappings(processFields(fields)); const templateName = generateTemplateName(dataStream); const indexPatternName = generateIndexPatternName(dataStream); + const templatePriority = getTemplatePriority(dataStream); + let pipelineName; if (dataStream.ingest_pipeline) { pipelineName = getPipelineNameForInstallation({ @@ -316,12 +319,12 @@ export async function installTemplate({ const template = getTemplate({ type: dataStream.type, - templateName, indexPatternName, mappings, pipelineName, packageName, composedOfTemplates, + templatePriority, ilmPolicy: dataStream.ilm_policy, hidden: dataStream.hidden, }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 308d7dcca127ba..0740fad886f473 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -23,16 +23,15 @@ expect.addSnapshotSerializer({ }); test('get template', () => { - const templateName = 'logs-nginx-access-abcd'; const indexPatternName = 'logs-nginx-access-abcd-*'; const template = getTemplate({ type: 'logs', - templateName, indexPatternName, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], + templatePriority: 200, }); expect(template.index_patterns).toStrictEqual([indexPatternName]); }); @@ -42,11 +41,11 @@ test('adds composed_of correctly', () => { const template = getTemplate({ type: 'logs', - templateName: 'name', indexPatternName: 'name-*', packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates, + templatePriority: 200, }); expect(template.composed_of).toStrictEqual(composedOfTemplates); }); @@ -56,39 +55,36 @@ test('adds empty composed_of correctly', () => { const template = getTemplate({ type: 'logs', - templateName: 'name', indexPatternName: 'name-*', packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates, + templatePriority: 200, }); expect(template.composed_of).toStrictEqual(composedOfTemplates); }); test('adds hidden field correctly', () => { - const templateWithHiddenName = 'logs-nginx-access-abcd'; const indexPatternName = 'logs-nginx-access-abcd-*'; const templateWithHidden = getTemplate({ type: 'logs', - templateName: templateWithHiddenName, indexPatternName, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], + templatePriority: 200, hidden: true, }); expect(templateWithHidden.data_stream.hidden).toEqual(true); - const templateWithoutHiddenName = 'logs-nginx-access-efgh'; - const templateWithoutHidden = getTemplate({ type: 'logs', - templateName: templateWithoutHiddenName, indexPatternName, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], + templatePriority: 200, }); expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); @@ -102,11 +98,11 @@ test('tests loading base.yml', () => { const mappings = generateMappings(processedFields); const template = getTemplate({ type: 'logs', - templateName: 'foo', indexPatternName: 'foo-*', packageName: 'nginx', mappings, composedOfTemplates: [], + templatePriority: 200, }); expect(template).toMatchSnapshot(path.basename(ymlPath)); @@ -121,11 +117,11 @@ test('tests loading coredns.logs.yml', () => { const mappings = generateMappings(processedFields); const template = getTemplate({ type: 'logs', - templateName: 'foo', indexPatternName: 'foo-*', packageName: 'coredns', mappings, composedOfTemplates: [], + templatePriority: 200, }); expect(template).toMatchSnapshot(path.basename(ymlPath)); @@ -140,11 +136,11 @@ test('tests loading system.yml', () => { const mappings = generateMappings(processedFields); const template = getTemplate({ type: 'metrics', - templateName: 'whatsthis', indexPatternName: 'whatsthis-*', packageName: 'system', mappings, composedOfTemplates: [], + templatePriority: 200, }); expect(template).toMatchSnapshot(path.basename(ymlPath)); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index e1c9763214ea69..2c497755cc88cb 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -33,6 +33,10 @@ export interface CurrentDataStream { const DEFAULT_SCALING_FACTOR = 1000; const DEFAULT_IGNORE_ABOVE = 1024; +// see discussion in https://github.com/elastic/kibana/issues/88307 +const DEFAULT_TEMPLATE_PRIORITY = 200; +const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; + /** * getTemplate retrieves the default template but overwrites the index pattern with the given value. * @@ -40,32 +44,32 @@ const DEFAULT_IGNORE_ABOVE = 1024; */ export function getTemplate({ type, - templateName, indexPatternName, mappings, pipelineName, packageName, composedOfTemplates, + templatePriority, ilmPolicy, hidden, }: { type: string; - templateName: string; indexPatternName: string; mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; composedOfTemplates: string[]; + templatePriority: number; ilmPolicy?: string | undefined; hidden?: boolean; }): IndexTemplate { const template = getBaseTemplate( type, - templateName, indexPatternName, mappings, packageName, composedOfTemplates, + templatePriority, ilmPolicy, hidden ); @@ -255,6 +259,25 @@ export function generateIndexPatternName(dataStream: RegistryDataStream): string } } +// Template priorities are discussed in https://github.com/elastic/kibana/issues/88307 +// See also https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html +// +// Built-in templates like logs-*-* and metrics-*-* have priority 100 +// +// EPM generated templates for data streams have priority 200 (DEFAULT_TEMPLATE_PRIORITY) +// +// EPM generated templates for data streams with dataset_is_prefix: true have priority 150 (DATASET_IS_PREFIX_TEMPLATE_PRIORITY) + +export function getTemplatePriority(dataStream: RegistryDataStream): number { + // undefined or explicitly set to false + // See also https://github.com/elastic/package-spec/pull/102 + if (!dataStream.dataset_is_prefix) { + return DEFAULT_TEMPLATE_PRIORITY; + } else { + return DATASET_IS_PREFIX_TEMPLATE_PRIORITY; + } +} + /** * Returns a map of the data stream path fields to elasticsearch index pattern. * @param dataStreams an array of RegistryDataStream objects @@ -275,11 +298,11 @@ export function generateESIndexPatterns( function getBaseTemplate( type: string, - templateName: string, indexPatternName: string, mappings: IndexTemplateMappings, packageName: string, composedOfTemplates: string[], + templatePriority: number, ilmPolicy?: string | undefined, hidden?: boolean ): IndexTemplate { @@ -293,11 +316,7 @@ function getBaseTemplate( }; return { - // This takes precedence over all index templates installed by ES by default (logs-*-* and metrics-*-*) - // if this number is lower than the ES value (which is 100) this template will never be applied when a data stream - // is created. I'm using 200 here to give some room for users to create their own template and fit it between the - // default and the one the ingest manager uses. - priority: 200, + priority: templatePriority, // To be completed with the correct index patterns index_patterns: [indexPatternName], template: { diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index c6833ed0f1c06b..2ded0e61734bfb 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -25,11 +25,11 @@ export default function ({ getService }: FtrProviderContext) { it('can be loaded', async () => { const template = getTemplate({ type: 'logs', - templateName, indexPatternName, mappings, packageName: 'system', composedOfTemplates: [], + templatePriority: 200, }); // This test is not an API integration test with Kibana From 764b414252e23f352a62a08c1229b8cee6d773e6 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Wed, 3 Feb 2021 17:05:55 +0100 Subject: [PATCH 5/9] Refactor indexPatternName -> templateIndexPattern --- .../epm/elasticsearch/template/install.ts | 6 ++--- .../elasticsearch/template/template.test.ts | 22 +++++++++---------- .../epm/elasticsearch/template/template.ts | 14 ++++++------ .../apis/epm/template.ts | 8 ++++--- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index ccddd8f8ad9759..f5f1b4bea788d4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -18,9 +18,9 @@ import { CallESAsCurrentUser } from '../../../../types'; import { Field, loadFieldsFromYaml, processFields } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { - generateIndexPatternName, generateMappings, generateTemplateName, + generateTemplateIndexPattern, getTemplate, getTemplatePriority, } from './template'; @@ -299,7 +299,7 @@ export async function installTemplate({ }): Promise { const mappings = generateMappings(processFields(fields)); const templateName = generateTemplateName(dataStream); - const indexPatternName = generateIndexPatternName(dataStream); + const templateIndexPattern = generateTemplateIndexPattern(dataStream); const templatePriority = getTemplatePriority(dataStream); let pipelineName; @@ -319,7 +319,7 @@ export async function installTemplate({ const template = getTemplate({ type: dataStream.type, - indexPatternName, + templateIndexPattern, mappings, pipelineName, packageName, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 0740fad886f473..b2b428a3df6143 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -23,17 +23,17 @@ expect.addSnapshotSerializer({ }); test('get template', () => { - const indexPatternName = 'logs-nginx-access-abcd-*'; + const templateIndexPattern = 'logs-nginx-access-abcd-*'; const template = getTemplate({ type: 'logs', - indexPatternName, + templateIndexPattern, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], templatePriority: 200, }); - expect(template.index_patterns).toStrictEqual([indexPatternName]); + expect(template.index_patterns).toStrictEqual([templateIndexPattern]); }); test('adds composed_of correctly', () => { @@ -41,7 +41,7 @@ test('adds composed_of correctly', () => { const template = getTemplate({ type: 'logs', - indexPatternName: 'name-*', + templateIndexPattern: 'name-*', packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates, @@ -55,7 +55,7 @@ test('adds empty composed_of correctly', () => { const template = getTemplate({ type: 'logs', - indexPatternName: 'name-*', + templateIndexPattern: 'name-*', packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates, @@ -65,11 +65,11 @@ test('adds empty composed_of correctly', () => { }); test('adds hidden field correctly', () => { - const indexPatternName = 'logs-nginx-access-abcd-*'; + const templateIndexPattern = 'logs-nginx-access-abcd-*'; const templateWithHidden = getTemplate({ type: 'logs', - indexPatternName, + templateIndexPattern, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], @@ -80,7 +80,7 @@ test('adds hidden field correctly', () => { const templateWithoutHidden = getTemplate({ type: 'logs', - indexPatternName, + templateIndexPattern, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], @@ -98,7 +98,7 @@ test('tests loading base.yml', () => { const mappings = generateMappings(processedFields); const template = getTemplate({ type: 'logs', - indexPatternName: 'foo-*', + templateIndexPattern: 'foo-*', packageName: 'nginx', mappings, composedOfTemplates: [], @@ -117,7 +117,7 @@ test('tests loading coredns.logs.yml', () => { const mappings = generateMappings(processedFields); const template = getTemplate({ type: 'logs', - indexPatternName: 'foo-*', + templateIndexPattern: 'foo-*', packageName: 'coredns', mappings, composedOfTemplates: [], @@ -136,7 +136,7 @@ test('tests loading system.yml', () => { const mappings = generateMappings(processedFields); const template = getTemplate({ type: 'metrics', - indexPatternName: 'whatsthis-*', + templateIndexPattern: 'whatsthis-*', packageName: 'system', mappings, composedOfTemplates: [], diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 2c497755cc88cb..b86c989f8c24c8 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -44,7 +44,7 @@ const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; */ export function getTemplate({ type, - indexPatternName, + templateIndexPattern, mappings, pipelineName, packageName, @@ -54,7 +54,7 @@ export function getTemplate({ hidden, }: { type: string; - indexPatternName: string; + templateIndexPattern: string; mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; @@ -65,7 +65,7 @@ export function getTemplate({ }): IndexTemplate { const template = getBaseTemplate( type, - indexPatternName, + templateIndexPattern, mappings, packageName, composedOfTemplates, @@ -249,7 +249,7 @@ export function generateTemplateName(dataStream: RegistryDataStream): string { return getRegistryDataStreamAssetBaseName(dataStream); } -export function generateIndexPatternName(dataStream: RegistryDataStream): string { +export function generateTemplateIndexPattern(dataStream: RegistryDataStream): string { // undefined or explicitly set to false // See also https://github.com/elastic/package-spec/pull/102 if (!dataStream.dataset_is_prefix) { @@ -291,14 +291,14 @@ export function generateESIndexPatterns( const patterns: Record = {}; for (const dataStream of dataStreams) { - patterns[dataStream.path] = generateIndexPatternName(dataStream); + patterns[dataStream.path] = generateTemplateIndexPattern(dataStream); } return patterns; } function getBaseTemplate( type: string, - indexPatternName: string, + templateIndexPattern: string, mappings: IndexTemplateMappings, packageName: string, composedOfTemplates: string[], @@ -318,7 +318,7 @@ function getBaseTemplate( return { priority: templatePriority, // To be completed with the correct index patterns - index_patterns: [indexPatternName], + index_patterns: [templateIndexPattern], template: { settings: { index: { diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index 2ded0e61734bfb..d79452ca0eb6f2 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -11,7 +11,7 @@ import { getTemplate } from '../../../../plugins/fleet/server/services/epm/elast export default function ({ getService }: FtrProviderContext) { const templateName = 'bar'; - const indexPatternName = 'bar-*'; + const templateIndexPattern = 'bar-*'; const es = getService('es'); const mappings = { properties: { @@ -25,7 +25,7 @@ export default function ({ getService }: FtrProviderContext) { it('can be loaded', async () => { const template = getTemplate({ type: 'logs', - indexPatternName, + templateIndexPattern, mappings, packageName: 'system', composedOfTemplates: [], @@ -52,7 +52,9 @@ export default function ({ getService }: FtrProviderContext) { // Checks if the content of the template that was loaded is as expected // We already know based on the above test that the template was valid // but we check here also if we wrote the index pattern inside the template as expected - expect(response2.index_templates[0].index_template.index_patterns).to.eql([indexPatternName]); + expect(response2.index_templates[0].index_template.index_patterns).to.eql([ + templateIndexPattern, + ]); }); }); } From 47f7556c7abcbc471eaf6c2aa78eb0494ae79f20 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Fri, 5 Feb 2021 12:10:27 +0100 Subject: [PATCH 6/9] Add unit tests. --- .../elasticsearch/template/template.test.ts | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index b2b428a3df6143..4b319264de4934 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -8,8 +8,14 @@ import { readFileSync } from 'fs'; import { safeLoad } from 'js-yaml'; import path from 'path'; +import { RegistryDataStream } from '../../../../types'; import { Field, processFields } from '../../fields/field'; -import { generateMappings, getTemplate } from './template'; +import { + generateMappings, + getTemplate, + getTemplatePriority, + generateTemplateIndexPattern, +} from './template'; // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ @@ -526,3 +532,62 @@ test('tests constant_keyword field type handling', () => { const mappings = generateMappings(processedFields); expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); }); + +test('tests priority and index pattern for data stream without dataset_is_prefix', () => { + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset); +}); + +test('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse); +}); + +test('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue); +}); From 890e2de04e2b67e36b254cb18faf1e51b85ebc9b Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Fri, 5 Feb 2021 13:07:57 +0100 Subject: [PATCH 7/9] Use more realistic index pattern in test. --- .../services/epm/elasticsearch/template/template.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 4b319264de4934..56d752c0c3e266 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -29,7 +29,7 @@ expect.addSnapshotSerializer({ }); test('get template', () => { - const templateIndexPattern = 'logs-nginx-access-abcd-*'; + const templateIndexPattern = 'logs-nginx.access-abcd-*'; const template = getTemplate({ type: 'logs', @@ -71,7 +71,7 @@ test('adds empty composed_of correctly', () => { }); test('adds hidden field correctly', () => { - const templateIndexPattern = 'logs-nginx-access-abcd-*'; + const templateIndexPattern = 'logs-nginx.access-abcd-*'; const templateWithHidden = getTemplate({ type: 'logs', From 4e543cbd1629965962ce7476c717f5aedc8c1a65 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Fri, 5 Feb 2021 13:34:43 +0100 Subject: [PATCH 8/9] Fix unit test. --- .../server/services/epm/elasticsearch/template/template.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 56d752c0c3e266..a176805307845c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -584,7 +584,7 @@ test('tests priority and index pattern for data stream with dataset_is_prefix se dataset_is_prefix: true, } as RegistryDataStream; const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; - const templatePriorityDatasetIsPrefixTrue = 200; + const templatePriorityDatasetIsPrefixTrue = 150; const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue); const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue); From 036745417ac0ea1bef64057e2025a55237d5c8f2 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Mon, 8 Feb 2021 12:37:14 +0100 Subject: [PATCH 9/9] Add unit test for installTemplate(). --- .../elasticsearch/template/install.test.ts | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts new file mode 100644 index 00000000000000..be9213aff360d0 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RegistryDataStream } from '../../../../types'; +import { Field } from '../../fields/field'; + +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { installTemplate } from './install'; + +test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; + const fields: Field[] = []; + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixUnset, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[0][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); +}); + +test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; + const fields: Field[] = []; + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixFalse, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[0][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); +}); + +test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; + const fields: Field[] = []; + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 150; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixTrue, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[0][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); +});