diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 0f59befc2e4673..e7e5a931b7429f 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -221,6 +221,7 @@ export interface RegistryDataStream { path: string; ingest_pipeline: string; elasticsearch?: RegistryElasticsearch; + dataset_is_prefix?: boolean; } export interface RegistryElasticsearch { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts new file mode 100644 index 00000000000000..be9213aff360d0 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RegistryDataStream } from '../../../../types'; +import { Field } from '../../fields/field'; + +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { installTemplate } from './install'; + +test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; + const fields: Field[] = []; + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixUnset, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[0][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); +}); + +test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; + const fields: Field[] = []; + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixFalse, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[0][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); +}); + +test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; + const fields: Field[] = []; + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 150; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixTrue, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[0][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 10e94d93bbc8e2..f5f1b4bea788d4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -17,7 +17,13 @@ import { import { CallESAsCurrentUser } from '../../../../types'; import { Field, loadFieldsFromYaml, processFields } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; -import { generateMappings, generateTemplateName, getTemplate } from './template'; +import { + generateMappings, + generateTemplateName, + generateTemplateIndexPattern, + getTemplate, + getTemplatePriority, +} from './template'; import { getAsset, getPathParts } from '../../archive'; import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install'; @@ -293,6 +299,9 @@ export async function installTemplate({ }): Promise { const mappings = generateMappings(processFields(fields)); const templateName = generateTemplateName(dataStream); + const templateIndexPattern = generateTemplateIndexPattern(dataStream); + const templatePriority = getTemplatePriority(dataStream); + let pipelineName; if (dataStream.ingest_pipeline) { pipelineName = getPipelineNameForInstallation({ @@ -310,11 +319,12 @@ export async function installTemplate({ const template = getTemplate({ type: dataStream.type, - templateName, + templateIndexPattern, mappings, pipelineName, packageName, composedOfTemplates, + templatePriority, ilmPolicy: dataStream.ilm_policy, hidden: dataStream.hidden, }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 80386a2a0dd56c..a176805307845c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -8,8 +8,14 @@ import { readFileSync } from 'fs'; import { safeLoad } from 'js-yaml'; import path from 'path'; +import { RegistryDataStream } from '../../../../types'; import { Field, processFields } from '../../fields/field'; -import { generateMappings, getTemplate } from './template'; +import { + generateMappings, + getTemplate, + getTemplatePriority, + generateTemplateIndexPattern, +} from './template'; // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ @@ -23,16 +29,17 @@ expect.addSnapshotSerializer({ }); test('get template', () => { - const templateName = 'logs-nginx-access-abcd'; + const templateIndexPattern = 'logs-nginx.access-abcd-*'; const template = getTemplate({ type: 'logs', - templateName, + templateIndexPattern, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], + templatePriority: 200, }); - expect(template.index_patterns).toStrictEqual([`${templateName}-*`]); + expect(template.index_patterns).toStrictEqual([templateIndexPattern]); }); test('adds composed_of correctly', () => { @@ -40,10 +47,11 @@ test('adds composed_of correctly', () => { const template = getTemplate({ type: 'logs', - templateName: 'name', + templateIndexPattern: 'name-*', packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates, + templatePriority: 200, }); expect(template.composed_of).toStrictEqual(composedOfTemplates); }); @@ -53,35 +61,36 @@ test('adds empty composed_of correctly', () => { const template = getTemplate({ type: 'logs', - templateName: 'name', + templateIndexPattern: 'name-*', packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates, + templatePriority: 200, }); expect(template.composed_of).toStrictEqual(composedOfTemplates); }); test('adds hidden field correctly', () => { - const templateWithHiddenName = 'logs-nginx-access-abcd'; + const templateIndexPattern = 'logs-nginx.access-abcd-*'; const templateWithHidden = getTemplate({ type: 'logs', - templateName: templateWithHiddenName, + templateIndexPattern, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], + templatePriority: 200, hidden: true, }); expect(templateWithHidden.data_stream.hidden).toEqual(true); - const templateWithoutHiddenName = 'logs-nginx-access-efgh'; - const templateWithoutHidden = getTemplate({ type: 'logs', - templateName: templateWithoutHiddenName, + templateIndexPattern, packageName: 'nginx', mappings: { properties: {} }, composedOfTemplates: [], + templatePriority: 200, }); expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); @@ -95,10 +104,11 @@ test('tests loading base.yml', () => { const mappings = generateMappings(processedFields); const template = getTemplate({ type: 'logs', - templateName: 'foo', + templateIndexPattern: 'foo-*', packageName: 'nginx', mappings, composedOfTemplates: [], + templatePriority: 200, }); expect(template).toMatchSnapshot(path.basename(ymlPath)); @@ -113,10 +123,11 @@ test('tests loading coredns.logs.yml', () => { const mappings = generateMappings(processedFields); const template = getTemplate({ type: 'logs', - templateName: 'foo', + templateIndexPattern: 'foo-*', packageName: 'coredns', mappings, composedOfTemplates: [], + templatePriority: 200, }); expect(template).toMatchSnapshot(path.basename(ymlPath)); @@ -131,10 +142,11 @@ test('tests loading system.yml', () => { const mappings = generateMappings(processedFields); const template = getTemplate({ type: 'metrics', - templateName: 'whatsthis', + templateIndexPattern: 'whatsthis-*', packageName: 'system', mappings, composedOfTemplates: [], + templatePriority: 200, }); expect(template).toMatchSnapshot(path.basename(ymlPath)); @@ -520,3 +532,62 @@ test('tests constant_keyword field type handling', () => { const mappings = generateMappings(processedFields); expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); }); + +test('tests priority and index pattern for data stream without dataset_is_prefix', () => { + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset); +}); + +test('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse); +}); + +test('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 150; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index ea0bb5dc53a1e4..b86c989f8c24c8 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -33,6 +33,10 @@ export interface CurrentDataStream { const DEFAULT_SCALING_FACTOR = 1000; const DEFAULT_IGNORE_ABOVE = 1024; +// see discussion in https://github.com/elastic/kibana/issues/88307 +const DEFAULT_TEMPLATE_PRIORITY = 200; +const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; + /** * getTemplate retrieves the default template but overwrites the index pattern with the given value. * @@ -40,29 +44,32 @@ const DEFAULT_IGNORE_ABOVE = 1024; */ export function getTemplate({ type, - templateName, + templateIndexPattern, mappings, pipelineName, packageName, composedOfTemplates, + templatePriority, ilmPolicy, hidden, }: { type: string; - templateName: string; + templateIndexPattern: string; mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; composedOfTemplates: string[]; + templatePriority: number; ilmPolicy?: string | undefined; hidden?: boolean; }): IndexTemplate { const template = getBaseTemplate( type, - templateName, + templateIndexPattern, mappings, packageName, composedOfTemplates, + templatePriority, ilmPolicy, hidden ); @@ -242,6 +249,35 @@ export function generateTemplateName(dataStream: RegistryDataStream): string { return getRegistryDataStreamAssetBaseName(dataStream); } +export function generateTemplateIndexPattern(dataStream: RegistryDataStream): string { + // undefined or explicitly set to false + // See also https://github.com/elastic/package-spec/pull/102 + if (!dataStream.dataset_is_prefix) { + return getRegistryDataStreamAssetBaseName(dataStream) + '-*'; + } else { + return getRegistryDataStreamAssetBaseName(dataStream) + '.*-*'; + } +} + +// Template priorities are discussed in https://github.com/elastic/kibana/issues/88307 +// See also https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html +// +// Built-in templates like logs-*-* and metrics-*-* have priority 100 +// +// EPM generated templates for data streams have priority 200 (DEFAULT_TEMPLATE_PRIORITY) +// +// EPM generated templates for data streams with dataset_is_prefix: true have priority 150 (DATASET_IS_PREFIX_TEMPLATE_PRIORITY) + +export function getTemplatePriority(dataStream: RegistryDataStream): number { + // undefined or explicitly set to false + // See also https://github.com/elastic/package-spec/pull/102 + if (!dataStream.dataset_is_prefix) { + return DEFAULT_TEMPLATE_PRIORITY; + } else { + return DATASET_IS_PREFIX_TEMPLATE_PRIORITY; + } +} + /** * Returns a map of the data stream path fields to elasticsearch index pattern. * @param dataStreams an array of RegistryDataStream objects @@ -255,17 +291,18 @@ export function generateESIndexPatterns( const patterns: Record = {}; for (const dataStream of dataStreams) { - patterns[dataStream.path] = generateTemplateName(dataStream) + '-*'; + patterns[dataStream.path] = generateTemplateIndexPattern(dataStream); } return patterns; } function getBaseTemplate( type: string, - templateName: string, + templateIndexPattern: string, mappings: IndexTemplateMappings, packageName: string, composedOfTemplates: string[], + templatePriority: number, ilmPolicy?: string | undefined, hidden?: boolean ): IndexTemplate { @@ -279,13 +316,9 @@ function getBaseTemplate( }; return { - // This takes precedence over all index templates installed by ES by default (logs-*-* and metrics-*-*) - // if this number is lower than the ES value (which is 100) this template will never be applied when a data stream - // is created. I'm using 200 here to give some room for users to create their own template and fit it between the - // default and the one the ingest manager uses. - priority: 200, + priority: templatePriority, // To be completed with the correct index patterns - index_patterns: [`${templateName}-*`], + index_patterns: [templateIndexPattern], template: { settings: { index: { diff --git a/x-pack/test/fleet_api_integration/apis/epm/index.js b/x-pack/test/fleet_api_integration/apis/epm/index.js index 23b7464a317e90..0020e6bdf1bb01 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/index.js +++ b/x-pack/test/fleet_api_integration/apis/epm/index.js @@ -11,7 +11,7 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./setup')); loadTestFile(require.resolve('./get')); loadTestFile(require.resolve('./file')); - //loadTestFile(require.resolve('./template')); + loadTestFile(require.resolve('./template')); loadTestFile(require.resolve('./ilm')); loadTestFile(require.resolve('./install_by_upload')); loadTestFile(require.resolve('./install_overrides')); diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index c7e9e211552578..d79452ca0eb6f2 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -10,8 +10,8 @@ import { FtrProviderContext } from '../../../api_integration/ftr_provider_contex import { getTemplate } from '../../../../plugins/fleet/server/services/epm/elasticsearch/template/template'; export default function ({ getService }: FtrProviderContext) { - const indexPattern = 'foo'; const templateName = 'bar'; + const templateIndexPattern = 'bar-*'; const es = getService('es'); const mappings = { properties: { @@ -25,27 +25,36 @@ export default function ({ getService }: FtrProviderContext) { it('can be loaded', async () => { const template = getTemplate({ type: 'logs', - templateName, + templateIndexPattern, mappings, packageName: 'system', composedOfTemplates: [], + templatePriority: 200, }); // This test is not an API integration test with Kibana // We want to test here if the template is valid and for this we need a running ES instance. // If the ES instance takes the template, we assume it is a valid template. - const { body: response1 } = await es.indices.putTemplate({ - name: templateName, + const { body: response1 } = await es.transport.request({ + method: 'PUT', + path: `/_index_template/${templateName}`, body: template, }); + // Checks if template loading worked as expected expect(response1).to.eql({ acknowledged: true }); - const { body: response2 } = await es.indices.getTemplate({ name: templateName }); + const { body: response2 } = await es.transport.request({ + method: 'GET', + path: `/_index_template/${templateName}`, + }); + // Checks if the content of the template that was loaded is as expected // We already know based on the above test that the template was valid // but we check here also if we wrote the index pattern inside the template as expected - expect(response2[templateName].index_patterns).to.eql([`${indexPattern}-*`]); + expect(response2.index_templates[0].index_template.index_patterns).to.eql([ + templateIndexPattern, + ]); }); }); }