diff --git a/x-pack/plugins/fleet/server/mocks.ts b/x-pack/plugins/fleet/server/mocks/index.ts similarity index 82% rename from x-pack/plugins/fleet/server/mocks.ts rename to x-pack/plugins/fleet/server/mocks/index.ts index ea3ba20c59e9c8..cff80f533d5e39 100644 --- a/x-pack/plugins/fleet/server/mocks.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -4,22 +4,18 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { elasticsearchServiceMock, loggingSystemMock, savedObjectsServiceMock, -} from 'src/core/server/mocks'; - -import { coreMock } from '../../../../src/core/server/mocks'; -import { licensingMock } from '../../../plugins/licensing/server/mocks'; - -import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks'; -import { securityMock } from '../../security/server/mocks'; - -import type { FleetAppContext } from './plugin'; -import type { PackagePolicyServiceInterface } from './services/package_policy'; -import type { AgentPolicyServiceInterface, AgentService } from './services'; + coreMock, +} from '../../../../../src/core/server/mocks'; +import { licensingMock } from '../../../../plugins/licensing/server/mocks'; +import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; +import { securityMock } from '../../../security/server/mocks'; +import type { PackagePolicyServiceInterface } from '../services/package_policy'; +import type { AgentPolicyServiceInterface, AgentService } from '../services'; +import type { FleetAppContext } from '../plugin'; export const createAppContextStartContractMock = (): FleetAppContext => { return { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index 2d2478843c4541..65eec939d58502 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`tests loading base.yml: base.yml 1`] = ` +exports[`EPM template tests loading base.yml: base.yml 1`] = ` { "priority": 200, "index_patterns": [ @@ -20,12 +20,12 @@ exports[`tests loading base.yml: base.yml 1`] = ` }, "refresh_interval": "5s", "number_of_shards": "1", + "number_of_routing_shards": "30", "query": { "default_field": [ - "message" + "long.nested.foo" ] - }, - "number_of_routing_shards": "30" + } } }, "mappings": { @@ -109,7 +109,7 @@ exports[`tests loading base.yml: base.yml 1`] = ` } `; -exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` +exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` { "priority": 200, "index_patterns": [ @@ -129,12 +129,17 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` }, "refresh_interval": "5s", "number_of_shards": "1", + "number_of_routing_shards": "30", "query": { "default_field": [ - "message" + "coredns.id", + "coredns.query.class", + "coredns.query.name", + "coredns.query.type", + "coredns.response.code", + "coredns.response.flags" ] - }, - "number_of_routing_shards": "30" + } } }, "mappings": { @@ -218,7 +223,7 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` } `; -exports[`tests loading system.yml: system.yml 1`] = ` +exports[`EPM template tests loading system.yml: system.yml 1`] = ` { "priority": 200, "index_patterns": [ @@ -238,12 +243,45 @@ exports[`tests loading system.yml: system.yml 1`] = ` }, "refresh_interval": "5s", "number_of_shards": "1", + "number_of_routing_shards": "30", "query": { "default_field": [ - "message" + "system.diskio.name", + "system.diskio.serial_number", + "system.filesystem.device_name", + "system.filesystem.type", + "system.filesystem.mount_point", + "system.network.name", + "system.process.state", + "system.process.cmdline", + "system.process.cgroup.id", + "system.process.cgroup.path", + "system.process.cgroup.cpu.id", + "system.process.cgroup.cpu.path", + "system.process.cgroup.cpuacct.id", + "system.process.cgroup.cpuacct.path", + "system.process.cgroup.memory.id", + "system.process.cgroup.memory.path", + "system.process.cgroup.blkio.id", + "system.process.cgroup.blkio.path", + "system.raid.name", + "system.raid.status", + "system.raid.level", + "system.raid.sync_action", + "system.socket.remote.host", + "system.socket.remote.etld_plus_one", + "system.socket.remote.host_error", + "system.socket.process.cmdline", + "system.users.id", + "system.users.seat", + "system.users.path", + "system.users.type", + "system.users.service", + "system.users.state", + "system.users.scope", + "system.users.remote_host" ] - }, - "number_of_routing_shards": "30" + } } }, "mappings": { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index a580f58d4fed1b..d68d7715436a3a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -4,194 +4,206 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { createAppContextStartContractMock } from '../../../../mocks'; +import { appContextService } from '../../../../services'; + import type { RegistryDataStream } from '../../../../types'; import type { Field } from '../../fields/field'; import { installTemplate } from './install'; -test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } +describe('EPM install', () => { + beforeEach(async () => { + appContextService.start(createAppContextStartContractMock()); }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixUnset, - packageVersion: pkg.version, - packageName: pkg.name, - }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); -}); + it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { index_templates: [] }; + } + }); -test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } + const fields: Field[] = []; + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixUnset, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[1][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixFalse = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: false, - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixFalse = 200; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixFalse, - packageVersion: pkg.version, - packageName: pkg.name, - }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); -}); + it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { index_templates: [] }; + } + }); -test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } + const fields: Field[] = []; + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixFalse, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[1][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixTrue = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: true, - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; - const templatePriorityDatasetIsPrefixTrue = 150; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixTrue, - packageVersion: pkg.version, - packageName: pkg.name, + it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { index_templates: [] }; + } + }); + + const fields: Field[] = []; + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 150; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixTrue, + packageVersion: pkg.version, + packageName: pkg.name, + }); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[1][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); -}); -test('tests installPackage remove the aliases property if the property existed', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { - index_templates: [ - { - name: 'metrics-package.dataset', - index_template: { - index_patterns: ['metrics-package.dataset-*'], - template: { aliases: {} }, + it('tests installPackage remove the aliases property if the property existed', async () => { + const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() + .callAsCurrentUser; + callCluster.mockImplementation(async (_, params) => { + if ( + params && + params.method === 'GET' && + params.path === '/_index_template/metrics-package.dataset' + ) { + return { + index_templates: [ + { + name: 'metrics-package.dataset', + index_template: { + index_patterns: ['metrics-package.dataset-*'], + template: { aliases: {} }, + }, }, - }, - ], - }; - } - }); + ], + }; + } + }); - const fields: Field[] = []; - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - await installTemplate({ - callCluster, - fields, - dataStream: dataStreamDatasetIsPrefixUnset, - packageVersion: pkg.version, - packageName: pkg.name, - }); + const fields: Field[] = []; + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const pkg = { + name: 'package', + version: '0.0.1', + }; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + await installTemplate({ + callCluster, + fields, + dataStream: dataStreamDatasetIsPrefixUnset, + packageVersion: pkg.version, + packageName: pkg.name, + }); - // @ts-ignore - const removeAliases = callCluster.mock.calls[1][1].body; - expect(removeAliases.template.aliases).not.toBeDefined(); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[2][1].body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); - expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); + // @ts-ignore + const removeAliases = callCluster.mock.calls[1][1].body; + expect(removeAliases.template.aliases).not.toBeDefined(); + // @ts-ignore + const sentTemplate = callCluster.mock.calls[2][1].body; + expect(sentTemplate).toBeDefined(); + expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); + expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 28cfda3fd41891..2769b97fe48a1e 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -300,7 +300,8 @@ export async function installTemplate({ packageVersion: string; packageName: string; }): Promise { - const mappings = generateMappings(processFields(fields)); + const validFields = processFields(fields); + const mappings = generateMappings(validFields); const templateName = generateTemplateName(dataStream); const templateIndexPattern = generateTemplateIndexPattern(dataStream); const templatePriority = getTemplatePriority(dataStream); @@ -362,6 +363,7 @@ export async function installTemplate({ const template = getTemplate({ type: dataStream.type, templateIndexPattern, + fields: validFields, mappings, pipelineName, packageName, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 0f53d260592e70..df82aa90b5a131 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -10,6 +10,9 @@ import path from 'path'; import { safeLoad } from 'js-yaml'; +import { createAppContextStartContractMock } from '../../../../mocks'; +import { appContextService } from '../../../../services'; + import type { RegistryDataStream } from '../../../../types'; import { processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; @@ -32,132 +35,145 @@ expect.addSnapshotSerializer({ }, }); -test('get template', () => { - const templateIndexPattern = 'logs-nginx.access-abcd-*'; - - const template = getTemplate({ - type: 'logs', - templateIndexPattern, - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates: [], - templatePriority: 200, +describe('EPM template', () => { + beforeEach(async () => { + appContextService.start(createAppContextStartContractMock()); }); - expect(template.index_patterns).toStrictEqual([templateIndexPattern]); -}); - -test('adds composed_of correctly', () => { - const composedOfTemplates = ['component1', 'component2']; - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'name-*', - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates, - templatePriority: 200, + it('get template', () => { + const templateIndexPattern = 'logs-nginx.access-abcd-*'; + + const template = getTemplate({ + type: 'logs', + templateIndexPattern, + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates: [], + templatePriority: 200, + }); + expect(template.index_patterns).toStrictEqual([templateIndexPattern]); }); - expect(template.composed_of).toStrictEqual(composedOfTemplates); -}); - -test('adds empty composed_of correctly', () => { - const composedOfTemplates: string[] = []; - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'name-*', - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates, - templatePriority: 200, + it('adds composed_of correctly', () => { + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'name-*', + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates, + templatePriority: 200, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); }); - expect(template.composed_of).toStrictEqual(composedOfTemplates); -}); -test('adds hidden field correctly', () => { - const templateIndexPattern = 'logs-nginx.access-abcd-*'; - - const templateWithHidden = getTemplate({ - type: 'logs', - templateIndexPattern, - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates: [], - templatePriority: 200, - hidden: true, - }); - expect(templateWithHidden.data_stream.hidden).toEqual(true); - - const templateWithoutHidden = getTemplate({ - type: 'logs', - templateIndexPattern, - packageName: 'nginx', - mappings: { properties: {} }, - composedOfTemplates: [], - templatePriority: 200, + it('adds empty composed_of correctly', () => { + const composedOfTemplates: string[] = []; + + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'name-*', + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates, + templatePriority: 200, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); }); - expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); -}); -test('tests loading base.yml', () => { - const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); - const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = safeLoad(fieldsYML); - - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'nginx', - mappings, - composedOfTemplates: [], - templatePriority: 200, + it('adds hidden field correctly', () => { + const templateIndexPattern = 'logs-nginx.access-abcd-*'; + + const templateWithHidden = getTemplate({ + type: 'logs', + templateIndexPattern, + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates: [], + templatePriority: 200, + hidden: true, + }); + expect(templateWithHidden.data_stream.hidden).toEqual(true); + + const templateWithoutHidden = getTemplate({ + type: 'logs', + templateIndexPattern, + packageName: 'nginx', + fields: [], + mappings: { properties: {} }, + composedOfTemplates: [], + templatePriority: 200, + }); + expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); -}); - -test('tests loading coredns.logs.yml', () => { - const ymlPath = path.join(__dirname, '../../fields/tests/coredns.logs.yml'); - const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = safeLoad(fieldsYML); - - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'logs', - templateIndexPattern: 'foo-*', - packageName: 'coredns', - mappings, - composedOfTemplates: [], - templatePriority: 200, + it('tests loading base.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'foo-*', + packageName: 'nginx', + fields: processedFields, + mappings, + composedOfTemplates: [], + templatePriority: 200, + }); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); -}); - -test('tests loading system.yml', () => { - const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); - const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = safeLoad(fieldsYML); - - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - const template = getTemplate({ - type: 'metrics', - templateIndexPattern: 'whatsthis-*', - packageName: 'system', - mappings, - composedOfTemplates: [], - templatePriority: 200, + it('tests loading coredns.logs.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/coredns.logs.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate({ + type: 'logs', + templateIndexPattern: 'foo-*', + packageName: 'coredns', + fields: processedFields, + mappings, + composedOfTemplates: [], + templatePriority: 200, + }); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); }); - expect(template).toMatchSnapshot(path.basename(ymlPath)); -}); + it('tests loading system.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + const template = getTemplate({ + type: 'metrics', + templateIndexPattern: 'whatsthis-*', + packageName: 'system', + fields: processedFields, + mappings, + composedOfTemplates: [], + templatePriority: 200, + }); + + expect(template).toMatchSnapshot(path.basename(ymlPath)); + }); -test('tests processing text field with multi fields', () => { - const textWithMultiFieldsLiteralYml = ` + it('tests processing text field with multi fields', () => { + const textWithMultiFieldsLiteralYml = ` - name: textWithMultiFields type: text multi_fields: @@ -166,30 +182,30 @@ test('tests processing text field with multi fields', () => { - name: indexed type: text `; - const textWithMultiFieldsMapping = { - properties: { - textWithMultiFields: { - type: 'text', - fields: { - raw: { - ignore_above: 1024, - type: 'keyword', - }, - indexed: { - type: 'text', + const textWithMultiFieldsMapping = { + properties: { + textWithMultiFields: { + type: 'text', + fields: { + raw: { + ignore_above: 1024, + type: 'keyword', + }, + indexed: { + type: 'text', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(textWithMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(textWithMultiFieldsMapping); + }); -test('tests processing keyword field with multi fields', () => { - const keywordWithMultiFieldsLiteralYml = ` + it('tests processing keyword field with multi fields', () => { + const keywordWithMultiFieldsLiteralYml = ` - name: keywordWithMultiFields type: keyword multi_fields: @@ -199,31 +215,31 @@ test('tests processing keyword field with multi fields', () => { type: text `; - const keywordWithMultiFieldsMapping = { - properties: { - keywordWithMultiFields: { - ignore_above: 1024, - type: 'keyword', - fields: { - raw: { - ignore_above: 1024, - type: 'keyword', - }, - indexed: { - type: 'text', + const keywordWithMultiFieldsMapping = { + properties: { + keywordWithMultiFields: { + ignore_above: 1024, + type: 'keyword', + fields: { + raw: { + ignore_above: 1024, + type: 'keyword', + }, + indexed: { + type: 'text', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(keywordWithMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(keywordWithMultiFieldsMapping); + }); -test('tests processing keyword field with multi fields with analyzed text field', () => { - const keywordWithAnalyzedMultiFieldsLiteralYml = ` + it('tests processing keyword field with multi fields with analyzed text field', () => { + const keywordWithAnalyzedMultiFieldsLiteralYml = ` - name: keywordWithAnalyzedMultiField type: keyword multi_fields: @@ -233,29 +249,29 @@ test('tests processing keyword field with multi fields with analyzed text field' search_analyzer: standard `; - const keywordWithAnalyzedMultiFieldsMapping = { - properties: { - keywordWithAnalyzedMultiField: { - ignore_above: 1024, - type: 'keyword', - fields: { - analyzed: { - analyzer: 'autocomplete', - search_analyzer: 'standard', - type: 'text', + const keywordWithAnalyzedMultiFieldsMapping = { + properties: { + keywordWithAnalyzedMultiField: { + ignore_above: 1024, + type: 'keyword', + fields: { + analyzed: { + analyzer: 'autocomplete', + search_analyzer: 'standard', + type: 'text', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping); + }); -test('tests processing keyword field with multi fields with normalized keyword field', () => { - const keywordWithNormalizedMultiFieldsLiteralYml = ` + it('tests processing keyword field with multi fields with normalized keyword field', () => { + const keywordWithNormalizedMultiFieldsLiteralYml = ` - name: keywordWithNormalizedMultiField type: keyword multi_fields: @@ -264,235 +280,235 @@ test('tests processing keyword field with multi fields with normalized keyword f normalizer: lowercase `; - const keywordWithNormalizedMultiFieldsMapping = { - properties: { - keywordWithNormalizedMultiField: { - ignore_above: 1024, - type: 'keyword', - fields: { - normalized: { - type: 'keyword', - ignore_above: 1024, - normalizer: 'lowercase', + const keywordWithNormalizedMultiFieldsMapping = { + properties: { + keywordWithNormalizedMultiField: { + ignore_above: 1024, + type: 'keyword', + fields: { + normalized: { + type: 'keyword', + ignore_above: 1024, + normalizer: 'lowercase', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(keywordWithNormalizedMultiFieldsLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping); -}); + }; + const fields: Field[] = safeLoad(keywordWithNormalizedMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping); + }); -test('tests processing object field with no other attributes', () => { - const objectFieldLiteralYml = ` + it('tests processing object field with no other attributes', () => { + const objectFieldLiteralYml = ` - name: objectField type: object `; - const objectFieldMapping = { - properties: { - objectField: { - type: 'object', + const objectFieldMapping = { + properties: { + objectField: { + type: 'object', + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldMapping); + }); -test('tests processing object field with enabled set to false', () => { - const objectFieldEnabledFalseLiteralYml = ` + it('tests processing object field with enabled set to false', () => { + const objectFieldEnabledFalseLiteralYml = ` - name: objectField type: object enabled: false `; - const objectFieldEnabledFalseMapping = { - properties: { - objectField: { - type: 'object', - enabled: false, + const objectFieldEnabledFalseMapping = { + properties: { + objectField: { + type: 'object', + enabled: false, + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldEnabledFalseMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldEnabledFalseMapping); + }); -test('tests processing object field with dynamic set to false', () => { - const objectFieldDynamicFalseLiteralYml = ` + it('tests processing object field with dynamic set to false', () => { + const objectFieldDynamicFalseLiteralYml = ` - name: objectField type: object dynamic: false `; - const objectFieldDynamicFalseMapping = { - properties: { - objectField: { - type: 'object', - dynamic: false, + const objectFieldDynamicFalseMapping = { + properties: { + objectField: { + type: 'object', + dynamic: false, + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldDynamicFalseMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldDynamicFalseMapping); + }); -test('tests processing object field with dynamic set to true', () => { - const objectFieldDynamicTrueLiteralYml = ` + it('tests processing object field with dynamic set to true', () => { + const objectFieldDynamicTrueLiteralYml = ` - name: objectField type: object dynamic: true `; - const objectFieldDynamicTrueMapping = { - properties: { - objectField: { - type: 'object', - dynamic: true, + const objectFieldDynamicTrueMapping = { + properties: { + objectField: { + type: 'object', + dynamic: true, + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldDynamicTrueMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldDynamicTrueMapping); + }); -test('tests processing object field with dynamic set to strict', () => { - const objectFieldDynamicStrictLiteralYml = ` + it('tests processing object field with dynamic set to strict', () => { + const objectFieldDynamicStrictLiteralYml = ` - name: objectField type: object dynamic: strict `; - const objectFieldDynamicStrictMapping = { - properties: { - objectField: { - type: 'object', - dynamic: 'strict', + const objectFieldDynamicStrictMapping = { + properties: { + objectField: { + type: 'object', + dynamic: 'strict', + }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldDynamicStrictMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldDynamicStrictMapping); + }); -test('tests processing object field with property', () => { - const objectFieldWithPropertyLiteralYml = ` + it('tests processing object field with property', () => { + const objectFieldWithPropertyLiteralYml = ` - name: a type: object - name: a.b type: keyword `; - const objectFieldWithPropertyMapping = { - properties: { - a: { - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const objectFieldWithPropertyMapping = { + properties: { + a: { + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldWithPropertyMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldWithPropertyMapping); + }); -test('tests processing object field with property, reverse order', () => { - const objectFieldWithPropertyReversedLiteralYml = ` + it('tests processing object field with property, reverse order', () => { + const objectFieldWithPropertyReversedLiteralYml = ` - name: a.b type: keyword - name: a type: object dynamic: false `; - const objectFieldWithPropertyReversedMapping = { - properties: { - a: { - dynamic: false, - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const objectFieldWithPropertyReversedMapping = { + properties: { + a: { + dynamic: false, + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); -}); + }; + const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); + }); -test('tests processing nested field with property', () => { - const nestedYaml = ` + it('tests processing nested field with property', () => { + const nestedYaml = ` - name: a.b type: keyword - name: a type: nested dynamic: false `; - const expectedMapping = { - properties: { - a: { - dynamic: false, - type: 'nested', - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const expectedMapping = { + properties: { + a: { + dynamic: false, + type: 'nested', + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(nestedYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(expectedMapping); -}); + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); -test('tests processing nested field with property, nested field first', () => { - const nestedYaml = ` + it('tests processing nested field with property, nested field first', () => { + const nestedYaml = ` - name: a type: nested include_in_parent: true - name: a.b type: keyword `; - const expectedMapping = { - properties: { - a: { - include_in_parent: true, - type: 'nested', - properties: { - b: { - ignore_above: 1024, - type: 'keyword', + const expectedMapping = { + properties: { + a: { + include_in_parent: true, + type: 'nested', + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(nestedYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(expectedMapping); -}); + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); -test('tests processing nested leaf field with properties', () => { - const nestedYaml = ` + it('tests processing nested leaf field with properties', () => { + const nestedYaml = ` - name: a type: object dynamic: false @@ -500,98 +516,99 @@ test('tests processing nested leaf field with properties', () => { type: nested enabled: false `; - const expectedMapping = { - properties: { - a: { - dynamic: false, - properties: { - b: { - enabled: false, - type: 'nested', + const expectedMapping = { + properties: { + a: { + dynamic: false, + properties: { + b: { + enabled: false, + type: 'nested', + }, }, }, }, - }, - }; - const fields: Field[] = safeLoad(nestedYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(mappings).toEqual(expectedMapping); -}); + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); -test('tests constant_keyword field type handling', () => { - const constantKeywordLiteralYaml = ` + it('tests constant_keyword field type handling', () => { + const constantKeywordLiteralYaml = ` - name: constantKeyword type: constant_keyword `; - const constantKeywordMapping = { - properties: { - constantKeyword: { - type: 'constant_keyword', + const constantKeywordMapping = { + properties: { + constantKeyword: { + type: 'constant_keyword', + }, }, - }, - }; - const fields: Field[] = safeLoad(constantKeywordLiteralYaml); - const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); -}); + }; + const fields: Field[] = safeLoad(constantKeywordLiteralYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); + }); -test('tests priority and index pattern for data stream without dataset_is_prefix', () => { - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset); - const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset); - - expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset); - expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset); -}); + it('tests priority and index pattern for data stream without dataset_is_prefix', () => { + const dataStreamDatasetIsPrefixUnset = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixUnset = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset); + }); -test('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { - const dataStreamDatasetIsPrefixFalse = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: false, - } as RegistryDataStream; - const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixFalse = 200; - const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse); - const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse); - - expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse); - expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse); -}); + it('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => { + const dataStreamDatasetIsPrefixFalse = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: false, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; + const templatePriorityDatasetIsPrefixFalse = 200; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse); + }); -test('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { - const dataStreamDatasetIsPrefixTrue = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - dataset_is_prefix: true, - } as RegistryDataStream; - const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; - const templatePriorityDatasetIsPrefixTrue = 150; - const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue); - const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue); - - expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue); - expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue); + it('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => { + const dataStreamDatasetIsPrefixTrue = { + type: 'metrics', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + dataset_is_prefix: true, + } as RegistryDataStream; + const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; + const templatePriorityDatasetIsPrefixTrue = 150; + const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue); + const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue); + + expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue); + expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index a13ad007663d8b..01b9a92045b298 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -13,6 +13,7 @@ import type { IndexTemplate, IndexTemplateMappings, } from '../../../../types'; +import { appContextService } from '../../../'; import { getRegistryDataStreamAssetBaseName } from '../index'; interface Properties { @@ -37,6 +38,9 @@ const DEFAULT_IGNORE_ABOVE = 1024; const DEFAULT_TEMPLATE_PRIORITY = 200; const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; +const QUERY_DEFAULT_FIELD_TYPES = ['keyword', 'text']; +const QUERY_DEFAULT_FIELD_LIMIT = 1024; + /** * getTemplate retrieves the default template but overwrites the index pattern with the given value. * @@ -45,6 +49,7 @@ const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; export function getTemplate({ type, templateIndexPattern, + fields, mappings, pipelineName, packageName, @@ -55,6 +60,7 @@ export function getTemplate({ }: { type: string; templateIndexPattern: string; + fields: Fields; mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; @@ -66,6 +72,7 @@ export function getTemplate({ const template = getBaseTemplate( type, templateIndexPattern, + fields, mappings, packageName, composedOfTemplates, @@ -296,9 +303,28 @@ export function generateESIndexPatterns( return patterns; } +const flattenFieldsToNameAndType = ( + fields: Fields, + path: string = '' +): Array> => { + let newFields: Array> = []; + fields.forEach((field) => { + const fieldName = path ? `${path}.${field.name}` : field.name; + newFields.push({ + name: fieldName, + type: field.type, + }); + if (field.fields && field.fields.length) { + newFields = newFields.concat(flattenFieldsToNameAndType(field.fields, fieldName)); + } + }); + return newFields; +}; + function getBaseTemplate( type: string, templateIndexPattern: string, + fields: Fields, mappings: IndexTemplateMappings, packageName: string, composedOfTemplates: string[], @@ -306,6 +332,8 @@ function getBaseTemplate( ilmPolicy?: string | undefined, hidden?: boolean ): IndexTemplate { + const logger = appContextService.getLogger(); + // Meta information to identify Ingest Manager's managed templates and indices const _meta = { package: { @@ -315,6 +343,21 @@ function getBaseTemplate( managed: true, }; + // Find all field names to set `index.query.default_field` to, which will be + // the first 1024 keyword or text fields + const defaultFields = flattenFieldsToNameAndType(fields).filter( + (field) => field.type && QUERY_DEFAULT_FIELD_TYPES.includes(field.type) + ); + if (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT) { + logger.warn( + `large amount of default fields detected for index template ${templateIndexPattern} in package ${packageName}, applying the first ${QUERY_DEFAULT_FIELD_LIMIT} fields` + ); + } + const defaultFieldNames = (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT + ? defaultFields.slice(0, QUERY_DEFAULT_FIELD_LIMIT) + : defaultFields + ).map((field) => field.name); + return { priority: templatePriority, // To be completed with the correct index patterns @@ -338,13 +381,18 @@ function getBaseTemplate( refresh_interval: '5s', // Default in the stack now, still good to have it in number_of_shards: '1', - // All the default fields which should be queried have to be added here. - // So far we add all keyword and text fields here. - query: { - default_field: ['message'], - }, // We are setting 30 because it can be devided by several numbers. Useful when shrinking. number_of_routing_shards: '30', + // All the default fields which should be queried have to be added here. + // So far we add all keyword and text fields here if there are any, otherwise + // this setting is skipped. + ...(defaultFieldNames.length + ? { + query: { + default_field: defaultFieldNames, + }, + } + : {}), }, }, mappings: { diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index d79452ca0eb6f2..517d2c77d430de 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -7,6 +7,7 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { appContextService } from '../../../../plugins/fleet/server/services'; import { getTemplate } from '../../../../plugins/fleet/server/services/epm/elasticsearch/template/template'; export default function ({ getService }: FtrProviderContext) { @@ -20,12 +21,31 @@ export default function ({ getService }: FtrProviderContext) { }, }, }; + const fields = [ + { + name: 'foo', + type: 'keyword', + }, + ]; + // This test was inspired by https://github.com/elastic/kibana/blob/master/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js describe('EPM - template', async () => { + beforeEach(async () => { + appContextService.start({ + // @ts-ignore + elasticsearch: { client: {} }, + // @ts-ignore + logger: { + warn: () => {}, + }, + }); + }); + it('can be loaded', async () => { const template = getTemplate({ type: 'logs', templateIndexPattern, + fields, mappings, packageName: 'system', composedOfTemplates: [],