diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts index 162fe94ecad708..889d74a1c65037 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts @@ -17,6 +17,19 @@ describe('computeHasMetadataOperator', () => { expect(computeHasMetadataOperator('from test* | eval x="[metadata _id]"')).toBe(false); }); it('should be true if query has operator', () => { + expect(computeHasMetadataOperator('from test* metadata _id')).toBe(true); + expect(computeHasMetadataOperator('from test* metadata _id, _index')).toBe(true); + expect(computeHasMetadataOperator('from test* metadata _index, _id')).toBe(true); + expect(computeHasMetadataOperator('from test* metadata _id ')).toBe(true); + expect(computeHasMetadataOperator('from test* metadata _id | limit 10')).toBe(true); + expect( + computeHasMetadataOperator(`from packetbeat* metadata + + _id + | limit 100`) + ).toBe(true); + + // still validates deprecated square bracket syntax expect(computeHasMetadataOperator('from test* [metadata _id]')).toBe(true); expect(computeHasMetadataOperator('from test* [metadata _id, _index]')).toBe(true); expect(computeHasMetadataOperator('from test* [metadata _index, _id]')).toBe(true); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts index 9cadae108ac22a..e7a6e523965b22 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts @@ -36,10 +36,10 @@ const constructValidationError = (error: Error) => { }; /** - * checks whether query has [metadata _id] operator + * checks whether query has metadata _id operator */ export const computeHasMetadataOperator = (esqlQuery: string) => { - return /(? export const ESQL_VALIDATION_MISSING_ID_IN_QUERY_ERROR = i18n.translate( 'xpack.securitySolution.detectionEngine.esqlValidation.missingIdInQueryError', { - defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the [metadata _id, _version, _index] operator after the source command. For example: FROM logs* [metadata _id, _version, _index]. In addition, the metadata properties (_id, _version, and _index) must be returned in the query response.`, + defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index. In addition, the metadata properties (_id, _version, and _index) must be returned in the query response.`, } ); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_esql_index.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_esql_index.test.ts index cde92602611aca..dc4394be257e5b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_esql_index.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_esql_index.test.ts @@ -8,7 +8,7 @@ import { renderHook } from '@testing-library/react-hooks'; import { useEsqlIndex } from './use_esql_index'; -const validEsqlQuery = 'from auditbeat* [metadata _id, _index, _version]'; +const validEsqlQuery = 'from auditbeat* metadata _id, _index, _version'; describe('useEsqlIndex', () => { it('should return empty array if isQueryReadEnabled is undefined', () => { const { result } = renderHook(() => useEsqlIndex(validEsqlQuery, 'esql', undefined)); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts index 4819f87d5a41fd..6a159f87d89d8b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts @@ -28,7 +28,7 @@ const fetchFieldsFromESQLMock = fetchFieldsFromESQL as jest.Mock; const { wrapper } = createQueryWrapperMock(); -const mockEsqlQuery = 'from auditbeat* [metadata _id]'; +const mockEsqlQuery = 'from auditbeat* metadata _id'; const mockIndexPatternFields: DataViewFieldBase[] = [ { name: 'agent.name', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts index 7fd34aff676903..c629f5056ef3ab 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts @@ -66,7 +66,7 @@ export default ({ getService }: FtrProviderContext) => { const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z']; const doc1 = { agent: { name: 'test-1' } }; const doc2 = { agent: { name: 'test-2' } }; - const ruleQuery = `from ecs_compliant [metadata _id, _index, _version] ${internalIdPipe( + const ruleQuery = `from ecs_compliant metadata _id, _index, _version ${internalIdPipe( id )} | where agent.name=="test-1"`; const rule: EsqlRuleCreateProps = { @@ -243,7 +243,7 @@ export default ({ getService }: FtrProviderContext) => { const rule: EsqlRuleCreateProps = { ...getCreateEsqlRulesSchemaMock('rule-1', true), // only _id and agent.name is projected at the end of query pipeline - query: `from ecs_compliant [metadata _id] ${internalIdPipe(id)} | keep _id, agent.name`, + query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | keep _id, agent.name`, from: 'now-1h', interval: '1h', }; @@ -278,6 +278,44 @@ export default ({ getService }: FtrProviderContext) => { ); }); + it('should support deprecated [metadata _id] syntax', async () => { + const id = uuidv4(); + const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z']; + const doc1 = { + agent: { name: 'test-1', version: '2', type: 'auditbeat' }, + host: { name: 'my-host' }, + client: { ip: '127.0.0.1' }, + }; + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + // only _id and agent.name is projected at the end of query pipeline + query: `from ecs_compliant [metadata _id] ${internalIdPipe(id)} | keep _id, agent.name`, + from: 'now-1h', + interval: '1h', + }; + + await indexEnhancedDocuments({ + documents: [doc1], + interval, + id, + }); + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + }); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 10, + }); + + expect(previewAlerts.length).toBe(1); + }); + it('should deduplicate alerts correctly based on source document _id', async () => { const id = uuidv4(); // document will fall into 2 rule execution windows @@ -290,7 +328,7 @@ export default ({ getService }: FtrProviderContext) => { const rule: EsqlRuleCreateProps = { ...getCreateEsqlRulesSchemaMock('rule-1', true), // only _id and agent.name is projected at the end of query pipeline - query: `from ecs_compliant [metadata _id] ${internalIdPipe(id)} | keep _id, agent.name`, + query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | keep _id, agent.name`, from: 'now-45m', interval: '30m', }; @@ -725,7 +763,7 @@ export default ({ getService }: FtrProviderContext) => { const id = uuidv4(); const rule: EsqlRuleCreateProps = { ...getCreateEsqlRulesSchemaMock(`rule-${id}`, true), - query: `from ecs_compliant [metadata _id] ${internalIdPipe( + query: `from ecs_compliant metadata _id ${internalIdPipe( id )} | keep _id, agent.name | sort agent.name`, from: '2020-10-28T05:15:00.000Z', @@ -913,7 +951,7 @@ export default ({ getService }: FtrProviderContext) => { const rule: EsqlRuleCreateProps = { ...getCreateEsqlRulesSchemaMock('rule-1', true), - query: `from ecs_compliant [metadata _id] ${internalIdPipe( + query: `from ecs_compliant metadata _id ${internalIdPipe( id )} | where agent.name=="test-1"`, from: 'now-1h', @@ -956,7 +994,7 @@ export default ({ getService }: FtrProviderContext) => { const rule: EsqlRuleCreateProps = { ...getCreateEsqlRulesSchemaMock('rule-1', true), - query: `from ecs_compliant [metadata _id] ${internalIdPipe( + query: `from ecs_compliant metadata _id ${internalIdPipe( id )} | where agent.name=="test-1"`, from: 'now-1h', @@ -1021,7 +1059,7 @@ export default ({ getService }: FtrProviderContext) => { const rule: EsqlRuleCreateProps = { ...getCreateEsqlRulesSchemaMock('rule-1', true), - query: `from ecs_non_compliant [metadata _id] ${internalIdPipe(id)}`, + query: `from ecs_non_compliant metadata _id ${internalIdPipe(id)}`, from: 'now-1h', interval: '1h', }; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts index 945cf43967cbea..2e95bb19a04773 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts @@ -138,7 +138,7 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { cy.get(ESQL_QUERY_BAR).should('not.be.visible'); }); - it('shows error when non-aggregating ES|QL query does not [metadata] operator', function () { + it('shows error when non-aggregating ES|QL query does not have metadata operator', function () { workaroundForResizeObserver(); const invalidNonAggregatingQuery = 'from auditbeat* | limit 5'; @@ -148,7 +148,7 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { getDefineContinueButton().click(); cy.get(ESQL_QUERY_BAR).contains( - 'must include the [metadata _id, _version, _index] operator after the source command' + 'must include the "metadata _id, _version, _index" operator after the source command' ); }); @@ -156,7 +156,7 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { workaroundForResizeObserver(); const invalidNonAggregatingQuery = - 'from auditbeat* [metadata _id, _version, _index] | keep agent.* | limit 5'; + 'from auditbeat* metadata _id, _version, _index | keep agent.* | limit 5'; selectEsqlRuleType(); expandEsqlQueryBar(); @@ -164,14 +164,14 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { getDefineContinueButton().click(); cy.get(ESQL_QUERY_BAR).contains( - 'must include the [metadata _id, _version, _index] operator after the source command' + 'must include the "metadata _id, _version, _index" operator after the source command' ); }); it('shows error when ES|QL query is invalid', function () { workaroundForResizeObserver(); const invalidEsqlQuery = - 'from auditbeat* [metadata _id, _version, _index] | not_existing_operator'; + 'from auditbeat* metadata _id, _version, _index | not_existing_operator'; visit(CREATE_RULE_URL); selectEsqlRuleType(); @@ -191,7 +191,7 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { it('shows custom ES|QL field in investigation fields autocomplete and saves it in rule', function () { const CUSTOM_ESQL_FIELD = '_custom_agent_name'; const queryWithCustomFields = [ - `from auditbeat* [metadata _id, _version, _index]`, + `from auditbeat* metadata _id, _version, _index`, `eval ${CUSTOM_ESQL_FIELD} = agent.name`, `keep _id, _custom_agent_name`, `limit 5`,