[8.14] [Security Solution][Detection Engine] use ES|QL metadata opera…

…tor without square brackets (#182114) (#182223)

# Backport

This will backport the following commits from `main` to `8.14`:
- [[Security Solution][Detection Engine] use ES|QL metadata operator
without square brackets
(#182114)](#182114)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Vitalii
Dmyterko","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-05-01T08:16:34Z","message":"[Security
Solution][Detection Engine] use ES|QL metadata operator without square
brackets (#182114)\n\n## Summary\r\n\r\nES|QL metadata operator has
changed its
syntax:\r\nhttps://www.elastic.co/guide/en/elasticsearch/reference/current/esql-metadata-fields.html\r\n\r\nfrom
\r\n\r\n```\r\nFROM index [METADATA _index, _id]\r\n```\r\n\r\nto
\r\n\r\n```\r\nFROM index METADATA _index, _id\r\n```\r\n\r\nThis PR
removes square brackets as required symbols for metadata\r\noperator.
Deprecated one still supported by ES|QL, but shows
warning","sha":"1633c0b3a70194a45d126bcbc946a0810db5a897","branchLabelMapping":{"^v8.15.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Detections
and Resp","backport:prev-minor","Team:Detection
Engine","v8.14.0","v8.15.0"],"title":"[Security Solution][Detection
Engine] use ES|QL metadata operator without square
brackets","number":182114,"url":"https://github.com/elastic/kibana/pull/182114","mergeCommit":{"message":"[Security
Solution][Detection Engine] use ES|QL metadata operator without square
brackets (#182114)\n\n## Summary\r\n\r\nES|QL metadata operator has
changed its
syntax:\r\nhttps://www.elastic.co/guide/en/elasticsearch/reference/current/esql-metadata-fields.html\r\n\r\nfrom
\r\n\r\n```\r\nFROM index [METADATA _index, _id]\r\n```\r\n\r\nto
\r\n\r\n```\r\nFROM index METADATA _index, _id\r\n```\r\n\r\nThis PR
removes square brackets as required symbols for metadata\r\noperator.
Deprecated one still supported by ES|QL, but shows
warning","sha":"1633c0b3a70194a45d126bcbc946a0810db5a897"}},"sourceBranch":"main","suggestedTargetBranches":["8.14"],"targetPullRequestStates":[{"branch":"8.14","label":"v8.14.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.15.0","branchLabelMappingKey":"^v8.15.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/182114","number":182114,"mergeCommit":{"message":"[Security
Solution][Detection Engine] use ES|QL metadata operator without square
brackets (#182114)\n\n## Summary\r\n\r\nES|QL metadata operator has
changed its
syntax:\r\nhttps://www.elastic.co/guide/en/elasticsearch/reference/current/esql-metadata-fields.html\r\n\r\nfrom
\r\n\r\n```\r\nFROM index [METADATA _index, _id]\r\n```\r\n\r\nto
\r\n\r\n```\r\nFROM index METADATA _index, _id\r\n```\r\n\r\nThis PR
removes square brackets as required symbols for metadata\r\noperator.
Deprecated one still supported by ES|QL, but shows
warning","sha":"1633c0b3a70194a45d126bcbc946a0810db5a897"}}]}]
BACKPORT-->

Co-authored-by: Vitalii Dmyterko <[email protected]>

x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts

@@ -17,6 +17,19 @@ describe('computeHasMetadataOperator', () => {
     expect(computeHasMetadataOperator('from test* | eval x="[metadata _id]"')).toBe(false);
   });
   it('should be true if query has operator', () => {
+    expect(computeHasMetadataOperator('from test* metadata _id')).toBe(true);
+    expect(computeHasMetadataOperator('from test* metadata _id, _index')).toBe(true);
+    expect(computeHasMetadataOperator('from test* metadata _index, _id')).toBe(true);
+    expect(computeHasMetadataOperator('from test*  metadata _id ')).toBe(true);
+    expect(computeHasMetadataOperator('from test*  metadata _id | limit 10')).toBe(true);
+    expect(
+      computeHasMetadataOperator(`from packetbeat* metadata 
+
+        _id
+        | limit 100`)
+    ).toBe(true);
+
+    // still validates deprecated square bracket syntax
     expect(computeHasMetadataOperator('from test* [metadata _id]')).toBe(true);
     expect(computeHasMetadataOperator('from test* [metadata _id, _index]')).toBe(true);
     expect(computeHasMetadataOperator('from test* [metadata _index, _id]')).toBe(true);

x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts

@@ -36,10 +36,10 @@ const constructValidationError = (error: Error) => {
 };
 
 /**
- * checks whether query has [metadata _id] operator
+ * checks whether query has metadata _id operator
  */
 export const computeHasMetadataOperator = (esqlQuery: string) => {
-  return /(?<!\|[\s\S.]*)\[\s*metadata[\s\S.]*_id[\s\S.]*\]/i.test(esqlQuery);
+  return /(?<!\|[\s\S.]*)\s*metadata[\s\S.]*_id[\s\S.]*/i.test(esqlQuery?.split('|')?.[0]);
 };
 
 /**
@@ -63,7 +63,7 @@ export const esqlValidator = async (
 
     const isEsqlQueryAggregating = computeIsESQLQueryAggregating(query);
 
-    // non-aggregating query which does not have [metadata], is not a valid one
+    // non-aggregating query which does not have metadata, is not a valid one
     if (!isEsqlQueryAggregating && !computeHasMetadataOperator(query)) {
       return {
         code: ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT,

x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/translations.ts

@@ -23,6 +23,6 @@ export const esqlValidationErrorMessage = (message: string) =>
 export const ESQL_VALIDATION_MISSING_ID_IN_QUERY_ERROR = i18n.translate(
   'xpack.securitySolution.detectionEngine.esqlValidation.missingIdInQueryError',
   {
-    defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the [metadata _id, _version, _index] operator after the source command. For example: FROM logs* [metadata _id, _version, _index].  In addition, the metadata properties (_id, _version, and _index)  must be returned in the query response.`,
+    defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.  In addition, the metadata properties (_id, _version, and _index)  must be returned in the query response.`,
   }
 );

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_esql_index.test.ts

@@ -8,7 +8,7 @@ import { renderHook } from '@testing-library/react-hooks';
 
 import { useEsqlIndex } from './use_esql_index';
 
-const validEsqlQuery = 'from auditbeat* [metadata _id, _index, _version]';
+const validEsqlQuery = 'from auditbeat* metadata _id, _index, _version';
 describe('useEsqlIndex', () => {
   it('should return empty array if isQueryReadEnabled is undefined', () => {
     const { result } = renderHook(() => useEsqlIndex(validEsqlQuery, 'esql', undefined));

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts

@@ -28,7 +28,7 @@ const fetchFieldsFromESQLMock = fetchFieldsFromESQL as jest.Mock;
 
 const { wrapper } = createQueryWrapperMock();
 
-const mockEsqlQuery = 'from auditbeat* [metadata _id]';
+const mockEsqlQuery = 'from auditbeat* metadata _id';
 const mockIndexPatternFields: DataViewFieldBase[] = [
   {
     name: 'agent.name',

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts

@@ -66,7 +66,7 @@ export default ({ getService }: FtrProviderContext) => {
       const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z'];
       const doc1 = { agent: { name: 'test-1' } };
       const doc2 = { agent: { name: 'test-2' } };
-      const ruleQuery = `from ecs_compliant [metadata _id, _index, _version] ${internalIdPipe(
+      const ruleQuery = `from ecs_compliant metadata _id, _index, _version ${internalIdPipe(
         id
       )} | where agent.name=="test-1"`;
       const rule: EsqlRuleCreateProps = {
@@ -243,7 +243,7 @@ export default ({ getService }: FtrProviderContext) => {
         const rule: EsqlRuleCreateProps = {
           ...getCreateEsqlRulesSchemaMock('rule-1', true),
           // only _id and agent.name is projected at the end of query pipeline
-          query: `from ecs_compliant [metadata _id] ${internalIdPipe(id)} | keep _id, agent.name`,
+          query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | keep _id, agent.name`,
           from: 'now-1h',
           interval: '1h',
         };
@@ -278,6 +278,44 @@ export default ({ getService }: FtrProviderContext) => {
         );
       });
 
+      it('should support deprecated [metadata _id] syntax', async () => {
+        const id = uuidv4();
+        const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z'];
+        const doc1 = {
+          agent: { name: 'test-1', version: '2', type: 'auditbeat' },
+          host: { name: 'my-host' },
+          client: { ip: '127.0.0.1' },
+        };
+
+        const rule: EsqlRuleCreateProps = {
+          ...getCreateEsqlRulesSchemaMock('rule-1', true),
+          // only _id and agent.name is projected at the end of query pipeline
+          query: `from ecs_compliant [metadata _id] ${internalIdPipe(id)} | keep _id, agent.name`,
+          from: 'now-1h',
+          interval: '1h',
+        };
+
+        await indexEnhancedDocuments({
+          documents: [doc1],
+          interval,
+          id,
+        });
+
+        const { previewId } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+        });
+
+        const previewAlerts = await getPreviewAlerts({
+          es,
+          previewId,
+          size: 10,
+        });
+
+        expect(previewAlerts.length).toBe(1);
+      });
+
       it('should deduplicate alerts correctly based on source document _id', async () => {
         const id = uuidv4();
         // document will fall into 2 rule execution windows
@@ -290,7 +328,7 @@ export default ({ getService }: FtrProviderContext) => {
         const rule: EsqlRuleCreateProps = {
           ...getCreateEsqlRulesSchemaMock('rule-1', true),
           // only _id and agent.name is projected at the end of query pipeline
-          query: `from ecs_compliant [metadata _id] ${internalIdPipe(id)} | keep _id, agent.name`,
+          query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | keep _id, agent.name`,
           from: 'now-45m',
           interval: '30m',
         };
@@ -725,7 +763,7 @@ export default ({ getService }: FtrProviderContext) => {
         const id = uuidv4();
         const rule: EsqlRuleCreateProps = {
           ...getCreateEsqlRulesSchemaMock(`rule-${id}`, true),
-          query: `from ecs_compliant [metadata _id] ${internalIdPipe(
+          query: `from ecs_compliant metadata _id ${internalIdPipe(
             id
           )} | keep _id, agent.name | sort agent.name`,
           from: '2020-10-28T05:15:00.000Z',
@@ -913,7 +951,7 @@ export default ({ getService }: FtrProviderContext) => {
 
         const rule: EsqlRuleCreateProps = {
           ...getCreateEsqlRulesSchemaMock('rule-1', true),
-          query: `from ecs_compliant [metadata _id] ${internalIdPipe(
+          query: `from ecs_compliant metadata _id ${internalIdPipe(
             id
           )} | where agent.name=="test-1"`,
           from: 'now-1h',
@@ -956,7 +994,7 @@ export default ({ getService }: FtrProviderContext) => {
 
         const rule: EsqlRuleCreateProps = {
           ...getCreateEsqlRulesSchemaMock('rule-1', true),
-          query: `from ecs_compliant [metadata _id] ${internalIdPipe(
+          query: `from ecs_compliant metadata _id ${internalIdPipe(
             id
           )} | where agent.name=="test-1"`,
           from: 'now-1h',
@@ -1021,7 +1059,7 @@ export default ({ getService }: FtrProviderContext) => {
 
           const rule: EsqlRuleCreateProps = {
             ...getCreateEsqlRulesSchemaMock('rule-1', true),
-            query: `from ecs_non_compliant [metadata _id] ${internalIdPipe(id)}`,
+            query: `from ecs_non_compliant metadata _id ${internalIdPipe(id)}`,
             from: 'now-1h',
             interval: '1h',
           };

x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts

@@ -138,7 +138,7 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
       cy.get(ESQL_QUERY_BAR).should('not.be.visible');
     });
 
-    it('shows error when non-aggregating ES|QL query does not [metadata] operator', function () {
+    it('shows error when non-aggregating ES|QL query does not have metadata operator', function () {
       workaroundForResizeObserver();
 
       const invalidNonAggregatingQuery = 'from auditbeat* | limit 5';
@@ -148,30 +148,30 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
       getDefineContinueButton().click();
 
       cy.get(ESQL_QUERY_BAR).contains(
-        'must include the [metadata _id, _version, _index] operator after the source command'
+        'must include the "metadata _id, _version, _index" operator after the source command'
       );
     });
 
     it('shows error when non-aggregating ES|QL query does not return _id field', function () {
       workaroundForResizeObserver();
 
       const invalidNonAggregatingQuery =
-        'from auditbeat* [metadata _id, _version, _index] | keep agent.* | limit 5';
+        'from auditbeat* metadata _id, _version, _index | keep agent.* | limit 5';
 
       selectEsqlRuleType();
       expandEsqlQueryBar();
       fillEsqlQueryBar(invalidNonAggregatingQuery);
       getDefineContinueButton().click();
 
       cy.get(ESQL_QUERY_BAR).contains(
-        'must include the [metadata _id, _version, _index] operator after the source command'
+        'must include the "metadata _id, _version, _index" operator after the source command'
       );
     });
 
     it('shows error when ES|QL query is invalid', function () {
       workaroundForResizeObserver();
       const invalidEsqlQuery =
-        'from auditbeat* [metadata _id, _version, _index] | not_existing_operator';
+        'from auditbeat* metadata _id, _version, _index | not_existing_operator';
       visit(CREATE_RULE_URL);
 
       selectEsqlRuleType();
@@ -191,7 +191,7 @@ describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => {
     it('shows custom ES|QL field in investigation fields autocomplete and saves it in rule', function () {
       const CUSTOM_ESQL_FIELD = '_custom_agent_name';
       const queryWithCustomFields = [
-        `from auditbeat* [metadata _id, _version, _index]`,
+        `from auditbeat* metadata _id, _version, _index`,
         `eval ${CUSTOM_ESQL_FIELD} = agent.name`,
         `keep _id, _custom_agent_name`,
         `limit 5`,