From b9ed7aad8adf490c5e6b2b8b0681d0180d5eee1e Mon Sep 17 00:00:00 2001 From: jpdjere Date: Thu, 8 Aug 2024 14:48:03 +0200 Subject: [PATCH 01/33] Create map of upgradable rule fields by type --- .../model/rule_assets/prebuilt_rule_asset.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index fa6c78bb7a8c13..bd2a18034d8ce3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -83,3 +83,32 @@ export const PrebuiltRuleAsset = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PR version: RuleVersion, }) ); + +/** + * Creates a Map of the fields that are upgradable during the Upgrade workflow, by type. + * This function creates the map dynamically, so that we don't need to manually + * add rule types if they are added, or manually add or remove any fields if they are + * added or removed to a specific rule type, or if we decide that they should not be part + * of the upgradable fields, since it's based on BaseCreateProps and TypeSpecificFields. + */ +function createUpgradableRuleFieldsByTypeMap() { + const baseFields = Object.keys( + BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET).shape + ); + + // Since we used Zod's .transform() method over TypeSpecificCreateProps above, + // the type of TypeSpecificFields is actually z.ZodEffects, and therefore does + // not have access to an options property. We need to transform it back into + // a z.ZodDiscriminatedUnion, accessing its underlying schema. + const TypeSpecificFieldsSchema = TypeSpecificFields._def.schema; + + return new Map( + TypeSpecificFieldsSchema.options.map((option) => { + const typeName = option.shape.type.value; + const typeSpecificFields = Object.keys(option.shape); + return [typeName, [...baseFields, ...typeSpecificFields]]; + }) + ); +} + +export const UPGRADABLE_RULES_FIELDS_BY_TYPE_MAP = createUpgradableRuleFieldsByTypeMap(); From 502c98c6de63e0205ed0ef2d21a9c4b7e32f332b Mon Sep 17 00:00:00 2001 From: jpdjere Date: Thu, 8 Aug 2024 15:03:44 +0200 Subject: [PATCH 02/33] Change comment --- .../model/rule_assets/prebuilt_rule_asset.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index bd2a18034d8ce3..4f4147d098050c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -86,10 +86,10 @@ export const PrebuiltRuleAsset = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PR /** * Creates a Map of the fields that are upgradable during the Upgrade workflow, by type. - * This function creates the map dynamically, so that we don't need to manually - * add rule types if they are added, or manually add or remove any fields if they are - * added or removed to a specific rule type, or if we decide that they should not be part - * of the upgradable fields, since it's based on BaseCreateProps and TypeSpecificFields. + * Creating this Map dynamically, based on BaseCreateProps and TypeSpecificFields, ensures that we don't need to: + * - manually add rule types to this Map if they are created + * - manually add or remove any fields if they are added or removed to a specific rule type + * - manually add or remove any fields if we decide that they should not be part of the upgradable fields. */ function createUpgradableRuleFieldsByTypeMap() { const baseFields = Object.keys( From 570357c71460dbb76f25fe8c2319c00bff9fac97 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Fri, 9 Aug 2024 12:39:34 +0200 Subject: [PATCH 03/33] Rework --- .../model/rule_assets/prebuilt_rule_asset.ts | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 4f4147d098050c..0a14b98e4b39fa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -36,15 +36,20 @@ const BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET = zodMaskFor( * necessary fields in the rules types where they exist. Fields to extract: * - response_actions: from Query and SavedQuery rules */ +const TYPE_SPECIFIC_FIELDS_TO_OMIT = new Set(['response_actions']); + +const filterTypeSpecificFields = (props: TypeSpecificCreateProps) => + Object.fromEntries( + Object.entries(props).filter(([key]) => !TYPE_SPECIFIC_FIELDS_TO_OMIT.has(key)) + ); + const TypeSpecificFields = TypeSpecificCreateProps.transform((val) => { switch (val.type) { case 'query': { - const { response_actions: _, ...rest } = val; - return rest; + return filterTypeSpecificFields(val); } case 'saved_query': { - const { response_actions: _, ...rest } = val; - return rest; + return filterTypeSpecificFields(val); } default: return val; @@ -96,16 +101,13 @@ function createUpgradableRuleFieldsByTypeMap() { BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET).shape ); - // Since we used Zod's .transform() method over TypeSpecificCreateProps above, - // the type of TypeSpecificFields is actually z.ZodEffects, and therefore does - // not have access to an options property. We need to transform it back into - // a z.ZodDiscriminatedUnion, accessing its underlying schema. - const TypeSpecificFieldsSchema = TypeSpecificFields._def.schema; - return new Map( - TypeSpecificFieldsSchema.options.map((option) => { + TypeSpecificCreateProps.options.map((option) => { const typeName = option.shape.type.value; - const typeSpecificFields = Object.keys(option.shape); + const typeSpecificFields = Object.keys(option.shape).filter( + // Filter out type-specific fields that should not be part of the upgradable fields + (field) => !TYPE_SPECIFIC_FIELDS_TO_OMIT.has(field) + ); return [typeName, [...baseFields, ...typeSpecificFields]]; }) ); From e1641d00d143b8005fce36d2bf8704b602225eff Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 12 Aug 2024 15:02:33 +0200 Subject: [PATCH 04/33] Added tests --- .../rule_assets/prebuilt_rule_asset.test.ts | 9 +- .../model/rule_assets/prebuilt_rule_asset.ts | 84 +++++++++++++------ 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts index 9190bb873e81b0..58aa7dc11019e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts @@ -7,8 +7,15 @@ import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zod-helpers'; import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; -import { PrebuiltRuleAsset } from './prebuilt_rule_asset'; +import { PrebuiltRuleAsset, TypeSpecificFields } from './prebuilt_rule_asset'; import { getPrebuiltRuleMock, getPrebuiltThreatMatchRuleMock } from './prebuilt_rule_asset.mock'; +import { TypeSpecificRuleParams } from '../../../rule_schema'; + +describe('TypeSpecificFields', () => { + it('contains all the rule types that are supported by the prebuilt rule asset', () => { + expect(TypeSpecificRuleParams._type.type).toEqual(TypeSpecificFields._type.type); + }); +}); describe('Prebuilt rule asset schema', () => { test('empty objects do not validate', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 0a14b98e4b39fa..ee61175c840812 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -6,13 +6,30 @@ */ import * as z from '@kbn/zod'; +import type { IsEqual } from 'type-fest'; import { RuleSignatureId, RuleVersion, BaseCreateProps, TypeSpecificCreateProps, + EqlRuleCreateFields, + EsqlRuleCreateFields, + MachineLearningRuleCreateFields, + NewTermsRuleCreateFields, + QueryRuleCreateFields, + SavedQueryRuleCreateFields, + ThreatMatchRuleCreateFields, + ThresholdRuleCreateFields, } from '../../../../../../common/api/detection_engine/model/rule_schema'; +function zodMaskFor() { + return function (props: U[]): Record { + type PropObject = Record; + const propObjects: PropObject[] = props.map((p: U) => ({ [p]: true })); + return Object.assign({}, ...propObjects); + }; +} + /** * The PrebuiltRuleAsset schema is created based on the rule schema defined in our OpenAPI specs. * However, we don't need all the rule schema fields to be present in the PrebuiltRuleAsset. @@ -32,37 +49,37 @@ const BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET = zodMaskFor( /** * Aditionally remove fields which are part only of the optional fields in the rule types that make up - * the TypeSpecificCreateProps discriminatedUnion, by using a Zod transformation which extracts out the - * necessary fields in the rules types where they exist. Fields to extract: + * the TypeSpecificCreateProps discriminatedUnion, by recreating a discriminated union of the types, but + * with the necessary fields ommitted, in the types where they exist. Fields to extract: * - response_actions: from Query and SavedQuery rules */ -const TYPE_SPECIFIC_FIELDS_TO_OMIT = new Set(['response_actions']); +const TYPE_SPECIFIC_FIELDS_TO_OMIT = ['response_actions'] as const; -const filterTypeSpecificFields = (props: TypeSpecificCreateProps) => - Object.fromEntries( - Object.entries(props).filter(([key]) => !TYPE_SPECIFIC_FIELDS_TO_OMIT.has(key)) - ); +const TYPE_SPECIFIC_FIELDS_TO_OMIT_FROM_QUERY_RULES = zodMaskFor()([ + ...TYPE_SPECIFIC_FIELDS_TO_OMIT, +]); +const TYPE_SPECIFIC_FIELDS_TO_OMIT_FROM_SAVED_QUERY_RULES = + zodMaskFor()([...TYPE_SPECIFIC_FIELDS_TO_OMIT]); -const TypeSpecificFields = TypeSpecificCreateProps.transform((val) => { - switch (val.type) { - case 'query': { - return filterTypeSpecificFields(val); - } - case 'saved_query': { - return filterTypeSpecificFields(val); - } - default: - return val; - } -}); +export type TypeSpecificFields = z.infer; +export const TypeSpecificFields = z.discriminatedUnion('type', [ + EqlRuleCreateFields, + QueryRuleCreateFields.omit(TYPE_SPECIFIC_FIELDS_TO_OMIT_FROM_QUERY_RULES), + SavedQueryRuleCreateFields.omit(TYPE_SPECIFIC_FIELDS_TO_OMIT_FROM_SAVED_QUERY_RULES), + ThresholdRuleCreateFields, + ThreatMatchRuleCreateFields, + MachineLearningRuleCreateFields, + NewTermsRuleCreateFields, + EsqlRuleCreateFields, +]); -function zodMaskFor() { - return function (props: U[]): Record { - type PropObject = Record; - const propObjects: PropObject[] = props.map((p: U) => ({ [p]: true })); - return Object.assign({}, ...propObjects); - }; -} +// Make sure the type-specific fields contain all the same rule types as the type-specific rule params. +// TS will throw a type error if the types are not equal (for example, if a new rule type is added to +// the TypeSpecificCreateProps and the new type is not reflected in TypeSpecificFields). +export const areTypesEqual: IsEqual< + typeof TypeSpecificCreateProps._type.type, + typeof TypeSpecificFields._type.type +> = true; /** * Asset containing source content of a prebuilt Security detection rule. @@ -101,12 +118,14 @@ function createUpgradableRuleFieldsByTypeMap() { BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET).shape ); + const specificTypesToOmit: readonly string[] = TYPE_SPECIFIC_FIELDS_TO_OMIT; + return new Map( TypeSpecificCreateProps.options.map((option) => { const typeName = option.shape.type.value; const typeSpecificFields = Object.keys(option.shape).filter( // Filter out type-specific fields that should not be part of the upgradable fields - (field) => !TYPE_SPECIFIC_FIELDS_TO_OMIT.has(field) + (field) => !specificTypesToOmit.includes(field) ); return [typeName, [...baseFields, ...typeSpecificFields]]; }) @@ -114,3 +133,14 @@ function createUpgradableRuleFieldsByTypeMap() { } export const UPGRADABLE_RULES_FIELDS_BY_TYPE_MAP = createUpgradableRuleFieldsByTypeMap(); + +export const test: PrebuiltRuleAsset = { + name: 'Test', + description: 'Test', + risk_score: 1, + severity: 'low', + type: 'query', + version: 1, + rule_id: 'rule-1', + unknown_field: 'unknown_value', +}; From 999d22722de8b7c4cee44df326961532aa2ca2d9 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 12 Aug 2024 15:07:24 +0200 Subject: [PATCH 05/33] Delete test --- .../model/rule_assets/prebuilt_rule_asset.ts | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index ee61175c840812..388226443db2f3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -132,15 +132,4 @@ function createUpgradableRuleFieldsByTypeMap() { ); } -export const UPGRADABLE_RULES_FIELDS_BY_TYPE_MAP = createUpgradableRuleFieldsByTypeMap(); - -export const test: PrebuiltRuleAsset = { - name: 'Test', - description: 'Test', - risk_score: 1, - severity: 'low', - type: 'query', - version: 1, - rule_id: 'rule-1', - unknown_field: 'unknown_value', -}; +export const UPGRADABLE_RULES_FIELDS_BY_TYPE_MAP = createUpgradableRuleFieldsByTypeMap(); \ No newline at end of file From 2218f1c6fc4518cd9febfdb4bcd0b7ab2b5b5141 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 12 Aug 2024 15:07:40 +0200 Subject: [PATCH 06/33] lint --- .../prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 388226443db2f3..0dfa131cab7e8f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -132,4 +132,4 @@ function createUpgradableRuleFieldsByTypeMap() { ); } -export const UPGRADABLE_RULES_FIELDS_BY_TYPE_MAP = createUpgradableRuleFieldsByTypeMap(); \ No newline at end of file +export const UPGRADABLE_RULES_FIELDS_BY_TYPE_MAP = createUpgradableRuleFieldsByTypeMap(); From a1ca8eccc1d413b57d885fc3ed653ab137a66b0b Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 12 Aug 2024 16:58:26 +0200 Subject: [PATCH 07/33] Remove test --- .../model/rule_assets/prebuilt_rule_asset.test.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts index 58aa7dc11019e6..9190bb873e81b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts @@ -7,15 +7,8 @@ import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zod-helpers'; import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; -import { PrebuiltRuleAsset, TypeSpecificFields } from './prebuilt_rule_asset'; +import { PrebuiltRuleAsset } from './prebuilt_rule_asset'; import { getPrebuiltRuleMock, getPrebuiltThreatMatchRuleMock } from './prebuilt_rule_asset.mock'; -import { TypeSpecificRuleParams } from '../../../rule_schema'; - -describe('TypeSpecificFields', () => { - it('contains all the rule types that are supported by the prebuilt rule asset', () => { - expect(TypeSpecificRuleParams._type.type).toEqual(TypeSpecificFields._type.type); - }); -}); describe('Prebuilt rule asset schema', () => { test('empty objects do not validate', () => { From 46618fda58d985b5655bd4abbcc4f675cfd19ce7 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 12 Aug 2024 17:44:04 +0200 Subject: [PATCH 08/33] Fix test --- .../model/rule_assets/prebuilt_rule_asset.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts index 9190bb873e81b0..4349111809db64 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts @@ -7,8 +7,19 @@ import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zod-helpers'; import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; -import { PrebuiltRuleAsset } from './prebuilt_rule_asset'; +import { PrebuiltRuleAsset, TypeSpecificFields } from './prebuilt_rule_asset'; import { getPrebuiltRuleMock, getPrebuiltThreatMatchRuleMock } from './prebuilt_rule_asset.mock'; +import { TypeSpecificCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; + +describe('TypeSpecificFields', () => { + it.only('contains all the rule types that are supported by the prebuilt rule asset', () => { + const createPropsTypes = TypeSpecificCreateProps.options.map(option => option.shape.type.value); + const fieldsTypes = TypeSpecificFields.options.map(option => option.shape.type.value); + + expect(createPropsTypes).toHaveLength(fieldsTypes.length); + expect(new Set(createPropsTypes)).toEqual(new Set(fieldsTypes)); + }); +}); describe('Prebuilt rule asset schema', () => { test('empty objects do not validate', () => { From 4499eb5eeb9d5cddcd7b9f2f509e4ed47b97a922 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 12 Aug 2024 21:49:47 +0200 Subject: [PATCH 09/33] Address changes --- .../rule_assets/prebuilt_rule_asset.test.ts | 18 ++++++++++-------- .../model/rule_assets/prebuilt_rule_asset.ts | 18 +++++++----------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts index 4349111809db64..5a3a943f2313f7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts @@ -9,19 +9,21 @@ import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zo import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; import { PrebuiltRuleAsset, TypeSpecificFields } from './prebuilt_rule_asset'; import { getPrebuiltRuleMock, getPrebuiltThreatMatchRuleMock } from './prebuilt_rule_asset.mock'; -import { TypeSpecificCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; +import { TypeSpecificCreateProps } from '../../../../../../common/api/detection_engine'; -describe('TypeSpecificFields', () => { - it.only('contains all the rule types that are supported by the prebuilt rule asset', () => { - const createPropsTypes = TypeSpecificCreateProps.options.map(option => option.shape.type.value); - const fieldsTypes = TypeSpecificFields.options.map(option => option.shape.type.value); +describe('Prebuilt rule asset schema', () => { + it('can be of all rule types that are supported', () => { + // Check that the discriminated union TypeSpecificFields, which is used to create + // the PrebuiltRuleAsset schema, contains all the rule types that are supported. + const createPropsTypes = TypeSpecificCreateProps.options.map( + (option) => option.shape.type.value + ); + const fieldsTypes = TypeSpecificFields.options.map((option) => option.shape.type.value); expect(createPropsTypes).toHaveLength(fieldsTypes.length); expect(new Set(createPropsTypes)).toEqual(new Set(fieldsTypes)); }); -}); -describe('Prebuilt rule asset schema', () => { test('empty objects do not validate', () => { const payload: Partial = {}; @@ -43,7 +45,7 @@ describe('Prebuilt rule asset schema', () => { expect(result.data).toEqual(getPrebuiltRuleMock()); }); - describe('ommited fields from the rule schema are ignored', () => { + describe('omitted fields from the rule schema are ignored', () => { // The PrebuiltRuleAsset schema is built out of the rule schema, // but the following fields are manually omitted. // See: detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 0dfa131cab7e8f..6d0a7411000240 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -50,7 +50,7 @@ const BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET = zodMaskFor( /** * Aditionally remove fields which are part only of the optional fields in the rule types that make up * the TypeSpecificCreateProps discriminatedUnion, by recreating a discriminated union of the types, but - * with the necessary fields ommitted, in the types where they exist. Fields to extract: + * with the necessary fields omitted, in the types where they exist. Fields to extract: * - response_actions: from Query and SavedQuery rules */ const TYPE_SPECIFIC_FIELDS_TO_OMIT = ['response_actions'] as const; @@ -81,6 +81,8 @@ export const areTypesEqual: IsEqual< typeof TypeSpecificFields._type.type > = true; +const PrebuiltAssetBaseProps = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET); + /** * Asset containing source content of a prebuilt Security detection rule. * Is defined for each prebuilt rule in https://github.com/elastic/detection-rules. @@ -97,7 +99,7 @@ export const areTypesEqual: IsEqual< * - some fields are omitted because they are not present in https://github.com/elastic/detection-rules */ export type PrebuiltRuleAsset = z.infer; -export const PrebuiltRuleAsset = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET) +export const PrebuiltRuleAsset = PrebuiltAssetBaseProps .and(TypeSpecificFields) .and( z.object({ @@ -115,19 +117,13 @@ export const PrebuiltRuleAsset = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PR */ function createUpgradableRuleFieldsByTypeMap() { const baseFields = Object.keys( - BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET).shape + PrebuiltAssetBaseProps.shape ); - const specificTypesToOmit: readonly string[] = TYPE_SPECIFIC_FIELDS_TO_OMIT; - return new Map( - TypeSpecificCreateProps.options.map((option) => { + TypeSpecificFields.options.map((option) => { const typeName = option.shape.type.value; - const typeSpecificFields = Object.keys(option.shape).filter( - // Filter out type-specific fields that should not be part of the upgradable fields - (field) => !specificTypesToOmit.includes(field) - ); - return [typeName, [...baseFields, ...typeSpecificFields]]; + return [typeName, [...baseFields, ...Object.keys(option.shape)]]; }) ); } From 448549d3b70d7cd4454a1788909dfcf7f3c59c77 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 12 Aug 2024 21:57:04 +0200 Subject: [PATCH 10/33] Address feedback --- .../rule_assets/prebuilt_rule_asset.test.ts | 2 +- .../model/rule_assets/prebuilt_rule_asset.ts | 22 ++++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts index 5a3a943f2313f7..9c176c88abdafd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts @@ -13,7 +13,7 @@ import { TypeSpecificCreateProps } from '../../../../../../common/api/detection_ describe('Prebuilt rule asset schema', () => { it('can be of all rule types that are supported', () => { - // Check that the discriminated union TypeSpecificFields, which is used to create + // Check that the discriminated union TypeSpecificFields, which is used to create // the PrebuiltRuleAsset schema, contains all the rule types that are supported. const createPropsTypes = TypeSpecificCreateProps.options.map( (option) => option.shape.type.value diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 6d0a7411000240..9649c48107d047 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -81,7 +81,7 @@ export const areTypesEqual: IsEqual< typeof TypeSpecificFields._type.type > = true; -const PrebuiltAssetBaseProps = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET); +const PrebuiltAssetBaseProps = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET); /** * Asset containing source content of a prebuilt Security detection rule. @@ -99,14 +99,12 @@ const PrebuiltAssetBaseProps = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREB * - some fields are omitted because they are not present in https://github.com/elastic/detection-rules */ export type PrebuiltRuleAsset = z.infer; -export const PrebuiltRuleAsset = PrebuiltAssetBaseProps - .and(TypeSpecificFields) - .and( - z.object({ - rule_id: RuleSignatureId, - version: RuleVersion, - }) - ); +export const PrebuiltRuleAsset = PrebuiltAssetBaseProps.and(TypeSpecificFields).and( + z.object({ + rule_id: RuleSignatureId, + version: RuleVersion, + }) +); /** * Creates a Map of the fields that are upgradable during the Upgrade workflow, by type. @@ -116,14 +114,12 @@ export const PrebuiltRuleAsset = PrebuiltAssetBaseProps * - manually add or remove any fields if we decide that they should not be part of the upgradable fields. */ function createUpgradableRuleFieldsByTypeMap() { - const baseFields = Object.keys( - PrebuiltAssetBaseProps.shape - ); + const baseFields = Object.keys(PrebuiltAssetBaseProps.shape); return new Map( TypeSpecificFields.options.map((option) => { const typeName = option.shape.type.value; - return [typeName, [...baseFields, ...Object.keys(option.shape)]]; + return [typeName, [...baseFields, 'version', ...Object.keys(option.shape)]]; }) ); } From 77dbb2f2f39268b6402ca3b6abbc854cd0f62855 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 13 Aug 2024 12:24:18 +0200 Subject: [PATCH 11/33] Alternatives to generate fields prop --- .../perform_rule_upgrade_route.ts | 47 +++++++++++++ .../model/rule_assets/prebuilt_rule_asset.ts | 67 ++++++++++++++++++- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts index 0c290c9968caab..6a85bfd4636d50 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts @@ -59,6 +59,9 @@ import { } from '../../model/rule_schema/specific_attributes/new_terms_attributes.gen'; import { RuleResponse } from '../../model/rule_schema/rule_schemas.gen'; import { AggregatedPrebuiltRuleError } from '../model'; +import { + PrebuiltRuleAssetFieldsDictionary, +} from '@kbn/security-solution-plugin/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset'; export type PickVersionValues = z.infer; export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); @@ -78,12 +81,56 @@ const createUpgradeFieldSchema = (fieldSchema: T) => ]) .optional(); + +const createRuleSpecifierFieldsSchema = () => { + const entries = Object.entries(PrebuiltRuleAssetFieldsDictionary.shape).map(([fieldName, fieldSchema]) => { + return [fieldName, createUpgradeFieldSchema(fieldSchema)]; + }); + + debugger; + return z.object(Object.fromEntries(entries)); +}; + +const test: RuleUpgradeSpecifier = { + rule_id: 'rule-1', + revision: 1, + version: 1, + pick_version: 'BASE', + // @xcrzx dynamically generated version confuses TS + fieldsTest: { + name: { + pick_version: 'RESOLVED', + resolved_value: false + }, + description: { + pick_version: 'TARGET', + }, + unknown_field: { + pick_version: 'BASE', + }, + }, + // @xcrzx hardcoded version works as expected + fields: { + name: { + pick_version: 'BASE', + }, + unknown_field: { + pick_version: 'BASE', + }, + description: { + pick_version: 'RESOLVED', + resolved_value: 'false' + }, + }, +}; + export type RuleUpgradeSpecifier = z.infer; export const RuleUpgradeSpecifier = z.object({ rule_id: RuleSignatureId, revision: z.number(), version: RuleVersion, pick_version: PickVersionValues.optional(), + fieldsTest: createRuleSpecifierFieldsSchema(), // Fields that can be customized during the upgrade workflow // as decided in: https://github.com/elastic/kibana/issues/186544 fields: z diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 9649c48107d047..1d911a7ac934ec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -22,6 +22,11 @@ import { ThresholdRuleCreateFields, } from '../../../../../../common/api/detection_engine/model/rule_schema'; +export type PickVersionValues = z.infer; +export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); +export type PickVersionValuesEnum = typeof PickVersionValues.enum; +export const PickVersionValuesEnum = PickVersionValues.enum; + function zodMaskFor() { return function (props: U[]): Record { type PropObject = Record; @@ -81,7 +86,9 @@ export const areTypesEqual: IsEqual< typeof TypeSpecificFields._type.type > = true; -const PrebuiltAssetBaseProps = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET); +export const PrebuiltAssetBaseProps = BaseCreateProps.omit( + BASE_PROPS_REMOVED_FROM_PREBUILT_RULE_ASSET +); /** * Asset containing source content of a prebuilt Security detection rule. @@ -114,14 +121,68 @@ export const PrebuiltRuleAsset = PrebuiltAssetBaseProps.and(TypeSpecificFields). * - manually add or remove any fields if we decide that they should not be part of the upgradable fields. */ function createUpgradableRuleFieldsByTypeMap() { - const baseFields = Object.keys(PrebuiltAssetBaseProps.shape); + const baseFields = [...Object.keys(PrebuiltAssetBaseProps.shape), 'version', 'rule_id']; return new Map( TypeSpecificFields.options.map((option) => { const typeName = option.shape.type.value; - return [typeName, [...baseFields, 'version', ...Object.keys(option.shape)]]; + return [typeName, [...baseFields, ...Object.keys(option.shape)]]; }) ); } export const UPGRADABLE_RULES_FIELDS_BY_TYPE_MAP = createUpgradableRuleFieldsByTypeMap(); + +///-------------- Alternative 1 --- "flatten" all props + +function getAllFields(schema: T): T { + if (schema instanceof z.ZodIntersection) { + return { + ...getAllFields(schema._def.left), + ...getAllFields(schema._def.right), + } as T; + } else if (schema instanceof z.ZodObject) { + return schema.shape as T; + } else if (schema instanceof z.ZodDiscriminatedUnion) { + return schema.options.reduce((acc: Partial, option: z.ZodObject) => { + const { type, ...rest } = option.shape; + return { ...acc, ...rest }; + }, {}) as T; + } + return {} as T; +} + +export const PrebuiltRuleAllFields = getAllFields(PrebuiltRuleAsset); +PrebuiltRuleAllFields._output; + +//// --------------- Alternative 2 -- +// uses .merge() instead of .and() since it returns a more useful ZodObject +// instead of a ZodIntersection + +const allTypeSpecificFields = TypeSpecificFields.options.reduce((acc, option) => { + return { ...acc, ...option.shape }; +}, {}); + +export const PrebuiltRuleAsset2 = PrebuiltAssetBaseProps.merge( + z.object(allTypeSpecificFields) +).merge( + z.object({ + rule_id: RuleSignatureId, + version: RuleVersion, + }) +); + +export const test = PrebuiltRuleAsset2.keyof(); +export const test2 = PrebuiltRuleAsset2.shape; + + + +// ---------- Alternative 3 --- @xcrzx this works in runtime but TS doesn't understand the types + +export const PrebuiltRuleAssetFieldsDictionary = z + .object({ + ...PrebuiltAssetBaseProps.shape, + ...TypeSpecificFields.options.reduce((acc, option) => ({ ...acc, ...option.shape }), {}), + rule_id: RuleSignatureId, + version: RuleVersion, + }); From bacd2bb3371524ead33dc5095998be7809c9bdde Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 13 Aug 2024 15:53:17 +0200 Subject: [PATCH 12/33] Cleanup --- .../perform_rule_upgrade_route.ts | 47 --------------- .../model/rule_assets/prebuilt_rule_asset.ts | 59 ------------------- 2 files changed, 106 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts index 6a85bfd4636d50..0c290c9968caab 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts @@ -59,9 +59,6 @@ import { } from '../../model/rule_schema/specific_attributes/new_terms_attributes.gen'; import { RuleResponse } from '../../model/rule_schema/rule_schemas.gen'; import { AggregatedPrebuiltRuleError } from '../model'; -import { - PrebuiltRuleAssetFieldsDictionary, -} from '@kbn/security-solution-plugin/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset'; export type PickVersionValues = z.infer; export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); @@ -81,56 +78,12 @@ const createUpgradeFieldSchema = (fieldSchema: T) => ]) .optional(); - -const createRuleSpecifierFieldsSchema = () => { - const entries = Object.entries(PrebuiltRuleAssetFieldsDictionary.shape).map(([fieldName, fieldSchema]) => { - return [fieldName, createUpgradeFieldSchema(fieldSchema)]; - }); - - debugger; - return z.object(Object.fromEntries(entries)); -}; - -const test: RuleUpgradeSpecifier = { - rule_id: 'rule-1', - revision: 1, - version: 1, - pick_version: 'BASE', - // @xcrzx dynamically generated version confuses TS - fieldsTest: { - name: { - pick_version: 'RESOLVED', - resolved_value: false - }, - description: { - pick_version: 'TARGET', - }, - unknown_field: { - pick_version: 'BASE', - }, - }, - // @xcrzx hardcoded version works as expected - fields: { - name: { - pick_version: 'BASE', - }, - unknown_field: { - pick_version: 'BASE', - }, - description: { - pick_version: 'RESOLVED', - resolved_value: 'false' - }, - }, -}; - export type RuleUpgradeSpecifier = z.infer; export const RuleUpgradeSpecifier = z.object({ rule_id: RuleSignatureId, revision: z.number(), version: RuleVersion, pick_version: PickVersionValues.optional(), - fieldsTest: createRuleSpecifierFieldsSchema(), // Fields that can be customized during the upgrade workflow // as decided in: https://github.com/elastic/kibana/issues/186544 fields: z diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 1d911a7ac934ec..8fa9fbbc6f5a7f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -22,11 +22,6 @@ import { ThresholdRuleCreateFields, } from '../../../../../../common/api/detection_engine/model/rule_schema'; -export type PickVersionValues = z.infer; -export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); -export type PickVersionValuesEnum = typeof PickVersionValues.enum; -export const PickVersionValuesEnum = PickVersionValues.enum; - function zodMaskFor() { return function (props: U[]): Record { type PropObject = Record; @@ -132,57 +127,3 @@ function createUpgradableRuleFieldsByTypeMap() { } export const UPGRADABLE_RULES_FIELDS_BY_TYPE_MAP = createUpgradableRuleFieldsByTypeMap(); - -///-------------- Alternative 1 --- "flatten" all props - -function getAllFields(schema: T): T { - if (schema instanceof z.ZodIntersection) { - return { - ...getAllFields(schema._def.left), - ...getAllFields(schema._def.right), - } as T; - } else if (schema instanceof z.ZodObject) { - return schema.shape as T; - } else if (schema instanceof z.ZodDiscriminatedUnion) { - return schema.options.reduce((acc: Partial, option: z.ZodObject) => { - const { type, ...rest } = option.shape; - return { ...acc, ...rest }; - }, {}) as T; - } - return {} as T; -} - -export const PrebuiltRuleAllFields = getAllFields(PrebuiltRuleAsset); -PrebuiltRuleAllFields._output; - -//// --------------- Alternative 2 -- -// uses .merge() instead of .and() since it returns a more useful ZodObject -// instead of a ZodIntersection - -const allTypeSpecificFields = TypeSpecificFields.options.reduce((acc, option) => { - return { ...acc, ...option.shape }; -}, {}); - -export const PrebuiltRuleAsset2 = PrebuiltAssetBaseProps.merge( - z.object(allTypeSpecificFields) -).merge( - z.object({ - rule_id: RuleSignatureId, - version: RuleVersion, - }) -); - -export const test = PrebuiltRuleAsset2.keyof(); -export const test2 = PrebuiltRuleAsset2.shape; - - - -// ---------- Alternative 3 --- @xcrzx this works in runtime but TS doesn't understand the types - -export const PrebuiltRuleAssetFieldsDictionary = z - .object({ - ...PrebuiltAssetBaseProps.shape, - ...TypeSpecificFields.options.reduce((acc, option) => ({ ...acc, ...option.shape }), {}), - rule_id: RuleSignatureId, - version: RuleVersion, - }); From f3fccc2e62ff264d48649c699c265f874caeed91 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 13 Aug 2024 18:23:59 +0200 Subject: [PATCH 13/33] Update --- .../perform_rule_upgrade_route.test.ts | 12 +++ .../perform_rule_upgrade_route.ts | 99 ++++++++++--------- .../model/rule_assets/prebuilt_rule_asset.ts | 39 +++++++- 3 files changed, 99 insertions(+), 51 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts index b58a254f9dc495..4815c79e76a79e 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts @@ -9,11 +9,14 @@ import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zo import { PickVersionValues, RuleUpgradeSpecifier, + RuleUpgradeSpecifierFields, UpgradeSpecificRulesRequest, UpgradeAllRulesRequest, PerformRuleUpgradeResponseBody, PerformRuleUpgradeRequestBody, } from './perform_rule_upgrade_route'; +import { RULE_UPGRADE_SPECIFIER_FIELDS } from '../../../../../server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset'; + describe('Perform Rule Upgrade Route Schemas', () => { describe('PickVersionValues', () => { @@ -38,6 +41,15 @@ describe('Perform Rule Upgrade Route Schemas', () => { }); }); + describe('RuleUpgradeSpecifierFields', () => { + it.only('accepts all upgradable fields from the Prebuilt Rule Asset', () => { + const upgradeSpecifierFields = new Set(Object.keys(RuleUpgradeSpecifierFields.shape)); + + expect(upgradeSpecifierFields).toEqual(RULE_UPGRADE_SPECIFIER_FIELDS); + + }) + }) + describe('RuleUpgradeSpecifier', () => { const validSpecifier = { rule_id: 'rule-1', diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts index 0c290c9968caab..ed5b5ac8f4b041 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts @@ -13,6 +13,7 @@ import { RuleName, RuleTagArray, RuleDescription, + IsRuleEnabled, Severity, SeverityMapping, RiskScore, @@ -41,6 +42,7 @@ import { RuleFilterArray, SavedQueryId, KqlQueryLanguage, + InvestigationFields, } from '../../model/rule_schema/common_attributes.gen'; import { MachineLearningJobId, @@ -78,6 +80,56 @@ const createUpgradeFieldSchema = (fieldSchema: T) => ]) .optional(); +export const RuleUpgradeSpecifierFields = z + .object({ + name: createUpgradeFieldSchema(RuleName), + tags: createUpgradeFieldSchema(RuleTagArray), + description: createUpgradeFieldSchema(RuleDescription), + severity: createUpgradeFieldSchema(Severity), + severity_mapping: createUpgradeFieldSchema(SeverityMapping), + enabled: createUpgradeFieldSchema(IsRuleEnabled), + risk_score: createUpgradeFieldSchema(RiskScore), + risk_score_mapping: createUpgradeFieldSchema(RiskScoreMapping), + references: createUpgradeFieldSchema(RuleReferenceArray), + false_positives: createUpgradeFieldSchema(RuleFalsePositiveArray), + threat: createUpgradeFieldSchema(ThreatArray), + note: createUpgradeFieldSchema(InvestigationGuide), + setup: createUpgradeFieldSchema(SetupGuide), + investigation_fields: createUpgradeFieldSchema(InvestigationFields), + related_integrations: createUpgradeFieldSchema(RelatedIntegrationArray), + required_fields: createUpgradeFieldSchema(RequiredFieldArray), + max_signals: createUpgradeFieldSchema(MaxSignals), + building_block_type: createUpgradeFieldSchema(BuildingBlockType), + from: createUpgradeFieldSchema(RuleIntervalFrom), + interval: createUpgradeFieldSchema(RuleInterval), + exceptions_list: createUpgradeFieldSchema(RuleExceptionList), + rule_name_override: createUpgradeFieldSchema(RuleNameOverride), + timestamp_override: createUpgradeFieldSchema(TimestampOverride), + timestamp_override_fallback_disabled: createUpgradeFieldSchema( + TimestampOverrideFallbackDisabled + ), + timeline_id: createUpgradeFieldSchema(TimelineTemplateId), + timeline_title: createUpgradeFieldSchema(TimelineTemplateTitle), + index: createUpgradeFieldSchema(IndexPatternArray), + data_view_id: createUpgradeFieldSchema(DataViewId), + query: createUpgradeFieldSchema(RuleQuery), + language: createUpgradeFieldSchema(QueryLanguage), + filters: createUpgradeFieldSchema(RuleFilterArray), + saved_id: createUpgradeFieldSchema(SavedQueryId), + machine_learning_job_id: createUpgradeFieldSchema(MachineLearningJobId), + anomaly_threshold: createUpgradeFieldSchema(AnomalyThreshold), + threat_query: createUpgradeFieldSchema(ThreatQuery), + threat_mapping: createUpgradeFieldSchema(ThreatMapping), + threat_index: createUpgradeFieldSchema(ThreatIndex), + threat_filters: createUpgradeFieldSchema(ThreatFilters), + threat_indicator_path: createUpgradeFieldSchema(ThreatIndicatorPath), + threat_language: createUpgradeFieldSchema(KqlQueryLanguage), + new_terms_fields: createUpgradeFieldSchema(NewTermsFields), + history_window_start: createUpgradeFieldSchema(HistoryWindowStart), + }); + + RuleUpgradeSpecifierFields.shape + export type RuleUpgradeSpecifier = z.infer; export const RuleUpgradeSpecifier = z.object({ rule_id: RuleSignatureId, @@ -86,52 +138,7 @@ export const RuleUpgradeSpecifier = z.object({ pick_version: PickVersionValues.optional(), // Fields that can be customized during the upgrade workflow // as decided in: https://github.com/elastic/kibana/issues/186544 - fields: z - .object({ - name: createUpgradeFieldSchema(RuleName), - tags: createUpgradeFieldSchema(RuleTagArray), - description: createUpgradeFieldSchema(RuleDescription), - severity: createUpgradeFieldSchema(Severity), - severity_mapping: createUpgradeFieldSchema(SeverityMapping), - risk_score: createUpgradeFieldSchema(RiskScore), - risk_score_mapping: createUpgradeFieldSchema(RiskScoreMapping), - references: createUpgradeFieldSchema(RuleReferenceArray), - false_positives: createUpgradeFieldSchema(RuleFalsePositiveArray), - threat: createUpgradeFieldSchema(ThreatArray), - note: createUpgradeFieldSchema(InvestigationGuide), - setup: createUpgradeFieldSchema(SetupGuide), - related_integrations: createUpgradeFieldSchema(RelatedIntegrationArray), - required_fields: createUpgradeFieldSchema(RequiredFieldArray), - max_signals: createUpgradeFieldSchema(MaxSignals), - building_block_type: createUpgradeFieldSchema(BuildingBlockType), - from: createUpgradeFieldSchema(RuleIntervalFrom), - interval: createUpgradeFieldSchema(RuleInterval), - exceptions_list: createUpgradeFieldSchema(RuleExceptionList), - rule_name_override: createUpgradeFieldSchema(RuleNameOverride), - timestamp_override: createUpgradeFieldSchema(TimestampOverride), - timestamp_override_fallback_disabled: createUpgradeFieldSchema( - TimestampOverrideFallbackDisabled - ), - timeline_id: createUpgradeFieldSchema(TimelineTemplateId), - timeline_title: createUpgradeFieldSchema(TimelineTemplateTitle), - index: createUpgradeFieldSchema(IndexPatternArray), - data_view_id: createUpgradeFieldSchema(DataViewId), - query: createUpgradeFieldSchema(RuleQuery), - language: createUpgradeFieldSchema(QueryLanguage), - filters: createUpgradeFieldSchema(RuleFilterArray), - saved_id: createUpgradeFieldSchema(SavedQueryId), - machine_learning_job_id: createUpgradeFieldSchema(MachineLearningJobId), - anomaly_threshold: createUpgradeFieldSchema(AnomalyThreshold), - threat_query: createUpgradeFieldSchema(ThreatQuery), - threat_mapping: createUpgradeFieldSchema(ThreatMapping), - threat_index: createUpgradeFieldSchema(ThreatIndex), - threat_filters: createUpgradeFieldSchema(ThreatFilters), - threat_indicator_path: createUpgradeFieldSchema(ThreatIndicatorPath), - threat_language: createUpgradeFieldSchema(KqlQueryLanguage), - new_terms_fields: createUpgradeFieldSchema(NewTermsFields), - history_window_start: createUpgradeFieldSchema(HistoryWindowStart), - }) - .optional(), + fields: RuleUpgradeSpecifierFields.optional() }); export type UpgradeSpecificRulesRequest = z.infer; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 8fa9fbbc6f5a7f..fd93617e71b6ca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -109,21 +109,50 @@ export const PrebuiltRuleAsset = PrebuiltAssetBaseProps.and(TypeSpecificFields). ); /** - * Creates a Map of the fields that are upgradable during the Upgrade workflow, by type. + * Creates a Map of the fields payloads to be passed to the `upgradePrebuiltRules()` method during the + * Upgrade workflow (`/upgrade/_perform` endpoint) by type. + * * Creating this Map dynamically, based on BaseCreateProps and TypeSpecificFields, ensures that we don't need to: * - manually add rule types to this Map if they are created * - manually add or remove any fields if they are added or removed to a specific rule type * - manually add or remove any fields if we decide that they should not be part of the upgradable fields. + * + * Notice that this Map includes, for each rule type, all fields that are part of the BaseCreateProps and all fields that + * are part of the TypeSpecificFields, including those that are not part of RuleUpgradeSpecifierFields schema, where + * the user of the /upgrade/_perform endpoint can specify which fields to upgrade during the upgrade workflow. */ -function createUpgradableRuleFieldsByTypeMap() { - const baseFields = [...Object.keys(PrebuiltAssetBaseProps.shape), 'version', 'rule_id']; +function createUpgradableRuleFieldsPayloadByType() { + const baseFields = Object.keys(PrebuiltAssetBaseProps.shape); return new Map( TypeSpecificFields.options.map((option) => { const typeName = option.shape.type.value; - return [typeName, [...baseFields, ...Object.keys(option.shape)]]; + const typeSpecificFieldsForType = Object.keys(option.shape); + + return [typeName, [...baseFields, ...typeSpecificFieldsForType]]; }) ); } -export const UPGRADABLE_RULES_FIELDS_BY_TYPE_MAP = createUpgradableRuleFieldsByTypeMap(); +export const UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE = createUpgradableRuleFieldsPayloadByType(); + +const FIELDS_NOT_UPGRADABLE: string[] = [ + 'alert_suppression', + 'author', + 'license', + 'concurrent_searches', + 'items_per_search', + 'version', + 'type', +]; +function createRuleUpgradeSpecifierFields() { + const allUpgradableFields = new Set(); + for (const [_, upgradableFields] of UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE) { + upgradableFields.forEach((field) => allUpgradableFields.add(field)); + } + FIELDS_NOT_UPGRADABLE.forEach((field) => allUpgradableFields.delete(field)); + + return allUpgradableFields; +} + +export const RULE_UPGRADE_SPECIFIER_FIELDS = createRuleUpgradeSpecifierFields(); From ac5f6aef4e1c3fb3a6c2012f3a78bab679510905 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 13 Aug 2024 18:52:41 +0200 Subject: [PATCH 14/33] Adds comments --- .../perform_rule_upgrade_route.test.ts | 2 +- .../perform_rule_upgrade_route.ts | 115 +++++++++--------- .../model/rule_assets/prebuilt_rule_asset.ts | 58 ++++++--- 3 files changed, 95 insertions(+), 80 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts index 4815c79e76a79e..882ed73c2ca8ab 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts @@ -42,7 +42,7 @@ describe('Perform Rule Upgrade Route Schemas', () => { }); describe('RuleUpgradeSpecifierFields', () => { - it.only('accepts all upgradable fields from the Prebuilt Rule Asset', () => { + it('accepts all upgradable fields from the Prebuilt Rule Asset', () => { const upgradeSpecifierFields = new Set(Object.keys(RuleUpgradeSpecifierFields.shape)); expect(upgradeSpecifierFields).toEqual(RULE_UPGRADE_SPECIFIER_FIELDS); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts index ed5b5ac8f4b041..c9ba9be508546f 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts @@ -7,6 +7,8 @@ import { z } from '@kbn/zod'; +import { RuleResponse } from '../../model/rule_schema/rule_schemas.gen'; +import { AggregatedPrebuiltRuleError } from '../model'; import { RuleSignatureId, RuleVersion, @@ -43,24 +45,20 @@ import { SavedQueryId, KqlQueryLanguage, InvestigationFields, -} from '../../model/rule_schema/common_attributes.gen'; -import { MachineLearningJobId, AnomalyThreshold, -} from '../../model/rule_schema/specific_attributes/ml_attributes.gen'; -import { ThreatQuery, ThreatMapping, ThreatIndex, ThreatFilters, ThreatIndicatorPath, -} from '../../model/rule_schema/specific_attributes/threat_match_attributes.gen'; -import { NewTermsFields, HistoryWindowStart, -} from '../../model/rule_schema/specific_attributes/new_terms_attributes.gen'; -import { RuleResponse } from '../../model/rule_schema/rule_schemas.gen'; -import { AggregatedPrebuiltRuleError } from '../model'; + EventCategoryOverride, + TiebreakerField, + TimestampField, + Threshold, +} from '../../model'; export type PickVersionValues = z.infer; export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); @@ -80,55 +78,54 @@ const createUpgradeFieldSchema = (fieldSchema: T) => ]) .optional(); -export const RuleUpgradeSpecifierFields = z - .object({ - name: createUpgradeFieldSchema(RuleName), - tags: createUpgradeFieldSchema(RuleTagArray), - description: createUpgradeFieldSchema(RuleDescription), - severity: createUpgradeFieldSchema(Severity), - severity_mapping: createUpgradeFieldSchema(SeverityMapping), - enabled: createUpgradeFieldSchema(IsRuleEnabled), - risk_score: createUpgradeFieldSchema(RiskScore), - risk_score_mapping: createUpgradeFieldSchema(RiskScoreMapping), - references: createUpgradeFieldSchema(RuleReferenceArray), - false_positives: createUpgradeFieldSchema(RuleFalsePositiveArray), - threat: createUpgradeFieldSchema(ThreatArray), - note: createUpgradeFieldSchema(InvestigationGuide), - setup: createUpgradeFieldSchema(SetupGuide), - investigation_fields: createUpgradeFieldSchema(InvestigationFields), - related_integrations: createUpgradeFieldSchema(RelatedIntegrationArray), - required_fields: createUpgradeFieldSchema(RequiredFieldArray), - max_signals: createUpgradeFieldSchema(MaxSignals), - building_block_type: createUpgradeFieldSchema(BuildingBlockType), - from: createUpgradeFieldSchema(RuleIntervalFrom), - interval: createUpgradeFieldSchema(RuleInterval), - exceptions_list: createUpgradeFieldSchema(RuleExceptionList), - rule_name_override: createUpgradeFieldSchema(RuleNameOverride), - timestamp_override: createUpgradeFieldSchema(TimestampOverride), - timestamp_override_fallback_disabled: createUpgradeFieldSchema( - TimestampOverrideFallbackDisabled - ), - timeline_id: createUpgradeFieldSchema(TimelineTemplateId), - timeline_title: createUpgradeFieldSchema(TimelineTemplateTitle), - index: createUpgradeFieldSchema(IndexPatternArray), - data_view_id: createUpgradeFieldSchema(DataViewId), - query: createUpgradeFieldSchema(RuleQuery), - language: createUpgradeFieldSchema(QueryLanguage), - filters: createUpgradeFieldSchema(RuleFilterArray), - saved_id: createUpgradeFieldSchema(SavedQueryId), - machine_learning_job_id: createUpgradeFieldSchema(MachineLearningJobId), - anomaly_threshold: createUpgradeFieldSchema(AnomalyThreshold), - threat_query: createUpgradeFieldSchema(ThreatQuery), - threat_mapping: createUpgradeFieldSchema(ThreatMapping), - threat_index: createUpgradeFieldSchema(ThreatIndex), - threat_filters: createUpgradeFieldSchema(ThreatFilters), - threat_indicator_path: createUpgradeFieldSchema(ThreatIndicatorPath), - threat_language: createUpgradeFieldSchema(KqlQueryLanguage), - new_terms_fields: createUpgradeFieldSchema(NewTermsFields), - history_window_start: createUpgradeFieldSchema(HistoryWindowStart), - }); - - RuleUpgradeSpecifierFields.shape +export const RuleUpgradeSpecifierFields = z.object({ + name: createUpgradeFieldSchema(RuleName), + tags: createUpgradeFieldSchema(RuleTagArray), + description: createUpgradeFieldSchema(RuleDescription), + severity: createUpgradeFieldSchema(Severity), + severity_mapping: createUpgradeFieldSchema(SeverityMapping), + enabled: createUpgradeFieldSchema(IsRuleEnabled), + risk_score: createUpgradeFieldSchema(RiskScore), + risk_score_mapping: createUpgradeFieldSchema(RiskScoreMapping), + references: createUpgradeFieldSchema(RuleReferenceArray), + false_positives: createUpgradeFieldSchema(RuleFalsePositiveArray), + threat: createUpgradeFieldSchema(ThreatArray), + note: createUpgradeFieldSchema(InvestigationGuide), + setup: createUpgradeFieldSchema(SetupGuide), + investigation_fields: createUpgradeFieldSchema(InvestigationFields), + related_integrations: createUpgradeFieldSchema(RelatedIntegrationArray), + required_fields: createUpgradeFieldSchema(RequiredFieldArray), + max_signals: createUpgradeFieldSchema(MaxSignals), + building_block_type: createUpgradeFieldSchema(BuildingBlockType), + from: createUpgradeFieldSchema(RuleIntervalFrom), + interval: createUpgradeFieldSchema(RuleInterval), + exceptions_list: createUpgradeFieldSchema(RuleExceptionList), + rule_name_override: createUpgradeFieldSchema(RuleNameOverride), + timestamp_override: createUpgradeFieldSchema(TimestampOverride), + timestamp_override_fallback_disabled: createUpgradeFieldSchema(TimestampOverrideFallbackDisabled), + timeline_id: createUpgradeFieldSchema(TimelineTemplateId), + timeline_title: createUpgradeFieldSchema(TimelineTemplateTitle), + index: createUpgradeFieldSchema(IndexPatternArray), + data_view_id: createUpgradeFieldSchema(DataViewId), + query: createUpgradeFieldSchema(RuleQuery), + language: createUpgradeFieldSchema(QueryLanguage), + filters: createUpgradeFieldSchema(RuleFilterArray), + saved_id: createUpgradeFieldSchema(SavedQueryId), + machine_learning_job_id: createUpgradeFieldSchema(MachineLearningJobId), + anomaly_threshold: createUpgradeFieldSchema(AnomalyThreshold), + threat_query: createUpgradeFieldSchema(ThreatQuery), + threat_mapping: createUpgradeFieldSchema(ThreatMapping), + threat_index: createUpgradeFieldSchema(ThreatIndex), + threat_filters: createUpgradeFieldSchema(ThreatFilters), + threat_indicator_path: createUpgradeFieldSchema(ThreatIndicatorPath), + threat_language: createUpgradeFieldSchema(KqlQueryLanguage), + new_terms_fields: createUpgradeFieldSchema(NewTermsFields), + history_window_start: createUpgradeFieldSchema(HistoryWindowStart), + event_category_override: createUpgradeFieldSchema(EventCategoryOverride), + tiebreaker_field: createUpgradeFieldSchema(TiebreakerField), + timestamp_field: createUpgradeFieldSchema(TimestampField), + threshold: createUpgradeFieldSchema(Threshold), +}); export type RuleUpgradeSpecifier = z.infer; export const RuleUpgradeSpecifier = z.object({ @@ -138,7 +135,7 @@ export const RuleUpgradeSpecifier = z.object({ pick_version: PickVersionValues.optional(), // Fields that can be customized during the upgrade workflow // as decided in: https://github.com/elastic/kibana/issues/186544 - fields: RuleUpgradeSpecifierFields.optional() + fields: RuleUpgradeSpecifierFields.optional(), }); export type UpgradeSpecificRulesRequest = z.infer; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index fd93617e71b6ca..f7f65eb0ee38b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -108,19 +108,7 @@ export const PrebuiltRuleAsset = PrebuiltAssetBaseProps.and(TypeSpecificFields). }) ); -/** - * Creates a Map of the fields payloads to be passed to the `upgradePrebuiltRules()` method during the - * Upgrade workflow (`/upgrade/_perform` endpoint) by type. - * - * Creating this Map dynamically, based on BaseCreateProps and TypeSpecificFields, ensures that we don't need to: - * - manually add rule types to this Map if they are created - * - manually add or remove any fields if they are added or removed to a specific rule type - * - manually add or remove any fields if we decide that they should not be part of the upgradable fields. - * - * Notice that this Map includes, for each rule type, all fields that are part of the BaseCreateProps and all fields that - * are part of the TypeSpecificFields, including those that are not part of RuleUpgradeSpecifierFields schema, where - * the user of the /upgrade/_perform endpoint can specify which fields to upgrade during the upgrade workflow. - */ + function createUpgradableRuleFieldsPayloadByType() { const baseFields = Object.keys(PrebuiltAssetBaseProps.shape); @@ -134,9 +122,27 @@ function createUpgradableRuleFieldsPayloadByType() { ); } +/** + * Map of the fields payloads to be passed to the `upgradePrebuiltRules()` method during the + * Upgrade workflow (`/upgrade/_perform` endpoint) by type. + * + * Creating this Map dynamically, based on BaseCreateProps and TypeSpecificFields, ensures that we don't need to: + * - manually add rule types to this Map if they are created + * - manually add or remove any fields if they are added or removed to a specific rule type + * - manually add or remove any fields if we decide that they should not be part of the upgradable fields. + * + * Notice that this Map includes, for each rule type, all fields that are part of the BaseCreateProps and all fields that + * are part of the TypeSpecificFields, including those that are not part of RuleUpgradeSpecifierFields schema, where + * the user of the /upgrade/_perform endpoint can specify which fields to upgrade during the upgrade workflow. + */ export const UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE = createUpgradableRuleFieldsPayloadByType(); -const FIELDS_NOT_UPGRADABLE: string[] = [ + +/** + * Fields which are not part of the RuleUpgradeSpecifierFields schema, and are handled + * manually during the upgrade workflow. + */ +const NON_UPGRADABLE_FIELDS: string[] = [ 'alert_suppression', 'author', 'license', @@ -144,15 +150,27 @@ const FIELDS_NOT_UPGRADABLE: string[] = [ 'items_per_search', 'version', 'type', + 'to' ]; + function createRuleUpgradeSpecifierFields() { - const allUpgradableFields = new Set(); - for (const [_, upgradableFields] of UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE) { - upgradableFields.forEach((field) => allUpgradableFields.add(field)); - } - FIELDS_NOT_UPGRADABLE.forEach((field) => allUpgradableFields.delete(field)); + const allUpgradableFields = new Set( + Object.values(UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE).flatMap(fields => fields) + ); + + NON_UPGRADABLE_FIELDS.forEach(field => allUpgradableFields.delete(field)); return allUpgradableFields; } -export const RULE_UPGRADE_SPECIFIER_FIELDS = createRuleUpgradeSpecifierFields(); +/** + * List of fields that are part of the RuleUpgradeSpecifierFields schema, which is part of the + * /upgrade/_perform endpoint request payload. This list is used to test that all upgradable fields from + * the PrebuiltRuleAsset are part of the RuleUpgradeSpecifierFields schema. + * + * Note that some of the fields of the PrebuiltRuleAsset schema are not upgradable in the update workflow + * of the /upgrade/_perform endpoint (and therefore nor part of RuleUpgradeSpecifierFields) so they are + * manually excluded from the list of upgradable fields. + * + */ +export const RULE_UPGRADE_SPECIFIER_FIELDS = createRuleUpgradeSpecifierFields(); \ No newline at end of file From d762af3a0c225875ccf5f4feb2a17abdf8ec5a66 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 13 Aug 2024 18:54:06 +0200 Subject: [PATCH 15/33] Fix type --- .../prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index f7f65eb0ee38b8..e68e425908b5e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -155,7 +155,7 @@ const NON_UPGRADABLE_FIELDS: string[] = [ function createRuleUpgradeSpecifierFields() { const allUpgradableFields = new Set( - Object.values(UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE).flatMap(fields => fields) + Object.values(UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE).flatMap((fields: string[]) => fields) ); NON_UPGRADABLE_FIELDS.forEach(field => allUpgradableFields.delete(field)); From e0d998eb6daeec96e473241c14f04571f444d44a Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 14 Aug 2024 10:50:52 +0200 Subject: [PATCH 16/33] Lint --- .../model/rule_assets/prebuilt_rule_asset.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index e68e425908b5e0..97fe4399369587 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -108,7 +108,6 @@ export const PrebuiltRuleAsset = PrebuiltAssetBaseProps.and(TypeSpecificFields). }) ); - function createUpgradableRuleFieldsPayloadByType() { const baseFields = Object.keys(PrebuiltAssetBaseProps.shape); @@ -137,7 +136,6 @@ function createUpgradableRuleFieldsPayloadByType() { */ export const UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE = createUpgradableRuleFieldsPayloadByType(); - /** * Fields which are not part of the RuleUpgradeSpecifierFields schema, and are handled * manually during the upgrade workflow. @@ -150,7 +148,7 @@ const NON_UPGRADABLE_FIELDS: string[] = [ 'items_per_search', 'version', 'type', - 'to' + 'to', ]; function createRuleUpgradeSpecifierFields() { @@ -158,7 +156,7 @@ function createRuleUpgradeSpecifierFields() { Object.values(UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE).flatMap((fields: string[]) => fields) ); - NON_UPGRADABLE_FIELDS.forEach(field => allUpgradableFields.delete(field)); + NON_UPGRADABLE_FIELDS.forEach((field) => allUpgradableFields.delete(field)); return allUpgradableFields; } @@ -171,6 +169,6 @@ function createRuleUpgradeSpecifierFields() { * Note that some of the fields of the PrebuiltRuleAsset schema are not upgradable in the update workflow * of the /upgrade/_perform endpoint (and therefore nor part of RuleUpgradeSpecifierFields) so they are * manually excluded from the list of upgradable fields. - * + * */ -export const RULE_UPGRADE_SPECIFIER_FIELDS = createRuleUpgradeSpecifierFields(); \ No newline at end of file +export const RULE_UPGRADE_SPECIFIER_FIELDS = createRuleUpgradeSpecifierFields(); From b8f4ccd571ef888a8e66dce407a12055b1f629c6 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 14 Aug 2024 11:36:52 +0200 Subject: [PATCH 17/33] Fix lint --- .../prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 97fe4399369587..a60808406ce74e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -7,11 +7,11 @@ import * as z from '@kbn/zod'; import type { IsEqual } from 'type-fest'; +import type { TypeSpecificCreateProps } from '../../../../../../common/api/detection_engine/model/rule_schema'; import { RuleSignatureId, RuleVersion, BaseCreateProps, - TypeSpecificCreateProps, EqlRuleCreateFields, EsqlRuleCreateFields, MachineLearningRuleCreateFields, From 8a3b28607ac03311732ccd2f6e8db9780483249f Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 14 Aug 2024 11:38:40 +0200 Subject: [PATCH 18/33] lint --- .../perform_rule_upgrade/perform_rule_upgrade_route.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts index 882ed73c2ca8ab..22186957ea4e8c 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts @@ -17,7 +17,6 @@ import { } from './perform_rule_upgrade_route'; import { RULE_UPGRADE_SPECIFIER_FIELDS } from '../../../../../server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset'; - describe('Perform Rule Upgrade Route Schemas', () => { describe('PickVersionValues', () => { test('validates correct enum values', () => { @@ -46,9 +45,8 @@ describe('Perform Rule Upgrade Route Schemas', () => { const upgradeSpecifierFields = new Set(Object.keys(RuleUpgradeSpecifierFields.shape)); expect(upgradeSpecifierFields).toEqual(RULE_UPGRADE_SPECIFIER_FIELDS); - - }) - }) + }); + }); describe('RuleUpgradeSpecifier', () => { const validSpecifier = { From 2a07a5d70c895f7c104138a616abb422181f8922 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 14 Aug 2024 13:59:49 +0200 Subject: [PATCH 19/33] fix --- .../prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index a60808406ce74e..eb4b7143e7e31c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -153,9 +153,8 @@ const NON_UPGRADABLE_FIELDS: string[] = [ function createRuleUpgradeSpecifierFields() { const allUpgradableFields = new Set( - Object.values(UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE).flatMap((fields: string[]) => fields) + Array.from(UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE.values()).flat() ); - NON_UPGRADABLE_FIELDS.forEach((field) => allUpgradableFields.delete(field)); return allUpgradableFields; From 4b23cce4a1e24781a206781a1ea2facf1893b8e2 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Thu, 15 Aug 2024 18:30:47 +0200 Subject: [PATCH 20/33] Created Zod schema and types --- .../model/diff/diffable_rule/diffable_rule.ts | 88 +++++++++++++++---- .../model/rule_assets/prebuilt_rule_asset.ts | 1 - 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts index 0a85dddc897237..6ddad3e58a96fc 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts @@ -6,7 +6,7 @@ */ import { z } from '@kbn/zod'; - +import { mapValues } from 'lodash'; import { AnomalyThreshold, EventCategoryOverride, @@ -203,21 +203,79 @@ const DiffableRule = z.intersection( ]) ); +// TODO: Add unit test to assert that types match between DiffableRules +// and this schema +type DiffableRuleTypes = z.infer; +const DiffableRuleTypes = z.union([ + DiffableCustomQueryFields.shape.type, + DiffableSavedQueryFields.shape.type, + DiffableEqlFields.shape.type, + DiffableEsqlFields.shape.type, + DiffableThreatMatchFields.shape.type, + DiffableThresholdFields.shape.type, + DiffableMachineLearningFields.shape.type, + DiffableNewTermsFields.shape.type, +]); + /** * This is a merge of all fields from all rule types into a single TS type. * This is NOT a union discriminated by rule type, as DiffableRule is. */ -export type DiffableAllFields = DiffableCommonFields & - Omit & - Omit & - Omit & - Omit & - Omit & - Omit & - Omit & - Omit & - DiffableRuleTypeField; - -interface DiffableRuleTypeField { - type: DiffableRule['type']; -} +// export type DiffableAllFields = DiffableCommonFields & +// Omit & +// Omit & +// Omit & +// Omit & +// Omit & +// Omit & +// Omit & +// Omit & +// DiffableRuleTypeField; + +type DiffableAllFields = z.infer; +const DiffableAllFields = DiffableCommonFields.merge(DiffableCustomQueryFields.omit({ type: true })) + .merge(DiffableSavedQueryFields.omit({ type: true })) + .merge(DiffableEqlFields.omit({ type: true })) + .merge(DiffableEsqlFields.omit({ type: true })) + .merge(DiffableThreatMatchFields.omit({ type: true })) + .merge(DiffableThresholdFields.omit({ type: true })) + .merge(DiffableMachineLearningFields.omit({ type: true })) + .merge(DiffableNewTermsFields.omit({ type: true })) + .merge(z.object({ type: DiffableRuleTypes })); + +type DiffableUpgradableFields = z.infer; +const DiffableUpgradableFields = DiffableAllFields.omit({ + type: true, + rule_id: true, + version: true, +}); + +export type PickVersionValues = z.infer; +export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); + +type FieldUpgradeSpecifier = z.infer>>>; +const fieldUpgradeSpecifier = (fieldSchema: T) => + z.discriminatedUnion('pick_version', [ + z.object({ + pick_version: PickVersionValues, + }), + z.object({ + pick_version: z.literal('RESOLVED'), + resolved_value: fieldSchema, + }), + ]); + +type FieldUpgradeSpecifiers = Required<{ + [Field in keyof TFields]: FieldUpgradeSpecifier; +}>; + +type RuleFieldsToUpgrade = FieldUpgradeSpecifiers; +const RuleFieldsToUpgrade = z + .object( + mapValues(DiffableUpgradableFields.shape, (fieldSchema) => { + return fieldUpgradeSpecifier(fieldSchema); + }) + ) + .strict(); +//TODO: Add units tests +// TODO: Make sure that only the expected fields are listed inRuleFieldsToUpgrade diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index eb4b7143e7e31c..397c5b573f352e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -141,7 +141,6 @@ export const UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE = createUpgradableRuleFields * manually during the upgrade workflow. */ const NON_UPGRADABLE_FIELDS: string[] = [ - 'alert_suppression', 'author', 'license', 'concurrent_searches', From b69827399590b42c901b854cb9f1fbdc5b86d863 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 26 Aug 2024 14:29:15 +0200 Subject: [PATCH 21/33] Add unit tests --- .../diff/diffable_rule/diffable_rule.test.ts | 44 +++++++ .../model/diff/diffable_rule/diffable_rule.ts | 85 ++++++++----- .../perform_rule_upgrade_route.test.ts | 46 +++++-- .../perform_rule_upgrade_route.ts | 117 +----------------- .../model/rule_assets/prebuilt_rule_asset.ts | 35 ------ 5 files changed, 132 insertions(+), 195 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts new file mode 100644 index 00000000000000..3b3bc3dea5e4a9 --- /dev/null +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + RuleFieldsToUpgrade, + DiffableUpgradableFields, + DiffableRule, + DiffableAllFields, + DiffableRuleTypes, +} from './diffable_rule'; + +describe('Diffable rule schema', () => { + describe('DiffableAllFields', () => { + it('includes all possible rules types listed in the diffable rule schemas', () => { + const diffableAllFieldsRuleTypes = DiffableAllFields.shape.type.options.map((x) => x.value); + const diffableRuleTypes = DiffableRuleTypes.options.map((x) => x.value); + expect(diffableAllFieldsRuleTypes).toStrictEqual(diffableRuleTypes); + }); + }); + + describe('DiffableRule', () => { + it('includes all possible rules types listed in the diffable rule schemas', () => { + const diffableRuleTypes = DiffableRule._def.right._def.options.map((x) => x.shape.type.value); + const ruleTypes = DiffableRuleTypes.options.map((x) => x.value); + expect(diffableRuleTypes).toStrictEqual(ruleTypes); + }); + }); + + describe('RuleFieldsToUpgrade', () => { + it('contains only upgradable fields defined in the diffable rule schemas', () => { + expect(Object.keys(RuleFieldsToUpgrade.shape)).toStrictEqual( + Object.keys(DiffableUpgradableFields.shape) + ); + }); + + it('correctly validates valid and invalid inputs', () => { + + }) + }); +}); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts index 6ddad3e58a96fc..a8c32b396eaf53 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts @@ -8,10 +8,13 @@ import { z } from '@kbn/zod'; import { mapValues } from 'lodash'; import { + AlertSuppression, AnomalyThreshold, EventCategoryOverride, HistoryWindowStart, + InvestigationFields, InvestigationGuide, + KqlQueryLanguage, MachineLearningJobId, MaxSignals, NewTermsFields, @@ -37,6 +40,7 @@ import { ThreatIndicatorPath, ThreatMapping, Threshold, + ThresholdAlertSuppression, TiebreakerField, TimestampField, } from '../../../../model/rule_schema'; @@ -77,6 +81,7 @@ export const DiffableCommonFields = z.object({ threat: ThreatArray, note: InvestigationGuide, setup: SetupGuide, + investigation_fields: InvestigationFields, related_integrations: RelatedIntegrationArray, required_fields: RequiredFieldArray, author: RuleAuthorArray, @@ -99,6 +104,7 @@ export const DiffableCustomQueryFields = z.object({ type: z.literal('query'), kql_query: RuleKqlQuery, // NOTE: new field data_source: RuleDataSource.optional(), // NOTE: new field + alert_suppression: AlertSuppression.optional(), }); export type DiffableSavedQueryFields = z.infer; @@ -106,6 +112,7 @@ export const DiffableSavedQueryFields = z.object({ type: z.literal('saved_query'), kql_query: RuleKqlQuery, // NOTE: new field data_source: RuleDataSource.optional(), // NOTE: new field + alert_suppression: AlertSuppression.optional(), }); export type DiffableEqlFields = z.infer; @@ -116,6 +123,7 @@ export const DiffableEqlFields = z.object({ event_category_override: EventCategoryOverride.optional(), timestamp_field: TimestampField.optional(), tiebreaker_field: TiebreakerField.optional(), + alert_suppression: AlertSuppression.optional(), }); // this is a new type of rule, no prebuilt rules created yet. @@ -124,6 +132,7 @@ export type DiffableEsqlFields = z.infer; export const DiffableEsqlFields = z.object({ type: z.literal('esql'), esql_query: RuleEsqlQuery, // NOTE: new field + alert_suppression: AlertSuppression.optional(), }); export type DiffableThreatMatchFields = z.infer; @@ -135,6 +144,8 @@ export const DiffableThreatMatchFields = z.object({ threat_mapping: ThreatMapping, data_source: RuleDataSource.optional(), // NOTE: new field threat_indicator_path: ThreatIndicatorPath.optional(), + threat_language: KqlQueryLanguage.optional(), + alert_suppression: AlertSuppression.optional(), }); export type DiffableThresholdFields = z.infer; @@ -143,6 +154,7 @@ export const DiffableThresholdFields = z.object({ kql_query: RuleKqlQuery, // NOTE: new field threshold: Threshold, data_source: RuleDataSource.optional(), // NOTE: new field + alert_suppression: ThresholdAlertSuppression.optional(), }); export type DiffableMachineLearningFields = z.infer; @@ -150,6 +162,7 @@ export const DiffableMachineLearningFields = z.object({ type: z.literal('machine_learning'), machine_learning_job_id: MachineLearningJobId, anomaly_threshold: AnomalyThreshold, + alert_suppression: AlertSuppression.optional(), }); export type DiffableNewTermsFields = z.infer; @@ -159,6 +172,7 @@ export const DiffableNewTermsFields = z.object({ new_terms_fields: NewTermsFields, history_window_start: HistoryWindowStart, data_source: RuleDataSource.optional(), // NOTE: new field + alert_suppression: AlertSuppression.optional(), }); /** @@ -189,7 +203,7 @@ export const DiffableNewTermsFields = z.object({ */ export type DiffableRule = z.infer; -const DiffableRule = z.intersection( +export const DiffableRule = z.intersection( DiffableCommonFields, z.discriminatedUnion('type', [ DiffableCustomQueryFields, @@ -205,8 +219,8 @@ const DiffableRule = z.intersection( // TODO: Add unit test to assert that types match between DiffableRules // and this schema -type DiffableRuleTypes = z.infer; -const DiffableRuleTypes = z.union([ +export type DiffableRuleTypes = z.infer; +export const DiffableRuleTypes = z.union([ DiffableCustomQueryFields.shape.type, DiffableSavedQueryFields.shape.type, DiffableEqlFields.shape.type, @@ -221,19 +235,10 @@ const DiffableRuleTypes = z.union([ * This is a merge of all fields from all rule types into a single TS type. * This is NOT a union discriminated by rule type, as DiffableRule is. */ -// export type DiffableAllFields = DiffableCommonFields & -// Omit & -// Omit & -// Omit & -// Omit & -// Omit & -// Omit & -// Omit & -// Omit & -// DiffableRuleTypeField; - -type DiffableAllFields = z.infer; -const DiffableAllFields = DiffableCommonFields.merge(DiffableCustomQueryFields.omit({ type: true })) +export type DiffableAllFields = z.infer; +export const DiffableAllFields = DiffableCommonFields.merge( + DiffableCustomQueryFields.omit({ type: true }) +) .merge(DiffableSavedQueryFields.omit({ type: true })) .merge(DiffableEqlFields.omit({ type: true })) .merge(DiffableEsqlFields.omit({ type: true })) @@ -243,39 +248,51 @@ const DiffableAllFields = DiffableCommonFields.merge(DiffableCustomQueryFields.o .merge(DiffableNewTermsFields.omit({ type: true })) .merge(z.object({ type: DiffableRuleTypes })); -type DiffableUpgradableFields = z.infer; -const DiffableUpgradableFields = DiffableAllFields.omit({ +/** + * Fields upgradable by the /upgrade/_perform endpoint. + * Specific fields are omitted because they are not upgradeable, and + * handled under the hood by endpoint logic. + * See: https://github.com/elastic/kibana/issues/186544 + */ +export type DiffableUpgradableFields = z.infer; +export const DiffableUpgradableFields = DiffableAllFields.omit({ type: true, rule_id: true, version: true, + author: true, + license: true, }); export type PickVersionValues = z.infer; export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); -type FieldUpgradeSpecifier = z.infer>>>; +export type FieldUpgradeSpecifier = z.infer< + ReturnType>> +>; const fieldUpgradeSpecifier = (fieldSchema: T) => z.discriminatedUnion('pick_version', [ - z.object({ - pick_version: PickVersionValues, - }), - z.object({ - pick_version: z.literal('RESOLVED'), - resolved_value: fieldSchema, - }), + z + .object({ + pick_version: PickVersionValues, + }) + .strict(), + z + .object({ + pick_version: z.literal('RESOLVED'), + resolved_value: fieldSchema, + }) + .strict(), ]); -type FieldUpgradeSpecifiers = Required<{ - [Field in keyof TFields]: FieldUpgradeSpecifier; -}>; +type FieldUpgradeSpecifiers = { + [Field in keyof TFields]?: FieldUpgradeSpecifier; +}; -type RuleFieldsToUpgrade = FieldUpgradeSpecifiers; -const RuleFieldsToUpgrade = z +export type RuleFieldsToUpgrade = FieldUpgradeSpecifiers; +export const RuleFieldsToUpgrade = z .object( mapValues(DiffableUpgradableFields.shape, (fieldSchema) => { - return fieldUpgradeSpecifier(fieldSchema); + return fieldUpgradeSpecifier(fieldSchema).optional(); }) ) .strict(); -//TODO: Add units tests -// TODO: Make sure that only the expected fields are listed inRuleFieldsToUpgrade diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts index 22186957ea4e8c..e8f1d978837225 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts @@ -4,18 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zod-helpers'; import { PickVersionValues, RuleUpgradeSpecifier, - RuleUpgradeSpecifierFields, UpgradeSpecificRulesRequest, UpgradeAllRulesRequest, PerformRuleUpgradeResponseBody, PerformRuleUpgradeRequestBody, } from './perform_rule_upgrade_route'; -import { RULE_UPGRADE_SPECIFIER_FIELDS } from '../../../../../server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset'; describe('Perform Rule Upgrade Route Schemas', () => { describe('PickVersionValues', () => { @@ -40,14 +37,6 @@ describe('Perform Rule Upgrade Route Schemas', () => { }); }); - describe('RuleUpgradeSpecifierFields', () => { - it('accepts all upgradable fields from the Prebuilt Rule Asset', () => { - const upgradeSpecifierFields = new Set(Object.keys(RuleUpgradeSpecifierFields.shape)); - - expect(upgradeSpecifierFields).toEqual(RULE_UPGRADE_SPECIFIER_FIELDS); - }); - }); - describe('RuleUpgradeSpecifier', () => { const validSpecifier = { rule_id: 'rule-1', @@ -62,7 +51,7 @@ describe('Perform Rule Upgrade Route Schemas', () => { expect(result.data).toEqual(validSpecifier); }); - test('validates a valid upgrade specifier with a fields property', () => { + test('validates a valid upgrade specifier with a valid field property', () => { const specifierWithFields = { ...validSpecifier, fields: { @@ -76,6 +65,39 @@ describe('Perform Rule Upgrade Route Schemas', () => { expect(result.data).toEqual(specifierWithFields); }); + test('rejects an upgrade specifier with an invalid fields property', () => { + const specifierWithFields = { + ...validSpecifier, + fields: { + unknown_field: { + pick_version: 'CURRENT', + }, + }, + }; + const result = RuleUpgradeSpecifier.safeParse(specifierWithFields); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"fields: Unrecognized key(s) in object: 'unknown_field'"` + ); + }); + + test('rejects an upgrade specifier with a field property with an invalid type', () => { + const specifierWithFields = { + ...validSpecifier, + fields: { + name: { + pick_version: 'CURRENT', + resolved_value: 'My name', + }, + }, + }; + const result = RuleUpgradeSpecifier.safeParse(specifierWithFields); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"fields.name: Unrecognized key(s) in object: 'resolved_value'"` + ); + }); + test('rejects upgrade specifier with invalid pick_version rule_id', () => { const invalid = { ...validSpecifier, rule_id: 123 }; const result = RuleUpgradeSpecifier.safeParse(invalid); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts index c9ba9be508546f..df3dfba0845afe 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts @@ -8,125 +8,14 @@ import { z } from '@kbn/zod'; import { RuleResponse } from '../../model/rule_schema/rule_schemas.gen'; -import { AggregatedPrebuiltRuleError } from '../model'; -import { - RuleSignatureId, - RuleVersion, - RuleName, - RuleTagArray, - RuleDescription, - IsRuleEnabled, - Severity, - SeverityMapping, - RiskScore, - RiskScoreMapping, - RuleReferenceArray, - RuleFalsePositiveArray, - ThreatArray, - InvestigationGuide, - SetupGuide, - RelatedIntegrationArray, - RequiredFieldArray, - MaxSignals, - BuildingBlockType, - RuleIntervalFrom, - RuleInterval, - RuleExceptionList, - RuleNameOverride, - TimestampOverride, - TimestampOverrideFallbackDisabled, - TimelineTemplateId, - TimelineTemplateTitle, - IndexPatternArray, - DataViewId, - RuleQuery, - QueryLanguage, - RuleFilterArray, - SavedQueryId, - KqlQueryLanguage, - InvestigationFields, - MachineLearningJobId, - AnomalyThreshold, - ThreatQuery, - ThreatMapping, - ThreatIndex, - ThreatFilters, - ThreatIndicatorPath, - NewTermsFields, - HistoryWindowStart, - EventCategoryOverride, - TiebreakerField, - TimestampField, - Threshold, -} from '../../model'; +import { AggregatedPrebuiltRuleError, RuleFieldsToUpgrade } from '../model'; +import { RuleSignatureId, RuleVersion } from '../../model'; export type PickVersionValues = z.infer; export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); export type PickVersionValuesEnum = typeof PickVersionValues.enum; export const PickVersionValuesEnum = PickVersionValues.enum; -const createUpgradeFieldSchema = (fieldSchema: T) => - z - .discriminatedUnion('pick_version', [ - z.object({ - pick_version: PickVersionValues, - }), - z.object({ - pick_version: z.literal('RESOLVED'), - resolved_value: fieldSchema, - }), - ]) - .optional(); - -export const RuleUpgradeSpecifierFields = z.object({ - name: createUpgradeFieldSchema(RuleName), - tags: createUpgradeFieldSchema(RuleTagArray), - description: createUpgradeFieldSchema(RuleDescription), - severity: createUpgradeFieldSchema(Severity), - severity_mapping: createUpgradeFieldSchema(SeverityMapping), - enabled: createUpgradeFieldSchema(IsRuleEnabled), - risk_score: createUpgradeFieldSchema(RiskScore), - risk_score_mapping: createUpgradeFieldSchema(RiskScoreMapping), - references: createUpgradeFieldSchema(RuleReferenceArray), - false_positives: createUpgradeFieldSchema(RuleFalsePositiveArray), - threat: createUpgradeFieldSchema(ThreatArray), - note: createUpgradeFieldSchema(InvestigationGuide), - setup: createUpgradeFieldSchema(SetupGuide), - investigation_fields: createUpgradeFieldSchema(InvestigationFields), - related_integrations: createUpgradeFieldSchema(RelatedIntegrationArray), - required_fields: createUpgradeFieldSchema(RequiredFieldArray), - max_signals: createUpgradeFieldSchema(MaxSignals), - building_block_type: createUpgradeFieldSchema(BuildingBlockType), - from: createUpgradeFieldSchema(RuleIntervalFrom), - interval: createUpgradeFieldSchema(RuleInterval), - exceptions_list: createUpgradeFieldSchema(RuleExceptionList), - rule_name_override: createUpgradeFieldSchema(RuleNameOverride), - timestamp_override: createUpgradeFieldSchema(TimestampOverride), - timestamp_override_fallback_disabled: createUpgradeFieldSchema(TimestampOverrideFallbackDisabled), - timeline_id: createUpgradeFieldSchema(TimelineTemplateId), - timeline_title: createUpgradeFieldSchema(TimelineTemplateTitle), - index: createUpgradeFieldSchema(IndexPatternArray), - data_view_id: createUpgradeFieldSchema(DataViewId), - query: createUpgradeFieldSchema(RuleQuery), - language: createUpgradeFieldSchema(QueryLanguage), - filters: createUpgradeFieldSchema(RuleFilterArray), - saved_id: createUpgradeFieldSchema(SavedQueryId), - machine_learning_job_id: createUpgradeFieldSchema(MachineLearningJobId), - anomaly_threshold: createUpgradeFieldSchema(AnomalyThreshold), - threat_query: createUpgradeFieldSchema(ThreatQuery), - threat_mapping: createUpgradeFieldSchema(ThreatMapping), - threat_index: createUpgradeFieldSchema(ThreatIndex), - threat_filters: createUpgradeFieldSchema(ThreatFilters), - threat_indicator_path: createUpgradeFieldSchema(ThreatIndicatorPath), - threat_language: createUpgradeFieldSchema(KqlQueryLanguage), - new_terms_fields: createUpgradeFieldSchema(NewTermsFields), - history_window_start: createUpgradeFieldSchema(HistoryWindowStart), - event_category_override: createUpgradeFieldSchema(EventCategoryOverride), - tiebreaker_field: createUpgradeFieldSchema(TiebreakerField), - timestamp_field: createUpgradeFieldSchema(TimestampField), - threshold: createUpgradeFieldSchema(Threshold), -}); - export type RuleUpgradeSpecifier = z.infer; export const RuleUpgradeSpecifier = z.object({ rule_id: RuleSignatureId, @@ -135,7 +24,7 @@ export const RuleUpgradeSpecifier = z.object({ pick_version: PickVersionValues.optional(), // Fields that can be customized during the upgrade workflow // as decided in: https://github.com/elastic/kibana/issues/186544 - fields: RuleUpgradeSpecifierFields.optional(), + fields: RuleFieldsToUpgrade.optional(), }); export type UpgradeSpecificRulesRequest = z.infer; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index 397c5b573f352e..6267be09652e81 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -135,38 +135,3 @@ function createUpgradableRuleFieldsPayloadByType() { * the user of the /upgrade/_perform endpoint can specify which fields to upgrade during the upgrade workflow. */ export const UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE = createUpgradableRuleFieldsPayloadByType(); - -/** - * Fields which are not part of the RuleUpgradeSpecifierFields schema, and are handled - * manually during the upgrade workflow. - */ -const NON_UPGRADABLE_FIELDS: string[] = [ - 'author', - 'license', - 'concurrent_searches', - 'items_per_search', - 'version', - 'type', - 'to', -]; - -function createRuleUpgradeSpecifierFields() { - const allUpgradableFields = new Set( - Array.from(UPGRADABLE_FIELDS_PAYLOAD_BY_RULE_TYPE.values()).flat() - ); - NON_UPGRADABLE_FIELDS.forEach((field) => allUpgradableFields.delete(field)); - - return allUpgradableFields; -} - -/** - * List of fields that are part of the RuleUpgradeSpecifierFields schema, which is part of the - * /upgrade/_perform endpoint request payload. This list is used to test that all upgradable fields from - * the PrebuiltRuleAsset are part of the RuleUpgradeSpecifierFields schema. - * - * Note that some of the fields of the PrebuiltRuleAsset schema are not upgradable in the update workflow - * of the /upgrade/_perform endpoint (and therefore nor part of RuleUpgradeSpecifierFields) so they are - * manually excluded from the list of upgradable fields. - * - */ -export const RULE_UPGRADE_SPECIFIER_FIELDS = createRuleUpgradeSpecifierFields(); From 6d8e13814cc0246743ab5034c55e85bd6546896a Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 26 Aug 2024 16:01:28 +0200 Subject: [PATCH 22/33] Unit tests --- .../diff/diffable_rule/diffable_rule.test.ts | 124 +++++++++++++++++- 1 file changed, 120 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts index 3b3bc3dea5e4a9..e09c2fe3ce820f 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { RuleFieldsToUpgrade, DiffableUpgradableFields, @@ -12,6 +11,7 @@ import { DiffableAllFields, DiffableRuleTypes, } from './diffable_rule'; +import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zod-helpers'; describe('Diffable rule schema', () => { describe('DiffableAllFields', () => { @@ -37,8 +37,124 @@ describe('Diffable rule schema', () => { ); }); - it('correctly validates valid and invalid inputs', () => { - - }) + describe('correctly validates valid and invalid inputs', () => { + it('validates 5 valid cases: BASE, CURRENT, TARGET, MERGED, RESOLVED', () => { + const validInputs = [ + { + name: { + pick_version: 'BASE', + }, + }, + { + description: { + pick_version: 'CURRENT', + }, + }, + { + risk_score: { + pick_version: 'TARGET', + }, + }, + { + note: { + pick_version: 'MERGED', + }, + }, + { + references: { + pick_version: 'RESOLVED', + resolved_value: ['www.link1.com', 'www.link2.com'], + }, + }, + ]; + + validInputs.forEach((input) => { + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseSuccess(result); + expect(result.data).toEqual(input); + }); + }); + + it('does not validate invalid combination of pick_version and resolved_value', () => { + const input = { + references: { + pick_version: 'MERGED', + resolved_value: ['www.link1.com', 'www.link2.com'], + }, + }; + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"references: Unrecognized key(s) in object: 'resolved_value'"` + ); + }); + + it('invalidates incorrect types of resolved_values provided to multiple fields', () => { + const input = { + references: { + pick_version: 'RESOLVED', + resolved_value: 'www.link1.com', + }, + tags: { + pick_version: 'RESOLVED', + resolved_value: 4, + }, + name: { + pick_version: 'RESOLVED', + resolved_value: 'Valid type name', + }, + }; + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"tags.resolved_value: Expected array, received number, references.resolved_value: Expected array, received string"` + ); + }); + + it('invalidates unknown fields', () => { + const input = { + unknown_field: { + pick_version: 'CURRENT', + }, + }; + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"Unrecognized key(s) in object: 'unknown_field'"` + ); + }); + + it('invalidates rule fields not part of RuleFieldsToUpgrade', () => { + const input = { + type: { + pick_version: 'BASE', + }, + rule_id: { + pick_version: 'CURRENT', + }, + version: { + pick_version: 'TARGET', + }, + author: { + pick_version: 'MERGED', + }, + license: { + pick_version: 'RESOLVED', + resolved_value: 'Elastic License', + }, + concurrent_searches: { + pick_version: 'TARGET', + }, + items_per_search: { + pick_version: 'TARGET', + }, + }; + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"Unrecognized key(s) in object: 'type', 'rule_id', 'version', 'author', 'license', 'concurrent_searches', 'items_per_search'"` + ); + }); + }); }); }); From 5c409890a540bdf86835d0c2442d1c30f544273d Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 26 Aug 2024 16:02:23 +0200 Subject: [PATCH 23/33] remove comments --- .../prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts index a8c32b396eaf53..8ceafb65a950ea 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts @@ -217,8 +217,6 @@ export const DiffableRule = z.intersection( ]) ); -// TODO: Add unit test to assert that types match between DiffableRules -// and this schema export type DiffableRuleTypes = z.infer; export const DiffableRuleTypes = z.union([ DiffableCustomQueryFields.shape.type, From 57ae7dd4f8e15f284b08dd46d62592a4e1db6ed1 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 26 Aug 2024 17:08:19 +0200 Subject: [PATCH 24/33] remove export --- .../prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts index 8ceafb65a950ea..3c360b048f16d3 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts @@ -57,6 +57,7 @@ import { TimelineTemplateReference, TimestampOverrideObject, } from './diffable_field_types'; +import { PickVersionValues } from '../../../perform_rule_upgrade/perform_rule_upgrade_route'; export type DiffableCommonFields = z.infer; export const DiffableCommonFields = z.object({ @@ -261,9 +262,6 @@ export const DiffableUpgradableFields = DiffableAllFields.omit({ license: true, }); -export type PickVersionValues = z.infer; -export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); - export type FieldUpgradeSpecifier = z.infer< ReturnType>> >; From a7ba65c65751d0f9e12df0d0b6a82fb79842a08f Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 27 Aug 2024 00:00:25 +0200 Subject: [PATCH 25/33] fixes --- packages/kbn-eslint-config/.eslintrc.js | 1 - .../model/diff/diffable_rule/diffable_rule.ts | 6 +++++- .../perform_rule_upgrade/perform_rule_upgrade_route.ts | 7 +------ .../diff/calculation/calculate_rule_fields_diff.ts | 10 ++++++++++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/kbn-eslint-config/.eslintrc.js b/packages/kbn-eslint-config/.eslintrc.js index de3bb8e313cccc..a68dc6ecd949e0 100644 --- a/packages/kbn-eslint-config/.eslintrc.js +++ b/packages/kbn-eslint-config/.eslintrc.js @@ -314,7 +314,6 @@ module.exports = { '@kbn/eslint/no_constructor_args_in_property_initializers': 'error', '@kbn/eslint/no_this_in_property_initializers': 'error', '@kbn/eslint/no_unsafe_console': 'error', - '@kbn/eslint/no_unsafe_js_yaml': 'error', '@kbn/imports/no_unresolvable_imports': 'error', '@kbn/imports/uniform_imports': 'error', '@kbn/imports/no_unused_imports': 'error', diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts index 3c360b048f16d3..bcd179d0d4765d 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts @@ -57,7 +57,6 @@ import { TimelineTemplateReference, TimestampOverrideObject, } from './diffable_field_types'; -import { PickVersionValues } from '../../../perform_rule_upgrade/perform_rule_upgrade_route'; export type DiffableCommonFields = z.infer; export const DiffableCommonFields = z.object({ @@ -262,6 +261,11 @@ export const DiffableUpgradableFields = DiffableAllFields.omit({ license: true, }); +export type PickVersionValues = z.infer; +export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); +export type PickVersionValuesEnum = typeof PickVersionValues.enum; +export const PickVersionValuesEnum = PickVersionValues.enum; + export type FieldUpgradeSpecifier = z.infer< ReturnType>> >; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts index df3dfba0845afe..359f20d2c91c8a 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts @@ -8,14 +8,9 @@ import { z } from '@kbn/zod'; import { RuleResponse } from '../../model/rule_schema/rule_schemas.gen'; -import { AggregatedPrebuiltRuleError, RuleFieldsToUpgrade } from '../model'; +import { AggregatedPrebuiltRuleError, PickVersionValues, RuleFieldsToUpgrade } from '../model'; import { RuleSignatureId, RuleVersion } from '../../model'; -export type PickVersionValues = z.infer; -export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); -export type PickVersionValuesEnum = typeof PickVersionValues.enum; -export const PickVersionValuesEnum = PickVersionValues.enum; - export type RuleUpgradeSpecifier = z.infer; export const RuleUpgradeSpecifier = z.object({ rule_id: RuleSignatureId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff.ts index e5b3ba031809a9..4f8126d3150726 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff.ts @@ -199,6 +199,7 @@ const commonFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor timestamp_override: simpleDiffAlgorithm, timeline_template: simpleDiffAlgorithm, building_block: simpleDiffAlgorithm, + investigation_fields: simpleDiffAlgorithm, }; const calculateCustomQueryFieldsDiff = ( @@ -211,6 +212,7 @@ const customQueryFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { event_category_override: singleLineStringDiffAlgorithm, timestamp_field: singleLineStringDiffAlgorithm, tiebreaker_field: singleLineStringDiffAlgorithm, + alert_suppression: simpleDiffAlgorithm, }; const calculateEsqlFieldsDiff = ( @@ -249,6 +253,7 @@ const calculateEsqlFieldsDiff = ( const esqlFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor = { type: simpleDiffAlgorithm, esql_query: simpleDiffAlgorithm, + alert_suppression: simpleDiffAlgorithm, }; const calculateThreatMatchFieldsDiff = ( @@ -265,6 +270,8 @@ const threatMatchFieldsDiffAlgorithms: FieldsDiffAlgorithmsFor Date: Tue, 27 Aug 2024 00:02:58 +0200 Subject: [PATCH 26/33] remove change --- packages/kbn-eslint-config/.eslintrc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-eslint-config/.eslintrc.js b/packages/kbn-eslint-config/.eslintrc.js index a68dc6ecd949e0..de3bb8e313cccc 100644 --- a/packages/kbn-eslint-config/.eslintrc.js +++ b/packages/kbn-eslint-config/.eslintrc.js @@ -314,6 +314,7 @@ module.exports = { '@kbn/eslint/no_constructor_args_in_property_initializers': 'error', '@kbn/eslint/no_this_in_property_initializers': 'error', '@kbn/eslint/no_unsafe_console': 'error', + '@kbn/eslint/no_unsafe_js_yaml': 'error', '@kbn/imports/no_unresolvable_imports': 'error', '@kbn/imports/uniform_imports': 'error', '@kbn/imports/no_unused_imports': 'error', From 5406026098727d380858b3affa49e64424ee85c3 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 27 Aug 2024 11:34:57 +0200 Subject: [PATCH 27/33] Fix moved import --- .../perform_rule_upgrade/perform_rule_upgrade_route.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts index e8f1d978837225..0310eb416c95b6 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts @@ -6,13 +6,13 @@ */ import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zod-helpers'; import { - PickVersionValues, RuleUpgradeSpecifier, UpgradeSpecificRulesRequest, UpgradeAllRulesRequest, PerformRuleUpgradeResponseBody, PerformRuleUpgradeRequestBody, } from './perform_rule_upgrade_route'; +import { PickVersionValues } from '../model'; describe('Perform Rule Upgrade Route Schemas', () => { describe('PickVersionValues', () => { From d76ad7e64bf6b0dc9a578ddc0bb3a864456ea0b6 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 27 Aug 2024 16:22:25 +0200 Subject: [PATCH 28/33] Fix types --- .../model/diff/diffable_rule/diffable_rule.ts | 2 +- .../prebuilt_rules/diff/convert_rule_to_diffable.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts index bcd179d0d4765d..9df6e01c2d4c12 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts @@ -81,7 +81,6 @@ export const DiffableCommonFields = z.object({ threat: ThreatArray, note: InvestigationGuide, setup: SetupGuide, - investigation_fields: InvestigationFields, related_integrations: RelatedIntegrationArray, required_fields: RequiredFieldArray, author: RuleAuthorArray, @@ -93,6 +92,7 @@ export const DiffableCommonFields = z.object({ max_signals: MaxSignals, // Optional fields + investigation_fields: InvestigationFields.optional(), rule_name_override: RuleNameOverrideObject.optional(), // NOTE: new field timestamp_override: TimestampOverrideObject.optional(), // NOTE: new field timeline_template: TimelineTemplateReference.optional(), // NOTE: new field diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts index f19d8b41be40bf..45b4612e83c8e9 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts @@ -142,6 +142,7 @@ const extractDiffableCommonFields = ( max_signals: rule.max_signals ?? DEFAULT_MAX_SIGNALS, // --------------------- OPTIONAL FIELDS + investigation_fields: rule.investigation_fields, rule_name_override: extractRuleNameOverrideObject(rule), timestamp_override: extractTimestampOverrideObject(rule), timeline_template: extractTimelineTemplateReference(rule), @@ -156,6 +157,7 @@ const extractDiffableCustomQueryFields = ( type: rule.type, kql_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), data_source: extractRuleDataSource(rule.index, rule.data_view_id), + alert_suppression: rule.alert_suppression, }; }; @@ -166,6 +168,7 @@ const extractDiffableSavedQueryFieldsFromRuleObject = ( type: rule.type, kql_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), data_source: extractRuleDataSource(rule.index, rule.data_view_id), + alert_suppression: rule.alert_suppression, }; }; @@ -179,6 +182,7 @@ const extractDiffableEqlFieldsFromRuleObject = ( event_category_override: rule.event_category_override, timestamp_field: rule.timestamp_field, tiebreaker_field: rule.tiebreaker_field, + alert_suppression: rule.alert_suppression, }; }; @@ -188,6 +192,7 @@ const extractDiffableEsqlFieldsFromRuleObject = ( return { type: rule.type, esql_query: extractRuleEsqlQuery(rule.query, rule.language), + alert_suppression: rule.alert_suppression, }; }; @@ -206,6 +211,8 @@ const extractDiffableThreatMatchFieldsFromRuleObject = ( threat_index: rule.threat_index, threat_mapping: rule.threat_mapping, threat_indicator_path: rule.threat_indicator_path, + threat_language: rule.threat_language, + alert_suppression: rule.alert_suppression, }; }; @@ -217,6 +224,7 @@ const extractDiffableThresholdFieldsFromRuleObject = ( kql_query: extractRuleKqlQuery(rule.query, rule.language, rule.filters, rule.saved_id), data_source: extractRuleDataSource(rule.index, rule.data_view_id), threshold: rule.threshold, + alert_suppression: rule.alert_suppression, }; }; @@ -227,6 +235,7 @@ const extractDiffableMachineLearningFieldsFromRuleObject = ( type: rule.type, machine_learning_job_id: rule.machine_learning_job_id, anomaly_threshold: rule.anomaly_threshold, + alert_suppression: rule.alert_suppression, }; }; @@ -239,5 +248,6 @@ const extractDiffableNewTermsFieldsFromRuleObject = ( data_source: extractRuleDataSource(rule.index, rule.data_view_id), new_terms_fields: rule.new_terms_fields, history_window_start: rule.history_window_start, + alert_suppression: rule.alert_suppression, }; }; From 870a2d779359c8e8497537a78520959ee24f8d0f Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 10 Sep 2024 17:34:00 -0300 Subject: [PATCH 29/33] Fix incorrect mock values for tests --- .../model/diff/diffable_rule/diffable_rule.test.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts index e09c2fe3ce820f..147f8fac51c199 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts @@ -63,7 +63,7 @@ describe('Diffable rule schema', () => { { references: { pick_version: 'RESOLVED', - resolved_value: ['www.link1.com', 'www.link2.com'], + resolved_value: ['www.example.com'], }, }, ]; @@ -79,7 +79,7 @@ describe('Diffable rule schema', () => { const input = { references: { pick_version: 'MERGED', - resolved_value: ['www.link1.com', 'www.link2.com'], + resolved_value: ['www.example.com'], }, }; const result = RuleFieldsToUpgrade.safeParse(input); @@ -93,16 +93,12 @@ describe('Diffable rule schema', () => { const input = { references: { pick_version: 'RESOLVED', - resolved_value: 'www.link1.com', + resolved_value: 'www.example.com', }, tags: { pick_version: 'RESOLVED', resolved_value: 4, }, - name: { - pick_version: 'RESOLVED', - resolved_value: 'Valid type name', - }, }; const result = RuleFieldsToUpgrade.safeParse(input); expectParseError(result); From cac237a630074b7df663a3efb5f33d798f1563fb Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 10 Sep 2024 18:10:59 -0300 Subject: [PATCH 30/33] refactored code --- .../diff/diffable_rule/diffable_rule.test.ts | 135 +------------ .../model/diff/diffable_rule/diffable_rule.ts | 77 ++----- .../perform_rule_upgrade_route.test.ts | 188 +++++++++++++++--- .../perform_rule_upgrade_route.ts | 55 ++++- 4 files changed, 224 insertions(+), 231 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts index 147f8fac51c199..dd84d769606c20 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.test.ts @@ -4,14 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { - RuleFieldsToUpgrade, - DiffableUpgradableFields, - DiffableRule, - DiffableAllFields, - DiffableRuleTypes, -} from './diffable_rule'; -import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zod-helpers'; +import { DiffableFieldsByTypeUnion, DiffableAllFields, DiffableRuleTypes } from './diffable_rule'; describe('Diffable rule schema', () => { describe('DiffableAllFields', () => { @@ -24,133 +17,9 @@ describe('Diffable rule schema', () => { describe('DiffableRule', () => { it('includes all possible rules types listed in the diffable rule schemas', () => { - const diffableRuleTypes = DiffableRule._def.right._def.options.map((x) => x.shape.type.value); + const diffableRuleTypes = DiffableFieldsByTypeUnion.options.map((x) => x.shape.type.value); const ruleTypes = DiffableRuleTypes.options.map((x) => x.value); expect(diffableRuleTypes).toStrictEqual(ruleTypes); }); }); - - describe('RuleFieldsToUpgrade', () => { - it('contains only upgradable fields defined in the diffable rule schemas', () => { - expect(Object.keys(RuleFieldsToUpgrade.shape)).toStrictEqual( - Object.keys(DiffableUpgradableFields.shape) - ); - }); - - describe('correctly validates valid and invalid inputs', () => { - it('validates 5 valid cases: BASE, CURRENT, TARGET, MERGED, RESOLVED', () => { - const validInputs = [ - { - name: { - pick_version: 'BASE', - }, - }, - { - description: { - pick_version: 'CURRENT', - }, - }, - { - risk_score: { - pick_version: 'TARGET', - }, - }, - { - note: { - pick_version: 'MERGED', - }, - }, - { - references: { - pick_version: 'RESOLVED', - resolved_value: ['www.example.com'], - }, - }, - ]; - - validInputs.forEach((input) => { - const result = RuleFieldsToUpgrade.safeParse(input); - expectParseSuccess(result); - expect(result.data).toEqual(input); - }); - }); - - it('does not validate invalid combination of pick_version and resolved_value', () => { - const input = { - references: { - pick_version: 'MERGED', - resolved_value: ['www.example.com'], - }, - }; - const result = RuleFieldsToUpgrade.safeParse(input); - expectParseError(result); - expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"references: Unrecognized key(s) in object: 'resolved_value'"` - ); - }); - - it('invalidates incorrect types of resolved_values provided to multiple fields', () => { - const input = { - references: { - pick_version: 'RESOLVED', - resolved_value: 'www.example.com', - }, - tags: { - pick_version: 'RESOLVED', - resolved_value: 4, - }, - }; - const result = RuleFieldsToUpgrade.safeParse(input); - expectParseError(result); - expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"tags.resolved_value: Expected array, received number, references.resolved_value: Expected array, received string"` - ); - }); - - it('invalidates unknown fields', () => { - const input = { - unknown_field: { - pick_version: 'CURRENT', - }, - }; - const result = RuleFieldsToUpgrade.safeParse(input); - expectParseError(result); - expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"Unrecognized key(s) in object: 'unknown_field'"` - ); - }); - - it('invalidates rule fields not part of RuleFieldsToUpgrade', () => { - const input = { - type: { - pick_version: 'BASE', - }, - rule_id: { - pick_version: 'CURRENT', - }, - version: { - pick_version: 'TARGET', - }, - author: { - pick_version: 'MERGED', - }, - license: { - pick_version: 'RESOLVED', - resolved_value: 'Elastic License', - }, - concurrent_searches: { - pick_version: 'TARGET', - }, - items_per_search: { - pick_version: 'TARGET', - }, - }; - const result = RuleFieldsToUpgrade.safeParse(input); - expectParseError(result); - expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"Unrecognized key(s) in object: 'type', 'rule_id', 'version', 'author', 'license', 'concurrent_searches', 'items_per_search'"` - ); - }); - }); - }); }); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts index 9df6e01c2d4c12..23fca59e8c8649 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts @@ -6,7 +6,6 @@ */ import { z } from '@kbn/zod'; -import { mapValues } from 'lodash'; import { AlertSuppression, AnomalyThreshold, @@ -202,20 +201,19 @@ export const DiffableNewTermsFields = z.object({ * top-level fields. */ +export const DiffableFieldsByTypeUnion = z.discriminatedUnion('type', [ + DiffableCustomQueryFields, + DiffableSavedQueryFields, + DiffableEqlFields, + DiffableEsqlFields, + DiffableThreatMatchFields, + DiffableThresholdFields, + DiffableMachineLearningFields, + DiffableNewTermsFields, +]); + export type DiffableRule = z.infer; -export const DiffableRule = z.intersection( - DiffableCommonFields, - z.discriminatedUnion('type', [ - DiffableCustomQueryFields, - DiffableSavedQueryFields, - DiffableEqlFields, - DiffableEsqlFields, - DiffableThreatMatchFields, - DiffableThresholdFields, - DiffableMachineLearningFields, - DiffableNewTermsFields, - ]) -); +export const DiffableRule = z.intersection(DiffableCommonFields, DiffableFieldsByTypeUnion); export type DiffableRuleTypes = z.infer; export const DiffableRuleTypes = z.union([ @@ -245,54 +243,3 @@ export const DiffableAllFields = DiffableCommonFields.merge( .merge(DiffableMachineLearningFields.omit({ type: true })) .merge(DiffableNewTermsFields.omit({ type: true })) .merge(z.object({ type: DiffableRuleTypes })); - -/** - * Fields upgradable by the /upgrade/_perform endpoint. - * Specific fields are omitted because they are not upgradeable, and - * handled under the hood by endpoint logic. - * See: https://github.com/elastic/kibana/issues/186544 - */ -export type DiffableUpgradableFields = z.infer; -export const DiffableUpgradableFields = DiffableAllFields.omit({ - type: true, - rule_id: true, - version: true, - author: true, - license: true, -}); - -export type PickVersionValues = z.infer; -export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); -export type PickVersionValuesEnum = typeof PickVersionValues.enum; -export const PickVersionValuesEnum = PickVersionValues.enum; - -export type FieldUpgradeSpecifier = z.infer< - ReturnType>> ->; -const fieldUpgradeSpecifier = (fieldSchema: T) => - z.discriminatedUnion('pick_version', [ - z - .object({ - pick_version: PickVersionValues, - }) - .strict(), - z - .object({ - pick_version: z.literal('RESOLVED'), - resolved_value: fieldSchema, - }) - .strict(), - ]); - -type FieldUpgradeSpecifiers = { - [Field in keyof TFields]?: FieldUpgradeSpecifier; -}; - -export type RuleFieldsToUpgrade = FieldUpgradeSpecifiers; -export const RuleFieldsToUpgrade = z - .object( - mapValues(DiffableUpgradableFields.shape, (fieldSchema) => { - return fieldUpgradeSpecifier(fieldSchema).optional(); - }) - ) - .strict(); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts index 0310eb416c95b6..7f7a64006adbf3 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.test.ts @@ -11,8 +11,10 @@ import { UpgradeAllRulesRequest, PerformRuleUpgradeResponseBody, PerformRuleUpgradeRequestBody, + RuleFieldsToUpgrade, + DiffableUpgradableFields, + PickVersionValues, } from './perform_rule_upgrade_route'; -import { PickVersionValues } from '../model'; describe('Perform Rule Upgrade Route Schemas', () => { describe('PickVersionValues', () => { @@ -37,6 +39,130 @@ describe('Perform Rule Upgrade Route Schemas', () => { }); }); + describe('RuleFieldsToUpgrade', () => { + it('contains only upgradable fields defined in the diffable rule schemas', () => { + expect(Object.keys(RuleFieldsToUpgrade.shape)).toStrictEqual( + Object.keys(DiffableUpgradableFields.shape) + ); + }); + + describe('correctly validates valid and invalid inputs', () => { + it('validates 5 valid cases: BASE, CURRENT, TARGET, MERGED, RESOLVED', () => { + const validInputs = [ + { + name: { + pick_version: 'BASE', + }, + }, + { + description: { + pick_version: 'CURRENT', + }, + }, + { + risk_score: { + pick_version: 'TARGET', + }, + }, + { + note: { + pick_version: 'MERGED', + }, + }, + { + references: { + pick_version: 'RESOLVED', + resolved_value: ['www.example.com'], + }, + }, + ]; + + validInputs.forEach((input) => { + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseSuccess(result); + expect(result.data).toEqual(input); + }); + }); + + it('does not validate invalid combination of pick_version and resolved_value', () => { + const input = { + references: { + pick_version: 'MERGED', + resolved_value: ['www.example.com'], + }, + }; + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"references: Unrecognized key(s) in object: 'resolved_value'"` + ); + }); + + it('invalidates incorrect types of resolved_values provided to multiple fields', () => { + const input = { + references: { + pick_version: 'RESOLVED', + resolved_value: 'www.example.com', + }, + tags: { + pick_version: 'RESOLVED', + resolved_value: 4, + }, + }; + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"tags.resolved_value: Expected array, received number, references.resolved_value: Expected array, received string"` + ); + }); + + it('invalidates unknown fields', () => { + const input = { + unknown_field: { + pick_version: 'CURRENT', + }, + }; + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"Unrecognized key(s) in object: 'unknown_field'"` + ); + }); + + it('invalidates rule fields not part of RuleFieldsToUpgrade', () => { + const input = { + type: { + pick_version: 'BASE', + }, + rule_id: { + pick_version: 'CURRENT', + }, + version: { + pick_version: 'TARGET', + }, + author: { + pick_version: 'MERGED', + }, + license: { + pick_version: 'RESOLVED', + resolved_value: 'Elastic License', + }, + concurrent_searches: { + pick_version: 'TARGET', + }, + items_per_search: { + pick_version: 'TARGET', + }, + }; + const result = RuleFieldsToUpgrade.safeParse(input); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot( + `"Unrecognized key(s) in object: 'type', 'rule_id', 'version', 'author', 'license', 'concurrent_searches', 'items_per_search'"` + ); + }); + }); + }); + describe('RuleUpgradeSpecifier', () => { const validSpecifier = { rule_id: 'rule-1', @@ -199,38 +325,38 @@ describe('Perform Rule Upgrade Route Schemas', () => { ); }); }); -}); -describe('PerformRuleUpgradeResponseBody', () => { - const validResponse = { - summary: { - total: 1, - succeeded: 1, - skipped: 0, - failed: 0, - }, - results: { - updated: [], - skipped: [], - }, - errors: [], - }; - - test('validates a correct perform rule upgrade response', () => { - const result = PerformRuleUpgradeResponseBody.safeParse(validResponse); - expectParseSuccess(result); - expect(result.data).toEqual(validResponse); - }); + describe('PerformRuleUpgradeResponseBody', () => { + const validResponse = { + summary: { + total: 1, + succeeded: 1, + skipped: 0, + failed: 0, + }, + results: { + updated: [], + skipped: [], + }, + errors: [], + }; - test('rejects missing required fields', () => { - const propsToDelete = Object.keys(validResponse); - propsToDelete.forEach((deletedProp) => { - const invalidResponse = Object.fromEntries( - Object.entries(validResponse).filter(([key]) => key !== deletedProp) - ); - const result = PerformRuleUpgradeResponseBody.safeParse(invalidResponse); - expectParseError(result); - expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"${deletedProp}: Required"`); + test('validates a correct perform rule upgrade response', () => { + const result = PerformRuleUpgradeResponseBody.safeParse(validResponse); + expectParseSuccess(result); + expect(result.data).toEqual(validResponse); + }); + + test('rejects missing required fields', () => { + const propsToDelete = Object.keys(validResponse); + propsToDelete.forEach((deletedProp) => { + const invalidResponse = Object.fromEntries( + Object.entries(validResponse).filter(([key]) => key !== deletedProp) + ); + const result = PerformRuleUpgradeResponseBody.safeParse(invalidResponse); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"${deletedProp}: Required"`); + }); }); }); }); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts index 359f20d2c91c8a..c7d3227ef03f32 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts @@ -6,11 +6,62 @@ */ import { z } from '@kbn/zod'; - +import { mapValues } from 'lodash'; import { RuleResponse } from '../../model/rule_schema/rule_schemas.gen'; -import { AggregatedPrebuiltRuleError, PickVersionValues, RuleFieldsToUpgrade } from '../model'; +import { AggregatedPrebuiltRuleError, DiffableAllFields } from '../model'; import { RuleSignatureId, RuleVersion } from '../../model'; +export type PickVersionValues = z.infer; +export const PickVersionValues = z.enum(['BASE', 'CURRENT', 'TARGET', 'MERGED']); +export type PickVersionValuesEnum = typeof PickVersionValues.enum; +export const PickVersionValuesEnum = PickVersionValues.enum; + +/** + * Fields upgradable by the /upgrade/_perform endpoint. + * Specific fields are omitted because they are not upgradeable, and + * handled under the hood by endpoint logic. + * See: https://github.com/elastic/kibana/issues/186544 + */ +export type DiffableUpgradableFields = z.infer; +export const DiffableUpgradableFields = DiffableAllFields.omit({ + type: true, + rule_id: true, + version: true, + author: true, + license: true, +}); + +export type FieldUpgradeSpecifier = z.infer< + ReturnType>> +>; +const fieldUpgradeSpecifier = (fieldSchema: T) => + z.discriminatedUnion('pick_version', [ + z + .object({ + pick_version: PickVersionValues, + }) + .strict(), + z + .object({ + pick_version: z.literal('RESOLVED'), + resolved_value: fieldSchema, + }) + .strict(), + ]); + +type FieldUpgradeSpecifiers = { + [Field in keyof TFields]?: FieldUpgradeSpecifier; +}; + +export type RuleFieldsToUpgrade = FieldUpgradeSpecifiers; +export const RuleFieldsToUpgrade = z + .object( + mapValues(DiffableUpgradableFields.shape, (fieldSchema) => { + return fieldUpgradeSpecifier(fieldSchema).optional(); + }) + ) + .strict(); + export type RuleUpgradeSpecifier = z.infer; export const RuleUpgradeSpecifier = z.object({ rule_id: RuleSignatureId, From 11b5857b1351a788030f096ab0d6d5e92f1971d4 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 10 Sep 2024 18:25:02 -0300 Subject: [PATCH 31/33] Add comment --- .../prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts index 23fca59e8c8649..d0a4aa12533e00 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts @@ -215,6 +215,9 @@ export const DiffableFieldsByTypeUnion = z.discriminatedUnion('type', [ export type DiffableRule = z.infer; export const DiffableRule = z.intersection(DiffableCommonFields, DiffableFieldsByTypeUnion); +/** + * Union of all possible rule types + */ export type DiffableRuleTypes = z.infer; export const DiffableRuleTypes = z.union([ DiffableCustomQueryFields.shape.type, From 5b3e6c150750746cc05c018397fdeac36b2d12b7 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 11 Sep 2024 10:18:20 -0300 Subject: [PATCH 32/33] Export internal types --- .../templates/zod_operation_schema.handlebars | 2 +- .../model/rule_schema/rule_schemas.gen.ts | 14 +++++++------- .../bulk_actions/bulk_actions_route.gen.ts | 2 +- .../model/rule_assets/prebuilt_rule_asset.test.ts | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/kbn-openapi-generator/src/template_service/templates/zod_operation_schema.handlebars b/packages/kbn-openapi-generator/src/template_service/templates/zod_operation_schema.handlebars index 80ca766585c03c..759b0d9294b8a2 100644 --- a/packages/kbn-openapi-generator/src/template_service/templates/zod_operation_schema.handlebars +++ b/packages/kbn-openapi-generator/src/template_service/templates/zod_operation_schema.handlebars @@ -34,7 +34,7 @@ export const {{@key}}: z.ZodType<{{@key}}, ZodTypeDef, {{@key}}Input> = {{> zod_ {{#if (shouldCastExplicitly this)}} {{!-- We need this temporary type to infer from it below, but in the end we want to export as a casted {{@key}} type --}} {{!-- error TS7056: The inferred type of this node exceeds the maximum length the compiler will serialize. An explicit type annotation is needed. --}} -const {{@key}}Internal = {{> zod_schema_item}}; +export const {{@key}}Internal = {{> zod_schema_item}}; export type {{@key}} = z.infer; export const {{@key}} = {{@key}}Internal as z.ZodType<{{@key}}>; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index c207f86639b73a..2d3dbcd3f436ff 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -597,7 +597,7 @@ export const EsqlRuleUpdateProps = SharedUpdateProps.merge(EsqlRuleCreateFields) export type EsqlRulePatchProps = z.infer; export const EsqlRulePatchProps = SharedPatchProps.merge(EsqlRulePatchFields.partial()); -const TypeSpecificCreatePropsInternal = z.discriminatedUnion('type', [ +export const TypeSpecificCreatePropsInternal = z.discriminatedUnion('type', [ EqlRuleCreateFields, QueryRuleCreateFields, SavedQueryRuleCreateFields, @@ -612,7 +612,7 @@ export type TypeSpecificCreateProps = z.infer; -const TypeSpecificPatchPropsInternal = z.union([ +export const TypeSpecificPatchPropsInternal = z.union([ EqlRulePatchFields, QueryRulePatchFields, SavedQueryRulePatchFields, @@ -627,7 +627,7 @@ export type TypeSpecificPatchProps = z.infer; -const TypeSpecificResponseInternal = z.discriminatedUnion('type', [ +export const TypeSpecificResponseInternal = z.discriminatedUnion('type', [ EqlRuleResponseFields, QueryRuleResponseFields, SavedQueryRuleResponseFields, @@ -641,7 +641,7 @@ const TypeSpecificResponseInternal = z.discriminatedUnion('type', [ export type TypeSpecificResponse = z.infer; export const TypeSpecificResponse = TypeSpecificResponseInternal as z.ZodType; -const RuleCreatePropsInternal = z.discriminatedUnion('type', [ +export const RuleCreatePropsInternal = z.discriminatedUnion('type', [ EqlRuleCreateProps, QueryRuleCreateProps, SavedQueryRuleCreateProps, @@ -655,7 +655,7 @@ const RuleCreatePropsInternal = z.discriminatedUnion('type', [ export type RuleCreateProps = z.infer; export const RuleCreateProps = RuleCreatePropsInternal as z.ZodType; -const RuleUpdatePropsInternal = z.discriminatedUnion('type', [ +export const RuleUpdatePropsInternal = z.discriminatedUnion('type', [ EqlRuleUpdateProps, QueryRuleUpdateProps, SavedQueryRuleUpdateProps, @@ -669,7 +669,7 @@ const RuleUpdatePropsInternal = z.discriminatedUnion('type', [ export type RuleUpdateProps = z.infer; export const RuleUpdateProps = RuleUpdatePropsInternal as z.ZodType; -const RulePatchPropsInternal = z.union([ +export const RulePatchPropsInternal = z.union([ EqlRulePatchProps, QueryRulePatchProps, SavedQueryRulePatchProps, @@ -683,7 +683,7 @@ const RulePatchPropsInternal = z.union([ export type RulePatchProps = z.infer; export const RulePatchProps = RulePatchPropsInternal as z.ZodType; -const RuleResponseInternal = z.discriminatedUnion('type', [ +export const RuleResponseInternal = z.discriminatedUnion('type', [ EqlRule, QueryRule, SavedQueryRule, diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/bulk_actions/bulk_actions_route.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/bulk_actions/bulk_actions_route.gen.ts index 52a52442df42ff..0f5b191a0fcd93 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/bulk_actions/bulk_actions_route.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/bulk_actions/bulk_actions_route.gen.ts @@ -284,7 +284,7 @@ export const BulkActionEditPayloadTimeline = z.object({ }), }); -const BulkActionEditPayloadInternal = z.union([ +export const BulkActionEditPayloadInternal = z.union([ BulkActionEditPayloadTags, BulkActionEditPayloadIndexPatterns, BulkActionEditPayloadInvestigationFields, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts index 9c176c88abdafd..ee028cfc7f2035 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.test.ts @@ -9,13 +9,13 @@ import { expectParseError, expectParseSuccess, stringifyZodError } from '@kbn/zo import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; import { PrebuiltRuleAsset, TypeSpecificFields } from './prebuilt_rule_asset'; import { getPrebuiltRuleMock, getPrebuiltThreatMatchRuleMock } from './prebuilt_rule_asset.mock'; -import { TypeSpecificCreateProps } from '../../../../../../common/api/detection_engine'; +import { TypeSpecificCreatePropsInternal } from '../../../../../../common/api/detection_engine'; describe('Prebuilt rule asset schema', () => { it('can be of all rule types that are supported', () => { // Check that the discriminated union TypeSpecificFields, which is used to create // the PrebuiltRuleAsset schema, contains all the rule types that are supported. - const createPropsTypes = TypeSpecificCreateProps.options.map( + const createPropsTypes = TypeSpecificCreatePropsInternal.options.map( (option) => option.shape.type.value ); const fieldsTypes = TypeSpecificFields.options.map((option) => option.shape.type.value); From d5c0618c2f98cce4a98c2362bd7069d3584b5443 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:12:29 +0000 Subject: [PATCH 33/33] [CI] Auto-commit changed files from 'yarn openapi:generate' --- .../api/model/exception_list_item_entry.gen.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-securitysolution-exceptions-common/api/model/exception_list_item_entry.gen.ts b/packages/kbn-securitysolution-exceptions-common/api/model/exception_list_item_entry.gen.ts index 6b6a76defde58f..0b7f0233ba429d 100644 --- a/packages/kbn-securitysolution-exceptions-common/api/model/exception_list_item_entry.gen.ts +++ b/packages/kbn-securitysolution-exceptions-common/api/model/exception_list_item_entry.gen.ts @@ -60,7 +60,7 @@ export const ExceptionListItemEntryExists = z.object({ operator: ExceptionListItemEntryOperator, }); -const ExceptionListItemEntryNestedEntryItemInternal = z.union([ +export const ExceptionListItemEntryNestedEntryItemInternal = z.union([ ExceptionListItemEntryMatch, ExceptionListItemEntryMatchAny, ExceptionListItemEntryExists, @@ -89,7 +89,7 @@ export const ExceptionListItemEntryMatchWildcard = z.object({ operator: ExceptionListItemEntryOperator, }); -const ExceptionListItemEntryInternal = z.discriminatedUnion('type', [ +export const ExceptionListItemEntryInternal = z.discriminatedUnion('type', [ ExceptionListItemEntryMatch, ExceptionListItemEntryMatchAny, ExceptionListItemEntryList,