diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index d4551f76ae390f5..3ea849ce4982b55 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -10,6 +10,7 @@ "data", "dataEnhanced", "embeddable", + "eventLog", "features", "taskManager", "inspector", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/ecs_event.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/ecs_event.ts new file mode 100644 index 000000000000000..c5d279fb9e4edea --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/ecs_event.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEvent as IEventLogEvent } from '../../../../../../event_log/server'; + +// https://www.elastic.co/guide/en/ecs/1.9/ecs-guidelines.html +// https://www.elastic.co/guide/en/ecs/1.9/ecs-category-field-values-reference.html +// https://www.elastic.co/guide/en/ecs/1.9/ecs-field-reference.html + +export type IEcsEvent = IEventLogEvent & IEcsAdditionalFields; + +interface IEcsAdditionalFields { + // https://www.elastic.co/guide/en/ecs/1.9/ecs-event.html + event?: { + dataset?: string; + created?: string; + kind?: string; + type?: string[]; + severity?: number; + sequence?: number; + }; + + // https://www.elastic.co/guide/en/ecs/1.9/ecs-log.html + log?: { + logger?: string; + level?: string; + }; + + // https://www.elastic.co/guide/en/ecs/1.9/ecs-rule.html + rule?: { + id?: string; + }; + + // custom fields + kibana?: { + detection_engine?: { + rule_status?: string; + rule_status_severity?: number; + }; + }; +} + +export type EcsEventKey = keyof IEcsEvent; +export type EcsEventBaseKey = '@timestamp' | 'message' | 'tags'; +export type EcsEventObjectKey = Exclude; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/ecs_event_builder.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/ecs_event_builder.ts new file mode 100644 index 000000000000000..24b941822e7565d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/ecs_event_builder.ts @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EcsEventObjectKey, IEcsEvent } from './ecs_event'; +import { RuleExecutionEventLevel, getLevelSeverity } from './rule_execution_event_levels'; +import { RuleExecutionStatus, getStatusSeverity } from './rule_execution_statuses'; + +const EVENT_LOG_PROVIDER = 'detection-engine'; // TODO: "siem", "siem-detection-engine", "security-solution", other? +const EVENT_LOG_NAME = 'rule-execution-log'; // TODO: A more generic rule-log? A separate rule-management (rule-audit) log? + +export class EcsEventBuilder { + private _result: IEcsEvent = {}; + + constructor() { + // TODO: Which version does event_log use? Should it be specified here or inside the event log itself? + this.ecs('1.9.0'); + this.logger(EVENT_LOG_PROVIDER, EVENT_LOG_NAME); + } + + /** + * Sets "@timestamp", message. + * https://www.elastic.co/guide/en/ecs/1.9/ecs-base.html + * @param eventDate When the event happened (not captured or created). Example: new Date(). + * @param eventMessage Example: "Machine learning job is not started". + */ + public baseFields(eventDate: Date, eventMessage: string): EcsEventBuilder { + return this.base({ + '@timestamp': eventDate.toISOString(), + message: eventMessage, + }); + } + + /** + * Sets event.provider, event.dataset, log.logger. + * https://www.elastic.co/guide/en/ecs/1.9/ecs-event.html + * https://www.elastic.co/guide/en/ecs/1.9/ecs-log.html + * @param logProvider 1st-level category (plugin, subsystem). Example: "detection-engine". + * @param logName 2nd-level category (feature, module). Example: "rule-execution-log". + */ + public logger(logProvider: string, logName: string): EcsEventBuilder { + return this.nested('event', { + provider: logProvider, + dataset: `${logProvider}.${logName}`, + }).nested('log', { + logger: `${logProvider}.${logName}`, + }); + } + + /** + * Sets log.level, event.severity. + * https://www.elastic.co/guide/en/ecs/1.9/ecs-event.html + * https://www.elastic.co/guide/en/ecs/1.9/ecs-log.html + * @param eventLevel Mapped to log.level. Example: "info", "error". + */ + public level(eventLevel: RuleExecutionEventLevel): EcsEventBuilder { + return this.nested('event', { + severity: getLevelSeverity(eventLevel), + }).nested('log', { + level: eventLevel, + }); + } + + /** + * Sets categorization fields: event.kind, event.type, event.action. + * https://www.elastic.co/guide/en/ecs/1.9/ecs-category-field-values-reference.html + * https://www.elastic.co/guide/en/ecs/1.9/ecs-event.html + * @param eventAction Actual event type. Example: "status-changed". + */ + public typeChange(eventAction: string): EcsEventBuilder { + return this.nested('event', { + kind: 'event', + type: ['change'], + action: eventAction, + }); + } + + /** + * Sets categorization fields: event.kind, event.type, event.action. + * https://www.elastic.co/guide/en/ecs/1.9/ecs-category-field-values-reference.html + * https://www.elastic.co/guide/en/ecs/1.9/ecs-event.html + * @param eventAction Actual event type. Example: "metric-search-duration-max", "metric-indexing-lookback". + */ + public typeMetric(eventAction: string): EcsEventBuilder { + return this.nested('event', { + kind: 'metric', + type: ['info'], + action: eventAction, + }); + } + + /** + * Sets any of the event.* fields. + * https://www.elastic.co/guide/en/ecs/1.9/ecs-event.html + */ + public event(fields: NonNullable): EcsEventBuilder { + return this.nested('event', fields); + } + + /** + * https://www.elastic.co/guide/en/ecs/1.9/ecs-rule.html + * @param ruleId Dynamic rule id (alert id in the Alerting framework terminology). + * @param spaceId Kibana space id. + */ + public rule(ruleId: string, spaceId?: string): EcsEventBuilder { + const existingSavedObjectRefs = this._result.kibana?.saved_objects ?? []; + const newSavedObjectRefs = existingSavedObjectRefs.concat({ + type: 'alert', + id: ruleId, + namespace: spaceId, + }); + + return this.nested('rule', { + id: ruleId, // TODO: "id" or "uuid"? + }).nested('kibana', { + saved_objects: newSavedObjectRefs, + }); + } + + /** + * Sets custom fields representing rule execution status: + * kibana.detection_engine.{rule_status, rule_status_severity} + * @param status Execution status of the rule. + */ + public ruleStatus(status: RuleExecutionStatus): EcsEventBuilder { + return this.nested('kibana', { + detection_engine: { + rule_status: status, + rule_status_severity: getStatusSeverity(status), + }, + }); + } + + /** + * Sets ecs.version. + * https://www.elastic.co/guide/en/ecs/1.9/ecs-ecs.html + * @param version Example: 1.7.0 + */ + public ecs(version: string): EcsEventBuilder { + return this.nested('ecs', { + version, + }); + } + + /** + * Builds and returns the final ECS event. + */ + public build(): IEcsEvent { + this.event({ + created: new Date().toISOString(), // TODO: del or use eventDate? + }); + return this._result; + } + + private base(fields: IEcsEvent): EcsEventBuilder { + this._result = { ...this._result, ...fields }; + return this; + } + + private nested( + key: K, + fields: V + ): EcsEventBuilder { + this._result[key] = { + ...this._result[key], + ...fields, + }; + return this; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/index.ts new file mode 100644 index 000000000000000..938931b4b34fc38 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './ecs_event'; +export * from './ecs_event_builder'; +export * from './rule_execution_event_levels'; +export * from './rule_execution_statuses'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/rule_execution_event_levels.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/rule_execution_event_levels.ts new file mode 100644 index 000000000000000..210aa26549fbc02 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/rule_execution_event_levels.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// ----------------------------------------------------------------------------- +// Levels + +export const RuleExecutionEventLevel = { + INFO: 'info', + WARNING: 'warning', + ERROR: 'error', +} as const; + +export type RuleExecutionEventLevel = typeof RuleExecutionEventLevel[keyof typeof RuleExecutionEventLevel]; + +// ----------------------------------------------------------------------------- +// Level severities + +type LevelMappingTo = Readonly>; + +const levelSeverityByLevel: LevelMappingTo = Object.freeze({ + [RuleExecutionEventLevel.INFO]: 10, + [RuleExecutionEventLevel.WARNING]: 20, + [RuleExecutionEventLevel.ERROR]: 30, +}); + +export const getLevelSeverity = (level: RuleExecutionEventLevel): number => { + return levelSeverityByLevel[level] ?? 0; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/rule_execution_statuses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/rule_execution_statuses.ts new file mode 100644 index 000000000000000..35ed3441f9a1bb4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/common_model/rule_execution_statuses.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { JobStatus } from '../../../../../common/detection_engine/schemas/common/schemas'; + +export type RuleExecutionStatus = JobStatus; + +type StatusMappingTo = Readonly>; + +const statusSeverityByStatus: StatusMappingTo = Object.freeze({ + succeeded: 0, + 'going to run': 10, + warning: 20, + 'partial failure': 20, + failed: 30, +}); + +export const getStatusSeverity = (status: RuleExecutionStatus): number => { + return statusSeverityByStatus[status] ?? 0; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/dev_tools_events.txt b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/dev_tools_events.txt new file mode 100644 index 000000000000000..99045811bf205d9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/dev_tools_events.txt @@ -0,0 +1,344 @@ +POST /try-ecs-event-log/_doc +{ + "ecs": { + "version": "1.9.0" + }, + "event": { + "provider": "detection-engine", + "dataset": "detection-engine.rule-execution-log", + "severity": 10, + "kind": "metric", + "type": [ + "info" + ], + "action": "metric-execution-gap", + "duration": 600000000000, + "created": "2021-03-09T10:55:54.568Z", + "sequence": 0 + }, + "log": { + "logger": "detection-engine.rule-execution-log", + "level": "info" + }, + "@timestamp": "2021-03-09T10:55:54.567Z", + "message": "Missing read privileges on indices [your-index-1, your-index-2]", + "rule": { + "id": "1234-56789-dfgdfhgfgh-122346567" + }, + "kibana": { + "saved_objects": [ + { + "type": "alert", + "id": "1234-56789-dfgdfhgfgh-122346567", + "namespace": "some-space" + } + ] + } +} + +POST /try-ecs-event-log/_doc +{ + "ecs": { + "version": "1.9.0" + }, + "event": { + "provider": "detection-engine", + "dataset": "detection-engine.rule-execution-log", + "severity": 10, + "kind": "metric", + "type": [ + "info" + ], + "action": "metric-search-duration-min", + "duration": 1250000, + "created": "2021-03-09T10:55:54.568Z", + "sequence": 1 + }, + "log": { + "logger": "detection-engine.rule-execution-log", + "level": "info" + }, + "@timestamp": "2021-03-09T10:55:54.567Z", + "message": "Missing read privileges on indices [your-index-1, your-index-2]", + "rule": { + "id": "1234-56789-dfgdfhgfgh-122346567" + }, + "kibana": { + "saved_objects": [ + { + "type": "alert", + "id": "1234-56789-dfgdfhgfgh-122346567", + "namespace": "some-space" + } + ] + } +} + +POST /try-ecs-event-log/_doc +{ + "ecs": { + "version": "1.9.0" + }, + "event": { + "provider": "detection-engine", + "dataset": "detection-engine.rule-execution-log", + "severity": 10, + "kind": "metric", + "type": [ + "info" + ], + "action": "metric-search-duration-max", + "duration": 3140000, + "created": "2021-03-09T10:55:54.568Z", + "sequence": 2 + }, + "log": { + "logger": "detection-engine.rule-execution-log", + "level": "info" + }, + "@timestamp": "2021-03-09T10:55:54.567Z", + "message": "Missing read privileges on indices [your-index-1, your-index-2]", + "rule": { + "id": "1234-56789-dfgdfhgfgh-122346567" + }, + "kibana": { + "saved_objects": [ + { + "type": "alert", + "id": "1234-56789-dfgdfhgfgh-122346567", + "namespace": "some-space" + } + ] + } +} + +POST /try-ecs-event-log/_doc +{ + "ecs": { + "version": "1.9.0" + }, + "event": { + "provider": "detection-engine", + "dataset": "detection-engine.rule-execution-log", + "severity": 10, + "kind": "metric", + "type": [ + "info" + ], + "action": "metric-search-duration-sum", + "duration": 4390000, + "created": "2021-03-09T10:55:54.568Z", + "sequence": 3 + }, + "log": { + "logger": "detection-engine.rule-execution-log", + "level": "info" + }, + "@timestamp": "2021-03-09T10:55:54.567Z", + "message": "Missing read privileges on indices [your-index-1, your-index-2]", + "rule": { + "id": "1234-56789-dfgdfhgfgh-122346567" + }, + "kibana": { + "saved_objects": [ + { + "type": "alert", + "id": "1234-56789-dfgdfhgfgh-122346567", + "namespace": "some-space" + } + ] + } +} + +POST /try-ecs-event-log/_doc +{ + "ecs": { + "version": "1.9.0" + }, + "event": { + "provider": "detection-engine", + "dataset": "detection-engine.rule-execution-log", + "severity": 10, + "kind": "metric", + "type": [ + "info" + ], + "action": "metric-indexing-duration-min", + "duration": 5450000, + "created": "2021-03-09T10:55:54.568Z", + "sequence": 4 + }, + "log": { + "logger": "detection-engine.rule-execution-log", + "level": "info" + }, + "@timestamp": "2021-03-09T10:55:54.567Z", + "message": "Missing read privileges on indices [your-index-1, your-index-2]", + "rule": { + "id": "1234-56789-dfgdfhgfgh-122346567" + }, + "kibana": { + "saved_objects": [ + { + "type": "alert", + "id": "1234-56789-dfgdfhgfgh-122346567", + "namespace": "some-space" + } + ] + } +} + +POST /try-ecs-event-log/_doc +{ + "ecs": { + "version": "1.9.0" + }, + "event": { + "provider": "detection-engine", + "dataset": "detection-engine.rule-execution-log", + "severity": 10, + "kind": "metric", + "type": [ + "info" + ], + "action": "metric-indexing-duration-max", + "duration": 7620000, + "created": "2021-03-09T10:55:54.568Z", + "sequence": 5 + }, + "log": { + "logger": "detection-engine.rule-execution-log", + "level": "info" + }, + "@timestamp": "2021-03-09T10:55:54.567Z", + "message": "Missing read privileges on indices [your-index-1, your-index-2]", + "rule": { + "id": "1234-56789-dfgdfhgfgh-122346567" + }, + "kibana": { + "saved_objects": [ + { + "type": "alert", + "id": "1234-56789-dfgdfhgfgh-122346567", + "namespace": "some-space" + } + ] + } +} + +POST /try-ecs-event-log/_doc +{ + "ecs": { + "version": "1.9.0" + }, + "event": { + "provider": "detection-engine", + "dataset": "detection-engine.rule-execution-log", + "severity": 10, + "kind": "metric", + "type": [ + "info" + ], + "action": "metric-indexing-duration-sum", + "duration": 13070000, + "created": "2021-03-09T10:55:54.568Z", + "sequence": 6 + }, + "log": { + "logger": "detection-engine.rule-execution-log", + "level": "info" + }, + "@timestamp": "2021-03-09T10:55:54.567Z", + "message": "Missing read privileges on indices [your-index-1, your-index-2]", + "rule": { + "id": "1234-56789-dfgdfhgfgh-122346567" + }, + "kibana": { + "saved_objects": [ + { + "type": "alert", + "id": "1234-56789-dfgdfhgfgh-122346567", + "namespace": "some-space" + } + ] + } +} + +POST /try-ecs-event-log/_doc +{ + "ecs": { + "version": "1.9.0" + }, + "event": { + "provider": "detection-engine", + "dataset": "detection-engine.rule-execution-log", + "severity": 10, + "kind": "metric", + "type": [ + "info" + ], + "action": "metric-indexing-lookback", + "end": "2021-03-09T10:55:54.567Z", + "created": "2021-03-09T10:55:54.568Z", + "sequence": 7 + }, + "log": { + "logger": "detection-engine.rule-execution-log", + "level": "info" + }, + "@timestamp": "2021-03-09T10:55:54.567Z", + "message": "Missing read privileges on indices [your-index-1, your-index-2]", + "rule": { + "id": "1234-56789-dfgdfhgfgh-122346567" + }, + "kibana": { + "saved_objects": [ + { + "type": "alert", + "id": "1234-56789-dfgdfhgfgh-122346567", + "namespace": "some-space" + } + ] + } +} + +POST /try-ecs-event-log/_doc +{ + "ecs": { + "version": "1.9.0" + }, + "event": { + "provider": "detection-engine", + "dataset": "detection-engine.rule-execution-log", + "severity": 20, + "kind": "event", + "type": [ + "change" + ], + "action": "status-changed", + "created": "2021-03-09T10:55:54.568Z", + "sequence": 8 + }, + "log": { + "logger": "detection-engine.rule-execution-log", + "level": "warning" + }, + "@timestamp": "2021-03-09T10:55:54.567Z", + "message": "Missing read privileges on indices [your-index-1, your-index-2]", + "rule": { + "id": "1234-56789-dfgdfhgfgh-122346567" + }, + "kibana": { + "saved_objects": [ + { + "type": "alert", + "id": "1234-56789-dfgdfhgfgh-122346567", + "namespace": "some-space" + } + ], + "detection_engine": { + "rule_status": "warning", + "rule_status_severity": 20 + } + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/dev_tools_index.txt b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/dev_tools_index.txt new file mode 100644 index 000000000000000..75b3b4a95d90fe7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/dev_tools_index.txt @@ -0,0 +1,3659 @@ +DELETE /try-ecs-event-log + +PUT /try-ecs-event-log +{ + "mappings": { + "_meta": { + "version": "1.9.0-dev" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "build": { + "properties": { + "original": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "postal_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "postal_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dll": { + "properties": { + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "signing_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "team_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dns": { + "properties": { + "answers": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "data": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "header_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "op_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "question": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "resolved_ip": { + "type": "ip" + }, + "response_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "stack_trace": { + "doc_values": false, + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "file": { + "properties": { + "accessed": { + "type": "date" + }, + "attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "signing_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "team_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "created": { + "type": "date" + }, + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "directory": { + "ignore_above": 1024, + "type": "keyword" + }, + "drive_letter": { + "ignore_above": 1, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "size": { + "type": "long" + }, + "target_path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "cpu": { + "properties": { + "usage": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "disk": { + "properties": { + "read": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "write": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "postal_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "network": { + "properties": { + "egress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "ingress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "properties": { + "file": { + "properties": { + "line": { + "type": "integer" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "function": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "syslog": { + "properties": { + "facility": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "priority": { + "type": "long" + }, + "severity": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "inner": { + "properties": { + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "observer": { + "properties": { + "egress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "postal_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "package": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "build_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "checksum": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "install_scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "installed": { + "type": "date" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "signing_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "team_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "signing_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "team_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssdeep": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "registry": { + "properties": { + "data": { + "properties": { + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "strings": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hive": { + "ignore_above": 1024, + "type": "keyword" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "related": { + "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "postal_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "postal_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "span": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "subtechnique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "tls": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "supported_ciphers": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "established": { + "type": "boolean" + }, + "next_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "resumed": { + "type": "boolean" + }, + "server": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3s": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "trace": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "transaction": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "changes": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "effective": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + }, + "target": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vulnerability": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "enumeration": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "report_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner": { + "properties": { + "vendor": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "score": { + "properties": { + "base": { + "type": "float" + }, + "environmental": { + "type": "float" + }, + "temporal": { + "type": "float" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kibana": { + "properties" : { + "detection_engine" : { + "properties" : { + "rule_status" : { + "type" : "keyword", + "ignore_above" : 1024 + }, + "rule_status_severity" : { + "type" : "long" + } + } + }, + "saved_objects" : { + "properties" : { + "id" : { + "type" : "keyword", + "ignore_above" : 1024 + }, + "namespace" : { + "type" : "keyword", + "ignore_above" : 1024 + }, + "type" : { + "type" : "keyword", + "ignore_above" : 1024 + } + } + } + } + } + } + }, + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": 10000 + } + }, + "refresh_interval": "5s" + } + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/dev_tools_queries.txt b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/dev_tools_queries.txt new file mode 100644 index 000000000000000..c84ea64985446d2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/dev_tools_queries.txt @@ -0,0 +1,230 @@ +# ----------------------------------------------------------------------------------------------------------- +# Queries + +# Whole execution log of the rule, events sorted from new to old +GET /try-ecs-event-log/_search +{ + "query": { + "bool": { + "filter": [ + { "term": { "kibana.saved_objects.type": "alert" } }, + { "term": { "kibana.saved_objects.namespace": "some-space" } }, + { "term": { "kibana.saved_objects.id": "1234-56789-dfgdfhgfgh-122346567" } } + ] + } + }, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ] +} + +# Rule details page: execution log +# Last 50 events of the rule which are actual events, not metrics +GET /try-ecs-event-log/_search +{ + "size": 50, + "query": { + "bool": { + "filter": [ + { "term": { "kibana.saved_objects.type": "alert" } }, + { "term": { "kibana.saved_objects.namespace": "some-space" } }, + { "term": { "kibana.saved_objects.id": "1234-56789-dfgdfhgfgh-122346567" } }, + { + "term": { "event.kind": "event" } + } + ] + } + }, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["@timestamp", "message", "log.level", "event.severity", "event.action"] +} + +# Rule details page: current execution status of the rule +GET /try-ecs-event-log/_search +{ + "size": 1, + "query": { + "bool": { + "filter": [ + { "term": { "kibana.saved_objects.type": "alert" } }, + { "term": { "kibana.saved_objects.namespace": "some-space" } }, + { "term": { "kibana.saved_objects.id": "1234-56789-dfgdfhgfgh-122346567" } }, + { + "term": { "event.action": "status-changed" } + } + ] + } + }, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ] +} + +# Rules management table: "Last run", "Last response" columns +# N current statuses of N rules. N=2 for simplicity +GET /try-ecs-event-log/_search +{ + "size": 0, + "query": { + "bool": { + "filter": [ + { "term": { "kibana.saved_objects.type": "alert" } }, + { "term": { "kibana.saved_objects.namespace": "some-space" } }, + { + "terms": { + "kibana.saved_objects.id": [ + "1234-56789-dfgdfhgfgh-122346567", + "1234-56789-dfgdfhgfgh-122346568" + ] + } + }, + { + "term": { "event.action": "status-changed" } + } + ] + } + }, + "aggs": { + "rules": { + "terms": { + "field": "kibana.saved_objects.id", + "size": 2 + }, + "aggs": { + "current_status": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["@timestamp", "kibana.detection_engine"] + } + } + } + } + } +} + +# Rules monitoring table: "Last run", "Last response", "Last Gap (if any)", "Query Time (ms)", "Indexing Time (ms)" columns +# N current statuses, last gaps, last max search durations, last max indexing durations. N=2 for simplicity +GET /try-ecs-event-log/_search +{ + "size": 0, + "query": { + "bool": { + "filter": [ + { "term": { "kibana.saved_objects.type": "alert" } }, + { "term": { "kibana.saved_objects.namespace": "some-space" } }, + { + "terms": { + "kibana.saved_objects.id": [ + "1234-56789-dfgdfhgfgh-122346567", + "1234-56789-dfgdfhgfgh-122346568" + ] + } + } + ] + } + }, + "aggs": { + "rules": { + "terms": { + "field": "kibana.saved_objects.id", + "size": 2 + }, + "aggs": { + "events_status_changed": { + "filter": { + "term": { "event.action": "status-changed" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["@timestamp", "kibana.detection_engine"] + } + } + } + }, + "metrics_execution_gap": { + "filter": { + "term": { "event.action": "metric-execution-gap" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["event.duration"] + } + } + } + }, + "metrics_search_duration_max": { + "filter": { + "term": { "event.action": "metric-search-duration-max" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["event.duration"] + } + } + } + }, + "metrics_indexing_duration_max": { + "filter": { + "term": { "event.action": "metric-indexing-duration-max" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["event.duration"] + } + } + } + }, + "metrics_indexing_lookback": { + "filter": { + "term": { "event.action": "metric-indexing-lookback" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["event.end"] + } + } + } + } + } + } + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/generate_events.js b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/generate_events.js new file mode 100644 index 000000000000000..64af626bd6f3d2d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/generate_events.js @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// NOTE: This is a temporary script for testing the idea of using Event Log +// for storing rule execution events and metrics. + +// TODO: delete it. + +require('../../../../../../../src/setup_node_env'); +require('./generate_events.ts'); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/generate_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/generate_events.ts new file mode 100644 index 000000000000000..d7179e137192e13 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/generate_events.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// NOTE: This is a temporary script for testing the idea of using Event Log +// for storing rule execution events and metrics. + +// TODO: delete it. + +import fs from 'fs'; +import { RuleExecutionEventLevel } from './common_model'; +import { StatusChangedEvent } from './write_model/status_changed_event'; + +const generate = () => { + const event: StatusChangedEvent = { + ruleId: '1234-56789-dfgdfhgfgh-122346567', + spaceId: 'some-space', + eventDate: new Date(), + eventLevel: RuleExecutionEventLevel.WARNING, + eventMessage: 'Missing read privileges on indices [your-index-1, your-index-2]', + eventType: 'status-changed', + eventPayload: { + status: 'warning', + executionGap: 10 * 60 * 1000, // 10 min in ms + searchDurations: [1.25, 3.14], // in milliseconds + indexingDurations: [5.45, 7.62], // in milliseconds + lastLookBackDate: new Date().toISOString(), + }, + }; + + const ecsEvents = StatusChangedEvent.toEcsEvents(event); + const fileContent = toDevToolsText(ecsEvents); + writeFile('dev_tools_events.txt', fileContent); +}; + +const toDevToolsText = (ecsEvents: unknown[]): string => { + const items = ecsEvents.map(e => { + const json = JSON.stringify(e, null, 2); + return `POST /try-ecs-event-log/_doc\n${json}\n`; + }); + return items.join('\n'); +}; + +const writeFile = (filename: string, content: string): void => { + const filepath = `${__dirname}/${filename}`; + fs.writeFileSync(filepath, content); +}; + +generate(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/proposal.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/proposal.md new file mode 100644 index 000000000000000..28bc8b03343f5b9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/proposal.md @@ -0,0 +1,389 @@ +## Proposal: building Rule Execution Log on top of Event Log and ECS + +### Overview + +We're going to get rid of storing rule execution statuses and additional data in custom "sidecar" saved objects. Those are objects stored in `.kibana` index and having type = `siem-detection-engine-rule-status`. The corresponding SO attributes are: + +```ts +export interface IRuleStatusSOAttributes extends Record { + alertId: string; // created alert id. + statusDate: StatusDate; + lastFailureAt: LastFailureAt | null | undefined; + lastFailureMessage: LastFailureMessage | null | undefined; + lastSuccessAt: LastSuccessAt | null | undefined; + lastSuccessMessage: LastSuccessMessage | null | undefined; + status: JobStatus | null | undefined; + lastLookBackDate: string | null | undefined; + gap: string | null | undefined; + bulkCreateTimeDurations: string[] | null | undefined; + searchAfterTimeDurations: string[] | null | undefined; +} +``` + +We're going to start using the Event Log ([`event_log`](https://github.com/elastic/kibana/tree/master/x-pack/plugins/event_log) plugin built by the Alerting team). + +For more context, please read https://github.com/elastic/kibana/issues/91265#issuecomment-781363642 + +Regarding software design: + +- We will be storing rule status updates and all the additional metrics data (search durations, gaps, etc) in event log during the rule execution. +- We will query event log on the Rule Details page to fetch the current execution status and the log itself. Eventually the idea is to have an "Execution Log" tab instead of "Failure History" and fetch let's say last 50 events of different types. We will be able to log generic info messages, generic errors/exceptions, add special types of events. +- We will query event log on the Rules Management page to render our rule management and monitoring tables. We will need to be able to aggregate log events to fetch M*N metrics of N rules in a single of a few queries (where M is the number of metrics we show, e.g. "Last gap", "Indexing time", etc). +- We will have 3 data models representing the "rule execution log": + - an underlying ECS model for events stored in Event Log + - a write model where we will have our rule execution events in terms of Detection Engine, mapping of these events to their ECS form, services to write them (something like Rule Execution Logger) + - a read model where we will have our rule execution events in terms of app user (Rule Details page), aggregation results (Rules Monitoring table), services to read them +- Services will wrap the underlying Event Log. +- In the beginning, Rule Execution Logger (the writer) will be encapsulated in the `RuleStatusService`, and the only execution event we will have is a Status Changed event. That means the Detection Engine will stay mostly untouched. The idea is to reduce the amount of refactoring. The disadvantage is that time-wise events in the log might be written not exactly when they happen. We will probably address that later, when implementing enhancements in the UI ("Execution Log" tab), so that all the logged events can have precise timestamps and show when exactly any of the events happened. Also, we'll be able to split a single fat Status Changed event into separate dedicated events, and use the Status Changed event only for cases when the status actually needs to be changed. +- Each Status Changed event will be mapped to a series of ECS events. +- In our API route handlers we will replace requests to `siem-detection-engine-rule-status` saved objects with requests to the new rule execution log service (read model). +- Our API schema will not change, except later in order to show the full "Execution Log" instead of "Failure History" we might want to introduce a non-breaking change. + +### What to review + +Please take a look at the code submitted in this PR: + +- `rule_execution_log/common_model` contains the ECS model of events for Event Log, common types and constants, a builder for creating ECS events for this particular log +- `rule_execution_log/write_model` contains the definition of rule execution events, `StatusChangedEvent` in particular, and mapping of it to a series of ECS events + +Please check the ECS events we're going to store and queries we will need to execute. You can play with it in Kibana Dev Tools: + +1. Use `dev_tools_index.txt` to create a test index. This file contains Elasticsearch index mappings for ECS version `1.9.0` (https://github.com/elastic/ecs/blob/1.9/generated/elasticsearch/7/template.json) adjusted with a custom mapping of `kibana.detection_engine` field. +2. Index events using `dev_tools_events.txt`. It contains ECS events as text that you can copy-paste into Kibana Dev Tools. +3. Run queries from `dev_tools_queries.txt`. It contains queries to event log index we will need to be able to execute. +4. Optionally, play with data in `generate_events.ts`. It transforms `StatusChangedEvent` to ECS events and writes them to `dev_tools_events.txt`. Run it with `node x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/generate_events.js`. + +### What's missing in the current Event Log API + +In short: + +- extended support for ECS - more fields and field sets, in particular: + - standard `event.*`, `log.*`, `rule.*` + - custom `kibana.detection_engine.*` +- aggregation queries +- sorting by multiple fields +- nice-to-have: custom ES DSL filters (`term`, `terms`) instead of string KQL filter +- limiting source fields to return: `"_source": ["@timestamp", "message", "log.level", "event.action"]` + +#### Extended support for ECS + +In this proposal, I'm designing our rule execution log events using a few additional fields. These fields are not currently supported by Event Log. + +```ts +interface IEcsAdditionalFields { + // https://www.elastic.co/guide/en/ecs/1.9/ecs-event.html + event?: { + dataset?: string; + created?: string; + kind?: string; + type?: string[]; + severity?: number; + sequence?: number; + }; + + // https://www.elastic.co/guide/en/ecs/1.9/ecs-log.html + log?: { + logger?: string; + level?: string; + }; + + // https://www.elastic.co/guide/en/ecs/1.9/ecs-rule.html + rule?: { + id?: string; + }; + + // custom fields + kibana?: { + detection_engine?: { + rule_status?: string; + rule_status_severity?: number; + }; + }; +} +``` + +My suggestion would be to add support for all the standard `event.*`, `log.*`, `rule.*` fields (at least these field sets), as well as for our custom `kibana.detection_engine.*` fields. If it's easy to add support for the whole ECS, I'd say let's do it. + +It's not super clear to me how exactly we're gonna specify `kibana.detection_engine` both in terms of TypeScript API and ES mapping. Should it be delegated to Event Log's clients with some registration API rather than hardcoded in Event Log itself? + +#### Aggregation queries + +We need to be able to execute aggregation queries with arbitrary aggs, like for example specified in this example query: + +```json +{ + "size": 0, + "query": { + "bool": { + "filter": [ + { "term": { "kibana.saved_objects.type": "alert" } }, + { "term": { "kibana.saved_objects.namespace": "some-space" } }, + { + "terms": { + "kibana.saved_objects.id": [ + "1234-56789-dfgdfhgfgh-122346567", + "1234-56789-dfgdfhgfgh-122346568" + ] + } + } + ] + } + }, + "aggs": { + "rules": { + "terms": { + "field": "kibana.saved_objects.id", + "size": 2 + }, + "aggs": { + "events_status_changed": { + "filter": { + "term": { "event.action": "status-changed" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["@timestamp", "kibana.detection_engine"] + } + } + } + }, + "metrics_execution_gap": { + "filter": { + "term": { "event.action": "metric-execution-gap" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["event.duration"] + } + } + } + }, + "metrics_search_duration_max": { + "filter": { + "term": { "event.action": "metric-search-duration-max" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["event.duration"] + } + } + } + }, + "metrics_indexing_duration_max": { + "filter": { + "term": { "event.action": "metric-indexing-duration-max" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["event.duration"] + } + } + } + }, + "metrics_indexing_lookback": { + "filter": { + "term": { "event.action": "metric-indexing-lookback" } + }, + "aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["event.end"] + } + } + } + } + } + } + } +} +``` + +We need a freedom to pack multiple aggs in a single query or to split it into several queries. + +We'd also like to be able to combine aggs with top-level filters, pagination, sorting and other options if we need so. See pre-filtering with `"term": { "event.action": "status-changed" }`: + +```json +{ + "size": 0, + "query": { + "bool": { + "filter": [ + { "term": { "kibana.saved_objects.type": "alert" } }, + { "term": { "kibana.saved_objects.namespace": "some-space" } }, + { + "terms": { + "kibana.saved_objects.id": [ + "1234-56789-dfgdfhgfgh-122346567", + "1234-56789-dfgdfhgfgh-122346568" + ] + } + }, + { + "term": { "event.action": "status-changed" } + } + ] + } + }, + "aggs": { + "rules": { + "terms": { + "field": "kibana.saved_objects.id", + "size": 2 + }, + "aggs": { + "current_status": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["@timestamp", "kibana.detection_engine"] + } + } + } + } + } +} +``` + +#### Sorting by multiple fields + +The current API allows to specify only a single sort field and restricts the fields that can be used to sort events. + +```ts + sort_field: schema.oneOf( + [ + schema.literal('@timestamp'), + schema.literal('event.start'), + schema.literal('event.end'), + schema.literal('event.provider'), + schema.literal('event.duration'), + schema.literal('event.action'), + schema.literal('message'), + ], + { + defaultValue: '@timestamp', + } + ), + sort_order: schema.oneOf([schema.literal('asc'), schema.literal('desc')], { + defaultValue: 'asc', + }), +``` + +We need to be able to sort by multiple fields both in normal queries and within an aggregation scope. + +```json +"sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ] +``` + +In the future we will need to sort by arbitrary fields in order to implement, for example, sorting by the current rule execution status (`kibana.detection_engine.rule_status_severity`) in our rules monitoring table. + +#### Custom ES DSL filters + +Would be nice to have an option to specify custom filters with ES DSL, like for example here: + +```json +"query": { + "bool": { + "filter": [ + { "term": { "kibana.saved_objects.type": "alert" } }, + { "term": { "kibana.saved_objects.namespace": "some-space" } }, + { + "terms": { + "kibana.saved_objects.id": [ + "1234-56789-dfgdfhgfgh-122346567", + "1234-56789-dfgdfhgfgh-122346568" + ] + } + }, + // This one would be a custom term filter: + { + "term": { "event.action": "status-changed" } + } + ] + } +}, +``` + +With the current API it's possible to specify a string KQL filter. It's definitely a great option to have, but in cases like this one, where we exactly know the resulting query to build, we'd prefer to save some server-side CPU cycles by not parsing KQL. + +#### Limiting source fields to return + +Sometimes we don't need to fetch the full event document with all its ECS fields. We need to be able to restrict the source fields as we like, both within aggs and the query scope: + +```json +"aggs": { + "last_item": { + "top_hits": { + "size": 1, + "sort": [ + { "@timestamp": { "order": "desc" } }, + { "event.sequence": { "order": "desc" } } + ], + "_source": ["event.duration"] + } + } +} +``` + +```json +{ + "query": {...}, + "sort": [...], + "_source": ["@timestamp", "message", "log.level", "event.severity", "event.action"] +} +``` + +### More questions about using Event Log + +ECS version: + +- Which version of ECS does the Event Log support? Is it strictly `1.6.0` or several versions can be supported at the same time for different clients? +- Who/when/how will upgrade the version(s) supported by Event Log? +- Should (or can) clients specify `ecs.version` when logging events? + +ECS `event.provider`, `event.dataset`, `log.logger`: + +- With the RAC project going on, our team will be extracting the Detection Engine into a separate plugin. There's a feeling that this plugin might grow over time accumulating all the common functionality. +- I felt like it would be nice to be able to specify `event.provider: 'detection-engine'`, but have a way to have multiple logs within the provider. +- In this proposal I'm using: + - `event.dataset: 'detection-engine.rule-execution-log'` + - `log.logger: 'detection-engine.rule-execution-log'` +- Similarly, we'd need to be able to query only `detection-engine.rule-execution-log` where needed, and not the full log. +- In this model, each logger/dataset would specify its own set of `event.action`s. +- Who should set `event.provider`, `event.dataset`, `log.logger` fields for an event - a client of `IEventLogger` or the logger itself? + +Could we theoretically use an instance of `ElasticsearchClient` to query the event log index, rather than using `IEventLogClient`? My concern is: while writing to this index and managing the index should be done via dedicated APIs, adding read APIs can become a leaky abstraction, which reminds me of the issues we currently have with saved objects APIs like lack of aggs support etc. On the one hand, for us, application developers, it's hard to predict what read API we will need in the future; would be nice to have freedom to use any that Elasticsearch provides. On the other hand, adding support for all the APIs can be difficult and lead to maintenance issues. Maybe there's a way to provide a decorator on top of `ElasticsearchClient` that would preserve its API or a thin adapter that would just slightly change it? diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/read_model/status_as_is.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/read_model/status_as_is.ts new file mode 100644 index 000000000000000..711a72c0bf69fc2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/read_model/status_as_is.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + JobStatus, + LastFailureAt, + LastFailureMessage, + LastSuccessAt, + LastSuccessMessage, + StatusDate, +} from '../../../../../common/detection_engine/schemas/common/schemas'; + +// ----------------------------------------------------------------------------- +// AS IS: Existing custom status SO + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export interface IRuleStatusSOAttributes extends Record { + alertId: string; // created alert id. + statusDate: StatusDate; + lastFailureAt: LastFailureAt | null | undefined; + lastFailureMessage: LastFailureMessage | null | undefined; + lastSuccessAt: LastSuccessAt | null | undefined; + lastSuccessMessage: LastSuccessMessage | null | undefined; + status: JobStatus | null | undefined; + lastLookBackDate: string | null | undefined; + gap: string | null | undefined; + bulkCreateTimeDurations: string[] | null | undefined; + searchAfterTimeDurations: string[] | null | undefined; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/write_model/rule_execution_event.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/write_model/rule_execution_event.ts new file mode 100644 index 000000000000000..207bef911afa1ce --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/write_model/rule_execution_event.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RuleExecutionEventLevel } from '../common_model'; + +// ----------------------------------------------------------------------------- +// Types of rule execution events in the write model + +export const RuleExecutionEventType = { + GENERIC: 'generic', + STATUS_CHANGED: 'status-changed', +} as const; + +export type RuleExecutionEventType = typeof RuleExecutionEventType[keyof typeof RuleExecutionEventType]; + +// ----------------------------------------------------------------------------- +// Base interfaces of rule execution events + +export interface RuleExecutionEvent { + ruleId: string; // alert id in the Alerting framework terminology + spaceId: string; + eventDate: Date; + eventLevel: RuleExecutionEventLevel; + eventMessage: string; + eventType: TType; + eventPayload: TPayload; +} + +export type AnyRuleExecutionEvent = RuleExecutionEvent; + +export type GenericRuleExecutionEvent = RuleExecutionEvent<'generic', TPayload>; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/write_model/status_changed_event.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/write_model/status_changed_event.ts new file mode 100644 index 000000000000000..c0cde41fd17a738 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/write_model/status_changed_event.ts @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + IEcsEvent, + EcsEventBuilder, + RuleExecutionEventLevel, + RuleExecutionStatus, +} from '../common_model'; + +import { AnyRuleExecutionEvent, RuleExecutionEvent } from './rule_execution_event'; +import { maxOf, minOf, sumOf, toNano } from './utils'; + +// ----------------------------------------------------------------------------- +// Interface of the event + +export type StatusChangedEvent = RuleExecutionEvent<'status-changed', StatusPayload>; + +export interface StatusPayload { + status: RuleExecutionStatus; + + // Additional optional payload + executionGap?: number; // in milliseconds + searchDurations?: number[]; // in milliseconds + indexingDurations?: number[]; // in milliseconds + lastLookBackDate?: string; // TODO: do we need it? not shown in the UI (commented out) +} + +export const StatusChangedEvent = { + toEcsEvents: (event: StatusChangedEvent): IEcsEvent[] => { + const ecsEvents = mapStatusChangedEventToEcsEvents(event); + + // TODO: move to a service implementing the rule execution log + // All generated ecsEvents have the same @timestamp, so we need an additional field for deterministic ordering. + // https://www.elastic.co/guide/en/ecs/1.8/ecs-event.html#field-event-sequence + let seq = 0; + ecsEvents.forEach((e) => { + e.event = e.event ?? {}; + e.event.sequence = seq++; + }); + + return ecsEvents; + }, +}; + +// ----------------------------------------------------------------------------- +// Mapping of the event to ECS events + +const mapStatusChangedEventToEcsEvents = (event: StatusChangedEvent): IEcsEvent[] => { + return [ + ...ecsExecutionGapMetric(event), + ...ecsSearchDurationMetrics(event), + ...ecsIndexingDurationMetrics(event), + ...ecsIndexingLookbackMetric(event), + ecsStatusChangedEvent(event), + ]; +}; + +/** + * Execution gap metric. + * -> event.action='metric-execution-gap', event.kind='metric', event.duration=executionGap + */ +const ecsExecutionGapMetric = (event: StatusChangedEvent): IEcsEvent[] => { + const { executionGap } = event.eventPayload; + const result = []; + + if (executionGap !== undefined) { + result.push( + ecsMetric(event, 'metric-execution-gap') + .event({ duration: toNano(executionGap) }) + .build() + ); + } + + return result; +}; + +/** + * Search duration metrics. + * -> event.action='metric-search-duration-min', event.kind='metric', event.duration=MIN(searchDurations) + * -> event.action='metric-search-duration-max', event.kind='metric', event.duration=MAX(searchDurations) + * -> event.action='metric-search-duration-sum', event.kind='metric', event.duration=SUM(searchDurations) + */ +const ecsSearchDurationMetrics = (event: StatusChangedEvent): IEcsEvent[] => { + const { searchDurations } = event.eventPayload; + const result = []; + + if (searchDurations && searchDurations.length > 0) { + const min = toNano(minOf(searchDurations)); + const max = toNano(maxOf(searchDurations)); + const sum = toNano(sumOf(searchDurations)); + + result.push( + ecsMetric(event, 'metric-search-duration-min').event({ duration: min }).build(), + ecsMetric(event, 'metric-search-duration-max').event({ duration: max }).build(), + ecsMetric(event, 'metric-search-duration-sum').event({ duration: sum }).build() + ); + } + + return result; +}; + +/** + * Indexing duration metrics. + * -> event.action='metric-indexing-duration-min', event.kind='metric', event.duration=MIN(indexingDurations) + * -> event.action='metric-indexing-duration-max', event.kind='metric', event.duration=MAX(indexingDurations) + * -> event.action='metric-indexing-duration-sum', event.kind='metric', event.duration=SUM(indexingDurations) + */ +const ecsIndexingDurationMetrics = (event: StatusChangedEvent): IEcsEvent[] => { + const { indexingDurations } = event.eventPayload; + const result = []; + + if (indexingDurations && indexingDurations.length > 0) { + const min = toNano(minOf(indexingDurations)); + const max = toNano(maxOf(indexingDurations)); + const sum = toNano(sumOf(indexingDurations)); + + result.push( + ecsMetric(event, 'metric-indexing-duration-min').event({ duration: min }).build(), + ecsMetric(event, 'metric-indexing-duration-max').event({ duration: max }).build(), + ecsMetric(event, 'metric-indexing-duration-sum').event({ duration: sum }).build() + ); + } + + return result; +}; + +/** + * Indexing lookback metric. + * -> event.action='metric-indexing-lookback', event.kind='metric', event.end=lastLookBackDate + */ +const ecsIndexingLookbackMetric = (event: StatusChangedEvent): IEcsEvent[] => { + const { lastLookBackDate } = event.eventPayload; + const result = []; + + if (lastLookBackDate) { + result.push( + ecsMetric(event, 'metric-indexing-lookback').event({ end: lastLookBackDate }).build() + ); + } + + return result; +}; + +/** + * Status changed event. + * -> event.action='status-changed', event.kind='event', kibana.detection_engine.{rule_status, rule_status_severity} + */ +const ecsStatusChangedEvent = (event: StatusChangedEvent): IEcsEvent => { + const { eventType, eventPayload } = event; + const { status } = eventPayload; + + return ecsEvent(event).typeChange(eventType).ruleStatus(status).build(); +}; + +const ecsEvent = (event: AnyRuleExecutionEvent): EcsEventBuilder => { + const { ruleId, spaceId, eventDate, eventLevel, eventMessage } = event; + + return new EcsEventBuilder() + .baseFields(eventDate, eventMessage) + .level(eventLevel) + .rule(ruleId, spaceId); +}; + +const ecsMetric = (event: AnyRuleExecutionEvent, eventAction: string): EcsEventBuilder => { + return ecsEvent(event).typeMetric(eventAction).level(RuleExecutionEventLevel.INFO); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/write_model/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/write_model/utils.ts new file mode 100644 index 000000000000000..7570da9aa975737 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/write_model/utils.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Converts milliseconds to nanoseconds. + * NOTE: Math.round handles floating point precision errors. In JS 1.25 + 3.14 = 4.390000000000001. + * Having float nanoseconds doesn't make sense, also ECS event.duration is of type long. + * @param ms Number of milliseconds. + */ +export const toNano = (ms: number): number => Math.round(ms * 1000000); + +// NOTE: It's safer to use reduce rather than Math.max(...array). The latter won't handle large input. +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max +export const maxOf = (array: number[]): number => array.reduce((a, b) => Math.max(a, b)); +export const minOf = (array: number[]): number => array.reduce((a, b) => Math.min(a, b)); +export const sumOf = (array: number[]): number => array.reduce((a, b) => a + b, 0); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 65efd25c9fba2d3..1542c4be0cbc195 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -160,6 +160,9 @@ export const signalRulesAlertType = ({ alertId, ruleStatusClient, }); + + await ruleStatusService.goingToRun(); + const savedObject = await services.savedObjectsClient.get( 'alert', alertId @@ -187,7 +190,6 @@ export const signalRulesAlertType = ({ logger.debug(buildRuleMessage('[+] Starting Signal Rule execution')); logger.debug(buildRuleMessage(`interval: ${interval}`)); let wroteWarningStatus = false; - await ruleStatusService.goingToRun(); // check if rule has permissions to access given index pattern // move this collection of lines into a function in utils @@ -255,6 +257,7 @@ export const signalRulesAlertType = ({ hasError = true; await ruleStatusService.error(gapMessage, { gap: gapString }); } + try { const { listClient, exceptionsClient } = getListsClient({ services, @@ -300,6 +303,8 @@ export const signalRulesAlertType = ({ logger.warn(errorMessage); hasError = true; await ruleStatusService.error(errorMessage); + + // TODO: return? } const anomalyResults = await findMlSignals({ @@ -687,6 +692,9 @@ export const signalRulesAlertType = ({ `[+] Finished indexing ${result.createdSignalsCount} signals into ${outputIndex}` ) ); + + // NOTE: (hasError || wroteWarningStatus) means the status has already been written. + // BUT! We will not store bulkCreateTimeDurations, searchAfterTimeDurations, lastLookBackDate. if (!hasError && !wroteWarningStatus) { await ruleStatusService.success('succeeded', { bulkCreateTimeDurations: result.bulkCreateTimes,