diff --git a/x-pack/plugins/security_solution/common/search_strategy/timeline/events/details/index.ts b/x-pack/plugins/security_solution/common/search_strategy/timeline/events/details/index.ts index 5d6bc33ec49f8c..1f9820f8e5c2b0 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/timeline/events/details/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/timeline/events/details/index.ts @@ -16,6 +16,7 @@ export interface TimelineEventsDetailsItem { values?: Maybe; // eslint-disable-next-line @typescript-eslint/no-explicit-any originalValue?: Maybe; + isObjectArray: boolean; } export interface TimelineEventsDetailsStrategyResponse extends IEsSearchResponse { diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/__snapshots__/json_view.test.tsx.snap b/x-pack/plugins/security_solution/public/common/components/event_details/__snapshots__/json_view.test.tsx.snap index 2b681870e92fe4..0412b3074e3f17 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/__snapshots__/json_view.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/common/components/event_details/__snapshots__/json_view.test.tsx.snap @@ -15,8 +15,8 @@ exports[`JSON View rendering should match snapshot 1`] = ` value="{ \\"_id\\": \\"pEMaMmkBUV60JmNWmWVi\\", \\"_index\\": \\"filebeat-8.0.0-2019.02.19-000001\\", - \\"_type\\": \\"_doc\\", \\"_score\\": 1, + \\"_type\\": \\"_doc\\", \\"@timestamp\\": \\"2019-02-28T16:50:54.621Z\\", \\"agent\\": { \\"ephemeral_id\\": \\"9d391ef2-a734-4787-8891-67031178c641\\", diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/columns.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/columns.tsx index 8fc6633df247f8..a62b652492c5f4 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/columns.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/columns.tsx @@ -85,24 +85,28 @@ export const getColumns = ({ sortable: false, truncateText: false, width: '30px', - render: (field: string) => ( - - c.id === field) !== -1} - data-test-subj={`toggle-field-${field}`} - data-colindex={1} - id={field} - onChange={() => - toggleColumn({ - columnHeaderType: defaultColumnHeaderType, - id: field, - width: DEFAULT_COLUMN_MIN_WIDTH, - }) - } - /> - - ), + render: (field: string, data: EventFieldsData) => { + const label = data.isObjectArray ? i18n.NESTED_COLUMN(field) : i18n.VIEW_COLUMN(field); + return ( + + c.id === field) !== -1} + data-test-subj={`toggle-field-${field}`} + data-colindex={1} + id={field} + onChange={() => + toggleColumn({ + columnHeaderType: defaultColumnHeaderType, + id: field, + width: DEFAULT_COLUMN_MIN_WIDTH, + }) + } + disabled={data.isObjectArray && data.type !== 'geo_point'} + /> + + ); + }, }, { field: 'field', @@ -118,38 +122,42 @@ export const getColumns = ({ - ( -
- - - -
- )} - > - - + {data.isObjectArray && data.type !== 'geo_point' ? ( + <>{field} + ) : ( + ( +
+ + + +
+ )} + > + + + )} diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/event_fields_browser.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/event_fields_browser.tsx index 497768785735b9..93d0e6ccfbe3c2 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/event_fields_browser.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/event_fields_browser.tsx @@ -108,7 +108,6 @@ export const EventFieldsBrowser = React.memo( const columnHeaders = useDeepEqualSelector((state) => { const { columns } = getTimeline(state, timelineId) ?? timelineDefaults; - return getColumnHeaders(columns, browserFields); }); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx index 7c7b8ba70f9bdd..00e2ee276f1816 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx @@ -112,6 +112,7 @@ export const getIconFromType = (type: string | null) => { case 'date': return 'clock'; case 'ip': + case 'geo_point': return 'globe'; case 'object': return 'questionInCircle'; diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/json_view.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/json_view.tsx index 449010781d448b..c9ca93582cd9a2 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/json_view.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/json_view.tsx @@ -54,12 +54,14 @@ export const JsonView = React.memo(({ data }) => { JsonView.displayName = 'JsonView'; export const buildJsonView = (data: TimelineEventsDetailsItem[]) => - data.reduce( - (accumulator, item) => - set( - item.field, - Array.isArray(item.originalValue) ? item.originalValue.join() : item.originalValue, - accumulator - ), - {} - ); + data + .sort((a, b) => a.field.localeCompare(b.field)) + .reduce( + (accumulator, item) => + set( + item.field, + Array.isArray(item.originalValue) ? item.originalValue.join() : item.originalValue, + accumulator + ), + {} + ); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/translations.ts b/x-pack/plugins/security_solution/public/common/components/event_details/translations.ts index c2b7bb4587dbd3..3a599b174251ac 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/translations.ts +++ b/x-pack/plugins/security_solution/public/common/components/event_details/translations.ts @@ -61,3 +61,10 @@ export const VIEW_COLUMN = (field: string) => values: { field }, defaultMessage: 'View {field} column', }); + +export const NESTED_COLUMN = (field: string) => + i18n.translate('xpack.securitySolution.eventDetails.nestedColumnCheckboxAriaLabel', { + values: { field }, + defaultMessage: + 'The {field} field is an object, and is broken down into nested fields which can be added as column', + }); diff --git a/x-pack/plugins/security_solution/public/common/mock/mock_detail_item.ts b/x-pack/plugins/security_solution/public/common/mock/mock_detail_item.ts index ca84ef539bec39..198ab084ae0b87 100644 --- a/x-pack/plugins/security_solution/public/common/mock/mock_detail_item.ts +++ b/x-pack/plugins/security_solution/public/common/mock/mock_detail_item.ts @@ -14,105 +14,126 @@ export const mockDetailItemData: TimelineEventsDetailsItem[] = [ field: '_id', originalValue: 'pEMaMmkBUV60JmNWmWVi', values: ['pEMaMmkBUV60JmNWmWVi'], + isObjectArray: false, }, { field: '_index', originalValue: 'filebeat-8.0.0-2019.02.19-000001', values: ['filebeat-8.0.0-2019.02.19-000001'], + isObjectArray: false, }, { field: '_type', originalValue: '_doc', values: ['_doc'], + isObjectArray: false, }, { field: '_score', originalValue: 1, values: ['1'], + isObjectArray: false, }, { field: '@timestamp', originalValue: '2019-02-28T16:50:54.621Z', values: ['2019-02-28T16:50:54.621Z'], + isObjectArray: false, }, { field: 'agent.ephemeral_id', originalValue: '9d391ef2-a734-4787-8891-67031178c641', values: ['9d391ef2-a734-4787-8891-67031178c641'], + isObjectArray: false, }, { field: 'agent.hostname', originalValue: 'siem-kibana', values: ['siem-kibana'], + isObjectArray: false, }, { - field: 'agent.id', - originalValue: '5de03d5f-52f3-482e-91d4-853c7de073c3', - values: ['5de03d5f-52f3-482e-91d4-853c7de073c3'], + field: 'cloud.project.id', + originalValue: 'elastic-beats', + values: ['elastic-beats'], + isObjectArray: false, }, { - field: 'agent.type', - originalValue: 'filebeat', - values: ['filebeat'], + field: 'cloud.provider', + originalValue: 'gce', + values: ['gce'], + isObjectArray: false, }, { - field: 'agent.version', - originalValue: '8.0.0', - values: ['8.0.0'], + field: 'destination.bytes', + originalValue: 584, + values: ['584'], + isObjectArray: false, }, { - field: 'cloud.availability_zone', - originalValue: 'projects/189716325846/zones/us-east1-b', - values: ['projects/189716325846/zones/us-east1-b'], + field: 'destination.ip', + originalValue: '10.47.8.200', + values: ['10.47.8.200'], + isObjectArray: false, }, { - field: 'cloud.instance.id', - originalValue: '5412578377715150143', - values: ['5412578377715150143'], + field: 'agent.id', + originalValue: '5de03d5f-52f3-482e-91d4-853c7de073c3', + values: ['5de03d5f-52f3-482e-91d4-853c7de073c3'], + isObjectArray: false, }, { field: 'cloud.instance.name', originalValue: 'siem-kibana', values: ['siem-kibana'], + isObjectArray: false, }, { field: 'cloud.machine.type', originalValue: 'projects/189716325846/machineTypes/n1-standard-1', values: ['projects/189716325846/machineTypes/n1-standard-1'], + isObjectArray: false, }, { - field: 'cloud.project.id', - originalValue: 'elastic-beats', - values: ['elastic-beats'], - }, - { - field: 'cloud.provider', - originalValue: 'gce', - values: ['gce'], - }, - { - field: 'destination.bytes', - originalValue: 584, - values: ['584'], - }, - { - field: 'destination.ip', - originalValue: '10.47.8.200', - values: ['10.47.8.200'], + field: 'agent.type', + originalValue: 'filebeat', + values: ['filebeat'], + isObjectArray: false, }, { field: 'destination.packets', originalValue: 4, values: ['4'], + isObjectArray: false, }, { field: 'destination.port', originalValue: 902, values: ['902'], + isObjectArray: false, }, { field: 'event.kind', originalValue: 'event', values: ['event'], + isObjectArray: false, + }, + { + field: 'agent.version', + originalValue: '8.0.0', + values: ['8.0.0'], + isObjectArray: false, + }, + { + field: 'cloud.availability_zone', + originalValue: 'projects/189716325846/zones/us-east1-b', + values: ['projects/189716325846/zones/us-east1-b'], + isObjectArray: false, + }, + { + field: 'cloud.instance.id', + originalValue: '5412578377715150143', + values: ['5412578377715150143'], + isObjectArray: false, }, ]; diff --git a/x-pack/plugins/security_solution/public/common/mock/timeline_results.ts b/x-pack/plugins/security_solution/public/common/mock/timeline_results.ts index 70ed497ce0caca..26b30e0d1f89ac 100644 --- a/x-pack/plugins/security_solution/public/common/mock/timeline_results.ts +++ b/x-pack/plugins/security_solution/public/common/mock/timeline_results.ts @@ -2287,11 +2287,13 @@ export const mockTimelineDetails: TimelineEventsDetailsItem[] = [ field: 'host.name', values: ['apache'], originalValue: 'apache', + isObjectArray: false, }, { field: 'user.id', values: ['1'], originalValue: 1, + isObjectArray: false, }, ]; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/helpers.test.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/helpers.test.ts index 3eb00f8534979e..c296b75a0a253a 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/helpers.test.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/helpers.test.ts @@ -29,6 +29,7 @@ describe('helpers', () => { { field: 'x', values: ['The nickname of the developer we all :heart:'], + isObjectArray: false, originalValue: 'The nickname of the developer we all :heart:', }, ]); @@ -40,6 +41,7 @@ describe('helpers', () => { { field: 'x', values: ['The nickname of the developer we all :heart:'], + isObjectArray: false, originalValue: 'The nickname of the developer we all :heart:', }, ]); @@ -51,6 +53,7 @@ describe('helpers', () => { { field: 'x', values: ['The nickname of the developer we all :heart:', 'We are all made of stars'], + isObjectArray: false, originalValue: 'The nickname of the developer we all :heart:', }, ]); @@ -65,6 +68,7 @@ describe('helpers', () => { { field: 'x.y.z', values: ['zed'], + isObjectArray: false, originalValue: 'zed', }, ]); @@ -76,6 +80,7 @@ describe('helpers', () => { { field: 'x.y.z', values: ['zed'], + isObjectArray: false, originalValue: 'zed', }, ]); @@ -90,6 +95,7 @@ describe('helpers', () => { { field: 'a', values: (5 as unknown) as string[], + isObjectArray: false, originalValue: 'zed', }, ], @@ -104,7 +110,7 @@ describe('helpers', () => { 'when trying to access field:', 'a', 'from data object of:', - [{ field: 'a', originalValue: 'zed', values: 5 }] + [{ field: 'a', isObjectArray: false, originalValue: 'zed', values: 5 }] ); }); @@ -116,6 +122,7 @@ describe('helpers', () => { { field: 'a', values: (['hi', 5] as unknown) as string[], + isObjectArray: false, originalValue: 'zed', }, ], @@ -130,7 +137,7 @@ describe('helpers', () => { 'when trying to access field:', 'a', 'from data object of:', - [{ field: 'a', originalValue: 'zed', values: ['hi', 5] }] + [{ field: 'a', isObjectArray: false, originalValue: 'zed', values: ['hi', 5] }] ); }); }); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/header/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/header/index.tsx index bec241e10d6137..ece28faedb9511 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/header/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/header/index.tsx @@ -86,6 +86,7 @@ export const HeaderComponent: React.FC = ({ getManageTimelineById, timelineId, ]); + const showSortingCapability = !isEqlOn && !(header.subType && header.subType.nested); return ( <> @@ -94,7 +95,7 @@ export const HeaderComponent: React.FC = ({ isLoading={isLoading} isResizing={false} onClick={onColumnSort} - showSortingCapability={!isEqlOn} + showSortingCapability={showSortingCapability} sort={sort} > @@ -99,7 +99,7 @@ exports[`Columns it renders the expected columns 1`] = ` fieldFormat="" fieldName="event.action" fieldType="" - key="plain-column-renderer-formatted-field-value-test-event.action-1-event.action-Action" + key="plain-column-renderer-formatted-field-value-test-event.action-1-event.action-Action-0" truncate={true} value="Action" /> @@ -129,7 +129,7 @@ exports[`Columns it renders the expected columns 1`] = ` fieldFormat="" fieldName="host.name" fieldType="" - key="plain-column-renderer-formatted-field-value-test-host.name-1-host.name-apache" + key="plain-column-renderer-formatted-field-value-test-host.name-1-host.name-apache-0" truncate={true} value="apache" /> @@ -159,7 +159,7 @@ exports[`Columns it renders the expected columns 1`] = ` fieldFormat="" fieldName="source.ip" fieldType="" - key="plain-column-renderer-formatted-field-value-test-source.ip-1-source.ip-192.168.0.1" + key="plain-column-renderer-formatted-field-value-test-source.ip-1-source.ip-192.168.0.1-0" truncate={true} value="192.168.0.1" /> @@ -189,7 +189,7 @@ exports[`Columns it renders the expected columns 1`] = ` fieldFormat="" fieldName="destination.ip" fieldType="" - key="plain-column-renderer-formatted-field-value-test-destination.ip-1-destination.ip-192.168.0.3" + key="plain-column-renderer-formatted-field-value-test-destination.ip-1-destination.ip-192.168.0.3-0" truncate={true} value="192.168.0.3" /> @@ -219,7 +219,7 @@ exports[`Columns it renders the expected columns 1`] = ` fieldFormat="" fieldName="user.name" fieldType="" - key="plain-column-renderer-formatted-field-value-test-user.name-1-user.name-john.dee" + key="plain-column-renderer-formatted-field-value-test-user.name-1-user.name-john.dee-0" truncate={true} value="john.dee" /> diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/__snapshots__/get_column_renderer.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/__snapshots__/get_column_renderer.test.tsx.snap index e2c46a07af8cce..4da4e12e0f7b3c 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/__snapshots__/get_column_renderer.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/__snapshots__/get_column_renderer.test.tsx.snap @@ -8,7 +8,7 @@ exports[`get_column_renderer renders correctly against snapshot 1`] = ` fieldFormat="" fieldName="event.severity" fieldType="" - key="plain-column-renderer-formatted-field-value-test-event.severity-1-message-3" + key="plain-column-renderer-formatted-field-value-test-event.severity-1-message-3-0" value="3" /> diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/__snapshots__/plain_column_renderer.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/__snapshots__/plain_column_renderer.test.tsx.snap index 8ea7708bf5907e..13912e6ad3da92 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/__snapshots__/plain_column_renderer.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/__snapshots__/plain_column_renderer.test.tsx.snap @@ -8,7 +8,7 @@ exports[`plain_column_renderer rendering renders correctly against snapshot 1`] fieldFormat="" fieldName="event.category" fieldType="keyword" - key="plain-column-renderer-formatted-field-value-test-event.category-1-event.category-Access" + key="plain-column-renderer-formatted-field-value-test-event.category-1-event.category-Access-0" value="Access" /> diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx index fa3612f08204d7..3032f556251f30 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx @@ -41,14 +41,27 @@ const columnNamesNotDraggable = [MESSAGE_FIELD_NAME]; const FormattedFieldValueComponent: React.FC<{ contextId: string; eventId: string; + isObjectArray?: boolean; fieldFormat?: string; fieldName: string; fieldType: string; truncate?: boolean; value: string | number | undefined | null; linkValue?: string | null | undefined; -}> = ({ contextId, eventId, fieldFormat, fieldName, fieldType, truncate, value, linkValue }) => { - if (fieldType === IP_FIELD_TYPE) { +}> = ({ + contextId, + eventId, + fieldFormat, + fieldName, + fieldType, + isObjectArray = false, + truncate, + value, + linkValue, +}) => { + if (isObjectArray) { + return <>{value}; + } else if (fieldType === IP_FIELD_TYPE) { return ( values != null - ? values.map((value) => ( + ? values.map((value, i) => ( (value: T | T[] | null): T[] => Array.isArray(value) ? value : value == null ? [] : [value]; - export const toStringArray = (value: T | T[] | null): string[] => { if (Array.isArray(value)) { return value.reduce((acc, v) => { @@ -42,3 +41,47 @@ export const toStringArray = (value: T | T[] | null): string[] => { return [`${value}`]; } }; +export const toObjectArrayOfStrings = ( + value: T | T[] | null +): Array<{ + str: string; + isObjectArray?: boolean; +}> => { + if (Array.isArray(value)) { + return value.reduce< + Array<{ + str: string; + isObjectArray?: boolean; + }> + >((acc, v) => { + if (v != null) { + switch (typeof v) { + case 'number': + case 'boolean': + return [...acc, { str: v.toString() }]; + case 'object': + try { + return [...acc, { str: JSON.stringify(v), isObjectArray: true }]; // need to track when string is not a simple value + } catch { + return [...acc, { str: 'Invalid Object' }]; + } + case 'string': + return [...acc, { str: v }]; + default: + return [...acc, { str: `${v}` }]; + } + } + return acc; + }, []); + } else if (value == null) { + return []; + } else if (!Array.isArray(value) && typeof value === 'object') { + try { + return [{ str: JSON.stringify(value), isObjectArray: true }]; + } catch { + return [{ str: 'Invalid Object' }]; + } + } else { + return [{ str: `${value}` }]; + } +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts index 505f99dd284557..8b2397fd7fab07 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts @@ -11,7 +11,7 @@ import { hostFieldsMap } from '../../../../../../common/ecs/ecs_fields'; import { HostsEdges } from '../../../../../../common/search_strategy/security_solution/hosts'; import { HostAggEsItem, HostBuckets, HostValue } from '../../../../../lib/hosts/types'; -import { toStringArray } from '../../../../helpers/to_array'; +import { toObjectArrayOfStrings } from '../../../../helpers/to_array'; export const HOSTS_FIELDS: readonly string[] = [ '_id', @@ -33,7 +33,11 @@ export const formatHostEdgesData = ( flattenedFields.cursor.value = hostId || ''; const fieldValue = getHostFieldValue(fieldName, bucket); if (fieldValue != null) { - return set(`node.${fieldName}`, toStringArray(fieldValue), flattenedFields); + return set( + `node.${fieldName}`, + toObjectArrayOfStrings(fieldValue).map(({ str }) => str), + flattenedFields + ); } return flattenedFields; }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/helpers.ts index 06d81140f475e5..aeaefe690cbde3 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/helpers.ts @@ -8,7 +8,7 @@ import { get, getOr, isEmpty } from 'lodash/fp'; import { set } from '@elastic/safer-lodash-set/fp'; import { mergeFieldsWithHit } from '../../../../../utils/build_query'; -import { toStringArray } from '../../../../helpers/to_array'; +import { toObjectArrayOfStrings } from '../../../../helpers/to_array'; import { AuthenticationsEdges, AuthenticationHit, @@ -55,7 +55,11 @@ export const formatAuthenticationData = ( const fieldPath = `node.${fieldName}`; const fieldValue = get(fieldPath, mergedResult); if (!isEmpty(fieldValue)) { - return set(fieldPath, toStringArray(fieldValue), mergedResult); + return set( + fieldPath, + toObjectArrayOfStrings(fieldValue).map(({ str }) => str), + mergedResult + ); } else { return mergedResult; } diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts index 9c522bd704ef06..2b35517d693d51 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts @@ -9,7 +9,7 @@ import { set } from '@elastic/safer-lodash-set/fp'; import { get, has, head } from 'lodash/fp'; import { hostFieldsMap } from '../../../../../../common/ecs/ecs_fields'; import { HostItem } from '../../../../../../common/search_strategy/security_solution/hosts'; -import { toStringArray } from '../../../../helpers/to_array'; +import { toObjectArrayOfStrings } from '../../../../helpers/to_array'; import { HostAggEsItem, HostBuckets, HostValue } from '../../../../../lib/hosts/types'; @@ -42,7 +42,11 @@ export const formatHostItem = (bucket: HostAggEsItem): HostItem => if (fieldName === '_id') { return set('_id', fieldValue, flattenedFields); } - return set(fieldName, toStringArray(fieldValue), flattenedFields); + return set( + fieldName, + toObjectArrayOfStrings(fieldValue).map(({ str }) => str), + flattenedFields + ); } return flattenedFields; }, {}); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/helpers.ts index 7b01f4e7dc816a..fe202b48540d7b 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/helpers.ts @@ -14,7 +14,7 @@ import { HostsUncommonProcessesEdges, HostsUncommonProcessHit, } from '../../../../../../common/search_strategy/security_solution/hosts/uncommon_processes'; -import { toStringArray } from '../../../../helpers/to_array'; +import { toObjectArrayOfStrings } from '../../../../helpers/to_array'; import { HostHits } from '../../../../../../common/search_strategy'; export const uncommonProcessesFields = [ @@ -82,7 +82,11 @@ export const formatUncommonProcessesData = ( fieldPath = `node.hosts.0.name`; fieldValue = get(fieldPath, mergedResult); } - return set(fieldPath, toStringArray(fieldValue), mergedResult); + return set( + fieldPath, + toObjectArrayOfStrings(fieldValue).map(({ str }) => str), + mergedResult + ); }, { node: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/helpers.ts index 3b387597618e8d..8fc7ae0304a352 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/helpers.ts @@ -13,7 +13,7 @@ import { NetworkDetailsHostHit, NetworkHit, } from '../../../../../../common/search_strategy/security_solution/network'; -import { toStringArray } from '../../../../helpers/to_array'; +import { toObjectArrayOfStrings } from '../../../../helpers/to_array'; export const getNetworkDetailsAgg = (type: string, networkHit: NetworkHit | {}) => { const firstSeen = getOr(null, `firstSeen.value_as_string`, networkHit); @@ -53,7 +53,7 @@ const formatHostEcs = (data: Record | null): HostEcs | null => } return { ...acc, - [key]: toStringArray(value), + [key]: toObjectArrayOfStrings(value).map(({ str }) => str), }; }, {}); }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/helpers.ts index bdd3195b3b756d..b007307412e955 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/helpers.ts @@ -7,7 +7,7 @@ import { EqlSearchStrategyResponse } from '../../../../../data_enhanced/common'; import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../common/constants'; -import { EqlSearchResponse } from '../../../../common/detection_engine/types'; +import { EqlSearchResponse, EqlSequence } from '../../../../common/detection_engine/types'; import { EventHit, TimelineEdges } from '../../../../common/search_strategy'; import { TimelineEqlRequestOptions, @@ -56,51 +56,53 @@ export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record>, fieldRequested: string[]) => + sequences.reduce>(async (acc, sequence, sequenceIndex) => { + const sequenceParentId = sequence.events[0]?._id ?? null; + const data = await acc; + const allData = await Promise.all( + sequence.events.map(async (event, eventIndex) => { + const item = await formatTimelineData( + fieldRequested, + TIMELINE_EVENTS_FIELDS, + event as EventHit + ); + return Promise.resolve({ + ...item, + node: { + ...item.node, + ecs: { + ...item.node.ecs, + ...(sequenceParentId != null + ? { + eql: { + parentId: sequenceParentId, + sequenceNumber: `${sequenceIndex}-${eventIndex}`, + }, + } + : {}), + }, + }, + }); + }) + ); + return Promise.resolve([...data, ...allData]); + }, Promise.resolve([])); -export const parseEqlResponse = ( +export const parseEqlResponse = async ( options: TimelineEqlRequestOptions, response: EqlSearchStrategyResponse> ): Promise => { const { activePage, querySize } = options.pagination; - // const totalCount = response.rawResponse?.body?.hits?.total?.value ?? 0; let edges: TimelineEdges[] = []; + if (response.rawResponse.body.hits.sequences !== undefined) { - edges = response.rawResponse.body.hits.sequences.reduce( - (data, sequence, sequenceIndex) => { - const sequenceParentId = sequence.events[0]?._id ?? null; - return [ - ...data, - ...sequence.events.map((event, eventIndex) => { - const item = formatTimelineData( - options.fieldRequested, - TIMELINE_EVENTS_FIELDS, - event as EventHit - ); - return { - ...item, - node: { - ...item.node, - ecs: { - ...item.node.ecs, - ...(sequenceParentId != null - ? { - eql: { - parentId: sequenceParentId, - sequenceNumber: `${sequenceIndex}-${eventIndex}`, - }, - } - : {}), - }, - }, - }; - }), - ]; - }, - [] - ); + edges = await parseSequences(response.rawResponse.body.hits.sequences, options.fieldRequested); } else if (response.rawResponse.body.hits.events !== undefined) { - edges = response.rawResponse.body.hits.events.map((event) => - formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, event as EventHit) + edges = await Promise.all( + response.rawResponse.body.hits.events.map(async (event) => + formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, event as EventHit) + ) ); } diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/index.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/index.ts index cf7877e987acea..249f5582d2a398 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/index.ts @@ -38,7 +38,7 @@ export const securitySolutionTimelineEqlSearchStrategyProvider = ( }, }; }), - mergeMap((esSearchRes) => + mergeMap(async (esSearchRes) => parseEqlResponse( request, (esSearchRes as unknown) as EqlSearchStrategyResponse> diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts index 10bb606dc2387e..61af6a7664faa6 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts @@ -8,54 +8,23 @@ import { EventHit } from '../../../../../../common/search_strategy'; import { TIMELINE_EVENTS_FIELDS } from './constants'; import { formatTimelineData } from './helpers'; +import { eventHit } from '../mocks'; describe('#formatTimelineData', () => { - it('happy path', () => { - const response: EventHit = { - _index: 'auditbeat-7.8.0-2020.11.05-000003', - _id: 'tkCt1nUBaEgqnrVSZ8R_', - _score: 0, - _type: '', - fields: { - 'event.category': ['process'], - 'process.ppid': [3977], - 'user.name': ['jenkins'], - 'process.args': ['go', 'vet', './...'], - message: ['Process go (PID: 4313) by user jenkins STARTED'], - 'process.pid': [4313], - 'process.working_directory': [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', - ], - 'process.entity_id': ['Z59cIkAAIw8ZoK0H'], - 'host.ip': ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - 'process.name': ['go'], - 'event.action': ['process_started'], - 'agent.type': ['auditbeat'], - '@timestamp': ['2020-11-17T14:48:08.922Z'], - 'event.module': ['system'], - 'event.type': ['start'], - 'host.name': ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - 'process.hash.sha1': ['1eac22336a41e0660fb302add9d97daa2bcc7040'], - 'host.os.family': ['debian'], - 'event.kind': ['event'], - 'host.id': ['e59991e835905c65ed3e455b33e13bd6'], - 'event.dataset': ['process'], - 'process.executable': [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', - ], - }, - _source: {}, - sort: ['1605624488922', 'beats-ci-immutable-ubuntu-1804-1605624279743236239'], - aggregations: {}, - }; - - expect( - formatTimelineData( - ['@timestamp', 'host.name', 'destination.ip', 'source.ip'], - TIMELINE_EVENTS_FIELDS, - response - ) - ).toEqual({ + it('happy path', async () => { + const res = await formatTimelineData( + [ + '@timestamp', + 'host.name', + 'destination.ip', + 'source.ip', + 'source.geo.location', + 'threat.indicator.matched.field', + ], + TIMELINE_EVENTS_FIELDS, + eventHit + ); + expect(res).toEqual({ cursor: { tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239', value: '1605624488922', @@ -72,6 +41,14 @@ describe('#formatTimelineData', () => { field: 'host.name', value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], }, + { + field: 'source.geo.location', + value: [`{"lon":118.7778,"lat":32.0617}`], + }, + { + field: 'threat.indicator.matched.field', + value: ['matched_field', 'matched_field_2'], + }, ], ecs: { '@timestamp': ['2020-11-17T14:48:08.922Z'], @@ -122,7 +99,7 @@ describe('#formatTimelineData', () => { }); }); - it('rule signal results', () => { + it('rule signal results', async () => { const response: EventHit = { _index: '.siem-signals-patrykkopycinski-default-000007', _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562', @@ -290,7 +267,7 @@ describe('#formatTimelineData', () => { }; expect( - formatTimelineData( + await formatTimelineData( ['@timestamp', 'host.name', 'destination.ip', 'source.ip'], TIMELINE_EVENTS_FIELDS, response diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts index 1a83cbf40f1f4d..e5bb8cb7e14b73 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts @@ -6,9 +6,13 @@ */ import { get, has, merge, uniq } from 'lodash/fp'; -import { EventHit, TimelineEdges } from '../../../../../../common/search_strategy'; +import { + EventHit, + TimelineEdges, + TimelineNonEcsData, +} from '../../../../../../common/search_strategy'; import { toStringArray } from '../../../../helpers/to_array'; -import { formatGeoLocation, isGeoField } from '../details/helpers'; +import { getDataSafety, getDataFromFieldsHits } from '../details/helpers'; const getTimestamp = (hit: EventHit): string => { if (hit.fields && hit.fields['@timestamp']) { @@ -19,13 +23,14 @@ const getTimestamp = (hit: EventHit): string => { return ''; }; -export const formatTimelineData = ( +export const formatTimelineData = async ( dataFields: readonly string[], ecsFields: readonly string[], hit: EventHit ) => - uniq([...ecsFields, ...dataFields]).reduce( - (flattenedFields, fieldName) => { + uniq([...ecsFields, ...dataFields]).reduce>( + async (acc, fieldName) => { + const flattenedFields: TimelineEdges = await acc; flattenedFields.node._id = hit._id; flattenedFields.node._index = hit._index; flattenedFields.node.ecs._id = hit._id; @@ -35,30 +40,81 @@ export const formatTimelineData = ( flattenedFields.cursor.value = hit.sort[0]; flattenedFields.cursor.tiebreaker = hit.sort[1]; } - return mergeTimelineFieldsWithHit(fieldName, flattenedFields, hit, dataFields, ecsFields); + const waitForIt = await mergeTimelineFieldsWithHit( + fieldName, + flattenedFields, + hit, + dataFields, + ecsFields + ); + return Promise.resolve(waitForIt); }, - { + Promise.resolve({ node: { ecs: { _id: '' }, data: [], _id: '', _index: '' }, cursor: { value: '', tiebreaker: null, }, - } + }) ); const specialFields = ['_id', '_index', '_type', '_score']; -const mergeTimelineFieldsWithHit = ( +const getValuesFromFields = async ( + fieldName: string, + hit: EventHit, + nestedParentFieldName?: string +): Promise => { + if (specialFields.includes(fieldName)) { + return [{ field: fieldName, value: toStringArray(get(fieldName, hit)) }]; + } + + let fieldToEval; + if (has(fieldName, hit._source)) { + fieldToEval = { + [fieldName]: get(fieldName, hit._source), + }; + } else { + if (nestedParentFieldName == null || nestedParentFieldName === fieldName) { + fieldToEval = { + [fieldName]: hit.fields[fieldName], + }; + } else if (nestedParentFieldName != null) { + fieldToEval = { + [nestedParentFieldName]: hit.fields[nestedParentFieldName], + }; + } else { + // fallback, should never hit + fieldToEval = { + [fieldName]: [], + }; + } + } + const formattedData = await getDataSafety(getDataFromFieldsHits, fieldToEval); + return formattedData.reduce( + (acc: TimelineNonEcsData[], { field, values }) => + // nested fields return all field values, pick only the one we asked for + field.includes(fieldName) ? [...acc, { field, value: values }] : acc, + [] + ); +}; + +const mergeTimelineFieldsWithHit = async ( fieldName: string, flattenedFields: T, - hit: { _source: {}; fields: Record }, + hit: EventHit, dataFields: readonly string[], ecsFields: readonly string[] ) => { if (fieldName != null || dataFields.includes(fieldName)) { + const fieldNameAsArray = fieldName.split('.'); + const nestedParentFieldName = Object.keys(hit.fields ?? []).find((f) => { + return f === fieldNameAsArray.slice(0, f.split('.').length).join('.'); + }); if ( has(fieldName, hit._source) || has(fieldName, hit.fields) || + nestedParentFieldName != null || specialFields.includes(fieldName) ) { const objectWithProperty = { @@ -67,16 +123,7 @@ const mergeTimelineFieldsWithHit = ( data: dataFields.includes(fieldName) ? [ ...get('node.data', flattenedFields), - { - field: fieldName, - value: specialFields.includes(fieldName) - ? toStringArray(get(fieldName, hit)) - : isGeoField(fieldName) - ? formatGeoLocation(hit.fields[fieldName]) - : has(fieldName, hit._source) - ? toStringArray(get(fieldName, hit._source)) - : toStringArray(hit.fields[fieldName]), - }, + ...(await getValuesFromFields(fieldName, hit, nestedParentFieldName)), ] : get('node.data', flattenedFields), ecs: ecsFields.includes(fieldName) diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts index 93985baed770e9..05058e3ee7a2dc 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts @@ -40,8 +40,10 @@ export const timelineEventsAll: SecuritySolutionTimelineFactory - formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, hit as EventHit) + const edges: TimelineEdges[] = await Promise.all( + hits.map((hit) => + formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, hit as EventHit) + ) ); const inspect = { dsl: [inspectStringifyObject(buildTimelineEventsAllQuery(queryOptions))], diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.test.ts index ca9a4b708161de..dc3efc6909c634 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.test.ts @@ -5,150 +5,192 @@ * 2.0. */ -import { EventHit } from '../../../../../../common/search_strategy'; -import { getDataFromFieldsHits } from './helpers'; +import { EventHit, EventSource } from '../../../../../../common/search_strategy'; +import { getDataFromFieldsHits, getDataFromSourceHits, getDataSafety } from './helpers'; +import { eventDetailsFormattedFields, eventHit } from '../mocks'; -describe('#getDataFromFieldsHits', () => { - it('happy path', () => { - const response: EventHit = { - _index: 'auditbeat-7.8.0-2020.11.05-000003', - _id: 'tkCt1nUBaEgqnrVSZ8R_', - _score: 0, - _type: '', - fields: { - 'event.category': ['process'], - 'process.ppid': [3977], - 'user.name': ['jenkins'], - 'process.args': ['go', 'vet', './...'], - message: ['Process go (PID: 4313) by user jenkins STARTED'], - 'process.pid': [4313], - 'process.working_directory': [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', +describe('Events Details Helpers', () => { + const fields: EventHit['fields'] = eventHit.fields; + const resultFields = eventDetailsFormattedFields; + describe('#getDataFromFieldsHits', () => { + it('happy path', () => { + const result = getDataFromFieldsHits(fields); + expect(result).toEqual(resultFields); + }); + it('lets get weird', () => { + const whackFields = { + 'crazy.pants': [ + { + 'matched.field': ['matched_field'], + first_seen: ['2021-02-22T17:29:25.195Z'], + provider: ['yourself'], + type: ['custom'], + 'matched.atomic': ['matched_atomic'], + lazer: [ + { + 'great.field': ['grrrrr'], + lazer: [ + { + lazer: [ + { + cool: true, + lazer: [ + { + lazer: [ + { + lazer: [ + { + lazer: [ + { + whoa: false, + }, + ], + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + lazer: [ + { + cool: false, + }, + ], + }, + ], + }, + { + 'great.field': ['grrrrr_2'], + }, + ], + }, ], - 'process.entity_id': ['Z59cIkAAIw8ZoK0H'], - 'host.ip': ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - 'process.name': ['go'], - 'event.action': ['process_started'], - 'agent.type': ['auditbeat'], - '@timestamp': ['2020-11-17T14:48:08.922Z'], - 'event.module': ['system'], - 'event.type': ['start'], - 'host.name': ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - 'process.hash.sha1': ['1eac22336a41e0660fb302add9d97daa2bcc7040'], - 'host.os.family': ['debian'], - 'event.kind': ['event'], - 'host.id': ['e59991e835905c65ed3e455b33e13bd6'], - 'event.dataset': ['process'], - 'process.executable': [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', - ], - }, - _source: {}, - sort: ['1605624488922', 'beats-ci-immutable-ubuntu-1804-1605624279743236239'], - aggregations: {}, + }; + const whackResultFields = [ + { + category: 'crazy', + field: 'crazy.pants.matched.field', + values: ['matched_field'], + originalValue: ['matched_field'], + isObjectArray: false, + }, + { + category: 'crazy', + field: 'crazy.pants.first_seen', + values: ['2021-02-22T17:29:25.195Z'], + originalValue: ['2021-02-22T17:29:25.195Z'], + isObjectArray: false, + }, + { + category: 'crazy', + field: 'crazy.pants.provider', + values: ['yourself'], + originalValue: ['yourself'], + isObjectArray: false, + }, + { + category: 'crazy', + field: 'crazy.pants.type', + values: ['custom'], + originalValue: ['custom'], + isObjectArray: false, + }, + { + category: 'crazy', + field: 'crazy.pants.matched.atomic', + values: ['matched_atomic'], + originalValue: ['matched_atomic'], + isObjectArray: false, + }, + { + category: 'crazy', + field: 'crazy.pants.lazer.great.field', + values: ['grrrrr', 'grrrrr_2'], + originalValue: ['grrrrr', 'grrrrr_2'], + isObjectArray: false, + }, + { + category: 'crazy', + field: 'crazy.pants.lazer.lazer.lazer.cool', + values: ['true', 'false'], + originalValue: ['true', 'false'], + isObjectArray: false, + }, + { + category: 'crazy', + field: 'crazy.pants.lazer.lazer.lazer.lazer.lazer.lazer.lazer.whoa', + values: ['false'], + originalValue: ['false'], + isObjectArray: false, + }, + ]; + const result = getDataFromFieldsHits(whackFields); + expect(result).toEqual(whackResultFields); + }); + }); + it('#getDataFromSourceHits', () => { + const _source: EventSource = { + '@timestamp': '2021-02-24T00:41:06.527Z', + 'signal.status': 'open', + 'signal.rule.name': 'Rawr', + 'threat.indicator': [ + { + provider: 'yourself', + type: 'custom', + first_seen: ['2021-02-22T17:29:25.195Z'], + matched: { atomic: 'atom', field: 'field', type: 'type' }, + }, + { + provider: 'other_you', + type: 'custom', + first_seen: '2021-02-22T17:29:25.195Z', + matched: { atomic: 'atom', field: 'field', type: 'type' }, + }, + ], }; - - expect(getDataFromFieldsHits(response.fields)).toEqual([ - { - category: 'event', - field: 'event.category', - originalValue: ['process'], - values: ['process'], - }, - { category: 'process', field: 'process.ppid', originalValue: ['3977'], values: ['3977'] }, - { category: 'user', field: 'user.name', originalValue: ['jenkins'], values: ['jenkins'] }, - { - category: 'process', - field: 'process.args', - originalValue: ['go', 'vet', './...'], - values: ['go', 'vet', './...'], - }, - { - category: 'base', - field: 'message', - originalValue: ['Process go (PID: 4313) by user jenkins STARTED'], - values: ['Process go (PID: 4313) by user jenkins STARTED'], - }, - { category: 'process', field: 'process.pid', originalValue: ['4313'], values: ['4313'] }, - { - category: 'process', - field: 'process.working_directory', - originalValue: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', - ], - values: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', - ], - }, - { - category: 'process', - field: 'process.entity_id', - originalValue: ['Z59cIkAAIw8ZoK0H'], - values: ['Z59cIkAAIw8ZoK0H'], - }, - { - category: 'host', - field: 'host.ip', - originalValue: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - values: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - }, - { category: 'process', field: 'process.name', originalValue: ['go'], values: ['go'] }, - { - category: 'event', - field: 'event.action', - originalValue: ['process_started'], - values: ['process_started'], - }, - { - category: 'agent', - field: 'agent.type', - originalValue: ['auditbeat'], - values: ['auditbeat'], - }, + expect(getDataFromSourceHits(_source)).toEqual([ { category: 'base', field: '@timestamp', - originalValue: ['2020-11-17T14:48:08.922Z'], - values: ['2020-11-17T14:48:08.922Z'], + values: ['2021-02-24T00:41:06.527Z'], + originalValue: ['2021-02-24T00:41:06.527Z'], + isObjectArray: false, }, - { category: 'event', field: 'event.module', originalValue: ['system'], values: ['system'] }, - { category: 'event', field: 'event.type', originalValue: ['start'], values: ['start'] }, { - category: 'host', - field: 'host.name', - originalValue: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - values: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + category: 'signal', + field: 'signal.status', + values: ['open'], + originalValue: ['open'], + isObjectArray: false, }, { - category: 'process', - field: 'process.hash.sha1', - originalValue: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], - values: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], + category: 'signal', + field: 'signal.rule.name', + values: ['Rawr'], + originalValue: ['Rawr'], + isObjectArray: false, }, - { category: 'host', field: 'host.os.family', originalValue: ['debian'], values: ['debian'] }, - { category: 'event', field: 'event.kind', originalValue: ['event'], values: ['event'] }, { - category: 'host', - field: 'host.id', - originalValue: ['e59991e835905c65ed3e455b33e13bd6'], - values: ['e59991e835905c65ed3e455b33e13bd6'], - }, - { - category: 'event', - field: 'event.dataset', - originalValue: ['process'], - values: ['process'], - }, - { - category: 'process', - field: 'process.executable', - originalValue: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', - ], + category: 'threat', + field: 'threat.indicator', values: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', + '{"provider":"yourself","type":"custom","first_seen":["2021-02-22T17:29:25.195Z"],"matched":{"atomic":"atom","field":"field","type":"type"}}', + '{"provider":"other_you","type":"custom","first_seen":"2021-02-22T17:29:25.195Z","matched":{"atomic":"atom","field":"field","type":"type"}}', + ], + originalValue: [ + '{"provider":"yourself","type":"custom","first_seen":["2021-02-22T17:29:25.195Z"],"matched":{"atomic":"atom","field":"field","type":"type"}}', + '{"provider":"other_you","type":"custom","first_seen":"2021-02-22T17:29:25.195Z","matched":{"atomic":"atom","field":"field","type":"type"}}', ], + isObjectArray: true, }, ]); }); + it('#getDataSafety', async () => { + const result = await getDataSafety(getDataFromFieldsHits, fields); + expect(result).toEqual(resultFields); + }); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.ts index 779454e9474ee7..2fc729729e4351 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.ts @@ -7,8 +7,12 @@ import { get, isEmpty, isNumber, isObject, isString } from 'lodash/fp'; -import { EventSource, TimelineEventsDetailsItem } from '../../../../../../common/search_strategy'; -import { toStringArray } from '../../../../helpers/to_array'; +import { + EventHit, + EventSource, + TimelineEventsDetailsItem, +} from '../../../../../../common/search_strategy'; +import { toObjectArrayOfStrings, toStringArray } from '../../../../helpers/to_array'; export const baseCategoryFields = ['@timestamp', 'labels', 'message', 'tags']; @@ -24,7 +28,10 @@ export const formatGeoLocation = (item: unknown[]) => { const itemGeo = item.length > 0 ? (item[0] as { coordinates: number[] }) : null; if (itemGeo != null && !isEmpty(itemGeo.coordinates)) { try { - return toStringArray({ long: itemGeo.coordinates[0], lat: itemGeo.coordinates[1] }); + return toStringArray({ + lon: itemGeo.coordinates[0], + lat: itemGeo.coordinates[1], + }); } catch { return toStringArray(item); } @@ -46,13 +53,18 @@ export const getDataFromSourceHits = ( const field = path ? `${path}.${source}` : source; const fieldCategory = getFieldCategory(field); + const objArrStr = toObjectArrayOfStrings(item); + const strArr = objArrStr.map(({ str }) => str); + const isObjectArray = objArrStr.some((o) => o.isObjectArray); + return [ ...accumulator, { category: fieldCategory, field, - values: toStringArray(item), - originalValue: toStringArray(item), + values: strArr, + originalValue: strArr, + isObjectArray, } as TimelineEventsDetailsItem, ]; } else if (isObject(item)) { @@ -65,18 +77,81 @@ export const getDataFromSourceHits = ( }, []); export const getDataFromFieldsHits = ( - fields: Record + fields: EventHit['fields'], + prependField?: string, + prependFieldCategory?: string ): TimelineEventsDetailsItem[] => Object.keys(fields).reduce((accumulator, field) => { const item: unknown[] = fields[field]; - const fieldCategory = getFieldCategory(field); - return [ + + const fieldCategory = + prependFieldCategory != null ? prependFieldCategory : getFieldCategory(field); + if (isGeoField(field)) { + return [ + ...accumulator, + { + category: fieldCategory, + field, + values: formatGeoLocation(item), + originalValue: formatGeoLocation(item), + isObjectArray: true, // important for UI + }, + ]; + } + const objArrStr = toObjectArrayOfStrings(item); + const strArr = objArrStr.map(({ str }) => str); + const isObjectArray = objArrStr.some((o) => o.isObjectArray); + const dotField = prependField ? `${prependField}.${field}` : field; + + // return simple field value (non-object, non-array) + if (!isObjectArray) { + return [ + ...accumulator, + { + category: fieldCategory, + field: dotField, + values: strArr, + originalValue: strArr, + isObjectArray, + }, + ]; + } + + // format nested fields + const nestedFields = Array.isArray(item) + ? item + .reduce((acc, i) => [...acc, getDataFromFieldsHits(i, dotField, fieldCategory)], []) + .flat() + : getDataFromFieldsHits(item, prependField, fieldCategory); + + // combine duplicate fields + const flat: Record = [ ...accumulator, - { - category: fieldCategory, - field, - values: isGeoField(field) ? formatGeoLocation(item) : toStringArray(item), - originalValue: toStringArray(item), - } as TimelineEventsDetailsItem, - ]; + ...nestedFields, + ].reduce( + (acc, f) => ({ + ...acc, + // acc/flat is hashmap to determine if we already have the field or not without an array iteration + // its converted back to array in return with Object.values + ...(acc[f.field] != null + ? { + [f.field]: { + ...f, + originalValue: acc[f.field].originalValue.includes(f.originalValue[0]) + ? acc[f.field].originalValue + : [...acc[f.field].originalValue, ...f.originalValue], + values: acc[f.field].values.includes(f.values[0]) + ? acc[f.field].values + : [...acc[f.field].values, ...f.values], + }, + } + : { [f.field]: f }), + }), + {} + ); + + return Object.values(flat); }, []); + +export const getDataSafety = (fn: (args: A) => T, args: A): Promise => + new Promise((resolve) => setTimeout(() => resolve(fn(args)))); diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/index.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/index.ts index f5deb258fc1f48..7794de7f0f4119 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { cloneDeep, merge } from 'lodash/fp'; +import { cloneDeep, merge, unionBy } from 'lodash/fp'; import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; import { @@ -13,11 +13,13 @@ import { TimelineEventsQueries, TimelineEventsDetailsStrategyResponse, TimelineEventsDetailsRequestOptions, + TimelineEventsDetailsItem, + EventSource, } from '../../../../../../common/search_strategy'; import { inspectStringifyObject } from '../../../../../utils/build_query'; import { SecuritySolutionTimelineFactory } from '../../types'; import { buildTimelineDetailsQuery } from './query.events_details.dsl'; -import { getDataFromSourceHits } from './helpers'; +import { getDataFromFieldsHits, getDataFromSourceHits, getDataSafety } from './helpers'; export const timelineEventsDetails: SecuritySolutionTimelineFactory = { buildDsl: (options: TimelineEventsDetailsRequestOptions) => { @@ -29,11 +31,10 @@ export const timelineEventsDetails: SecuritySolutionTimelineFactory ): Promise => { const { indexName, eventId, docValueFields = [] } = options; - const { _source, ...hitsData } = cloneDeep(response.rawResponse.hits.hits[0] ?? {}); + const { _source, fields, ...hitsData } = cloneDeep(response.rawResponse.hits.hits[0] ?? {}); const inspect = { dsl: [inspectStringifyObject(buildTimelineDetailsQuery(indexName, eventId, docValueFields))], }; - if (response.isRunning) { return { ...response, @@ -41,12 +42,19 @@ export const timelineEventsDetails: SecuritySolutionTimelineFactory( + getDataFromSourceHits, + _source + ); + const fieldsData = await getDataSafety( + getDataFromFieldsHits, + merge(fields, hitsData) + ); - const sourceData = getDataFromSourceHits(merge(_source, hitsData)); - + const data = unionBy('field', fieldsData, sourceData); return { ...response, - data: sourceData, + data, inspect, }; }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts index 47716e21bca319..4545a3a3e136b3 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts @@ -24,6 +24,7 @@ describe('buildTimelineDetailsQuery', () => { Object { "allowNoIndices": true, "body": Object { + "_source": true, "docvalue_fields": Array [ Object { "field": "@timestamp", @@ -38,6 +39,9 @@ describe('buildTimelineDetailsQuery', () => { "field": "agent.name", }, ], + "fields": Array [ + "*", + ], "query": Object { "terms": Object { "_id": Array [ diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts index e8890072c1aff6..c624eb14ae969f 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts @@ -22,6 +22,8 @@ export const buildTimelineDetailsQuery = ( _id: [id], }, }, + fields: ['*'], + _source: true, }, size: 1, }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/mocks.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/mocks.ts new file mode 100644 index 00000000000000..13b7fe70512460 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/mocks.ts @@ -0,0 +1,329 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const eventHit = { + _index: 'auditbeat-7.8.0-2020.11.05-000003', + _id: 'tkCt1nUBaEgqnrVSZ8R_', + _score: 0, + _type: '', + fields: { + 'event.category': ['process'], + 'process.ppid': [3977], + 'user.name': ['jenkins'], + 'process.args': ['go', 'vet', './...'], + message: ['Process go (PID: 4313) by user jenkins STARTED'], + 'process.pid': [4313], + 'process.working_directory': [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', + ], + 'process.entity_id': ['Z59cIkAAIw8ZoK0H'], + 'host.ip': ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], + 'process.name': ['go'], + 'event.action': ['process_started'], + 'agent.type': ['auditbeat'], + '@timestamp': ['2020-11-17T14:48:08.922Z'], + 'event.module': ['system'], + 'event.type': ['start'], + 'host.name': ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + 'process.hash.sha1': ['1eac22336a41e0660fb302add9d97daa2bcc7040'], + 'host.os.family': ['debian'], + 'event.kind': ['event'], + 'host.id': ['e59991e835905c65ed3e455b33e13bd6'], + 'event.dataset': ['process'], + 'process.executable': [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', + ], + 'source.geo.location': [{ coordinates: [118.7778, 32.0617], type: 'Point' }], + 'threat.indicator': [ + { + 'matched.field': ['matched_field'], + first_seen: ['2021-02-22T17:29:25.195Z'], + provider: ['yourself'], + type: ['custom'], + 'matched.atomic': ['matched_atomic'], + lazer: [ + { + 'great.field': ['grrrrr'], + }, + { + 'great.field': ['grrrrr_2'], + }, + ], + }, + { + 'matched.field': ['matched_field_2'], + first_seen: ['2021-02-22T17:29:25.195Z'], + provider: ['other_you'], + type: ['custom'], + 'matched.atomic': ['matched_atomic_2'], + lazer: [ + { + 'great.field': [ + { + wowoe: [ + { + fooooo: ['grrrrr'], + }, + ], + astring: 'cool', + aNumber: 1, + anObject: { + neat: true, + }, + }, + ], + }, + ], + }, + ], + }, + _source: {}, + sort: ['1605624488922', 'beats-ci-immutable-ubuntu-1804-1605624279743236239'], + aggregations: {}, +}; + +export const eventDetailsFormattedFields = [ + { + category: 'event', + field: 'event.category', + isObjectArray: false, + originalValue: ['process'], + values: ['process'], + }, + { + category: 'process', + field: 'process.ppid', + isObjectArray: false, + originalValue: ['3977'], + values: ['3977'], + }, + { + category: 'user', + field: 'user.name', + isObjectArray: false, + originalValue: ['jenkins'], + values: ['jenkins'], + }, + { + category: 'process', + field: 'process.args', + isObjectArray: false, + originalValue: ['go', 'vet', './...'], + values: ['go', 'vet', './...'], + }, + { + category: 'base', + field: 'message', + isObjectArray: false, + originalValue: ['Process go (PID: 4313) by user jenkins STARTED'], + values: ['Process go (PID: 4313) by user jenkins STARTED'], + }, + { + category: 'process', + field: 'process.pid', + isObjectArray: false, + originalValue: ['4313'], + values: ['4313'], + }, + { + category: 'process', + field: 'process.working_directory', + isObjectArray: false, + originalValue: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', + ], + values: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', + ], + }, + { + category: 'process', + field: 'process.entity_id', + isObjectArray: false, + originalValue: ['Z59cIkAAIw8ZoK0H'], + values: ['Z59cIkAAIw8ZoK0H'], + }, + { + category: 'host', + field: 'host.ip', + isObjectArray: false, + originalValue: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], + values: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], + }, + { + category: 'process', + field: 'process.name', + isObjectArray: false, + originalValue: ['go'], + values: ['go'], + }, + { + category: 'event', + field: 'event.action', + isObjectArray: false, + originalValue: ['process_started'], + values: ['process_started'], + }, + { + category: 'agent', + field: 'agent.type', + isObjectArray: false, + originalValue: ['auditbeat'], + values: ['auditbeat'], + }, + { + category: 'base', + field: '@timestamp', + isObjectArray: false, + originalValue: ['2020-11-17T14:48:08.922Z'], + values: ['2020-11-17T14:48:08.922Z'], + }, + { + category: 'event', + field: 'event.module', + isObjectArray: false, + originalValue: ['system'], + values: ['system'], + }, + { + category: 'event', + field: 'event.type', + isObjectArray: false, + originalValue: ['start'], + values: ['start'], + }, + { + category: 'host', + field: 'host.name', + isObjectArray: false, + originalValue: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + values: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + }, + { + category: 'process', + field: 'process.hash.sha1', + isObjectArray: false, + originalValue: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], + values: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], + }, + { + category: 'host', + field: 'host.os.family', + isObjectArray: false, + originalValue: ['debian'], + values: ['debian'], + }, + { + category: 'event', + field: 'event.kind', + isObjectArray: false, + originalValue: ['event'], + values: ['event'], + }, + { + category: 'host', + field: 'host.id', + isObjectArray: false, + originalValue: ['e59991e835905c65ed3e455b33e13bd6'], + values: ['e59991e835905c65ed3e455b33e13bd6'], + }, + { + category: 'event', + field: 'event.dataset', + isObjectArray: false, + originalValue: ['process'], + values: ['process'], + }, + { + category: 'process', + field: 'process.executable', + isObjectArray: false, + originalValue: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', + ], + values: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', + ], + }, + { + category: 'source', + field: 'source.geo.location', + isObjectArray: true, + originalValue: [`{"lon":118.7778,"lat":32.0617}`], + values: [`{"lon":118.7778,"lat":32.0617}`], + }, + { + category: 'threat', + field: 'threat.indicator.matched.field', + values: ['matched_field', 'matched_field_2'], + originalValue: ['matched_field', 'matched_field_2'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.first_seen', + values: ['2021-02-22T17:29:25.195Z'], + originalValue: ['2021-02-22T17:29:25.195Z'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.provider', + values: ['yourself', 'other_you'], + originalValue: ['yourself', 'other_you'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.type', + values: ['custom'], + originalValue: ['custom'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.matched.atomic', + values: ['matched_atomic', 'matched_atomic_2'], + originalValue: ['matched_atomic', 'matched_atomic_2'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field', + values: ['grrrrr', 'grrrrr_2'], + originalValue: ['grrrrr', 'grrrrr_2'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field.wowoe.fooooo', + values: ['grrrrr'], + originalValue: ['grrrrr'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field.astring', + values: ['cool'], + originalValue: ['cool'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field.aNumber', + values: ['1'], + originalValue: ['1'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field.neat', + values: ['true'], + originalValue: ['true'], + isObjectArray: false, + }, +]; diff --git a/x-pack/test/api_integration/apis/security_solution/timeline_details.ts b/x-pack/test/api_integration/apis/security_solution/timeline_details.ts index 296e497eb4952d..d653528fd47e26 100644 --- a/x-pack/test/api_integration/apis/security_solution/timeline_details.ts +++ b/x-pack/test/api_integration/apis/security_solution/timeline_details.ts @@ -20,96 +20,133 @@ const EXPECTED_DATA = [ field: '@timestamp', values: ['2019-02-10T02:39:44.107Z'], originalValue: ['2019-02-10T02:39:44.107Z'], + isObjectArray: false, }, { category: '@version', field: '@version', values: ['1'], originalValue: ['1'], + isObjectArray: false, + }, + { + category: '_id', + field: '_id', + values: ['QRhG1WgBqd-n62SwZYDT'], + originalValue: ['QRhG1WgBqd-n62SwZYDT'], + isObjectArray: false, + }, + { + category: '_index', + field: '_index', + values: ['filebeat-7.0.0-iot-2019.06'], + originalValue: ['filebeat-7.0.0-iot-2019.06'], + isObjectArray: false, + }, + { + category: '_score', + field: '_score', + values: ['1'], + originalValue: ['1'], + isObjectArray: false, }, { category: 'agent', field: 'agent.ephemeral_id', values: ['909cd6a1-527d-41a5-9585-a7fb5386f851'], originalValue: ['909cd6a1-527d-41a5-9585-a7fb5386f851'], + isObjectArray: false, }, { category: 'agent', field: 'agent.hostname', values: ['raspberrypi'], originalValue: ['raspberrypi'], + isObjectArray: false, }, { category: 'agent', field: 'agent.id', values: ['4d3ea604-27e5-4ec7-ab64-44f82285d776'], originalValue: ['4d3ea604-27e5-4ec7-ab64-44f82285d776'], + isObjectArray: false, }, { category: 'agent', field: 'agent.type', values: ['filebeat'], originalValue: ['filebeat'], + isObjectArray: false, }, { category: 'agent', field: 'agent.version', values: ['7.0.0'], originalValue: ['7.0.0'], + isObjectArray: false, }, { category: 'destination', field: 'destination.domain', values: ['s3-iad-2.cf.dash.row.aiv-cdn.net'], originalValue: ['s3-iad-2.cf.dash.row.aiv-cdn.net'], + isObjectArray: false, }, { category: 'destination', field: 'destination.ip', values: ['10.100.7.196'], originalValue: ['10.100.7.196'], + isObjectArray: false, }, { category: 'destination', field: 'destination.port', values: ['40684'], originalValue: ['40684'], + isObjectArray: false, }, { category: 'ecs', field: 'ecs.version', values: ['1.0.0-beta2'], originalValue: ['1.0.0-beta2'], + isObjectArray: false, }, { category: 'event', field: 'event.dataset', values: ['suricata.eve'], originalValue: ['suricata.eve'], + isObjectArray: false, }, { category: 'event', field: 'event.end', values: ['2019-02-10T02:39:44.107Z'], originalValue: ['2019-02-10T02:39:44.107Z'], + isObjectArray: false, }, { category: 'event', field: 'event.kind', values: ['event'], originalValue: ['event'], + isObjectArray: false, }, { category: 'event', field: 'event.module', values: ['suricata'], originalValue: ['suricata'], + isObjectArray: false, }, { category: 'event', field: 'event.type', values: ['fileinfo'], originalValue: ['fileinfo'], + isObjectArray: false, }, { category: 'file', @@ -120,270 +157,484 @@ const EXPECTED_DATA = [ originalValue: [ '/dm/2$XTMWANo0Q2RZKlH-95UoAahZrOg~/0a9a/bf72/e1da/4c20-919e-0cbabcf7bfe8/75f50c57-d25f-4e97-9e37-01b9f5caa293_audio_13.mp4', ], + isObjectArray: false, }, { category: 'file', field: 'file.size', values: ['48277'], originalValue: ['48277'], + isObjectArray: false, }, { category: 'fileset', field: 'fileset.name', values: ['eve'], originalValue: ['eve'], + isObjectArray: false, }, { category: 'flow', field: 'flow.locality', values: ['public'], originalValue: ['public'], + isObjectArray: false, }, { category: 'host', field: 'host.architecture', values: ['armv7l'], originalValue: ['armv7l'], + isObjectArray: false, + }, + { + category: 'host', + field: 'host.containerized', + values: ['false'], + originalValue: ['false'], + isObjectArray: false, }, { category: 'host', field: 'host.hostname', values: ['raspberrypi'], originalValue: ['raspberrypi'], + isObjectArray: false, }, { category: 'host', field: 'host.id', values: ['b19a781f683541a7a25ee345133aa399'], originalValue: ['b19a781f683541a7a25ee345133aa399'], + isObjectArray: false, }, { category: 'host', field: 'host.name', values: ['raspberrypi'], originalValue: ['raspberrypi'], + isObjectArray: false, }, { category: 'host', field: 'host.os.codename', values: ['stretch'], originalValue: ['stretch'], + isObjectArray: false, }, { category: 'host', field: 'host.os.family', values: [''], originalValue: [''], + isObjectArray: false, }, { category: 'host', field: 'host.os.kernel', values: ['4.14.50-v7+'], originalValue: ['4.14.50-v7+'], + isObjectArray: false, }, { category: 'host', field: 'host.os.name', values: ['Raspbian GNU/Linux'], originalValue: ['Raspbian GNU/Linux'], + isObjectArray: false, }, { category: 'host', field: 'host.os.platform', values: ['raspbian'], originalValue: ['raspbian'], + isObjectArray: false, }, { category: 'host', field: 'host.os.version', values: ['9 (stretch)'], originalValue: ['9 (stretch)'], + isObjectArray: false, }, { category: 'http', field: 'http.request.method', values: ['get'], originalValue: ['get'], + isObjectArray: false, }, { category: 'http', field: 'http.response.body.bytes', values: ['48277'], originalValue: ['48277'], + isObjectArray: false, }, { category: 'http', field: 'http.response.status_code', values: ['206'], originalValue: ['206'], + isObjectArray: false, }, { category: 'input', field: 'input.type', values: ['log'], originalValue: ['log'], + isObjectArray: false, }, { category: 'base', field: 'labels.pipeline', values: ['filebeat-7.0.0-suricata-eve-pipeline'], originalValue: ['filebeat-7.0.0-suricata-eve-pipeline'], + isObjectArray: false, }, { category: 'log', field: 'log.file.path', values: ['/var/log/suricata/eve.json'], originalValue: ['/var/log/suricata/eve.json'], + isObjectArray: false, }, { category: 'log', field: 'log.offset', values: ['1856288115'], originalValue: ['1856288115'], + isObjectArray: false, }, { category: 'network', field: 'network.name', values: ['iot'], originalValue: ['iot'], + isObjectArray: false, }, { category: 'network', field: 'network.protocol', values: ['http'], originalValue: ['http'], + isObjectArray: false, }, { category: 'network', field: 'network.transport', values: ['tcp'], originalValue: ['tcp'], + isObjectArray: false, }, { category: 'service', field: 'service.type', values: ['suricata'], originalValue: ['suricata'], + isObjectArray: false, }, { category: 'source', field: 'source.as.num', values: ['16509'], originalValue: ['16509'], + isObjectArray: false, }, { category: 'source', field: 'source.as.org', values: ['Amazon.com, Inc.'], originalValue: ['Amazon.com, Inc.'], + isObjectArray: false, }, { category: 'source', field: 'source.domain', values: ['server-54-239-219-210.jfk51.r.cloudfront.net'], originalValue: ['server-54-239-219-210.jfk51.r.cloudfront.net'], + isObjectArray: false, }, { category: 'source', field: 'source.geo.city_name', values: ['Seattle'], originalValue: ['Seattle'], + isObjectArray: false, }, { category: 'source', field: 'source.geo.continent_name', values: ['North America'], originalValue: ['North America'], + isObjectArray: false, }, { category: 'source', field: 'source.geo.country_iso_code', values: ['US'], originalValue: ['US'], + isObjectArray: false, + }, + { + category: 'source', + field: 'source.geo.location', + values: ['{"lon":-122.3341,"lat":47.6103}'], + originalValue: ['{"lon":-122.3341,"lat":47.6103}'], + isObjectArray: true, }, { category: 'source', field: 'source.geo.location.lat', values: ['47.6103'], originalValue: ['47.6103'], + isObjectArray: false, }, { category: 'source', field: 'source.geo.location.lon', values: ['-122.3341'], originalValue: ['-122.3341'], + isObjectArray: false, }, { category: 'source', field: 'source.geo.region_iso_code', values: ['US-WA'], originalValue: ['US-WA'], + isObjectArray: false, }, { category: 'source', field: 'source.geo.region_name', values: ['Washington'], originalValue: ['Washington'], + isObjectArray: false, }, { category: 'source', field: 'source.ip', values: ['54.239.219.210'], originalValue: ['54.239.219.210'], + isObjectArray: false, }, { category: 'source', field: 'source.port', values: ['80'], originalValue: ['80'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.app_proto', + values: ['http'], + originalValue: ['http'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.dest_ip', + values: ['10.100.7.196'], + originalValue: ['10.100.7.196'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.dest_port', + values: ['40684'], + originalValue: ['40684'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.fileinfo.filename', + values: [ + '/dm/2$XTMWANo0Q2RZKlH-95UoAahZrOg~/0a9a/bf72/e1da/4c20-919e-0cbabcf7bfe8/75f50c57-d25f-4e97-9e37-01b9f5caa293_audio_13.mp4', + ], + originalValue: [ + '/dm/2$XTMWANo0Q2RZKlH-95UoAahZrOg~/0a9a/bf72/e1da/4c20-919e-0cbabcf7bfe8/75f50c57-d25f-4e97-9e37-01b9f5caa293_audio_13.mp4', + ], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.fileinfo.size', + values: ['48277'], + originalValue: ['48277'], + isObjectArray: false, }, { category: 'suricata', field: 'suricata.eve.fileinfo.state', values: ['CLOSED'], originalValue: ['CLOSED'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.fileinfo.stored', + values: ['false'], + originalValue: ['false'], + isObjectArray: false, }, { category: 'suricata', field: 'suricata.eve.fileinfo.tx_id', values: ['301'], originalValue: ['301'], + isObjectArray: false, }, { category: 'suricata', field: 'suricata.eve.flow_id', values: ['196625917175466'], originalValue: ['196625917175466'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.http.hostname', + values: ['s3-iad-2.cf.dash.row.aiv-cdn.net'], + originalValue: ['s3-iad-2.cf.dash.row.aiv-cdn.net'], + isObjectArray: false, }, { category: 'suricata', field: 'suricata.eve.http.http_content_type', values: ['video/mp4'], originalValue: ['video/mp4'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.http.http_method', + values: ['get'], + originalValue: ['get'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.http.length', + values: ['48277'], + originalValue: ['48277'], + isObjectArray: false, }, { category: 'suricata', field: 'suricata.eve.http.protocol', values: ['HTTP/1.1'], originalValue: ['HTTP/1.1'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.http.status', + values: ['206'], + originalValue: ['206'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.http.url', + values: [ + '/dm/2$XTMWANo0Q2RZKlH-95UoAahZrOg~/0a9a/bf72/e1da/4c20-919e-0cbabcf7bfe8/75f50c57-d25f-4e97-9e37-01b9f5caa293_audio_13.mp4', + ], + originalValue: [ + '/dm/2$XTMWANo0Q2RZKlH-95UoAahZrOg~/0a9a/bf72/e1da/4c20-919e-0cbabcf7bfe8/75f50c57-d25f-4e97-9e37-01b9f5caa293_audio_13.mp4', + ], + isObjectArray: false, }, { category: 'suricata', field: 'suricata.eve.in_iface', values: ['eth0'], originalValue: ['eth0'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.proto', + values: ['tcp'], + originalValue: ['tcp'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.src_ip', + values: ['54.239.219.210'], + originalValue: ['54.239.219.210'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.src_port', + values: ['80'], + originalValue: ['80'], + isObjectArray: false, + }, + { + category: 'suricata', + field: 'suricata.eve.timestamp', + values: ['2019-02-10T02:39:44.107Z'], + originalValue: ['2019-02-10T02:39:44.107Z'], + isObjectArray: false, }, { category: 'base', field: 'tags', values: ['suricata'], originalValue: ['suricata'], + isObjectArray: false, + }, + { + category: 'traefik', + field: 'traefik.access.geoip.city_name', + values: ['Seattle'], + originalValue: ['Seattle'], + isObjectArray: false, + }, + { + category: 'traefik', + field: 'traefik.access.geoip.continent_name', + values: ['North America'], + originalValue: ['North America'], + isObjectArray: false, + }, + { + category: 'traefik', + field: 'traefik.access.geoip.country_iso_code', + values: ['US'], + originalValue: ['US'], + isObjectArray: false, + }, + { + category: 'traefik', + field: 'traefik.access.geoip.location', + values: ['{"lon":-122.3341,"lat":47.6103}'], + originalValue: ['{"lon":-122.3341,"lat":47.6103}'], + isObjectArray: true, + }, + { + category: 'traefik', + field: 'traefik.access.geoip.region_iso_code', + values: ['US-WA'], + originalValue: ['US-WA'], + isObjectArray: false, + }, + { + category: 'traefik', + field: 'traefik.access.geoip.region_name', + values: ['Washington'], + originalValue: ['Washington'], + isObjectArray: false, }, { category: 'url', field: 'url.domain', values: ['s3-iad-2.cf.dash.row.aiv-cdn.net'], originalValue: ['s3-iad-2.cf.dash.row.aiv-cdn.net'], + isObjectArray: false, }, { category: 'url', @@ -394,6 +645,7 @@ const EXPECTED_DATA = [ originalValue: [ '/dm/2$XTMWANo0Q2RZKlH-95UoAahZrOg~/0a9a/bf72/e1da/4c20-919e-0cbabcf7bfe8/75f50c57-d25f-4e97-9e37-01b9f5caa293_audio_13.mp4', ], + isObjectArray: false, }, { category: 'url', @@ -404,27 +656,9 @@ const EXPECTED_DATA = [ originalValue: [ '/dm/2$XTMWANo0Q2RZKlH-95UoAahZrOg~/0a9a/bf72/e1da/4c20-919e-0cbabcf7bfe8/75f50c57-d25f-4e97-9e37-01b9f5caa293_audio_13.mp4', ], - }, - { - category: '_index', - field: '_index', - values: ['filebeat-7.0.0-iot-2019.06'], - originalValue: ['filebeat-7.0.0-iot-2019.06'], - }, - { - category: '_id', - field: '_id', - values: ['QRhG1WgBqd-n62SwZYDT'], - originalValue: ['QRhG1WgBqd-n62SwZYDT'], - }, - { - category: '_score', - field: '_score', - values: ['1'], - originalValue: ['1'], + isObjectArray: false, }, ]; - const EXPECTED_KPI_COUNTS = { destinationIpCount: 154, hostCount: 1, @@ -456,7 +690,7 @@ export default function ({ getService }: FtrProviderContext) { wait_for_completion_timeout: '10s', }) .expect(200); - expect(sortBy(detailsData, 'name')).to.eql(sortBy(EXPECTED_DATA, 'name')); + expect(sortBy(detailsData, 'field')).to.eql(sortBy(EXPECTED_DATA, 'field')); }); it('Make sure that we get kpi data', async () => {