Skip to content

Commit

Permalink
First pass at rebuilding nested object structure from fields response
Browse files Browse the repository at this point in the history 
* Always requests TIMELINE_CTI_FIELDS as part of request

This only works for one level of nesting; will be extending tests to
allow for multiple levels momentarily.
  • Loading branch information
@rylnd
rylnd committed Apr 2, 2021
1 parent eae160c commit 1cb213c
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 22 deletions.
10 changes: 10 additions & 0 deletions ...plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,15 @@
* 2.0.
*/

export const TIMELINE_CTI_FIELDS = [
'threat.indicator.event.dataset',
'threat.indicator.event.reference',
'threat.indicator.matched.atomic',
'threat.indicator.matched.field',
'threat.indicator.matched.type',
'threat.indicator.provider',
];

export const TIMELINE_EVENTS_FIELDS = [
'@timestamp',
'signal.status',
Expand Down Expand Up @@ -230,4 +239,5 @@ export const TIMELINE_EVENTS_FIELDS = [
'zeek.ssl.established',
'zeek.ssl.resumed',
'zeek.ssl.version',
...TIMELINE_CTI_FIELDS,
];
122 changes: 117 additions & 5 deletions ...gins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import { EventHit } from '../../../../../../common/search_strategy';
import { TIMELINE_EVENTS_FIELDS } from './constants';
import { formatTimelineData } from './helpers';
import { buildObjectFromField, formatTimelineData } from './helpers';
import { eventHit } from '../mocks';

describe('#formatTimelineData', () => {
Expand Down Expand Up @@ -42,12 +42,12 @@ describe('#formatTimelineData', () => {
value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
},
{
field: 'source.geo.location',
value: [`{"lon":118.7778,"lat":32.0617}`],
field: 'threat.indicator.matched.field',
value: ['matched_field', 'other_matched_field', 'matched_field_2'],
},
{
field: 'threat.indicator.matched.field',
value: ['matched_field', 'matched_field_2'],
field: 'source.geo.location',
value: [`{"lon":118.7778,"lat":32.0617}`],
},
],
ecs: {
Expand Down Expand Up @@ -94,6 +94,34 @@ describe('#formatTimelineData', () => {
user: {
name: ['jenkins'],
},
threat: {
indicator: [
{
event: {
dataset: [],
reference: [],
},
matched: {
atomic: ['matched_atomic'],
field: ['matched_field', 'other_matched_field'],
type: [],
},
provider: ['yourself'],
},
{
event: {
dataset: [],
reference: [],
},
matched: {
atomic: ['matched_atomic_2'],
field: ['matched_field_2'],
type: [],
},
provider: ['other_you'],
},
],
},
},
},
});
Expand Down Expand Up @@ -371,4 +399,88 @@ describe('#formatTimelineData', () => {
},
});
});

describe('buildObjectFromField', () => {
it('base case', () => {
expect(buildObjectFromField('@timestamp', eventHit)).toEqual({
'@timestamp': ['2020-11-17T14:48:08.922Z'],
});
});

it('another base case', () => {
const event = {
fields: {
foo: [{ bar: ['baz'] }],
},
};
// @ts-expect-error nestedEvent is minimal
expect(buildObjectFromField('foo.bar', event, 'foo')).toEqual({
foo: [{ bar: ['baz'] }],
});
});

it('simple nested', () => {
const nestedEvent = {
fields: {
'foo.bar': [
{
baz: ['host.name'],
},
],
},
};
// @ts-expect-error nestedEvent is minimal
expect(buildObjectFromField('foo.bar.baz', nestedEvent, 'foo.bar')).toEqual({
foo: {
bar: [
{
baz: ['host.name'],
},
],
},
});
});

it('nested field', () => {
expect(
buildObjectFromField('threat.indicator.matched.atomic', eventHit, 'threat.indicator')
).toEqual({
threat: {
indicator: [
{
matched: {
atomic: ['matched_atomic'],
},
},
{
matched: {
atomic: ['matched_atomic_2'],
},
},
],
},
});
});

it('nested field with multiple values', () => {
expect(
buildObjectFromField('threat.indicator.matched.field', eventHit, 'threat.indicator')
).toEqual({
threat: {
indicator: [
{
matched: {
field: ['matched_field', 'other_matched_field'],
},
},
{
matched: {
field: ['matched_field_2'],
},
},
],
},
});
});
});
});
55 changes: 39 additions & 16 deletions ...k/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
* 2.0.
*/

import { set } from '@elastic/safer-lodash-set';
import { get, has, merge, uniq } from 'lodash/fp';
import {
EventHit,
Expand Down Expand Up @@ -99,22 +100,53 @@ const getValuesFromFields = async (
);
};

export const buildObjectFromField = (
fieldPath: string,
hit: EventHit,
nestedParentPath?: string
) => {
if (nestedParentPath) {
const childPath = fieldPath.replace(`${nestedParentPath}.`, '');
return set(
{},
nestedParentPath,
(get(nestedParentPath, hit.fields) ?? []).map((nestedFields) => {
const value = get(childPath, nestedFields);
return set({}, childPath, toStringArray(value));
})
);
} else {
const value = has(fieldPath, hit._source)
? get(fieldPath, hit._source)
: get(fieldPath, hit.fields);
return set({}, fieldPath, toStringArray(value));
}
};

/**
* If a prefix of our full field path is present as a field, we know that our field is nested
*/
const getNestedParentPath = (fieldPath: string, fields: Record<string, unknown[]> = {}) => {
const fieldParts = fieldPath.split('.');
return Object.keys(fields).find(
(field) =>
field !== fieldPath && field === fieldParts.slice(0, field.split('.').length).join('.')
);
};

const mergeTimelineFieldsWithHit = async <T>(
fieldName: string,
flattenedFields: T,
hit: EventHit,
dataFields: readonly string[],
ecsFields: readonly string[]
) => {
const nestedParentPath = getNestedParentPath(fieldName, hit.fields);
if (fieldName != null || dataFields.includes(fieldName)) {
const fieldNameAsArray = fieldName.split('.');
const nestedParentFieldName = Object.keys(hit.fields ?? []).find((f) => {
return f === fieldNameAsArray.slice(0, f.split('.').length).join('.');
});
if (
has(fieldName, hit._source) ||
has(fieldName, hit.fields) ||
nestedParentFieldName != null ||
nestedParentPath != null ||
specialFields.includes(fieldName)
) {
const objectWithProperty = {
Expand All @@ -123,22 +155,13 @@ const mergeTimelineFieldsWithHit = async <T>(
data: dataFields.includes(fieldName)
? [
...get('node.data', flattenedFields),
...(await getValuesFromFields(fieldName, hit, nestedParentFieldName)),
...(await getValuesFromFields(fieldName, hit, nestedParentPath)),
]
: get('node.data', flattenedFields),
ecs: ecsFields.includes(fieldName)
? {
...get('node.ecs', flattenedFields),
// @ts-expect-error
...fieldName.split('.').reduceRight(
// @ts-expect-error
(obj, next) => ({ [next]: obj }),
toStringArray(
has(fieldName, hit._source)
? get(fieldName, hit._source)
: hit.fields[fieldName]
)
),
...buildObjectFromField(fieldName, hit, nestedParentPath),
}
: get('node.ecs', flattenedFields),
},
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/mocks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ export const eventHit = {
'source.geo.location': [{ coordinates: [118.7778, 32.0617], type: 'Point' }],
'threat.indicator': [
{
'matched.field': ['matched_field'],
'matched.field': ['matched_field', 'other_matched_field'],
first_seen: ['2021-02-22T17:29:25.195Z'],
provider: ['yourself'],
type: ['custom'],
Expand Down

0 comments on commit 1cb213c

Please sign in to comment.