Skip to content

Commit

Permalink
[Security Solution][Timeline] Rebuild nested fields structure from fi…
Browse files Browse the repository at this point in the history 
…elds response (elastic#96187)

* First pass at rebuilding nested object structure from fields response

* Always requests TIMELINE_CTI_FIELDS as part of request

This only works for one level of nesting; will be extending tests to
allow for multiple levels momentarily.

* Build objects from arbitrary levels of nesting

This is a recursive implementation, but recursion depth is limited to
the number of levels of nesting, with arguments reducing in size as we
go (i.e. logarithmic)

* Simplify parsing logic, perf improvements

* Order short-circuiting conditions by cost, ascending
* Simplify object building for non-nested objects from fields
  * The non-nested case is the same as the base recursive case, so
    always call our recursive function if building from .fields
* Simplify getNestedParentPath
  * We can do a few simple string comparison rather than building up
    multiple strings/arrays
* Don't call getNestedParentPath unnecessarily, only if we have a field

* Simplify if branching

By definition, nestedParentFieldName can never be equal to fieldName, which means
there are only two branches here.

* Declare/export a more accurate fields type

Each top-level field value can be either an array of leaf values
(unknown[]), or an array of nested fields.

* Remove unnecessary condition

If fieldName is null or undefined, there is no reason to search for it
in dataFields. Looking through the git history this looks to be dead
code as a result of refactoring, as opposed to a legitimate bugfix, so
I'm removing it.

* Fix failing tests

* one was a test failure due to my modifying mock data
* one may have been a legitimate bug where we don't handle a hit without
  a fields response; I need to follow up with Xavier to verify.

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
@rylnd @kibanamachine
rylnd and kibanamachine committed Apr 13, 2021
1 parent 0a6fd4e commit 585bbcb
Show file tree
Hide file tree
Showing 5 changed files with 260 additions and 34 deletions.
4 changes: 3 additions & 1 deletion ...curity_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@ export interface EventsActionGroupData {
doc_count: number;
}

export type Fields = Record<string, unknown[] | Fields[]>;

export interface EventHit extends SearchHit {
sort: string[];
_source: EventSource;
fields: Record<string, unknown[]>;
fields: Fields;
aggregations: {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
[agg: string]: any;
Expand Down
6 changes: 3 additions & 3 deletions x-pack/plugins/security_solution/common/utils/mock_event_details.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ export const eventHit = {
'source.geo.location': [{ coordinates: [118.7778, 32.0617], type: 'Point' }],
'threat.indicator': [
{
'matched.field': ['matched_field'],
'matched.field': ['matched_field', 'other_matched_field'],
first_seen: ['2021-02-22T17:29:25.195Z'],
provider: ['yourself'],
type: ['custom'],
Expand Down Expand Up @@ -259,8 +259,8 @@ export const eventDetailsFormattedFields = [
{
category: 'threat',
field: 'threat.indicator.matched.field',
values: ['matched_field', 'matched_field_2'],
originalValue: ['matched_field', 'matched_field_2'],
values: ['matched_field', 'other_matched_field', 'matched_field_2'],
originalValue: ['matched_field', 'other_matched_field', 'matched_field_2'],
isObjectArray: false,
},
{
Expand Down
10 changes: 10 additions & 0 deletions ...plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,15 @@
* 2.0.
*/

export const TIMELINE_CTI_FIELDS = [
'threat.indicator.event.dataset',
'threat.indicator.event.reference',
'threat.indicator.matched.atomic',
'threat.indicator.matched.field',
'threat.indicator.matched.type',
'threat.indicator.provider',
];

export const TIMELINE_EVENTS_FIELDS = [
'@timestamp',
'signal.status',
Expand Down Expand Up @@ -230,4 +239,5 @@ export const TIMELINE_EVENTS_FIELDS = [
'zeek.ssl.established',
'zeek.ssl.resumed',
'zeek.ssl.version',
...TIMELINE_CTI_FIELDS,
];
209 changes: 203 additions & 6 deletions ...gins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
* 2.0.
*/

import { eventHit } from '../../../../../../common/utils/mock_event_details';
import { EventHit } from '../../../../../../common/search_strategy';
import { TIMELINE_EVENTS_FIELDS } from './constants';
import { formatTimelineData } from './helpers';
import { eventHit } from '../../../../../../common/utils/mock_event_details';
import { buildObjectForFieldPath, formatTimelineData } from './helpers';

describe('#formatTimelineData', () => {
it('happy path', async () => {
Expand Down Expand Up @@ -42,12 +42,12 @@ describe('#formatTimelineData', () => {
value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
},
{
field: 'source.geo.location',
value: [`{"lon":118.7778,"lat":32.0617}`],
field: 'threat.indicator.matched.field',
value: ['matched_field', 'other_matched_field', 'matched_field_2'],
},
{
field: 'threat.indicator.matched.field',
value: ['matched_field', 'matched_field_2'],
field: 'source.geo.location',
value: [`{"lon":118.7778,"lat":32.0617}`],
},
],
ecs: {
Expand Down Expand Up @@ -94,6 +94,34 @@ describe('#formatTimelineData', () => {
user: {
name: ['jenkins'],
},
threat: {
indicator: [
{
event: {
dataset: [],
reference: [],
},
matched: {
atomic: ['matched_atomic'],
field: ['matched_field', 'other_matched_field'],
type: [],
},
provider: ['yourself'],
},
{
event: {
dataset: [],
reference: [],
},
matched: {
atomic: ['matched_atomic_2'],
field: ['matched_field_2'],
type: [],
},
provider: ['other_you'],
},
],
},
},
},
});
Expand Down Expand Up @@ -371,4 +399,173 @@ describe('#formatTimelineData', () => {
},
});
});

describe('buildObjectForFieldPath', () => {
it('builds an object from a single non-nested field', () => {
expect(buildObjectForFieldPath('@timestamp', eventHit)).toEqual({
'@timestamp': ['2020-11-17T14:48:08.922Z'],
});
});

it('builds an object with no fields response', () => {
const { fields, ...fieldLessHit } = eventHit;
// @ts-expect-error fieldLessHit is intentionally missing fields
expect(buildObjectForFieldPath('@timestamp', fieldLessHit)).toEqual({
'@timestamp': [],
});
});

it('does not misinterpret non-nested fields with a common prefix', () => {
// @ts-expect-error hit is minimal
const hit: EventHit = {
fields: {
'foo.bar': ['baz'],
'foo.barBaz': ['foo'],
},
};

expect(buildObjectForFieldPath('foo.barBaz', hit)).toEqual({
foo: { barBaz: ['foo'] },
});
});

it('builds an array of objects from a nested field', () => {
// @ts-expect-error hit is minimal
const hit: EventHit = {
fields: {
foo: [{ bar: ['baz'] }],
},
};
expect(buildObjectForFieldPath('foo.bar', hit)).toEqual({
foo: [{ bar: ['baz'] }],
});
});

it('builds intermediate objects for nested fields', () => {
// @ts-expect-error nestedHit is minimal
const nestedHit: EventHit = {
fields: {
'foo.bar': [
{
baz: ['host.name'],
},
],
},
};
expect(buildObjectForFieldPath('foo.bar.baz', nestedHit)).toEqual({
foo: {
bar: [
{
baz: ['host.name'],
},
],
},
});
});

it('builds intermediate objects at multiple levels', () => {
expect(buildObjectForFieldPath('threat.indicator.matched.atomic', eventHit)).toEqual({
threat: {
indicator: [
{
matched: {
atomic: ['matched_atomic'],
},
},
{
matched: {
atomic: ['matched_atomic_2'],
},
},
],
},
});
});

it('preserves multiple values for a single leaf', () => {
expect(buildObjectForFieldPath('threat.indicator.matched.field', eventHit)).toEqual({
threat: {
indicator: [
{
matched: {
field: ['matched_field', 'other_matched_field'],
},
},
{
matched: {
field: ['matched_field_2'],
},
},
],
},
});
});

describe('multiple levels of nested fields', () => {
let nestedHit: EventHit;

beforeEach(() => {
// @ts-expect-error nestedHit is minimal
nestedHit = {
fields: {
'nested_1.foo': [
{
'nested_2.bar': [
{ leaf: ['leaf_value'], leaf_2: ['leaf_2_value'] },
{ leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
],
},
{
'nested_2.bar': [
{ leaf: ['leaf_value_2'], leaf_2: ['leaf_2_value_4'] },
{ leaf: ['leaf_value_3'], leaf_2: ['leaf_2_value_5'] },
],
},
],
},
};
});

it('includes objects without the field', () => {
expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf', nestedHit)).toEqual({
nested_1: {
foo: [
{
nested_2: {
bar: [{ leaf: ['leaf_value'] }, { leaf: [] }],
},
},
{
nested_2: {
bar: [{ leaf: ['leaf_value_2'] }, { leaf: ['leaf_value_3'] }],
},
},
],
},
});
});

it('groups multiple leaf values', () => {
expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf_2', nestedHit)).toEqual({
nested_1: {
foo: [
{
nested_2: {
bar: [
{ leaf_2: ['leaf_2_value'] },
{ leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
],
},
},
{
nested_2: {
bar: [{ leaf_2: ['leaf_2_value_4'] }, { leaf_2: ['leaf_2_value_5'] }],
},
},
],
},
});
});
});
});
});
Loading

0 comments on commit 585bbcb

Please sign in to comment.