Skip to content

Commit

Permalink
re-re-revert? - went back to older commit that accomadated signal clo…
Browse files Browse the repository at this point in the history 
…sure better, unit tests need updating
  • Loading branch information
@yctercero
yctercero committed Jul 15, 2020
1 parent 6f941a2 commit 89b26ab
Show file tree
Hide file tree
Showing 7 changed files with 197 additions and 131 deletions.
54 changes: 23 additions & 31 deletions x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1103,6 +1103,12 @@ describe('build_exceptions_query', () => {
expect(query).toEqual([]);
});

test('it returns empty array if lists is undefined', () => {
const query = buildQueryExceptions({ language: 'kuery', lists: undefined });

expect(query).toEqual([]);
});

test('it returns expected query when lists exist and language is "kuery"', () => {
const payload = getExceptionListItemSchemaMock();
const payload2 = getExceptionListItemSchemaMock();
Expand All @@ -1112,43 +1118,33 @@ describe('build_exceptions_query', () => {
field: 'parent',
type: 'nested',
entries: [
makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }),
makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }),
makeMatchEntry({ field: 'c', operator: 'included', value: 'valueC' }),
makeMatchEntry({ field: 'd', operator: 'included', value: 'valueD' }),
],
},
makeMatchAnyEntry({ field: 'e' }),
makeMatchAnyEntry({ field: 'e', operator: 'excluded' }),
];
const query = buildQueryExceptions({
language: 'kuery',
lists: [payload, payload2],
});
const expectedQuery =
'some.parentField:{ nested.field:"some value" } and not some.not.nested.field:"some value" and not b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and not e:("value-1" or "value-2")';
'not ((some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value") or (b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and not e:("value-1" or "value-2")))';

expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]);
});

test('it returns expected query when lists exist and language is "lucene"', () => {
const payload = getExceptionListItemSchemaMock();
payload.entries = [makeMatchAnyEntry({ field: 'a' }), makeMatchAnyEntry({ field: 'b' })];
const payload2 = getExceptionListItemSchemaMock();
payload2.entries = [
makeMatchAnyEntry({ field: 'b' }),
{
field: 'parent',
type: 'nested',
entries: [
makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }),
makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }),
],
},
makeMatchAnyEntry({ field: 'e' }),
];
payload2.entries = [makeMatchAnyEntry({ field: 'c' }), makeMatchAnyEntry({ field: 'd' })];
const query = buildQueryExceptions({
language: 'lucene',
lists: [payload, payload2],
});
const expectedQuery =
'some.parentField:{ nested.field:"some value" } AND NOT some.not.nested.field:"some value" AND NOT b:("value-1" OR "value-2") AND parent:{ c:"valueC" AND d:"valueD" } AND NOT e:("value-1" OR "value-2")';
'NOT ((a:("value-1" OR "value-2") AND b:("value-1" OR "value-2")) OR (c:("value-1" OR "value-2") AND d:("value-1" OR "value-2")))';

expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]);
});
Expand All @@ -1168,6 +1164,12 @@ describe('build_exceptions_query', () => {
expect(query).toEqual([]);
});

test('it returns empty array if lists is undefined', () => {
const query = buildQueryExceptions({ language: 'kuery', lists: undefined, exclude });

expect(query).toEqual([]);
});

test('it returns expected query when lists exist and language is "kuery"', () => {
const payload = getExceptionListItemSchemaMock();
const payload2 = getExceptionListItemSchemaMock();
Expand All @@ -1189,33 +1191,23 @@ describe('build_exceptions_query', () => {
exclude,
});
const expectedQuery =
'some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value" and b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and e:("value-1" or "value-2")';
'(some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value") or (b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and e:("value-1" or "value-2"))';

expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]);
});

test('it returns expected query when lists exist and language is "lucene"', () => {
const payload = getExceptionListItemSchemaMock();
payload.entries = [makeMatchAnyEntry({ field: 'a' }), makeMatchAnyEntry({ field: 'b' })];
const payload2 = getExceptionListItemSchemaMock();
payload2.entries = [
makeMatchAnyEntry({ field: 'b' }),
{
field: 'parent',
type: 'nested',
entries: [
makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }),
makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }),
],
},
makeMatchAnyEntry({ field: 'e' }),
];
payload2.entries = [makeMatchAnyEntry({ field: 'c' }), makeMatchAnyEntry({ field: 'd' })];
const query = buildQueryExceptions({
language: 'lucene',
lists: [payload, payload2],
exclude,
});
const expectedQuery =
'some.parentField:{ nested.field:"some value" } AND some.not.nested.field:"some value" AND b:("value-1" OR "value-2") AND parent:{ c:"valueC" AND d:"valueD" } AND e:("value-1" OR "value-2")';
'(a:("value-1" OR "value-2") AND b:("value-1" OR "value-2")) OR (c:("value-1" OR "value-2") AND d:("value-1" OR "value-2"))';

expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]);
});
Expand Down
89 changes: 43 additions & 46 deletions x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
CreateExceptionListItemSchema,
} from '../shared_imports';
import { Language } from './schemas/common/schemas';
import { hasLargeValueList } from './utils';

type Operators = 'and' | 'or' | 'not';
type LuceneOperators = 'AND' | 'OR' | 'NOT';
Expand All @@ -46,18 +47,16 @@ export const getLanguageBooleanOperator = ({
export const operatorBuilder = ({
operator,
language,
exclude,
}: {
operator: Operator;
language: Language;
exclude: boolean;
}): string => {
const not = getLanguageBooleanOperator({
language,
value: 'not',
});

if ((exclude && operator === 'included') || (!exclude && operator === 'excluded')) {
if (operator === 'excluded') {
return `${not} `;
} else {
return '';
Expand All @@ -67,14 +66,12 @@ export const operatorBuilder = ({
export const buildExists = ({
item,
language,
exclude,
}: {
item: EntryExists;
language: Language;
exclude: boolean;
}): string => {
const { operator, field } = item;
const exceptionOperator = operatorBuilder({ operator, language, exclude });
const exceptionOperator = operatorBuilder({ operator, language });

switch (language) {
case 'kuery':
Expand All @@ -89,26 +86,22 @@ export const buildExists = ({
export const buildMatch = ({
item,
language,
exclude,
}: {
item: EntryMatch;
language: Language;
exclude: boolean;
}): string => {
const { value, operator, field } = item;
const exceptionOperator = operatorBuilder({ operator, language, exclude });
const exceptionOperator = operatorBuilder({ operator, language });

return `${exceptionOperator}${field}:"${value}"`;
};

export const buildMatchAny = ({
item,
language,
exclude,
}: {
item: EntryMatchAny;
language: Language;
exclude: boolean;
}): string => {
const { value, operator, field } = item;

Expand All @@ -117,7 +110,7 @@ export const buildMatchAny = ({
return '';
default:
const or = getLanguageBooleanOperator({ language, value: 'or' });
const exceptionOperator = operatorBuilder({ operator, language, exclude });
const exceptionOperator = operatorBuilder({ operator, language });
const matchAnyValues = value.map((v) => `"${v}"`);

return `${exceptionOperator}${field}:(${matchAnyValues.join(` ${or} `)})`;
Expand All @@ -141,18 +134,16 @@ export const buildNested = ({
export const evaluateValues = ({
item,
language,
exclude,
}: {
item: Entry | EntryNested;
language: Language;
exclude: boolean;
}): string => {
if (entriesExists.is(item)) {
return buildExists({ item, language, exclude });
return buildExists({ item, language });
} else if (entriesMatch.is(item)) {
return buildMatch({ item, language, exclude });
return buildMatch({ item, language });
} else if (entriesMatchAny.is(item)) {
return buildMatchAny({ item, language, exclude });
return buildMatchAny({ item, language });
} else if (entriesNested.is(item)) {
return buildNested({ item, language });
} else {
Expand All @@ -163,40 +154,40 @@ export const evaluateValues = ({
export const formatQuery = ({
exceptions,
language,
exclude,
}: {
exceptions: string[];
language: Language;
exclude: boolean;
}): string => {
const and = getLanguageBooleanOperator({ language, value: 'and' });
const formattedExceptions = exceptions.map((exception, index) => {
const or = getLanguageBooleanOperator({ language, value: 'or' });
const not = getLanguageBooleanOperator({ language, value: 'not' });
const formattedExceptionItems = exceptions.map((exceptionItem, index) => {
if (index === 0) {
return exception;
return `(${exceptionItem})`;
}

return `${and} ${exception}`;
return `${or} (${exceptionItem})`;
});

return formattedExceptions.join(' ');
const exceptionItemsQuery = formattedExceptionItems.join(' ');
return exclude ? `${not} (${exceptionItemsQuery})` : exceptionItemsQuery;
};

export const buildExceptionItemEntries = ({
lists,
entries,
language,
exclude,
}: {
lists: EntriesArray;
entries: EntriesArray;
language: Language;
exclude: boolean;
}): string => {
const and = getLanguageBooleanOperator({ language, value: 'and' });
const exceptionItem = lists
.filter(({ type }) => type !== 'list')
.reduce<string[]>((accum, listItem) => {
const exceptionSegment = evaluateValues({ item: listItem, language, exclude });
return [...accum, exceptionSegment];
}, []);

return exceptionItem.join(` ${and} `);
const exceptionItemEntries = entries.reduce<string[]>((accum, listItem) => {
const exceptionSegment = evaluateValues({ item: listItem, language });
return [...accum, exceptionSegment];
}, []);

return exceptionItemEntries.join(` ${and} `);
};

export const buildQueryExceptions = ({
Expand All @@ -208,23 +199,29 @@ export const buildQueryExceptions = ({
lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema> | undefined;
exclude?: boolean;
}): DataQuery[] => {
if (lists != null && lists.length > 0) {
const exceptions = lists.reduce<string[]>((acc, exceptionItem) => {
return [
...acc,
...(exceptionItem.entries !== undefined
? [buildExceptionItemEntries({ lists: exceptionItem.entries, language, exclude })]
: []),
];
}, []);
const formattedQuery = formatQuery({ exceptions, language });
if (lists == null || (lists != null && lists.length === 0)) {
return [];
}

const exceptionItems = lists.reduce<string[]>((acc, exceptionItem) => {
const { entries } = exceptionItem;

if (entries != null && entries.length > 0 && !hasLargeValueList(entries)) {
return [...acc, buildExceptionItemEntries({ entries, language })];
} else {
return acc;
}
}, []);

if (exceptionItems.length === 0) {
return [];
} else {
const formattedQuery = formatQuery({ exceptions: exceptionItems, language, exclude });
return [
{
query: formattedQuery,
language,
},
];
} else {
return [];
}
};
7 changes: 7 additions & 0 deletions x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@ export const getQueryFilter = (
};

const initialQuery = [{ query, language }];
/*
* Pinning exceptions to 'kuery' because lucene
* does not support nested queries, while our exceptions
* UI does, since we can pass both lucene and kql into
* buildEsQuery, this allows us to offer nested queries
* regardless
*/
const exceptions = buildQueryExceptions({ language: 'kuery', lists, exclude: excludeExceptions });
const queries: DataQuery[] = [...initialQuery, ...exceptions];

Expand Down
Loading

0 comments on commit 89b26ab

Please sign in to comment.