From 761cfb3546eaefeaed8430b97779734a1612b162 Mon Sep 17 00:00:00 2001 From: Angela Chuang <6295984+angorayc@users.noreply.github.com> Date: Tue, 24 Nov 2020 15:06:21 +0000 Subject: [PATCH 1/2] [Security Solution] Fix incorrect time for dns histogram (#83532) * getSuitableUnit * update dns histogram query * update dns query * update dns histogram query * fix type error * fix lint error * remove unused comments * fix histogram query size * revert change * fix unit test * fix dns request options * clean up * cleanup types * fix dependency * review * review * revert * restore docValueFields * fix unit test * cleanup * restore docValueFields for dns histogram * review * review * lint Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> # Conflicts: # x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx # x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts --- .../matrix_histogram/index.ts | 1 + .../common/components/charts/barchart.tsx | 7 +- .../components/matrix_histogram/index.tsx | 4 + .../components/matrix_histogram/types.ts | 3 + .../containers/matrix_histogram/index.ts | 20 +- .../containers/network_dns/histogram.ts | 65 -- .../network/containers/network_dns/index.tsx | 31 +- .../pages/navigation/dns_query_tab_body.tsx | 15 +- .../pages/navigation/network_routes.tsx | 3 +- .../public/network/pages/navigation/types.ts | 6 +- .../public/network/pages/network.tsx | 1 + .../server/lib/hosts/query.hosts.dsl.ts | 2 +- .../hosts/query.last_first_seen_host.dsl.ts | 2 +- .../factory/hosts/all/__mocks__/index.ts | 56 +- .../factory/hosts/all/query.all_hosts.dsl.ts | 2 +- .../hosts/authentications/__mocks__/index.ts | 2 + .../hosts/authentications/dsl/query.dsl.ts | 2 +- .../hosts/last_first_seen/__mocks__/index.ts | 28 +- .../query.last_first_seen_host.dsl.ts | 2 +- .../matrix_histogram/__mocks__/index.ts | 929 ++++++++++++++---- .../matrix_histogram/dns/__mocks__/index.ts | 53 +- .../factory/matrix_histogram/dns/helpers.ts | 7 +- .../matrix_histogram/dns/index.test.ts | 4 +- .../factory/matrix_histogram/dns/index.ts | 4 +- .../dns/query.dns_histogram.dsl.ts | 121 ++- .../network/details/__mocks__/index.ts | 97 +- .../details/query.details_network.dsl.ts | 2 +- .../network/dns/query.dns_network.dsl.ts | 2 +- .../factory/network/http/__mocks__/index.ts | 6 +- .../events/all/query.events_all.dsl.ts | 2 +- .../query.events_last_event_time.dsl.ts | 6 +- 31 files changed, 1103 insertions(+), 382 deletions(-) delete mode 100644 x-pack/plugins/security_solution/public/network/containers/network_dns/histogram.ts diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts index 0217c48668fb90..f7ada91784ab42 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts @@ -37,6 +37,7 @@ export interface MatrixHistogramRequestOptions extends RequestBasicOptions { stackByField: string; threshold?: { field: string | undefined; value: number } | undefined; inspect?: Maybe; + isPtrIncluded?: boolean; } export interface MatrixHistogramStrategyResponse extends IEsSearchResponse { diff --git a/x-pack/plugins/security_solution/public/common/components/charts/barchart.tsx b/x-pack/plugins/security_solution/public/common/components/charts/barchart.tsx index fb1ed956dfc52c..5a50442f8dd5f6 100644 --- a/x-pack/plugins/security_solution/public/common/components/charts/barchart.tsx +++ b/x-pack/plugins/security_solution/public/common/components/charts/barchart.tsx @@ -47,6 +47,9 @@ const checkIfAnyValidSeriesExist = ( !checkIfAllValuesAreZero(data) && data.some(checkIfAllTheDataInTheSeriesAreValid); +const yAccessors = ['y']; +const splitSeriesAccessors = ['g']; + // Bar chart rotation: https://ela.st/chart-rotations export const BarChartBaseComponent = ({ data, @@ -86,9 +89,9 @@ export const BarChartBaseComponent = ({ xScaleType={getOr(ScaleType.Linear, 'configs.series.xScaleType', chartConfigs)} yScaleType={getOr(ScaleType.Linear, 'configs.series.yScaleType', chartConfigs)} xAccessor="x" - yAccessors={['y']} + yAccessors={yAccessors} timeZone={timeZone} - splitSeriesAccessors={['g']} + splitSeriesAccessors={splitSeriesAccessors} data={series.value!} stackAccessors={get('configs.series.stackAccessors', chartConfigs)} color={series.color ? series.color : undefined} diff --git a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx index e7d7e60a3c4088..5f567508a40116 100644 --- a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx @@ -64,6 +64,7 @@ const HistogramPanel = styled(Panel)<{ height?: number }>` export const MatrixHistogramComponent: React.FC = ({ chartHeight, defaultStackByOption, + docValueFields, endDate, errorMessage, filterQuery, @@ -72,6 +73,7 @@ export const MatrixHistogramComponent: React.FC = hideHistogramIfEmpty = false, id, indexNames, + isPtrIncluded, legendPosition, mapping, panelHeight = DEFAULT_PANEL_HEIGHT, @@ -138,6 +140,8 @@ export const MatrixHistogramComponent: React.FC = indexNames, startDate, stackByField: selectedStackByOption.value, + isPtrIncluded, + docValueFields, }); const titleWithStackByField = useMemo( diff --git a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts index 828cadd90bb135..88ad83d8d0627b 100644 --- a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts @@ -12,6 +12,7 @@ import { InputsModelId } from '../../store/inputs/constants'; import { MatrixHistogramType } from '../../../../common/search_strategy/security_solution'; import { UpdateDateRange } from '../charts/common'; import { GlobalTimeArgs } from '../../containers/use_global_time'; +import { DocValueFields } from '../../../../common/search_strategy'; export type MatrixHistogramMappingTypes = Record< string, @@ -57,6 +58,7 @@ interface MatrixHistogramBasicProps { } export interface MatrixHistogramQueryProps { + docValueFields?: DocValueFields[]; endDate: string; errorMessage: string; indexNames: string[]; @@ -72,6 +74,7 @@ export interface MatrixHistogramQueryProps { histogramType: MatrixHistogramType; threshold?: { field: string | undefined; value: number } | undefined; skip?: boolean; + isPtrIncluded?: boolean; } export interface MatrixHistogramProps extends MatrixHistogramBasicProps { diff --git a/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts b/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts index 8db513da1f3a1c..9317f6ba4b86da 100644 --- a/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts @@ -5,7 +5,7 @@ */ import deepEqual from 'fast-deep-equal'; -import { getOr, noop } from 'lodash/fp'; +import { getOr, isEmpty, noop } from 'lodash/fp'; import { useCallback, useEffect, useRef, useState } from 'react'; import { MatrixHistogramQueryProps } from '../../components/matrix_histogram/types'; @@ -46,11 +46,13 @@ export interface UseMatrixHistogramArgs { } export const useMatrixHistogram = ({ + docValueFields, endDate, errorMessage, filterQuery, histogramType, indexNames, + isPtrIncluded, stackByField, startDate, threshold, @@ -78,6 +80,8 @@ export const useMatrixHistogram = ({ }, stackByField, threshold, + ...(isPtrIncluded != null ? { isPtrIncluded } : {}), + ...(!isEmpty(docValueFields) ? { docValueFields } : {}), }); const [matrixHistogramResponse, setMatrixHistogramResponse] = useState({ @@ -169,13 +173,25 @@ export const useMatrixHistogram = ({ }, stackByField, threshold, + ...(isPtrIncluded != null ? { isPtrIncluded } : {}), + ...(!isEmpty(docValueFields) ? { docValueFields } : {}), }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [indexNames, endDate, filterQuery, startDate, stackByField, histogramType, threshold]); + }, [ + indexNames, + endDate, + filterQuery, + startDate, + stackByField, + histogramType, + threshold, + isPtrIncluded, + docValueFields, + ]); useEffect(() => { if (!skip) { diff --git a/x-pack/plugins/security_solution/public/network/containers/network_dns/histogram.ts b/x-pack/plugins/security_solution/public/network/containers/network_dns/histogram.ts deleted file mode 100644 index dce0c3bd2b30d4..00000000000000 --- a/x-pack/plugins/security_solution/public/network/containers/network_dns/histogram.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React from 'react'; -import { connect } from 'react-redux'; -import { compose } from 'redux'; -import { DocumentNode } from 'graphql'; -import { ScaleType } from '@elastic/charts'; - -import { MatrixHistogram } from '../../../common/components/matrix_histogram'; -import { - MatrixHistogramOption, - GetSubTitle, -} from '../../../common/components/matrix_histogram/types'; -import { UpdateDateRange } from '../../../common/components/charts/common'; -import { GlobalTimeArgs } from '../../../common/containers/use_global_time'; -import { withKibana } from '../../../common/lib/kibana'; -import { QueryTemplatePaginatedProps } from '../../../common/containers/query_template_paginated'; -import { DEFAULT_TABLE_ACTIVE_PAGE, DEFAULT_TABLE_LIMIT } from '../../../common/store/constants'; -import { networkModel, networkSelectors } from '../../store'; -import { State, inputsSelectors } from '../../../common/store'; - -export const HISTOGRAM_ID = 'networkDnsHistogramQuery'; - -interface DnsHistogramOwnProps extends QueryTemplatePaginatedProps { - dataKey: string | string[]; - defaultStackByOption: MatrixHistogramOption; - errorMessage: string; - isDnsHistogram?: boolean; - query: DocumentNode; - scaleType: ScaleType; - setQuery: GlobalTimeArgs['setQuery']; - showLegend?: boolean; - stackByOptions: MatrixHistogramOption[]; - subtitle?: string | GetSubTitle; - title: string; - type: networkModel.NetworkType; - updateDateRange: UpdateDateRange; - yTickFormatter?: (value: number) => string; -} - -const makeMapHistogramStateToProps = () => { - const getNetworkDnsSelector = networkSelectors.dnsSelector(); - const getQuery = inputsSelectors.globalQueryByIdSelector(); - const mapStateToProps = (state: State, { id = HISTOGRAM_ID }: DnsHistogramOwnProps) => { - const { isInspected } = getQuery(state, id); - return { - ...getNetworkDnsSelector(state), - activePage: DEFAULT_TABLE_ACTIVE_PAGE, - limit: DEFAULT_TABLE_LIMIT, - isInspected, - id, - }; - }; - - return mapStateToProps; -}; - -export const NetworkDnsHistogramQuery = compose>( - connect(makeMapHistogramStateToProps), - withKibana -)(MatrixHistogram); diff --git a/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx index 92a8f8c49dfc62..f2ed032835e295 100644 --- a/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx @@ -13,26 +13,22 @@ import { inputsModel } from '../../../common/store'; import { useShallowEqualSelector } from '../../../common/hooks/use_selector'; import { useKibana } from '../../../common/lib/kibana'; import { createFilter } from '../../../common/containers/helpers'; -import { NetworkDnsEdges, PageInfoPaginated } from '../../../../common/search_strategy'; import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; import { networkModel, networkSelectors } from '../../store'; import { + DocValueFields, NetworkQueries, NetworkDnsRequestOptions, NetworkDnsStrategyResponse, MatrixOverOrdinalHistogramData, -} from '../../../../common/search_strategy/security_solution/network'; -import { - AbortError, - isCompleteResponse, - isErrorResponse, -} from '../../../../../../../src/plugins/data/common'; + NetworkDnsEdges, + PageInfoPaginated, +} from '../../../../common/search_strategy'; +import { AbortError, isCompleteResponse, isErrorResponse } from '../../../../../../../src/plugins/data/common'; import * as i18n from './translations'; import { getInspectResponse } from '../../../helpers'; import { InspectResponse } from '../../../types'; -export * from './histogram'; - const ID = 'networkDnsQuery'; export interface NetworkDnsArgs { @@ -50,6 +46,7 @@ export interface NetworkDnsArgs { interface UseNetworkDns { id?: string; + docValueFields: DocValueFields[]; indexNames: string[]; type: networkModel.NetworkType; filterQuery?: ESTermQuery | string; @@ -59,6 +56,7 @@ interface UseNetworkDns { } export const useNetworkDns = ({ + docValueFields, endDate, filterQuery, indexNames, @@ -77,6 +75,7 @@ export const useNetworkDns = ({ !skip ? { defaultIndex: indexNames, + docValueFields: docValueFields ?? [], factoryQueryType: NetworkQueries.dns, filterQuery: createFilter(filterQuery), isPtrIncluded, @@ -193,6 +192,7 @@ export const useNetworkDns = ({ const myRequest = { ...(prevRequest ?? {}), defaultIndex: indexNames, + docValueFields: docValueFields ?? [], isPtrIncluded, factoryQueryType: NetworkQueries.dns, filterQuery: createFilter(filterQuery), @@ -209,7 +209,18 @@ export const useNetworkDns = ({ } return prevRequest; }); - }, [activePage, indexNames, endDate, filterQuery, limit, startDate, sort, skip, isPtrIncluded]); + }, [ + activePage, + indexNames, + endDate, + filterQuery, + limit, + startDate, + sort, + skip, + isPtrIncluded, + docValueFields, + ]); useEffect(() => { networkDnsSearch(networkDnsRequest); diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/dns_query_tab_body.tsx b/x-pack/plugins/security_solution/public/network/pages/navigation/dns_query_tab_body.tsx index a8bae2509e0d6e..8d850a926f0937 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/dns_query_tab_body.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/dns_query_tab_body.tsx @@ -8,7 +8,7 @@ import React, { useEffect, useCallback, useMemo } from 'react'; import { getOr } from 'lodash/fp'; import { NetworkDnsTable } from '../../components/network_dns_table'; -import { useNetworkDns, HISTOGRAM_ID } from '../../containers/network_dns'; +import { useNetworkDns } from '../../containers/network_dns'; import { manageQuery } from '../../../common/components/page/manage_query'; import { NetworkComponentQueryProps } from './types'; @@ -20,6 +20,10 @@ import { import * as i18n from '../translations'; import { MatrixHistogram } from '../../../common/components/matrix_histogram'; import { MatrixHistogramType } from '../../../../common/search_strategy/security_solution'; +import { networkSelectors } from '../../store'; +import { useShallowEqualSelector } from '../../../common/hooks/use_selector'; + +const HISTOGRAM_ID = 'networkDnsHistogramQuery'; const NetworkDnsTableManage = manageQuery(NetworkDnsTable); @@ -43,6 +47,7 @@ export const histogramConfigs: Omit = { const DnsQueryTabBodyComponent: React.FC = ({ deleteQuery, + docValueFields, endDate, filterQuery, indexNames, @@ -51,6 +56,9 @@ const DnsQueryTabBodyComponent: React.FC = ({ setQuery, type, }) => { + const getNetworkDnsSelector = networkSelectors.dnsSelector(); + const { isPtrIncluded } = useShallowEqualSelector(getNetworkDnsSelector); + useEffect(() => { return () => { if (deleteQuery) { @@ -63,6 +71,7 @@ const DnsQueryTabBodyComponent: React.FC = ({ loading, { totalCount, networkDns, pageInfo, loadPage, id, inspect, isInspected, refetch }, ] = useNetworkDns({ + docValueFields: docValueFields ?? [], endDate, filterQuery, indexNames, @@ -87,9 +96,11 @@ const DnsQueryTabBodyComponent: React.FC = ({ return ( <> ( ({ networkPagePath, + docValueFields, type, to, filterQuery, @@ -107,7 +108,7 @@ export const NetworkRoutes = React.memo( return ( - + <> diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/types.ts b/x-pack/plugins/security_solution/public/network/pages/navigation/types.ts index ed04fd01a7b891..ef8cc4079e9a5d 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/types.ts +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/types.ts @@ -14,6 +14,7 @@ import { GlobalTimeArgs } from '../../../common/containers/use_global_time'; import { SetAbsoluteRangeDatePicker } from '../types'; import { NarrowDateRange } from '../../../common/components/ml/types'; +import { DocValueFields } from '../../../common/containers/source'; interface QueryTabBodyProps extends Pick { skip: boolean; @@ -25,7 +26,9 @@ interface QueryTabBodyProps extends Pick( = { isPartial: false, isRunning: false, rawResponse: { - took: 150, + took: 36, timed_out: false, - _shards: { total: 21, successful: 21, skipped: 0, failed: 0 }, - hits: { total: 0, max_score: 0, hits: [] }, + _shards: { + total: 55, + successful: 55, + skipped: 38, + failed: 0, + }, + hits: { + max_score: 0, + hits: [], + total: 0, + }, aggregations: { - NetworkDns: { + dns_count: { + value: 3, + }, + dns_name_query_count: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, buckets: [ { - key_as_string: '2020-09-08T15:00:00.000Z', - key: 1599577200000, - doc_count: 7083, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T15:45:00.000Z', - key: 1599579900000, - doc_count: 146148, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T16:30:00.000Z', - key: 1599582600000, - doc_count: 65025, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T17:15:00.000Z', - key: 1599585300000, - doc_count: 62317, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T18:00:00.000Z', - key: 1599588000000, - doc_count: 58223, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T18:45:00.000Z', - key: 1599590700000, - doc_count: 55712, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T19:30:00.000Z', - key: 1599593400000, - doc_count: 55328, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T20:15:00.000Z', - key: 1599596100000, - doc_count: 63878, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T21:00:00.000Z', - key: 1599598800000, - doc_count: 54151, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T21:45:00.000Z', - key: 1599601500000, - doc_count: 55170, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T22:30:00.000Z', - key: 1599604200000, - doc_count: 43115, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-08T23:15:00.000Z', - key: 1599606900000, - doc_count: 52204, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T00:00:00.000Z', - key: 1599609600000, - doc_count: 43609, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T00:45:00.000Z', - key: 1599612300000, - doc_count: 44825, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T01:30:00.000Z', - key: 1599615000000, - doc_count: 52374, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T02:15:00.000Z', - key: 1599617700000, - doc_count: 44667, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T03:00:00.000Z', - key: 1599620400000, - doc_count: 45231, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T03:45:00.000Z', - key: 1599623100000, - doc_count: 42871, - dns: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, + key: 'google.com', + doc_count: 1, + unique_domains: { + value: 1, + }, + dns_question_name: { buckets: [ - { key: 'google.com', doc_count: 1, orderAgg: { value: 1 } }, - { key: 'google.internal', doc_count: 1, orderAgg: { value: 1 } }, + { + key_as_string: '2020-11-12T01:13:31.395Z', + key: 1605143611395, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:21:48.492Z', + key: 1605144108492, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:30:05.589Z', + key: 1605144605589, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:38:22.686Z', + key: 1605145102686, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:46:39.783Z', + key: 1605145599783, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:54:56.880Z', + key: 1605146096880, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:03:13.977Z', + key: 1605146593977, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:11:31.074Z', + key: 1605147091074, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:19:48.171Z', + key: 1605147588171, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:28:05.268Z', + key: 1605148085268, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:36:22.365Z', + key: 1605148582365, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:44:39.462Z', + key: 1605149079462, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:52:56.559Z', + key: 1605149576559, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:01:13.656Z', + key: 1605150073656, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:09:30.753Z', + key: 1605150570753, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:17:47.850Z', + key: 1605151067850, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:26:04.947Z', + key: 1605151564947, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:34:22.044Z', + key: 1605152062044, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:42:39.141Z', + key: 1605152559141, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:50:56.238Z', + key: 1605153056238, + doc_count: 1, + }, + { + key_as_string: '2020-11-12T03:59:13.335Z', + key: 1605153553335, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:07:30.432Z', + key: 1605154050432, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:15:47.529Z', + key: 1605154547529, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:24:04.626Z', + key: 1605155044626, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:32:21.723Z', + key: 1605155541723, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:40:38.820Z', + key: 1605156038820, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:48:55.917Z', + key: 1605156535917, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:57:13.014Z', + key: 1605157033014, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:05:30.111Z', + key: 1605157530111, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:13:47.208Z', + key: 1605158027208, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:22:04.305Z', + key: 1605158524305, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:30:21.402Z', + key: 1605159021402, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:38:38.499Z', + key: 1605159518499, + doc_count: 0, + }, ], }, + dns_bytes_in: { + value: 0, + }, + dns_bytes_out: { + value: 0, + }, }, { - key_as_string: '2020-09-09T04:30:00.000Z', - key: 1599625800000, - doc_count: 41327, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T05:15:00.000Z', - key: 1599628500000, - doc_count: 39860, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T06:00:00.000Z', - key: 1599631200000, - doc_count: 44061, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T06:45:00.000Z', - key: 1599633900000, - doc_count: 39193, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T07:30:00.000Z', - key: 1599636600000, - doc_count: 40909, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T08:15:00.000Z', - key: 1599639300000, - doc_count: 43293, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T09:00:00.000Z', - key: 1599642000000, - doc_count: 47640, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T09:45:00.000Z', - key: 1599644700000, - doc_count: 48605, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T10:30:00.000Z', - key: 1599647400000, - doc_count: 42072, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T11:15:00.000Z', - key: 1599650100000, - doc_count: 46398, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T12:00:00.000Z', - key: 1599652800000, - doc_count: 49378, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T12:45:00.000Z', - key: 1599655500000, - doc_count: 51171, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T13:30:00.000Z', - key: 1599658200000, - doc_count: 57911, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, - }, - { - key_as_string: '2020-09-09T14:15:00.000Z', - key: 1599660900000, - doc_count: 58909, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, + key: 'google.internal', + doc_count: 1, + unique_domains: { + value: 1, + }, + dns_question_name: { + buckets: [ + { + key_as_string: '2020-11-12T01:13:31.395Z', + key: 1605143611395, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:21:48.492Z', + key: 1605144108492, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:30:05.589Z', + key: 1605144605589, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:38:22.686Z', + key: 1605145102686, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:46:39.783Z', + key: 1605145599783, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:54:56.880Z', + key: 1605146096880, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:03:13.977Z', + key: 1605146593977, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:11:31.074Z', + key: 1605147091074, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:19:48.171Z', + key: 1605147588171, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:28:05.268Z', + key: 1605148085268, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:36:22.365Z', + key: 1605148582365, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:44:39.462Z', + key: 1605149079462, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:52:56.559Z', + key: 1605149576559, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:01:13.656Z', + key: 1605150073656, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:09:30.753Z', + key: 1605150570753, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:17:47.850Z', + key: 1605151067850, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:26:04.947Z', + key: 1605151564947, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:34:22.044Z', + key: 1605152062044, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:42:39.141Z', + key: 1605152559141, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:50:56.238Z', + key: 1605153056238, + doc_count: 1, + }, + { + key_as_string: '2020-11-12T03:59:13.335Z', + key: 1605153553335, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:07:30.432Z', + key: 1605154050432, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:15:47.529Z', + key: 1605154547529, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:24:04.626Z', + key: 1605155044626, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:32:21.723Z', + key: 1605155541723, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:40:38.820Z', + key: 1605156038820, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:48:55.917Z', + key: 1605156535917, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:57:13.014Z', + key: 1605157033014, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:05:30.111Z', + key: 1605157530111, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:13:47.208Z', + key: 1605158027208, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:22:04.305Z', + key: 1605158524305, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:30:21.402Z', + key: 1605159021402, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:38:38.499Z', + key: 1605159518499, + doc_count: 0, + }, + ], + }, + dns_bytes_in: { + value: 0, + }, + dns_bytes_out: { + value: 0, + }, }, { - key_as_string: '2020-09-09T15:00:00.000Z', - key: 1599663600000, - doc_count: 62358, - dns: { doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [] }, + key: 'windows.net', + doc_count: 1, + unique_domains: { + value: 1, + }, + dns_question_name: { + buckets: [ + { + key_as_string: '2020-11-12T01:13:31.395Z', + key: 1605143611395, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:21:48.492Z', + key: 1605144108492, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:30:05.589Z', + key: 1605144605589, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:38:22.686Z', + key: 1605145102686, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:46:39.783Z', + key: 1605145599783, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T01:54:56.880Z', + key: 1605146096880, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:03:13.977Z', + key: 1605146593977, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:11:31.074Z', + key: 1605147091074, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:19:48.171Z', + key: 1605147588171, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:28:05.268Z', + key: 1605148085268, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:36:22.365Z', + key: 1605148582365, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:44:39.462Z', + key: 1605149079462, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T02:52:56.559Z', + key: 1605149576559, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:01:13.656Z', + key: 1605150073656, + doc_count: 1, + }, + { + key_as_string: '2020-11-12T03:09:30.753Z', + key: 1605150570753, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:17:47.850Z', + key: 1605151067850, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:26:04.947Z', + key: 1605151564947, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:34:22.044Z', + key: 1605152062044, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:42:39.141Z', + key: 1605152559141, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:50:56.238Z', + key: 1605153056238, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T03:59:13.335Z', + key: 1605153553335, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:07:30.432Z', + key: 1605154050432, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:15:47.529Z', + key: 1605154547529, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:24:04.626Z', + key: 1605155044626, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:32:21.723Z', + key: 1605155541723, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:40:38.820Z', + key: 1605156038820, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:48:55.917Z', + key: 1605156535917, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T04:57:13.014Z', + key: 1605157033014, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:05:30.111Z', + key: 1605157530111, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:13:47.208Z', + key: 1605158027208, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:22:04.305Z', + key: 1605158524305, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:30:21.402Z', + key: 1605159021402, + doc_count: 0, + }, + { + key_as_string: '2020-11-12T05:38:38.499Z', + key: 1605159518499, + doc_count: 0, + }, + ], + }, + dns_bytes_in: { + value: 0, + }, + dns_bytes_out: { + value: 0, + }, }, ], }, @@ -1566,6 +1921,7 @@ export const formattedDnsSearchStrategyResponse: MatrixHistogramStrategyResponse dsl: [ JSON.stringify( { + allowNoIndices: true, index: [ 'apm-*-transaction*', 'auditbeat-*', @@ -1575,20 +1931,50 @@ export const formattedDnsSearchStrategyResponse: MatrixHistogramStrategyResponse 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, ignoreUnavailable: true, body: { aggregations: { - NetworkDns: { - date_histogram: { field: '@timestamp', fixed_interval: '2700000ms' }, + dns_count: { + cardinality: { + field: 'dns.question.registered_domain', + }, + }, + dns_name_query_count: { + terms: { + field: 'dns.question.registered_domain', + size: 1000000, + }, aggs: { - dns: { - terms: { - field: 'dns.question.registered_domain', - order: { orderAgg: 'desc' }, + dns_question_name: { + date_histogram: { + field: '@timestamp', + fixed_interval: '2700000ms', + min_doc_count: 0, + extended_bounds: { min: 1599579675528, max: 1599666075529 }, + }, + }, + bucket_sort: { + bucket_sort: { + sort: [ + { + unique_domains: { + order: 'desc', + }, + }, + { + _key: { + order: 'asc', + }, + }, + ], + from: 0, size: 10, }, - aggs: { orderAgg: { cardinality: { field: 'dns.question.name' } } }, + }, + unique_domains: { + cardinality: { + field: 'dns.question.name', + }, }, }, }, @@ -1596,7 +1982,18 @@ export const formattedDnsSearchStrategyResponse: MatrixHistogramStrategyResponse query: { bool: { filter: [ - { bool: { must: [], filter: [{ match_all: {} }], should: [], must_not: [] } }, + { + bool: { + must: [], + filter: [ + { + match_all: {}, + }, + ], + should: [], + must_not: [], + }, + }, { range: { '@timestamp': { @@ -1607,11 +2004,20 @@ export const formattedDnsSearchStrategyResponse: MatrixHistogramStrategyResponse }, }, ], + must_not: [ + { + term: { + 'dns.question.type': { + value: 'PTR', + }, + }, + }, + ], }, }, - size: 0, - track_total_hits: true, }, + size: 0, + track_total_hits: false, }, null, 2 @@ -1619,8 +2025,105 @@ export const formattedDnsSearchStrategyResponse: MatrixHistogramStrategyResponse ], }, matrixHistogramData: [ - { x: 1599623100000, y: 1, g: 'google.com' }, - { x: 1599623100000, y: 1, g: 'google.internal' }, + { x: 1605143611395, y: 0, g: 'google.com' }, + { x: 1605144108492, y: 0, g: 'google.com' }, + { x: 1605144605589, y: 0, g: 'google.com' }, + { x: 1605145102686, y: 0, g: 'google.com' }, + { x: 1605145599783, y: 0, g: 'google.com' }, + { x: 1605146096880, y: 0, g: 'google.com' }, + { x: 1605146593977, y: 0, g: 'google.com' }, + { x: 1605147091074, y: 0, g: 'google.com' }, + { x: 1605147588171, y: 0, g: 'google.com' }, + { x: 1605148085268, y: 0, g: 'google.com' }, + { x: 1605148582365, y: 0, g: 'google.com' }, + { x: 1605149079462, y: 0, g: 'google.com' }, + { x: 1605149576559, y: 0, g: 'google.com' }, + { x: 1605150073656, y: 0, g: 'google.com' }, + { x: 1605150570753, y: 0, g: 'google.com' }, + { x: 1605151067850, y: 0, g: 'google.com' }, + { x: 1605151564947, y: 0, g: 'google.com' }, + { x: 1605152062044, y: 0, g: 'google.com' }, + { x: 1605152559141, y: 0, g: 'google.com' }, + { x: 1605153056238, y: 1, g: 'google.com' }, + { x: 1605153553335, y: 0, g: 'google.com' }, + { x: 1605154050432, y: 0, g: 'google.com' }, + { x: 1605154547529, y: 0, g: 'google.com' }, + { x: 1605155044626, y: 0, g: 'google.com' }, + { x: 1605155541723, y: 0, g: 'google.com' }, + { x: 1605156038820, y: 0, g: 'google.com' }, + { x: 1605156535917, y: 0, g: 'google.com' }, + { x: 1605157033014, y: 0, g: 'google.com' }, + { x: 1605157530111, y: 0, g: 'google.com' }, + { x: 1605158027208, y: 0, g: 'google.com' }, + { x: 1605158524305, y: 0, g: 'google.com' }, + { x: 1605159021402, y: 0, g: 'google.com' }, + { x: 1605159518499, y: 0, g: 'google.com' }, + { x: 1605143611395, y: 0, g: 'google.internal' }, + { x: 1605144108492, y: 0, g: 'google.internal' }, + { x: 1605144605589, y: 0, g: 'google.internal' }, + { x: 1605145102686, y: 0, g: 'google.internal' }, + { x: 1605145599783, y: 0, g: 'google.internal' }, + { x: 1605146096880, y: 0, g: 'google.internal' }, + { x: 1605146593977, y: 0, g: 'google.internal' }, + { x: 1605147091074, y: 0, g: 'google.internal' }, + { x: 1605147588171, y: 0, g: 'google.internal' }, + { x: 1605148085268, y: 0, g: 'google.internal' }, + { x: 1605148582365, y: 0, g: 'google.internal' }, + { x: 1605149079462, y: 0, g: 'google.internal' }, + { x: 1605149576559, y: 0, g: 'google.internal' }, + { x: 1605150073656, y: 0, g: 'google.internal' }, + { x: 1605150570753, y: 0, g: 'google.internal' }, + { x: 1605151067850, y: 0, g: 'google.internal' }, + { x: 1605151564947, y: 0, g: 'google.internal' }, + { x: 1605152062044, y: 0, g: 'google.internal' }, + { x: 1605152559141, y: 0, g: 'google.internal' }, + { x: 1605153056238, y: 1, g: 'google.internal' }, + { x: 1605153553335, y: 0, g: 'google.internal' }, + { x: 1605154050432, y: 0, g: 'google.internal' }, + { x: 1605154547529, y: 0, g: 'google.internal' }, + { x: 1605155044626, y: 0, g: 'google.internal' }, + { x: 1605155541723, y: 0, g: 'google.internal' }, + { x: 1605156038820, y: 0, g: 'google.internal' }, + { x: 1605156535917, y: 0, g: 'google.internal' }, + { x: 1605157033014, y: 0, g: 'google.internal' }, + { x: 1605157530111, y: 0, g: 'google.internal' }, + { x: 1605158027208, y: 0, g: 'google.internal' }, + { x: 1605158524305, y: 0, g: 'google.internal' }, + { x: 1605159021402, y: 0, g: 'google.internal' }, + { x: 1605159518499, y: 0, g: 'google.internal' }, + { x: 1605143611395, y: 0, g: 'windows.net' }, + { x: 1605144108492, y: 0, g: 'windows.net' }, + { x: 1605144605589, y: 0, g: 'windows.net' }, + { x: 1605145102686, y: 0, g: 'windows.net' }, + { x: 1605145599783, y: 0, g: 'windows.net' }, + { x: 1605146096880, y: 0, g: 'windows.net' }, + { x: 1605146593977, y: 0, g: 'windows.net' }, + { x: 1605147091074, y: 0, g: 'windows.net' }, + { x: 1605147588171, y: 0, g: 'windows.net' }, + { x: 1605148085268, y: 0, g: 'windows.net' }, + { x: 1605148582365, y: 0, g: 'windows.net' }, + { x: 1605149079462, y: 0, g: 'windows.net' }, + { x: 1605149576559, y: 0, g: 'windows.net' }, + { x: 1605150073656, y: 1, g: 'windows.net' }, + { x: 1605150570753, y: 0, g: 'windows.net' }, + { x: 1605151067850, y: 0, g: 'windows.net' }, + { x: 1605151564947, y: 0, g: 'windows.net' }, + { x: 1605152062044, y: 0, g: 'windows.net' }, + { x: 1605152559141, y: 0, g: 'windows.net' }, + { x: 1605153056238, y: 0, g: 'windows.net' }, + { x: 1605153553335, y: 0, g: 'windows.net' }, + { x: 1605154050432, y: 0, g: 'windows.net' }, + { x: 1605154547529, y: 0, g: 'windows.net' }, + { x: 1605155044626, y: 0, g: 'windows.net' }, + { x: 1605155541723, y: 0, g: 'windows.net' }, + { x: 1605156038820, y: 0, g: 'windows.net' }, + { x: 1605156535917, y: 0, g: 'windows.net' }, + { x: 1605157033014, y: 0, g: 'windows.net' }, + { x: 1605157530111, y: 0, g: 'windows.net' }, + { x: 1605158027208, y: 0, g: 'windows.net' }, + { x: 1605158524305, y: 0, g: 'windows.net' }, + { x: 1605159021402, y: 0, g: 'windows.net' }, + { x: 1605159518499, y: 0, g: 'windows.net' }, ], totalCount: 0, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts index 3a769127bbe853..6f1e593ca2edc5 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts @@ -18,55 +18,66 @@ export const mockOptions = { ], filterQuery: '{"bool":{"must":[],"filter":[{"match_all":{}}],"should":[],"must_not":[]}}', histogramType: MatrixHistogramType.dns, + isPtrIncluded: false, timerange: { interval: '12h', from: '2020-09-08T15:41:15.528Z', to: '2020-09-09T15:41:15.529Z' }, stackByField: 'dns.question.registered_domain', }; export const expectedDsl = { - index: [ - 'apm-*-transaction*', - 'auditbeat-*', - 'endgame-*', - 'filebeat-*', - 'logs-*', - 'packetbeat-*', - 'winlogbeat-*', - ], allowNoIndices: true, - ignoreUnavailable: true, body: { aggregations: { - NetworkDns: { - date_histogram: { field: '@timestamp', fixed_interval: '2700000ms' }, + dns_count: { cardinality: { field: 'dns.question.registered_domain' } }, + dns_name_query_count: { aggs: { - dns: { - terms: { - field: 'dns.question.registered_domain', - order: { orderAgg: 'desc' }, + bucket_sort: { + bucket_sort: { + from: 0, size: 10, + sort: [{ unique_domains: { order: 'desc' } }, { _key: { order: 'asc' } }], + }, + }, + dns_question_name: { + date_histogram: { + extended_bounds: { max: 1599666075529, min: 1599579675528 }, + field: '@timestamp', + fixed_interval: '2700000ms', + min_doc_count: 0, }, - aggs: { orderAgg: { cardinality: { field: 'dns.question.name' } } }, }, + unique_domains: { cardinality: { field: 'dns.question.name' } }, }, + terms: { field: 'dns.question.registered_domain', size: 1000000 }, }, }, query: { bool: { filter: [ - { bool: { must: [], filter: [{ match_all: {} }], should: [], must_not: [] } }, + { bool: { filter: [{ match_all: {} }], must: [], must_not: [], should: [] } }, { range: { '@timestamp': { + format: 'strict_date_optional_time', gte: '2020-09-08T15:41:15.528Z', lte: '2020-09-09T15:41:15.529Z', - format: 'strict_date_optional_time', }, }, }, ], + must_not: [{ term: { 'dns.question.type': { value: 'PTR' } } }], }, }, - size: 0, - track_total_hits: true, }, + ignoreUnavailable: true, + index: [ + 'apm-*-transaction*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'packetbeat-*', + 'winlogbeat-*', + ], + size: 0, + track_total_hits: false, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/helpers.ts index d0fff848b426a9..9131a9c4be87c9 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/helpers.ts @@ -17,15 +17,16 @@ export const getDnsParsedData = ( ): MatrixHistogramData[] => { let result: MatrixHistogramData[] = []; data.forEach((bucketData: unknown) => { - const time = get('key', bucketData); + const questionName = get('key', bucketData); const histData = getOr([], keyBucket, bucketData).map( // eslint-disable-next-line @typescript-eslint/naming-convention ({ key, doc_count }: DnsHistogramSubBucket) => ({ - x: time, + x: key, y: doc_count, - g: key, + g: questionName, }) ); + result = [...result, ...histData]; }); return result; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/index.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/index.test.ts index 8afc764d97f87b..fcdd13d42c91f8 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/index.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/index.test.ts @@ -19,8 +19,8 @@ jest.mock('./helpers', () => ({ describe('dnsMatrixHistogramConfig', () => { test('should export dnsMatrixHistogramConfig corrrectly', () => { expect(dnsMatrixHistogramConfig).toEqual({ - aggName: 'aggregations.NetworkDns.buckets', - parseKey: 'dns.buckets', + aggName: 'aggregations.dns_name_query_count.buckets', + parseKey: 'dns_question_name.buckets', buildDsl: buildDnsHistogramQuery, parser: getDnsParsedData, }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/index.ts index 557e2ebf759e68..d592348de7afd5 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/index.ts @@ -9,7 +9,7 @@ import { getDnsParsedData } from './helpers'; export const dnsMatrixHistogramConfig = { buildDsl: buildDnsHistogramQuery, - aggName: 'aggregations.NetworkDns.buckets', - parseKey: 'dns.buckets', + aggName: 'aggregations.dns_name_query_count.buckets', + parseKey: 'dns_question_name.buckets', parser: getDnsParsedData, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/query.dns_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/query.dns_histogram.dsl.ts index 08a080865dfc06..4374d6b0da896f 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/query.dns_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/query.dns_histogram.dsl.ts @@ -4,17 +4,65 @@ * you may not use this file except in compliance with the Elastic License. */ +import { isEmpty } from 'lodash/fp'; + +import moment from 'moment'; + +import { Direction, MatrixHistogramRequestOptions } from '../../../../../../common/search_strategy'; import { - createQueryFilterClauses, calculateTimeSeriesInterval, + createQueryFilterClauses, } from '../../../../../utils/build_query'; -import { MatrixHistogramRequestOptions } from '../../../../../../common/search_strategy/security_solution/matrix_histogram'; + +const HUGE_QUERY_SIZE = 1000000; + +const getCountAgg = () => ({ + dns_count: { + cardinality: { + field: 'dns.question.registered_domain', + }, + }, +}); + +const createIncludePTRFilter = (isPtrIncluded: boolean) => + isPtrIncluded + ? {} + : { + must_not: [ + { + term: { + 'dns.question.type': { + value: 'PTR', + }, + }, + }, + ], + }; + +const getHistogramAggregation = ({ from, to }: { from: string; to: string }) => { + const interval = calculateTimeSeriesInterval(from, to); + const histogramTimestampField = '@timestamp'; + + return { + date_histogram: { + field: histogramTimestampField, + fixed_interval: interval, + min_doc_count: 0, + extended_bounds: { + min: moment(from).valueOf(), + max: moment(to).valueOf(), + }, + }, + }; +}; export const buildDnsHistogramQuery = ({ + defaultIndex, + docValueFields, filterQuery, + isPtrIncluded = false, + stackByField = 'dns.question.registered_domain', timerange: { from, to }, - defaultIndex, - stackByField, }: MatrixHistogramRequestOptions) => { const filter = [ ...createQueryFilterClauses(filterQuery), @@ -29,55 +77,48 @@ export const buildDnsHistogramQuery = ({ }, ]; - const getHistogramAggregation = () => { - const interval = calculateTimeSeriesInterval(from, to); - const histogramTimestampField = '@timestamp'; - const dateHistogram = { - date_histogram: { - field: histogramTimestampField, - fixed_interval: interval, - }, - }; - - return { - NetworkDns: { - ...dateHistogram, - aggs: { - dns: { - terms: { - field: stackByField, - order: { - orderAgg: 'desc', + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + aggregations: { + ...getCountAgg(), + dns_name_query_count: { + terms: { + field: stackByField, + size: HUGE_QUERY_SIZE, + }, + aggs: { + dns_question_name: getHistogramAggregation({ from, to }), + bucket_sort: { + bucket_sort: { + sort: [ + { unique_domains: { order: Direction.desc } }, + { _key: { order: Direction.asc } }, + ], + from: 0, + size: 10, }, - size: 10, }, - aggs: { - orderAgg: { - cardinality: { - field: 'dns.question.name', - }, + unique_domains: { + cardinality: { + field: 'dns.question.name', }, }, }, }, }, - }; - }; - - const dslQuery = { - index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, - body: { - aggregations: getHistogramAggregation(), query: { bool: { filter, + ...createIncludePTRFilter(isPtrIncluded), }, }, - size: 0, - track_total_hits: true, }, + size: 0, + track_total_hits: false, }; return dslQuery; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts index fbe007622005c1..0379fa3d32edba 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts @@ -263,7 +263,101 @@ export const formattedSearchStrategyResponse = { ...mockSearchStrategyResponse, inspect: { dsl: [ - '{\n "allowNoIndices": true,\n "index": [\n "apm-*-transaction*",\n "auditbeat-*",\n "endgame-*",\n "filebeat-*",\n "logs-*",\n "packetbeat-*",\n "winlogbeat-*"\n ],\n "ignoreUnavailable": true,\n "body": {\n "aggs": {\n "source": {\n "filter": {\n "term": {\n "source.ip": "35.196.65.164"\n }\n },\n "aggs": {\n "firstSeen": {\n "min": {\n "field": "@timestamp"\n }\n },\n "lastSeen": {\n "max": {\n "field": "@timestamp"\n }\n },\n "as": {\n "filter": {\n "exists": {\n "field": "source.as"\n }\n },\n "aggs": {\n "results": {\n "top_hits": {\n "size": 1,\n "_source": [\n "source.as"\n ],\n "sort": [\n {\n "@timestamp": "desc"\n }\n ]\n }\n }\n }\n },\n "geo": {\n "filter": {\n "exists": {\n "field": "source.geo"\n }\n },\n "aggs": {\n "results": {\n "top_hits": {\n "size": 1,\n "_source": [\n "source.geo"\n ],\n "sort": [\n {\n "@timestamp": "desc"\n }\n ]\n }\n }\n }\n }\n }\n },\n "destination": {\n "filter": {\n "term": {\n "destination.ip": "35.196.65.164"\n }\n },\n "aggs": {\n "firstSeen": {\n "min": {\n "field": "@timestamp"\n }\n },\n "lastSeen": {\n "max": {\n "field": "@timestamp"\n }\n },\n "as": {\n "filter": {\n "exists": {\n "field": "destination.as"\n }\n },\n "aggs": {\n "results": {\n "top_hits": {\n "size": 1,\n "_source": [\n "destination.as"\n ],\n "sort": [\n {\n "@timestamp": "desc"\n }\n ]\n }\n }\n }\n },\n "geo": {\n "filter": {\n "exists": {\n "field": "destination.geo"\n }\n },\n "aggs": {\n "results": {\n "top_hits": {\n "size": 1,\n "_source": [\n "destination.geo"\n ],\n "sort": [\n {\n "@timestamp": "desc"\n }\n ]\n }\n }\n }\n }\n }\n },\n "host": {\n "filter": {\n "term": {\n "host.ip": "35.196.65.164"\n }\n },\n "aggs": {\n "results": {\n "top_hits": {\n "size": 1,\n "_source": [\n "host"\n ],\n "sort": [\n {\n "@timestamp": "desc"\n }\n ]\n }\n }\n }\n }\n },\n "query": {\n "bool": {\n "should": []\n }\n },\n "size": 0,\n "track_total_hits": false\n }\n}', + JSON.stringify( + { + allowNoIndices: true, + index: [ + 'apm-*-transaction*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'packetbeat-*', + 'winlogbeat-*', + ], + ignoreUnavailable: true, + body: { + docvalue_fields: mockOptions.docValueFields, + aggs: { + source: { + filter: { term: { 'source.ip': '35.196.65.164' } }, + aggs: { + firstSeen: { min: { field: '@timestamp' } }, + lastSeen: { max: { field: '@timestamp' } }, + as: { + filter: { exists: { field: 'source.as' } }, + aggs: { + results: { + top_hits: { + size: 1, + _source: ['source.as'], + sort: [{ '@timestamp': 'desc' }], + }, + }, + }, + }, + geo: { + filter: { exists: { field: 'source.geo' } }, + aggs: { + results: { + top_hits: { + size: 1, + _source: ['source.geo'], + sort: [{ '@timestamp': 'desc' }], + }, + }, + }, + }, + }, + }, + destination: { + filter: { term: { 'destination.ip': '35.196.65.164' } }, + aggs: { + firstSeen: { min: { field: '@timestamp' } }, + lastSeen: { max: { field: '@timestamp' } }, + as: { + filter: { exists: { field: 'destination.as' } }, + aggs: { + results: { + top_hits: { + size: 1, + _source: ['destination.as'], + sort: [{ '@timestamp': 'desc' }], + }, + }, + }, + }, + geo: { + filter: { exists: { field: 'destination.geo' } }, + aggs: { + results: { + top_hits: { + size: 1, + _source: ['destination.geo'], + sort: [{ '@timestamp': 'desc' }], + }, + }, + }, + }, + }, + }, + host: { + filter: { term: { 'host.ip': '35.196.65.164' } }, + aggs: { + results: { + top_hits: { size: 1, _source: ['host'], sort: [{ '@timestamp': 'desc' }] }, + }, + }, + }, + }, + query: { bool: { should: [] } }, + size: 0, + track_total_hits: false, + }, + }, + null, + 2 + ), ], }, networkDetails: { @@ -370,6 +464,7 @@ export const expectedDsl = { }, }, }, + docvalue_fields: mockOptions.docValueFields, query: { bool: { should: [] } }, size: 0, track_total_hits: false, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts index 67aeba60c4d2f3..9661915be6f3bb 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts @@ -106,7 +106,7 @@ export const buildNetworkDetailsQuery = ({ index: defaultIndex, ignoreUnavailable: true, body: { - ...(isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggs: { ...getAggs('source', ip), ...getAggs('destination', ip), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts index 85b9051189bfe9..ae1ec34b394412 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts @@ -89,7 +89,7 @@ export const buildDnsQuery = ({ index: defaultIndex, ignoreUnavailable: true, body: { - ...(isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { ...getCountAgg(), dns_name_query_count: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts index d105cb621cf465..8a3947924797fa 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts @@ -9,12 +9,12 @@ import { IEsSearchResponse } from '../../../../../../../../../../src/plugins/dat import { Direction, NetworkDnsFields, - NetworkDnsRequestOptions, + NetworkHttpRequestOptions, NetworkQueries, SortField, } from '../../../../../../../common/search_strategy'; -export const mockOptions: NetworkDnsRequestOptions = { +export const mockOptions: NetworkHttpRequestOptions = { defaultIndex: [ 'apm-*-transaction*', 'auditbeat-*', @@ -29,7 +29,7 @@ export const mockOptions: NetworkDnsRequestOptions = { pagination: { activePage: 0, cursorStart: 0, fakePossibleCount: 50, querySize: 10 }, sort: { direction: Direction.desc } as SortField, timerange: { interval: '12h', from: '2020-09-13T09:00:43.249Z', to: '2020-09-14T09:00:43.249Z' }, -} as NetworkDnsRequestOptions; +} as NetworkHttpRequestOptions; export const mockSearchStrategyResponse: IEsSearchResponse = { isPartial: false, diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/query.events_all.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/query.events_all.dsl.ts index 09551dd0693062..a5a0c877ecdd38 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/query.events_all.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/query.events_all.dsl.ts @@ -58,7 +58,7 @@ export const buildTimelineEventsAllQuery = ({ index: defaultIndex, ignoreUnavailable: true, body: { - ...(isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), query: { bool: { filter, diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts index 0f4eabf692919b..17563bfdbe249e 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts @@ -40,7 +40,7 @@ export const buildLastEventTimeQuery = ({ index: indicesToQuery.network, ignoreUnavailable: true, body: { - ...(isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { last_seen_event: { max: { field: '@timestamp' } }, }, @@ -58,7 +58,7 @@ export const buildLastEventTimeQuery = ({ index: indicesToQuery.hosts, ignoreUnavailable: true, body: { - ...(isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { last_seen_event: { max: { field: '@timestamp' } }, }, @@ -76,7 +76,7 @@ export const buildLastEventTimeQuery = ({ index: indicesToQuery[indexKey], ignoreUnavailable: true, body: { - ...(isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { last_seen_event: { max: { field: '@timestamp' } }, }, From 1256e8994a86b21ef5b0b0ec7d9c7e2d76298827 Mon Sep 17 00:00:00 2001 From: Angela Chuang Date: Tue, 24 Nov 2020 16:33:14 +0000 Subject: [PATCH 2/2] eslint --- .../public/network/containers/network_dns/index.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx index f2ed032835e295..bbbebc875ab8f7 100644 --- a/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx @@ -24,7 +24,11 @@ import { NetworkDnsEdges, PageInfoPaginated, } from '../../../../common/search_strategy'; -import { AbortError, isCompleteResponse, isErrorResponse } from '../../../../../../../src/plugins/data/common'; +import { + AbortError, + isCompleteResponse, + isErrorResponse, +} from '../../../../../../../src/plugins/data/common'; import * as i18n from './translations'; import { getInspectResponse } from '../../../helpers'; import { InspectResponse } from '../../../types';