From 0685b41ba2a2de20c4be391396a27b6846af9a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 10:59:41 +0200 Subject: [PATCH 01/15] Tweak the typing of the fetcher functions --- .../infra/public/utils/logs_overview_fetchers.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 46a0edf75b7566..005c38c97cdc7d 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -5,9 +5,11 @@ */ import { InfraClientCoreSetup } from '../types'; -import { LogsFetchDataResponse } from '../../../observability/public'; +import { FetchData, LogsFetchDataResponse, HasData } from '../../../observability/public'; -export function getLogsHasDataFetcher(getStartServices: InfraClientCoreSetup['getStartServices']) { +export function getLogsHasDataFetcher( + getStartServices: InfraClientCoreSetup['getStartServices'] +): HasData { return async () => { // if you need the data plugin, this is how you get it // const [, startPlugins] = await getStartServices(); @@ -22,9 +24,8 @@ export function getLogsHasDataFetcher(getStartServices: InfraClientCoreSetup['ge export function getLogsOverviewDataFetcher( getStartServices: InfraClientCoreSetup['getStartServices'] -) { - return async (): Promise => { - // if you need the data plugin, this is how you get it +): FetchData { + return async () => { // const [, startPlugins] = await getStartServices(); // const { data } = startPlugins; From 6209620c1de20be1bec5c63ab3fab9b0cbe51b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 11:05:17 +0200 Subject: [PATCH 02/15] Extract mocked stats and series --- .../public/utils/logs_overview_fetchers.ts | 127 ++++++++++-------- 1 file changed, 68 insertions(+), 59 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 005c38c97cdc7d..a4f0b0f95e4462 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -7,6 +7,8 @@ import { InfraClientCoreSetup } from '../types'; import { FetchData, LogsFetchDataResponse, HasData } from '../../../observability/public'; +type StatsAndSeries = Pick; + export function getLogsHasDataFetcher( getStartServices: InfraClientCoreSetup['getStartServices'] ): HasData { @@ -25,70 +27,77 @@ export function getLogsHasDataFetcher( export function getLogsOverviewDataFetcher( getStartServices: InfraClientCoreSetup['getStartServices'] ): FetchData { - return async () => { - // const [, startPlugins] = await getStartServices(); - // const { data } = startPlugins; + return async (params) => { + const [, startPlugins] = await getStartServices(); + const { data } = startPlugins; - // if you need a core dep, we need to pass in more than just getStartServices + const { stats, series } = await fetchLogsOverview(); - // perform query return { title: 'Log rate', appLink: 'TBD', // TODO: what format should this be in, relative I assume? - stats: { - nginx: { - type: 'number', - label: 'nginx', - value: 345341, - }, - 'elasticsearch.audit': { - type: 'number', - label: 'elasticsearch.audit', - value: 164929, - }, - 'haproxy.log': { - type: 'number', - label: 'haproxy.log', - value: 51101, - }, - }, - // Note: My understanding is that these series coordinates will be - // combined into objects that look like: - // { x: timestamp, y: value, g: label (e.g. nginx) } - // so they fit the stacked bar chart API - // https://elastic.github.io/elastic-charts/?path=/story/bar-chart--stacked-with-axis-and-legend - series: { - nginx: { - label: 'nginx', - coordinates: [ - { x: 1593000000000, y: 10014 }, - { x: 1593000900000, y: 12827 }, - { x: 1593001800000, y: 2946 }, - { x: 1593002700000, y: 14298 }, - { x: 1593003600000, y: 4096 }, - ], - }, - 'elasticsearch.audit': { - label: 'elasticsearch.audit', - coordinates: [ - { x: 1593000000000, y: 5676 }, - { x: 1593000900000, y: 6783 }, - { x: 1593001800000, y: 2394 }, - { x: 1593002700000, y: 4554 }, - { x: 1593003600000, y: 5659 }, - ], - }, - 'haproxy.log': { - label: 'haproxy.log', - coordinates: [ - { x: 1593000000000, y: 9085 }, - { x: 1593000900000, y: 9002 }, - { x: 1593001800000, y: 3940 }, - { x: 1593002700000, y: 5451 }, - { x: 1593003600000, y: 9133 }, - ], - }, - }, + stats, + series, }; }; } + +async function fetchLogsOverview(): Promise { + return Promise.resolve({ + stats: { + nginx: { + type: 'number', + label: 'nginx', + value: 345341, + }, + 'elasticsearch.audit': { + type: 'number', + label: 'elasticsearch.audit', + value: 164929, + }, + 'haproxy.log': { + type: 'number', + label: 'haproxy.log', + value: 51101, + }, + }, + + // Note: My understanding is that these series coordinates will be + // combined into objects that look like: + // { x: timestamp, y: value, g: label (e.g. nginx) } + // so they fit the stacked bar chart API + // https://elastic.github.io/elastic-charts/?path=/story/bar-chart--stacked-with-axis-and-legend + series: { + nginx: { + label: 'nginx', + coordinates: [ + { x: 1593000000000, y: 10014 }, + { x: 1593000900000, y: 12827 }, + { x: 1593001800000, y: 2946 }, + { x: 1593002700000, y: 14298 }, + { x: 1593003600000, y: 4096 }, + ], + }, + 'elasticsearch.audit': { + label: 'elasticsearch.audit', + coordinates: [ + { x: 1593000000000, y: 5676 }, + { x: 1593000900000, y: 6783 }, + { x: 1593001800000, y: 2394 }, + { x: 1593002700000, y: 4554 }, + { x: 1593003600000, y: 5659 }, + ], + }, + 'haproxy.log': { + label: 'haproxy.log', + coordinates: [ + { x: 1593000000000, y: 9085 }, + { x: 1593000900000, y: 9002 }, + { x: 1593001800000, y: 3940 }, + { x: 1593002700000, y: 5451 }, + { x: 1593003600000, y: 9133 }, + ], + }, + }, + }); +} From bd39e103f5402c3d76addc800b4de7f441b81826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 11:07:42 +0200 Subject: [PATCH 03/15] Pass arguments to `fetchLogsOverview` --- .../public/utils/logs_overview_fetchers.ts | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index a4f0b0f95e4462..fe0ea18f98aec9 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -4,8 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -import { InfraClientCoreSetup } from '../types'; -import { FetchData, LogsFetchDataResponse, HasData } from '../../../observability/public'; +import { InfraClientCoreSetup, InfraClientStartDeps } from '../types'; +import { + FetchData, + LogsFetchDataResponse, + HasData, + FetchDataParams, +} from '../../../observability/public'; + +interface LogParams { + index: string; + timestampField: string; +} type StatsAndSeries = Pick; @@ -31,7 +41,12 @@ export function getLogsOverviewDataFetcher( const [, startPlugins] = await getStartServices(); const { data } = startPlugins; - const { stats, series } = await fetchLogsOverview(); + // FIXME figure out how to get these from the sourceConfiguration + const { stats, series } = await fetchLogsOverview( + { index: 'filebeat-*', timestampField: '@timestamp' }, + params, + data + ); return { title: 'Log rate', @@ -42,7 +57,11 @@ export function getLogsOverviewDataFetcher( }; } -async function fetchLogsOverview(): Promise { +async function fetchLogsOverview( + logParams: LogParams, + params: FetchDataParams, + dataPlugin: InfraClientStartDeps['data'] +): Promise { return Promise.resolve({ stats: { nginx: { From 69abef13fb2cf01ae7effa7bcf7cf5ff3d4dd2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 11:24:36 +0200 Subject: [PATCH 04/15] Query `data` plugin and get the required aggregations --- .../public/utils/logs_overview_fetchers.ts | 158 ++++++++++++------ 1 file changed, 103 insertions(+), 55 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index fe0ea18f98aec9..9c2af2988bad6d 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -62,61 +62,109 @@ async function fetchLogsOverview( params: FetchDataParams, dataPlugin: InfraClientStartDeps['data'] ): Promise { - return Promise.resolve({ - stats: { - nginx: { - type: 'number', - label: 'nginx', - value: 345341, - }, - 'elasticsearch.audit': { - type: 'number', - label: 'elasticsearch.audit', - value: 164929, - }, - 'haproxy.log': { - type: 'number', - label: 'haproxy.log', - value: 51101, - }, - }, + const esSearcher = dataPlugin.search.getSearchStrategy('es'); + return new Promise((resolve, reject) => { + esSearcher + .search({ + params: { + index: logParams.index, + body: { + size: 0, + query: { + range: { + [logParams.timestampField]: { + gt: params.startTime, + lte: params.endTime, + format: 'strict_date_optional_time', + }, + }, + }, + aggs: { + stats: { + terms: { + field: 'event.dataset', + size: 4, + }, + }, + series: { + date_histogram: { + field: logParams.timestampField, + fixed_interval: params.bucketSize, + }, + aggs: { + dataset: { + terms: { + field: 'event.dataset', + size: 4, + }, + }, + }, + }, + }, + }, + }, + }) + .subscribe( + () => { + resolve({ + stats: { + nginx: { + type: 'number', + label: 'nginx', + value: 345341, + }, + 'elasticsearch.audit': { + type: 'number', + label: 'elasticsearch.audit', + value: 164929, + }, + 'haproxy.log': { + type: 'number', + label: 'haproxy.log', + value: 51101, + }, + }, - // Note: My understanding is that these series coordinates will be - // combined into objects that look like: - // { x: timestamp, y: value, g: label (e.g. nginx) } - // so they fit the stacked bar chart API - // https://elastic.github.io/elastic-charts/?path=/story/bar-chart--stacked-with-axis-and-legend - series: { - nginx: { - label: 'nginx', - coordinates: [ - { x: 1593000000000, y: 10014 }, - { x: 1593000900000, y: 12827 }, - { x: 1593001800000, y: 2946 }, - { x: 1593002700000, y: 14298 }, - { x: 1593003600000, y: 4096 }, - ], - }, - 'elasticsearch.audit': { - label: 'elasticsearch.audit', - coordinates: [ - { x: 1593000000000, y: 5676 }, - { x: 1593000900000, y: 6783 }, - { x: 1593001800000, y: 2394 }, - { x: 1593002700000, y: 4554 }, - { x: 1593003600000, y: 5659 }, - ], - }, - 'haproxy.log': { - label: 'haproxy.log', - coordinates: [ - { x: 1593000000000, y: 9085 }, - { x: 1593000900000, y: 9002 }, - { x: 1593001800000, y: 3940 }, - { x: 1593002700000, y: 5451 }, - { x: 1593003600000, y: 9133 }, - ], - }, - }, + // Note: My understanding is that these series coordinates will be + // combined into objects that look like: + // { x: timestamp, y: value, g: label (e.g. nginx) } + // so they fit the stacked bar chart API + // https://elastic.github.io/elastic-charts/?path=/story/bar-chart--stacked-with-axis-and-legend + series: { + nginx: { + label: 'nginx', + coordinates: [ + { x: 1593000000000, y: 10014 }, + { x: 1593000900000, y: 12827 }, + { x: 1593001800000, y: 2946 }, + { x: 1593002700000, y: 14298 }, + { x: 1593003600000, y: 4096 }, + ], + }, + 'elasticsearch.audit': { + label: 'elasticsearch.audit', + coordinates: [ + { x: 1593000000000, y: 5676 }, + { x: 1593000900000, y: 6783 }, + { x: 1593001800000, y: 2394 }, + { x: 1593002700000, y: 4554 }, + { x: 1593003600000, y: 5659 }, + ], + }, + 'haproxy.log': { + label: 'haproxy.log', + coordinates: [ + { x: 1593000000000, y: 9085 }, + { x: 1593000900000, y: 9002 }, + { x: 1593001800000, y: 3940 }, + { x: 1593002700000, y: 5451 }, + { x: 1593003600000, y: 9133 }, + ], + }, + }, + }); + }, + (error) => reject(error) + ); }); } From 8261fa4603467ed2e1400743d6447dee712e8686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 11:31:02 +0200 Subject: [PATCH 05/15] Process aggregations into a consumable format --- .../public/utils/logs_overview_fetchers.ts | 114 +++++++++--------- 1 file changed, 56 insertions(+), 58 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 9c2af2988bad6d..4a6f8f7bc60fbb 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -12,6 +12,19 @@ import { FetchDataParams, } from '../../../observability/public'; +interface StatsAggregation { + buckets: Array<{ key: string; doc_count: number }>; +} + +interface SeriesAggregation { + buckets: Array<{ + key_as_string: string; + key: number; + doc_count: number; + dataset: StatsAggregation; + }>; +} + interface LogParams { index: string; timestampField: string; @@ -105,66 +118,51 @@ async function fetchLogsOverview( }, }) .subscribe( - () => { - resolve({ - stats: { - nginx: { - type: 'number', - label: 'nginx', - value: 345341, - }, - 'elasticsearch.audit': { - type: 'number', - label: 'elasticsearch.audit', - value: 164929, - }, - 'haproxy.log': { - type: 'number', - label: 'haproxy.log', - value: 51101, - }, - }, - - // Note: My understanding is that these series coordinates will be - // combined into objects that look like: - // { x: timestamp, y: value, g: label (e.g. nginx) } - // so they fit the stacked bar chart API - // https://elastic.github.io/elastic-charts/?path=/story/bar-chart--stacked-with-axis-and-legend - series: { - nginx: { - label: 'nginx', - coordinates: [ - { x: 1593000000000, y: 10014 }, - { x: 1593000900000, y: 12827 }, - { x: 1593001800000, y: 2946 }, - { x: 1593002700000, y: 14298 }, - { x: 1593003600000, y: 4096 }, - ], - }, - 'elasticsearch.audit': { - label: 'elasticsearch.audit', - coordinates: [ - { x: 1593000000000, y: 5676 }, - { x: 1593000900000, y: 6783 }, - { x: 1593001800000, y: 2394 }, - { x: 1593002700000, y: 4554 }, - { x: 1593003600000, y: 5659 }, - ], - }, - 'haproxy.log': { - label: 'haproxy.log', - coordinates: [ - { x: 1593000000000, y: 9085 }, - { x: 1593000900000, y: 9002 }, - { x: 1593001800000, y: 3940 }, - { x: 1593002700000, y: 5451 }, - { x: 1593003600000, y: 9133 }, - ], - }, - }, - }); + (response) => { + if (response.rawResponse.aggregations) { + resolve(processLogsOverviewAggregations(response.rawResponse.aggregations)); + } else { + resolve({ stats: {}, series: {} }); + } }, (error) => reject(error) ); }); } + +function processLogsOverviewAggregations(aggregations: { + stats: StatsAggregation; + series: SeriesAggregation; +}): StatsAndSeries { + const processedStats = aggregations.stats.buckets.reduce( + (result, bucket) => { + result[bucket.key] = { + type: 'number', + label: bucket.key, + value: bucket.doc_count, + }; + + return result; + }, + {} + ); + + const processedSeries = aggregations.series.buckets.reduce( + (result, bucket) => { + const x = bucket.key; // the timestamp of the bucket + bucket.dataset.buckets.forEach((b) => { + const label = b.key; + result[label] = result[label] || { label, coordinates: [] }; + result[label].coordinates.push({ x, y: b.doc_count }); + }); + + return result; + }, + {} + ); + + return { + stats: processedStats, + series: processedSeries, + }; +} From 62884e02080e27e06be365adeba4567bf7339cf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 11:38:59 +0200 Subject: [PATCH 06/15] Extract query parts onto builder functions --- .../public/utils/logs_overview_fetchers.ts | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 4a6f8f7bc60fbb..375ea85e466bff 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -83,37 +83,8 @@ async function fetchLogsOverview( index: logParams.index, body: { size: 0, - query: { - range: { - [logParams.timestampField]: { - gt: params.startTime, - lte: params.endTime, - format: 'strict_date_optional_time', - }, - }, - }, - aggs: { - stats: { - terms: { - field: 'event.dataset', - size: 4, - }, - }, - series: { - date_histogram: { - field: logParams.timestampField, - fixed_interval: params.bucketSize, - }, - aggs: { - dataset: { - terms: { - field: 'event.dataset', - size: 4, - }, - }, - }, - }, - }, + query: buildLogOverviewQuery(logParams, params), + aggs: buildLogOverviewAggregations(logParams, params), }, }, }) @@ -130,6 +101,43 @@ async function fetchLogsOverview( }); } +function buildLogOverviewQuery(logParams: LogParams, params: FetchDataParams) { + return { + range: { + [logParams.timestampField]: { + gt: params.startTime, + lte: params.endTime, + format: 'strict_date_optional_time', + }, + }, + }; +} + +function buildLogOverviewAggregations(logParams: LogParams, params: FetchDataParams) { + return { + stats: { + terms: { + field: 'event.dataset', + size: 4, + }, + }, + series: { + date_histogram: { + field: logParams.timestampField, + fixed_interval: params.bucketSize, + }, + aggs: { + dataset: { + terms: { + field: 'event.dataset', + size: 4, + }, + }, + }, + }, + }; +} + function processLogsOverviewAggregations(aggregations: { stats: StatsAggregation; series: SeriesAggregation; From 92607df97030d07e0bdcf86a1a0eb55c41d915e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 11:50:29 +0200 Subject: [PATCH 07/15] Perform `hasData` check --- .../public/utils/logs_overview_fetchers.ts | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 375ea85e466bff..74aa3f0adb3ffa 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -36,14 +36,10 @@ export function getLogsHasDataFetcher( getStartServices: InfraClientCoreSetup['getStartServices'] ): HasData { return async () => { - // if you need the data plugin, this is how you get it - // const [, startPlugins] = await getStartServices(); - // const { data } = startPlugins; - - // if you need a core dep, we need to pass in more than just getStartServices + const [, startPlugins] = await getStartServices(); + const { data } = startPlugins; - // perform query - return true; + return await hasLogsOverview('filebeat-*', data); }; } @@ -70,6 +66,30 @@ export function getLogsOverviewDataFetcher( }; } +async function hasLogsOverview( + index: string, + dataPlugin: InfraClientStartDeps['data'] +): Promise { + const esSearcher = dataPlugin.search.getSearchStrategy('es'); + return new Promise((resolve, reject) => { + esSearcher + .search({ + params: { + index, + body: { + size: 0, + }, + }, + }) + .subscribe( + (response) => { + resolve(response.rawResponse.hits.total > 0); + }, + (error) => reject(error) + ); + }); +} + async function fetchLogsOverview( logParams: LogParams, params: FetchDataParams, From f6215c0097f36eabe61f2e5a045be5c36565d157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 16:39:16 +0200 Subject: [PATCH 08/15] Fetch metadata from the source configuration --- .../infra/public/utils/logs_overview_fetchers.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 74aa3f0adb3ffa..05d4c7e5b58926 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -11,6 +11,7 @@ import { HasData, FetchDataParams, } from '../../../observability/public'; +import { callFetchLogSourceConfigurationAPI } from '../containers/logs/log_source/api/fetch_log_source_configuration'; interface StatsAggregation { buckets: Array<{ key: string; doc_count: number }>; @@ -47,12 +48,19 @@ export function getLogsOverviewDataFetcher( getStartServices: InfraClientCoreSetup['getStartServices'] ): FetchData { return async (params) => { - const [, startPlugins] = await getStartServices(); + const [core, startPlugins] = await getStartServices(); const { data } = startPlugins; - // FIXME figure out how to get these from the sourceConfiguration + const sourceConfiguration = await callFetchLogSourceConfigurationAPI( + 'default', + core.http.fetch + ); + const { stats, series } = await fetchLogsOverview( - { index: 'filebeat-*', timestampField: '@timestamp' }, + { + index: sourceConfiguration.data.configuration.logAlias, + timestampField: sourceConfiguration.data.configuration.fields.timestamp, + }, params, data ); From 966a7e7249da0405ab46eb92bbe58a0bbaacee0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 16:40:29 +0200 Subject: [PATCH 09/15] Use source status API to check for data presence --- .../public/utils/logs_overview_fetchers.ts | 32 +++---------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 05d4c7e5b58926..2c1ed52d8e6e23 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -12,6 +12,7 @@ import { FetchDataParams, } from '../../../observability/public'; import { callFetchLogSourceConfigurationAPI } from '../containers/logs/log_source/api/fetch_log_source_configuration'; +import { callFetchLogSourceStatusAPI } from '../containers/logs/log_source/api/fetch_log_source_status'; interface StatsAggregation { buckets: Array<{ key: string; doc_count: number }>; @@ -37,10 +38,9 @@ export function getLogsHasDataFetcher( getStartServices: InfraClientCoreSetup['getStartServices'] ): HasData { return async () => { - const [, startPlugins] = await getStartServices(); - const { data } = startPlugins; - - return await hasLogsOverview('filebeat-*', data); + const [core] = await getStartServices(); + const sourceStatus = await callFetchLogSourceStatusAPI('default', core.http.fetch); + return sourceStatus.data.logIndexNames.length > 0; }; } @@ -74,30 +74,6 @@ export function getLogsOverviewDataFetcher( }; } -async function hasLogsOverview( - index: string, - dataPlugin: InfraClientStartDeps['data'] -): Promise { - const esSearcher = dataPlugin.search.getSearchStrategy('es'); - return new Promise((resolve, reject) => { - esSearcher - .search({ - params: { - index, - body: { - size: 0, - }, - }, - }) - .subscribe( - (response) => { - resolve(response.rawResponse.hits.total > 0); - }, - (error) => reject(error) - ); - }); -} - async function fetchLogsOverview( logParams: LogParams, params: FetchDataParams, From 2b10df8a383eca819a7e219ce6d25eed6dcd6d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 16:41:36 +0200 Subject: [PATCH 10/15] Extract default sourceId to a constant --- x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 2c1ed52d8e6e23..dd2cb058d7f057 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -34,12 +34,14 @@ interface LogParams { type StatsAndSeries = Pick; +const SOURCE_ID = 'default'; + export function getLogsHasDataFetcher( getStartServices: InfraClientCoreSetup['getStartServices'] ): HasData { return async () => { const [core] = await getStartServices(); - const sourceStatus = await callFetchLogSourceStatusAPI('default', core.http.fetch); + const sourceStatus = await callFetchLogSourceStatusAPI(SOURCE_ID, core.http.fetch); return sourceStatus.data.logIndexNames.length > 0; }; } @@ -52,7 +54,7 @@ export function getLogsOverviewDataFetcher( const { data } = startPlugins; const sourceConfiguration = await callFetchLogSourceConfigurationAPI( - 'default', + SOURCE_ID, core.http.fetch ); From 0dfddfda861ccab5842daa8e149a224eeee37ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 1 Jul 2020 21:03:17 +0200 Subject: [PATCH 11/15] Extract DEFAULT_SOURCE_ID constant to a shared location --- x-pack/plugins/infra/common/constants.ts | 7 +++++++ .../plugins/infra/public/utils/logs_overview_fetchers.ts | 7 +++---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/infra/common/constants.ts diff --git a/x-pack/plugins/infra/common/constants.ts b/x-pack/plugins/infra/common/constants.ts new file mode 100644 index 00000000000000..65dcb2e43c6f7f --- /dev/null +++ b/x-pack/plugins/infra/common/constants.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const DEFAULT_SOURCE_ID = 'default'; diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index dd2cb058d7f057..c667210d123b70 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { DEFAULT_SOURCE_ID } from '../../common/constants'; import { InfraClientCoreSetup, InfraClientStartDeps } from '../types'; import { FetchData, @@ -34,14 +35,12 @@ interface LogParams { type StatsAndSeries = Pick; -const SOURCE_ID = 'default'; - export function getLogsHasDataFetcher( getStartServices: InfraClientCoreSetup['getStartServices'] ): HasData { return async () => { const [core] = await getStartServices(); - const sourceStatus = await callFetchLogSourceStatusAPI(SOURCE_ID, core.http.fetch); + const sourceStatus = await callFetchLogSourceStatusAPI(DEFAULT_SOURCE_ID, core.http.fetch); return sourceStatus.data.logIndexNames.length > 0; }; } @@ -54,7 +53,7 @@ export function getLogsOverviewDataFetcher( const { data } = startPlugins; const sourceConfiguration = await callFetchLogSourceConfigurationAPI( - SOURCE_ID, + DEFAULT_SOURCE_ID, core.http.fetch ); From 865942e90e5e6ccfbbeba09bea0d86aca21df4fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 3 Jul 2020 11:46:43 +0200 Subject: [PATCH 12/15] Add test for `hasData` --- .../utils/logs_overview_fetches.test.ts | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 x-pack/plugins/infra/public/utils/logs_overview_fetches.test.ts diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetches.test.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetches.test.ts new file mode 100644 index 00000000000000..53c593b079ad5c --- /dev/null +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetches.test.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { coreMock } from 'src/core/public/mocks'; +import { dataPluginMock } from 'src/plugins/data/public/mocks'; +import { CoreStart } from 'kibana/public'; +import { getLogsHasDataFetcher } from './logs_overview_fetchers'; +import { InfraClientStartDeps, InfraClientStartExports } from '../types'; +import { callFetchLogSourceStatusAPI } from '../containers/logs/log_source/api/fetch_log_source_status'; + +// Note +// Calls to `.mock*` functions will fail the typecheck because how jest does the mocking. +// The calls will be preluded with a `@ts-expect-error` +jest.mock('../containers/logs/log_source/api/fetch_log_source_status'); + +function setup() { + const core = coreMock.createStart(); + const data = dataPluginMock.createStartContract(); + + const mockedGetStartServices = jest.fn(() => { + const deps = { data }; + return Promise.resolve([ + core as CoreStart, + deps as InfraClientStartDeps, + void 0 as InfraClientStartExports, + ]) as Promise<[CoreStart, InfraClientStartDeps, InfraClientStartExports]>; + }); + return { core, mockedGetStartServices }; +} + +describe('Logs UI Observability Homepage Functions', () => { + describe('getLogsHasDataFetcher()', () => { + beforeEach(() => { + // @ts-expect-error + callFetchLogSourceStatusAPI.mockReset(); + }); + it('should return true when some index is present', async () => { + const { mockedGetStartServices } = setup(); + + // @ts-expect-error + callFetchLogSourceStatusAPI.mockResolvedValue({ + data: { logIndexFields: [], logIndexNames: ['filebeat'] }, + }); + + const hasData = getLogsHasDataFetcher(mockedGetStartServices); + const response = await hasData(); + + expect(callFetchLogSourceStatusAPI).toHaveBeenCalledTimes(1); + expect(response).toBe(true); + }); + + it('should return false when no index is present', async () => { + const { mockedGetStartServices } = setup(); + + // @ts-expect-error + callFetchLogSourceStatusAPI.mockResolvedValue({ + data: { logIndexFields: [], logIndexNames: [] }, + }); + + const hasData = getLogsHasDataFetcher(mockedGetStartServices); + const response = await hasData(); + + expect(callFetchLogSourceStatusAPI).toHaveBeenCalledTimes(1); + expect(response).toBe(false); + }); + }); + + describe('getLogsOverviewDataFetcher()', () => { + it.skip('should work', async () => { + // Pending + }); + }); +}); From 929afde9415560f6c4ae4dc3eb85cb88485f7752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 3 Jul 2020 12:59:50 +0200 Subject: [PATCH 13/15] Normalize values The graphs need to show the log rate per minute instead of the log count. This commit adds two helper functions to normalize those values. --- .../public/utils/logs_overview_fetchers.ts | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index c667210d123b70..e0d8048469d397 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -66,11 +66,14 @@ export function getLogsOverviewDataFetcher( data ); + const timeSpanInMinutes = + (Date.parse(params.endTime).valueOf() - Date.parse(params.startTime).valueOf()) / (1000 * 60); + return { title: 'Log rate', appLink: 'TBD', // TODO: what format should this be in, relative I assume? - stats, - series, + stats: normalizeStats(stats, timeSpanInMinutes), + series: normalizeSeries(series), }; }; } @@ -179,3 +182,36 @@ function processLogsOverviewAggregations(aggregations: { series: processedSeries, }; } + +function normalizeStats( + stats: LogsFetchDataResponse['stats'], + timeSpanInMinutes: number +): LogsFetchDataResponse['stats'] { + return Object.keys(stats).reduce((normalized, key) => { + normalized[key] = { + ...stats[key], + value: stats[key].value / timeSpanInMinutes, + }; + return normalized; + }, {}); +} + +function normalizeSeries(series: LogsFetchDataResponse['series']): LogsFetchDataResponse['series'] { + const seriesKeys = Object.keys(series); + const timestamps = seriesKeys.flatMap((key) => series[key].coordinates.map((c) => c.x)); + const [first, second] = [...new Set(timestamps)].sort(); + const timeSpanInMinutes = (second - first) / (1000 * 60); + + return seriesKeys.reduce((normalized, key) => { + normalized[key] = { + ...series[key], + coordinates: series[key].coordinates.map((c) => { + if (c.y) { + return { ...c, y: c.y / timeSpanInMinutes }; + } + return c; + }), + }; + return normalized; + }, {}); +} From 2ec953feb1b912cc80105c1c0097eaa65679b9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 3 Jul 2020 13:47:30 +0200 Subject: [PATCH 14/15] Pass application path --- x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index e0d8048469d397..9df4452eae7a1e 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { encode } from 'rison-node'; import { DEFAULT_SOURCE_ID } from '../../common/constants'; import { InfraClientCoreSetup, InfraClientStartDeps } from '../types'; import { @@ -71,7 +72,9 @@ export function getLogsOverviewDataFetcher( return { title: 'Log rate', - appLink: 'TBD', // TODO: what format should this be in, relative I assume? + appLink: `/app/logs/stream?logPosition=(end:${encode(params.endTime)},start:${encode( + params.startTime + )})`, stats: normalizeStats(stats, timeSpanInMinutes), series: normalizeSeries(series), }; From c2d130c45cd9e7e8e36c3181df20cc6272333dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Fri, 3 Jul 2020 15:00:17 +0200 Subject: [PATCH 15/15] Fix types after merge from master --- .../plugins/infra/public/utils/logs_overview_fetchers.ts | 7 +++++-- .../infra/public/utils/logs_overview_fetches.test.ts | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 9df4452eae7a1e..65ea53a8465bbf 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -5,6 +5,7 @@ */ import { encode } from 'rison-node'; +import { i18n } from '@kbn/i18n'; import { DEFAULT_SOURCE_ID } from '../../common/constants'; import { InfraClientCoreSetup, InfraClientStartDeps } from '../types'; import { @@ -42,7 +43,7 @@ export function getLogsHasDataFetcher( return async () => { const [core] = await getStartServices(); const sourceStatus = await callFetchLogSourceStatusAPI(DEFAULT_SOURCE_ID, core.http.fetch); - return sourceStatus.data.logIndexNames.length > 0; + return sourceStatus.data.logIndicesExist; }; } @@ -71,7 +72,9 @@ export function getLogsOverviewDataFetcher( (Date.parse(params.endTime).valueOf() - Date.parse(params.startTime).valueOf()) / (1000 * 60); return { - title: 'Log rate', + title: i18n.translate('xpack.infra.logs.logOverview.logOverviewTitle', { + defaultMessage: 'Logs', + }), appLink: `/app/logs/stream?logPosition=(end:${encode(params.endTime)},start:${encode( params.startTime )})`, diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetches.test.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetches.test.ts index 53c593b079ad5c..6f9e41fbd08f3a 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetches.test.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetches.test.ts @@ -41,7 +41,7 @@ describe('Logs UI Observability Homepage Functions', () => { // @ts-expect-error callFetchLogSourceStatusAPI.mockResolvedValue({ - data: { logIndexFields: [], logIndexNames: ['filebeat'] }, + data: { logIndexFields: [], logIndicesExist: true }, }); const hasData = getLogsHasDataFetcher(mockedGetStartServices); @@ -56,7 +56,7 @@ describe('Logs UI Observability Homepage Functions', () => { // @ts-expect-error callFetchLogSourceStatusAPI.mockResolvedValue({ - data: { logIndexFields: [], logIndexNames: [] }, + data: { logIndexFields: [], logIndicesExist: false }, }); const hasData = getLogsHasDataFetcher(mockedGetStartServices);