From 49da11f2443d7d1353f177b9bd488a4cf1a06325 Mon Sep 17 00:00:00 2001 From: Mathis Date: Tue, 7 Jul 2020 17:41:10 +0200 Subject: [PATCH 1/9] [APM] Quote trace id to ensure a word is searched (#69500) (#69504) * [APM] Quote trace id to ensure a word is searched (#69500) Signed-off-by: Mathis Raguin * [APM] Fix TransactionActionMenu tests (one test was not updated) Co-authored-by: Elastic Machine --- .../__test__/TransactionActionMenu.test.tsx | 2 +- .../shared/TransactionActionMenu/__test__/sections.test.ts | 6 +++--- .../components/shared/TransactionActionMenu/sections.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/TransactionActionMenu.test.tsx b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/TransactionActionMenu.test.tsx index ec546b5c6280fc3..0a2cb90fdd5dafa 100644 --- a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/TransactionActionMenu.test.tsx +++ b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/TransactionActionMenu.test.tsx @@ -94,7 +94,7 @@ describe('TransactionActionMenu component', () => { expect(mock.core.application.navigateToApp).toHaveBeenCalledWith('logs', { path: - 'link-to/logs?time=1545092070952&filter=trace.id:%228b60bd32ecc6e1506735a8b6cfcf175c%22%20OR%208b60bd32ecc6e1506735a8b6cfcf175c', + 'link-to/logs?time=1545092070952&filter=trace.id:%228b60bd32ecc6e1506735a8b6cfcf175c%22%20OR%20%228b60bd32ecc6e1506735a8b6cfcf175c%22', }); }); diff --git a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/sections.test.ts b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/sections.test.ts index b2adc6cdac4a67d..50325e0b9d60441 100644 --- a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/sections.test.ts +++ b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/sections.test.ts @@ -42,7 +42,7 @@ describe('Transaction action menu', () => { key: 'traceLogs', label: 'Trace logs', href: - 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20123', + 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20%22123%22', condition: true, }, ], @@ -113,7 +113,7 @@ describe('Transaction action menu', () => { key: 'traceLogs', label: 'Trace logs', href: - 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20123', + 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20%22123%22', condition: true, }, ], @@ -183,7 +183,7 @@ describe('Transaction action menu', () => { key: 'traceLogs', label: 'Trace logs', href: - 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20123', + 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20%22123%22', condition: true, }, ], diff --git a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/sections.ts b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/sections.ts index d3a9ade3925a129..5ca0285eb4eeb7b 100644 --- a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/sections.ts +++ b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/sections.ts @@ -180,7 +180,7 @@ export const getSections = ({ path: `/link-to/logs`, query: { time, - filter: `trace.id:"${transaction.trace.id}" OR ${transaction.trace.id}`, + filter: `trace.id:"${transaction.trace.id}" OR "${transaction.trace.id}"`, }, }), condition: true, From e41692e0e7276a2dd845e419a08230546b05263d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez?= Date: Tue, 7 Jul 2020 17:42:45 +0200 Subject: [PATCH 2/9] [Logs UI] Process long running requests in logs overview (#70791) --- .../public/utils/logs_overview_fetchers.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 65ea53a8465bbfc..5a0a996287959cd 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -6,6 +6,7 @@ import { encode } from 'rison-node'; import { i18n } from '@kbn/i18n'; +import { SearchResponse } from 'src/plugins/data/public'; import { DEFAULT_SOURCE_ID } from '../../common/constants'; import { InfraClientCoreSetup, InfraClientStartDeps } from '../types'; import { @@ -89,9 +90,10 @@ async function fetchLogsOverview( params: FetchDataParams, dataPlugin: InfraClientStartDeps['data'] ): Promise { - const esSearcher = dataPlugin.search.getSearchStrategy('es'); return new Promise((resolve, reject) => { - esSearcher + let esResponse: SearchResponse = {}; + + dataPlugin.search .search({ params: { index: logParams.index, @@ -103,14 +105,15 @@ async function fetchLogsOverview( }, }) .subscribe( - (response) => { - if (response.rawResponse.aggregations) { - resolve(processLogsOverviewAggregations(response.rawResponse.aggregations)); + (response) => (esResponse = response.rawResponse), + (error) => reject(error), + () => { + if (esResponse.aggregations) { + resolve(processLogsOverviewAggregations(esResponse.aggregations)); } else { resolve({ stats: {}, series: {} }); } - }, - (error) => reject(error) + } ); }); } From 735d3bae8c4e77e4990427a6fcd19630bc43c658 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Tue, 7 Jul 2020 11:19:59 -0500 Subject: [PATCH 3/9] [APM] Improvements to data telemetry (#70524) Make some changes to how we deal with data telemetry in APM and reduce the number of fields we're storing in Saved Objects in the .kibana index. Add a telemetry doc in dev_docs explaining how telemetry is collected and how to make updates. (In this PR the docs only cover data telemetry, but there's a space for the behavioral telemetry docs.) Stop storing the mapping for the data telemetry in the Saved Object but instead use `{ dynamic: false }`. This reduces the number of fields used by APM in the .kibana index (as requested in #43673.) Before: ```bash > curl -s -X GET "admin:changeme@localhost:9200/.kibana/_field_caps?fields=*&pretty=true" | jq '.fields|length' 653 ``` After: ```bash > curl -s -X GET "admin:changeme@localhost:9200/.kibana/_field_caps?fields=*&pretty=true" | jq '.fields|length' 415 ``` We don't need the mapping anymore for storing the saved object, but we still do need to update the telemetry repository when the mapping changes, and the `upload-telemetry-data` script uses that mapping when generating data. For these purposes the mapping in now defined in TypeScript in a function in common/apm_telemetry.ts. It's broken down into some variables that and put together as the same mapping object that was there before, but having it in this form should make it easier to update. A new script, `merge-telemetry-mapping`, takes the telemetry repository's xpack-phone-home.json mapping, merges in the result of our mapping and replaces the file. The result can be committed to the telemetry repo, making it easier to make changes to the mapping. References #61583 Fixes #67032 --- .../__snapshots__/apm_telemetry.test.ts.snap | 913 ++++++++++++++++++ x-pack/plugins/apm/common/agent_name.ts | 5 +- .../plugins/apm/common/apm_telemetry.test.ts | 51 + x-pack/plugins/apm/common/apm_telemetry.ts | 229 +++++ x-pack/plugins/apm/dev_docs/telemetry.md | 69 ++ x-pack/plugins/apm/public/plugin.ts | 5 +- x-pack/plugins/apm/readme.md | 1 + .../apm/scripts/merge-telemetry-mapping.js | 21 + .../scripts/merge-telemetry-mapping/index.ts | 30 + .../scripts/shared/create-or-update-index.ts | 11 +- .../scripts/upload-telemetry-data/index.ts | 30 +- .../apm/server/saved_objects/apm_telemetry.ts | 912 +---------------- 12 files changed, 1349 insertions(+), 928 deletions(-) create mode 100644 x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap create mode 100644 x-pack/plugins/apm/common/apm_telemetry.test.ts create mode 100644 x-pack/plugins/apm/common/apm_telemetry.ts create mode 100644 x-pack/plugins/apm/dev_docs/telemetry.md create mode 100644 x-pack/plugins/apm/scripts/merge-telemetry-mapping.js create mode 100644 x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap new file mode 100644 index 000000000000000..3ac20a05639fbf0 --- /dev/null +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -0,0 +1,913 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the telemetry mapping 1`] = ` +Object { + "properties": Object { + "agents": Object { + "properties": Object { + "dotnet": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "go": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "java": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "js-base": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "nodejs": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "python": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "ruby": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "rum-js": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + }, + }, + "cardinality": Object { + "properties": Object { + "transaction": Object { + "properties": Object { + "name": Object { + "properties": Object { + "all_agents": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "rum": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + "user_agent": Object { + "properties": Object { + "original": Object { + "properties": Object { + "all_agents": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "rum": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + }, + }, + "counts": Object { + "properties": Object { + "agent_configuration": Object { + "properties": Object { + "all": Object { + "type": "long", + }, + }, + }, + "error": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "max_error_groups_per_service": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "max_transaction_groups_per_service": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "metric": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "onboarding": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "services": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "sourcemap": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "span": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "traces": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "transaction": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + }, + }, + "has_any_services": Object { + "type": "boolean", + }, + "indices": Object { + "properties": Object { + "all": Object { + "properties": Object { + "total": Object { + "properties": Object { + "docs": Object { + "properties": Object { + "count": Object { + "type": "long", + }, + }, + }, + "store": Object { + "properties": Object { + "size_in_bytes": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + "shards": Object { + "properties": Object { + "total": Object { + "type": "long", + }, + }, + }, + }, + }, + "integrations": Object { + "properties": Object { + "ml": Object { + "properties": Object { + "all_jobs_count": Object { + "type": "long", + }, + }, + }, + }, + }, + "retainment": Object { + "properties": Object { + "error": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "metric": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "onboarding": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "span": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "transaction": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "services_per_agent": Object { + "properties": Object { + "dotnet": Object { + "null_value": 0, + "type": "long", + }, + "go": Object { + "null_value": 0, + "type": "long", + }, + "java": Object { + "null_value": 0, + "type": "long", + }, + "js-base": Object { + "null_value": 0, + "type": "long", + }, + "nodejs": Object { + "null_value": 0, + "type": "long", + }, + "python": Object { + "null_value": 0, + "type": "long", + }, + "ruby": Object { + "null_value": 0, + "type": "long", + }, + "rum-js": Object { + "null_value": 0, + "type": "long", + }, + }, + }, + "tasks": Object { + "properties": Object { + "agent_configuration": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "agents": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "cardinality": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "groupings": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "indices_stats": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "integrations": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "processor_events": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "services": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "versions": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + "version": Object { + "properties": Object { + "apm_server": Object { + "properties": Object { + "major": Object { + "type": "long", + }, + "minor": Object { + "type": "long", + }, + "patch": Object { + "type": "long", + }, + }, + }, + }, + }, + }, +} +`; diff --git a/x-pack/plugins/apm/common/agent_name.ts b/x-pack/plugins/apm/common/agent_name.ts index 9d462dad87ec071..8b479d1d82fe7c3 100644 --- a/x-pack/plugins/apm/common/agent_name.ts +++ b/x-pack/plugins/apm/common/agent_name.ts @@ -15,15 +15,14 @@ import { AgentName } from '../typings/es_schemas/ui/fields/agent'; */ export const AGENT_NAMES: AgentName[] = [ - 'java', - 'js-base', - 'rum-js', 'dotnet', 'go', 'java', + 'js-base', 'nodejs', 'python', 'ruby', + 'rum-js', ]; export function isAgentName(agentName: string): agentName is AgentName { diff --git a/x-pack/plugins/apm/common/apm_telemetry.test.ts b/x-pack/plugins/apm/common/apm_telemetry.test.ts new file mode 100644 index 000000000000000..1612716142ce70e --- /dev/null +++ b/x-pack/plugins/apm/common/apm_telemetry.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + getApmTelemetryMapping, + mergeApmTelemetryMapping, +} from './apm_telemetry'; + +describe('APM telemetry helpers', () => { + describe('getApmTelemetry', () => { + it('generates a JSON object with the telemetry mapping', () => { + expect(getApmTelemetryMapping()).toMatchSnapshot(); + }); + }); + + describe('mergeApmTelemetryMapping', () => { + describe('with an invalid mapping', () => { + it('throws an error', () => { + expect(() => mergeApmTelemetryMapping({})).toThrowError(); + }); + }); + + describe('with a valid mapping', () => { + it('merges the mapping', () => { + // This is "valid" in the sense that it has all of the deep fields + // needed to merge. It's not a valid mapping opbject. + const validTelemetryMapping = { + mappings: { + properties: { + stack_stats: { + properties: { + kibana: { + properties: { plugins: { properties: { apm: {} } } }, + }, + }, + }, + }, + }, + }; + + expect( + mergeApmTelemetryMapping(validTelemetryMapping)?.mappings.properties + .stack_stats.properties.kibana.properties.plugins.properties.apm + ).toEqual(getApmTelemetryMapping()); + }); + }); + }); +}); diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts new file mode 100644 index 000000000000000..1532058adf64ff6 --- /dev/null +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -0,0 +1,229 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { produce } from 'immer'; +import { AGENT_NAMES } from './agent_name'; + +/** + * Generate an object containing the mapping used for APM telemetry. Can be used + * with the `upload-telemetry-data` script or to update the mapping in the + * telemetry repository. + * + * This function breaks things up to make the mapping easier to understand. + */ +export function getApmTelemetryMapping() { + const keyword = { + type: 'keyword', + ignore_above: 1024, + }; + + const long = { + type: 'long', + }; + + const allProperties = { + properties: { + all: long, + }, + }; + + const oneDayProperties = { + properties: { + '1d': long, + }, + }; + + const oneDayAllProperties = { + properties: { + '1d': long, + all: long, + }, + }; + + const msProperties = { + properties: { + ms: long, + }, + }; + + const tookProperties = { + properties: { + took: msProperties, + }, + }; + + const compositeNameVersionProperties = { + properties: { + composite: keyword, + name: keyword, + version: keyword, + }, + }; + + const agentProperties = { + properties: { version: keyword }, + }; + + const serviceProperties = { + properties: { + framework: compositeNameVersionProperties, + language: compositeNameVersionProperties, + runtime: compositeNameVersionProperties, + }, + }; + + return { + properties: { + agents: { + properties: AGENT_NAMES.reduce>( + (previousValue, currentValue) => { + previousValue[currentValue] = { + properties: { + agent: agentProperties, + service: serviceProperties, + }, + }; + + return previousValue; + }, + {} + ), + }, + counts: { + properties: { + agent_configuration: allProperties, + error: oneDayAllProperties, + max_error_groups_per_service: oneDayProperties, + max_transaction_groups_per_service: oneDayProperties, + metric: oneDayAllProperties, + onboarding: oneDayAllProperties, + services: oneDayProperties, + sourcemap: oneDayAllProperties, + span: oneDayAllProperties, + traces: oneDayProperties, + transaction: oneDayAllProperties, + }, + }, + cardinality: { + properties: { + user_agent: { + properties: { + original: { + properties: { + all_agents: oneDayProperties, + rum: oneDayProperties, + }, + }, + }, + }, + transaction: { + properties: { + name: { + properties: { + all_agents: oneDayProperties, + rum: oneDayProperties, + }, + }, + }, + }, + }, + }, + has_any_services: { + type: 'boolean', + }, + indices: { + properties: { + all: { + properties: { + total: { + properties: { + docs: { + properties: { + count: long, + }, + }, + store: { + properties: { + size_in_bytes: long, + }, + }, + }, + }, + }, + }, + shards: { + properties: { + total: long, + }, + }, + }, + }, + integrations: { + properties: { + ml: { + properties: { + all_jobs_count: long, + }, + }, + }, + }, + retainment: { + properties: { + error: msProperties, + metric: msProperties, + onboarding: msProperties, + span: msProperties, + transaction: msProperties, + }, + }, + services_per_agent: { + properties: AGENT_NAMES.reduce>( + (previousValue, currentValue) => { + previousValue[currentValue] = { ...long, null_value: 0 }; + return previousValue; + }, + {} + ), + }, + tasks: { + properties: { + agent_configuration: tookProperties, + agents: tookProperties, + cardinality: tookProperties, + groupings: tookProperties, + indices_stats: tookProperties, + integrations: tookProperties, + processor_events: tookProperties, + services: tookProperties, + versions: tookProperties, + }, + }, + version: { + properties: { + apm_server: { + properties: { + major: long, + minor: long, + patch: long, + }, + }, + }, + }, + }, + }; +} + +/** + * Merge a telemetry mapping object (from https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json) + * with the output from `getApmTelemetryMapping`. + */ +export function mergeApmTelemetryMapping( + xpackPhoneHomeMapping: Record +) { + return produce(xpackPhoneHomeMapping, (draft: Record) => { + draft.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = getApmTelemetryMapping(); + return draft; + }); +} diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md new file mode 100644 index 000000000000000..9674d39e57177b4 --- /dev/null +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -0,0 +1,69 @@ +# APM Telemetry + +In order to learn about our customers' usage and experience of APM, we collect +two types of telemetry, which we'll refer to here as "Data Telemetry" and +"Behavioral Telemetry." + +This document will explain how they are collected and how to make changes to +them. + +[The telemetry repository has information about accessing the clusters](https://github.com/elastic/telemetry#kibana-access). +Telemetry data is uploaded to the "xpack-phone-home" indices. + +## Data Telemetry + +Information that can be derived from a cluster's APM indices is queried and sent +to the telemetry cluster using the +[Usage Collection plugin](../../../../src/plugins/usage_collection/README.md). + +During the APM server-side plugin's setup phase a +[Saved Object](https://www.elastic.co/guide/en/kibana/master/managing-saved-objects.html) +for APM telemetry is registered and a +[task manager](../../task_manager/server/README.md) task is registered and started. +The task periodically queries the APM indices and saves the results in the Saved +Object, and the usage collector periodically gets the data from the saved object +and uploads it to the telemetry cluster. + +Once uploaded to the telemetry cluster, the data telemetry is stored in +`stack_stats.kibana.plugins.apm` in the xpack-phone-home index. + +### Generating sample data + +The script in `scripts/upload-telemetry-data` can generate sample telemetry data and upload it to a cluster of your choosing. + +You'll need to set the `GITHUB_TOKEN` environment variable to a token that has `repo` scope so it can read from the +[elastic/telemetry](https://github.com/elastic/telemetry) repository. (You probably have a token that works for this in +~/.backport/config.json.) + +The script will run as the `elastic` user using the elasticsearch hosts and password settings from the config/kibana.yml +and/or config/kibana.dev.yml files. + +Running the script with `--clear` will delete the index first. + +After running the script you should see sample telemetry data in the "xpack-phone-home" index. + +### Updating Data Telemetry Mappings + +In order for fields to be searchable on the telemetry cluster, they need to be +added to the cluster's mapping. The mapping is defined in +[the telemetry repository's xpack-phone-home template](https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json). + +The mapping for the telemetry data is here under `stack_stats.kibana.plugins.apm`. + +The mapping used there can be generated with the output of the [`getTelemetryMapping`](../common/apm_telemetry.ts) function. + +To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository. + +If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm: + +```bash +node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates/xpack-phone-home.json +``` + +this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. + +## Behavioral Telemetry + +Behavioral telemetry is recorded with the ui_metrics and application_usage methods from the Usage Collection plugin. + +Please fill this in with more details. diff --git a/x-pack/plugins/apm/public/plugin.ts b/x-pack/plugins/apm/public/plugin.ts index d24cb29eaf24f95..f31ad83666a1760 100644 --- a/x-pack/plugins/apm/public/plugin.ts +++ b/x-pack/plugins/apm/public/plugin.ts @@ -66,8 +66,9 @@ export interface ApmPluginStartDeps { } export class ApmPlugin implements Plugin { - private readonly initializerContext: PluginInitializerContext; - constructor(initializerContext: PluginInitializerContext) { + constructor( + private readonly initializerContext: PluginInitializerContext + ) { this.initializerContext = initializerContext; } public setup(core: CoreSetup, plugins: ApmPluginSetupDeps) { diff --git a/x-pack/plugins/apm/readme.md b/x-pack/plugins/apm/readme.md index f460ff6ff9bf2c0..9b02972d353023f 100644 --- a/x-pack/plugins/apm/readme.md +++ b/x-pack/plugins/apm/readme.md @@ -162,3 +162,4 @@ You can access the development environment at http://localhost:9001. - [Cypress integration tests](./e2e/README.md) - [VSCode setup instructions](./dev_docs/vscode_setup.md) - [Github PR commands](./dev_docs/github_commands.md) +- [Telemetry](./dev_docs/telemetry.md) diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js b/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js new file mode 100644 index 000000000000000..741df981a9cb0a5 --- /dev/null +++ b/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// compile typescript on the fly +// eslint-disable-next-line import/no-extraneous-dependencies +require('@babel/register')({ + extensions: ['.ts'], + plugins: [ + '@babel/plugin-proposal-optional-chaining', + '@babel/plugin-proposal-nullish-coalescing-operator', + ], + presets: [ + '@babel/typescript', + ['@babel/preset-env', { targets: { node: 'current' } }], + ], +}); + +require('./merge-telemetry-mapping/index.ts'); diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts new file mode 100644 index 000000000000000..c06d4cec150dcfe --- /dev/null +++ b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { readFileSync, truncateSync, writeFileSync } from 'fs'; +import { resolve } from 'path'; +import { argv } from 'yargs'; +import { mergeApmTelemetryMapping } from '../../common/apm_telemetry'; + +function errorExit(error?: Error) { + console.error(`usage: ${argv.$0} /path/to/xpack-phone-home.json`); // eslint-disable-line no-console + if (error) { + throw error; + } + process.exit(1); +} + +try { + const filename = resolve(argv._[0]); + const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8')); + + const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping); + + truncateSync(filename); + writeFileSync(filename, JSON.stringify(newMapping, null, 2)); +} catch (error) { + errorExit(error); +} diff --git a/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts b/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts index 3f88b73f5598437..6d44e12fb00a2bd 100644 --- a/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts +++ b/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts @@ -30,6 +30,11 @@ export async function createOrUpdateIndex({ } } + // Some settings are non-updateable and need to be removed. + const settings = { ...template.settings }; + delete settings?.index?.number_of_shards; + delete settings?.index?.sort; + const indexExists = ( await client.indices.exists({ index: indexName, @@ -42,6 +47,7 @@ export async function createOrUpdateIndex({ body: template, }); } else { + await client.indices.close({ index: indexName }); await Promise.all([ template.mappings ? client.indices.putMapping({ @@ -49,12 +55,13 @@ export async function createOrUpdateIndex({ body: template.mappings, }) : Promise.resolve(undefined as any), - template.settings + settings ? client.indices.putSettings({ index: indexName, - body: template.settings, + body: settings, }) : Promise.resolve(undefined as any), ]); + await client.indices.open({ index: indexName }); } } diff --git a/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts b/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts index 5f9c72810fc91c4..a44fad82f20e648 100644 --- a/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts +++ b/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts @@ -11,7 +11,7 @@ // - Easier testing of the telemetry tasks // - Validate whether we can run the queries we want to on the telemetry data -import { merge, chunk, flatten } from 'lodash'; +import { merge, chunk, flatten, omit } from 'lodash'; import { Client } from '@elastic/elasticsearch'; import { argv } from 'yargs'; import { Logger } from 'kibana/server'; @@ -20,7 +20,7 @@ import { stampLogger } from '../shared/stamp-logger'; import { CollectTelemetryParams } from '../../server/lib/apm_telemetry/collect_data_telemetry'; import { downloadTelemetryTemplate } from '../shared/download-telemetry-template'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { apmTelemetry } from '../../server/saved_objects/apm_telemetry'; +import { mergeApmTelemetryMapping } from '../../common/apm_telemetry'; import { generateSampleDocuments } from './generate-sample-documents'; import { readKibanaConfig } from '../shared/read-kibana-config'; import { getHttpAuth } from '../shared/get-http-auth'; @@ -40,8 +40,6 @@ async function uploadData() { githubToken, }); - const kibanaMapping = apmTelemetry.mappings; - const config = readKibanaConfig(); const httpAuth = getHttpAuth(config); @@ -50,19 +48,25 @@ async function uploadData() { nodes: [config['elasticsearch.hosts']], ...(httpAuth ? { - auth: httpAuth, + auth: { ...httpAuth, username: 'elastic' }, } : {}), }); - const newTemplate = merge(telemetryTemplate, { - settings: { - index: { mapping: { total_fields: { limit: 10000 } } }, - }, - }); - - // override apm mapping instead of merging - newTemplate.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = kibanaMapping; + // The new template is the template downloaded from the telemetry repo, with + // our current telemetry mapping merged in, with the "index_patterns" key + // (which cannot be used when creating an index) removed. + const newTemplate = omit( + mergeApmTelemetryMapping( + merge(telemetryTemplate, { + index_patterns: undefined, + settings: { + index: { mapping: { total_fields: { limit: 10000 } } }, + }, + }) + ), + 'index_patterns' + ); await createOrUpdateIndex({ indexName: xpackTelemetryIndexName, diff --git a/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts b/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts index f741b61fc7a0404..411f453042b9351 100644 --- a/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts +++ b/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts @@ -4,918 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ import { SavedObjectsType } from 'src/core/server'; +import { APM_TELEMETRY_SAVED_OBJECT_ID } from '../../common/apm_saved_object_constants'; export const apmTelemetry: SavedObjectsType = { - name: 'apm-telemetry', + name: APM_TELEMETRY_SAVED_OBJECT_ID, hidden: false, namespaceType: 'agnostic', mappings: { - properties: { - agents: { - properties: { - dotnet: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - go: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - java: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - 'js-base': { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - nodejs: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - python: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - ruby: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - 'rum-js': { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - }, - }, - counts: { - properties: { - agent_configuration: { - properties: { - all: { - type: 'long', - }, - }, - }, - error: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - max_error_groups_per_service: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - max_transaction_groups_per_service: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - metric: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - onboarding: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - services: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - sourcemap: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - span: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - traces: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - transaction: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - }, - }, - cardinality: { - properties: { - user_agent: { - properties: { - original: { - properties: { - all_agents: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - rum: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - transaction: { - properties: { - name: { - properties: { - all_agents: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - rum: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - }, - }, - has_any_services: { - type: 'boolean', - }, - indices: { - properties: { - all: { - properties: { - total: { - properties: { - docs: { - properties: { - count: { - type: 'long', - }, - }, - }, - store: { - properties: { - size_in_bytes: { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - shards: { - properties: { - total: { - type: 'long', - }, - }, - }, - }, - }, - integrations: { - properties: { - ml: { - properties: { - all_jobs_count: { - type: 'long', - }, - }, - }, - }, - }, - retainment: { - properties: { - error: { - properties: { - ms: { - type: 'long', - }, - }, - }, - metric: { - properties: { - ms: { - type: 'long', - }, - }, - }, - onboarding: { - properties: { - ms: { - type: 'long', - }, - }, - }, - span: { - properties: { - ms: { - type: 'long', - }, - }, - }, - transaction: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - services_per_agent: { - properties: { - dotnet: { - type: 'long', - null_value: 0, - }, - go: { - type: 'long', - null_value: 0, - }, - java: { - type: 'long', - null_value: 0, - }, - 'js-base': { - type: 'long', - null_value: 0, - }, - nodejs: { - type: 'long', - null_value: 0, - }, - python: { - type: 'long', - null_value: 0, - }, - ruby: { - type: 'long', - null_value: 0, - }, - 'rum-js': { - type: 'long', - null_value: 0, - }, - }, - }, - tasks: { - properties: { - agent_configuration: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - agents: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - cardinality: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - groupings: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - indices_stats: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - integrations: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - processor_events: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - services: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - versions: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - version: { - properties: { - apm_server: { - properties: { - major: { - type: 'long', - }, - minor: { - type: 'long', - }, - patch: { - type: 'long', - }, - }, - }, - }, - }, - } as SavedObjectsType['mappings']['properties'], + dynamic: false, + properties: {}, }, }; From f946e8e2a932a91be0976031ac3e80b4d47e0270 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 7 Jul 2020 09:47:45 -0700 Subject: [PATCH 4/9] [Metrics UI] Set includeTimeseries on Observability fetchData request (#70735) Co-authored-by: Elastic Machine --- x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts | 1 + x-pack/plugins/infra/public/metrics_overview_fetchers.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts b/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts index 21946c7c5653ab5..11348dab18b8262 100644 --- a/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts +++ b/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts @@ -76,6 +76,7 @@ describe('Metrics UI Observability Homepage Functions', () => { metrics: [{ type: 'cpu' }, { type: 'memory' }, { type: 'rx' }, { type: 'tx' }], groupBy: [], nodeType: 'host', + includeTimeseries: true, timerange: { from: startTime.valueOf(), to: endTime.valueOf(), diff --git a/x-pack/plugins/infra/public/metrics_overview_fetchers.ts b/x-pack/plugins/infra/public/metrics_overview_fetchers.ts index d10ad5dda53204d..50b12696415100d 100644 --- a/x-pack/plugins/infra/public/metrics_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/metrics_overview_fetchers.ts @@ -88,6 +88,7 @@ export const createMetricsFetchData = ( metrics: ['cpu', 'memory', 'rx', 'tx'].map((type) => ({ type })) as SnapshotMetricInput[], groupBy: [], nodeType: 'host', + includeTimeseries: true, timerange: { from: moment(startTime).valueOf(), to: moment(endTime).valueOf(), From 50a2991312ad05a5ce78d988bdce35d27f0cb9fe Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Tue, 7 Jul 2020 19:22:58 +0200 Subject: [PATCH 5/9] [Ingest Pipeline] Processor Editor Item Styling tweak (#70786) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Small styling tweaks to processor items - Moved the move button to the before the processor name - Cancel button is still after description if there is one - Made inline text description a bit taller and changed border style * Commit code that moves the cancel move button 🤦🏼‍♂️ * Do not completely hide the move button, prevent ui from jumping * Update styling and UX of move button; EuiToggleButton - Bring the styling of the button more in line with this comment https://github.com/elastic/kibana/pull/70786#issuecomment-654222298 * use cross icon for cancelling move * replace hard values with EUI values in SCSS * Address rerendering triggered by context - also prevent re-renders basded on contstructing objects on each render * Similarly move use of context to settings form container We are only interested in the es docs path string in the settings form component so no need to render for other updates. Co-authored-by: Elastic Machine --- .../pipeline_processors_editor.helpers.tsx | 7 +- .../pipeline_processors_editor.test.tsx | 2 +- .../components/_shared.scss | 4 +- .../pipeline_processors_editor_item/index.ts | 4 +- ...eline_processors_editor_item.container.tsx | 28 ++++++ .../pipeline_processors_editor_item.scss | 32 ++++--- .../pipeline_processors_editor_item.tsx | 88 +++++++++++-------- .../pipeline_processors_editor_item/types.ts | 10 +++ .../processor_settings_form.container.tsx | 9 +- .../processor_settings_form.tsx | 8 +- .../processors_tree/components/tree_node.tsx | 6 +- .../pipeline_processors_editor/context.tsx | 58 ++++++------ .../pipeline_processors_editor/types.ts | 31 ++++++- 13 files changed, 184 insertions(+), 103 deletions(-) create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.container.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/types.ts diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.helpers.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.helpers.tsx index cc3817d92d5e3c0..e7258a74f473269 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.helpers.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.helpers.tsx @@ -112,7 +112,7 @@ const createActions = (testBed: TestBed) => { moveProcessor(processorSelector: string, dropZoneSelector: string) { act(() => { - find(`${processorSelector}.moveItemButton`).simulate('click'); + find(`${processorSelector}.moveItemButton`).simulate('change'); }); component.update(); act(() => { @@ -144,12 +144,13 @@ const createActions = (testBed: TestBed) => { startAndCancelMove(processorSelector: string) { act(() => { - find(`${processorSelector}.moveItemButton`).simulate('click'); + find(`${processorSelector}.moveItemButton`).simulate('change'); }); component.update(); act(() => { - find(`${processorSelector}.cancelMoveItemButton`).simulate('click'); + find(`${processorSelector}.cancelMoveItemButton`).simulate('change'); }); + component.update(); }, duplicateProcessor(processorSelector: string) { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.test.tsx index a4bbf840dff7101..acfa012990b21c4 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.test.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.test.tsx @@ -153,7 +153,7 @@ describe('Pipeline Editor', () => { const processorSelector = 'processors>0'; actions.startAndCancelMove(processorSelector); // Assert that we have exited move mode for this processor - expect(exists(`moveItemButton-${processorSelector}`)); + expect(exists(`${processorSelector}.moveItemButton`)).toBe(true); const [onUpdateResult] = onUpdate.mock.calls[onUpdate.mock.calls.length - 1]; const { processors } = onUpdateResult.getData(); // Assert that nothing has changed diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/_shared.scss b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/_shared.scss index 8d17a3970d94f85..c7c49c00bb5cfc8 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/_shared.scss +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/_shared.scss @@ -1,2 +1,2 @@ -$dropZoneZIndex: 1; /* Prevent the next item down from obscuring the button */ -$cancelButtonZIndex: 2; +$dropZoneZIndex: $euiZLevel1; /* Prevent the next item down from obscuring the button */ +$cancelButtonZIndex: $euiZLevel2; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/index.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/index.ts index 02bafdb32602444..fb3f513300c6a01 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/index.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/index.ts @@ -4,4 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -export { PipelineProcessorsEditorItem, Handlers } from './pipeline_processors_editor_item'; +export { PipelineProcessorsEditorItem } from './pipeline_processors_editor_item.container'; + +export { Handlers } from './types'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.container.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.container.tsx new file mode 100644 index 000000000000000..5201320e97d3a10 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.container.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent } from 'react'; + +import { usePipelineProcessorsContext } from '../../context'; + +import { + PipelineProcessorsEditorItem as ViewComponent, + Props as ViewComponentProps, +} from './pipeline_processors_editor_item'; + +type Props = Omit; + +export const PipelineProcessorsEditorItem: FunctionComponent = (props) => { + const { state } = usePipelineProcessorsContext(); + + return ( + + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.scss b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.scss index 6b5e11808460632..85a123b421975fb 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.scss +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.scss @@ -1,10 +1,12 @@ @import '../shared'; .pipelineProcessorsEditor__item { - transition: border-color 1s; + transition: border-color $euiAnimSpeedExtraSlow $euiAnimSlightResistance; min-height: 50px; + &--selected { - border: 1px solid $euiColorPrimary; + border: $euiBorderThin; + border-color: $euiColorPrimary; } &--displayNone { @@ -25,15 +27,14 @@ } &__textContainer { - padding: 4px; - border-radius: 2px; - - transition: border-color 0.3s; - border: 2px solid transparent; + cursor: text; + border-bottom: 1px dashed transparent; &--notEditing { + border-bottom: $euiBorderEditable; + border-width: $euiBorderWidthThin; &:hover { - border: 2px solid $euiColorLightShade; + border-color: $euiColorMediumShade; } } } @@ -46,12 +47,17 @@ } &__textInput { - height: 21px; - min-width: 150px; + height: $euiSizeL; + min-width: 200px; } - &__cancelMoveButton { - // Ensure that the cancel button is above the drop zones - z-index: $cancelButtonZIndex; + &__moveButton { + &:hover { + transform: none !important; + } + &--cancel { + // Ensure that the cancel button is above the drop zones + z-index: $cancelButtonZIndex; + } } } diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.tsx index 09c047d1d51b764..97b57a971ff7d6f 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.tsx @@ -8,7 +8,7 @@ import classNames from 'classnames'; import React, { FunctionComponent, memo } from 'react'; import { EuiButtonIcon, - EuiButton, + EuiButtonToggle, EuiFlexGroup, EuiFlexItem, EuiPanel, @@ -16,25 +16,23 @@ import { EuiToolTip, } from '@elastic/eui'; -import { ProcessorInternal, ProcessorSelector } from '../../types'; +import { ProcessorInternal, ProcessorSelector, ContextValueEditor } from '../../types'; import { selectorToDataTestSubject } from '../../utils'; +import { ProcessorsDispatch } from '../../processors_reducer'; -import { usePipelineProcessorsContext } from '../../context'; +import { ProcessorInfo } from '../processors_tree'; import './pipeline_processors_editor_item.scss'; import { InlineTextInput } from './inline_text_input'; import { ContextMenu } from './context_menu'; import { i18nTexts } from './i18n_texts'; -import { ProcessorInfo } from '../processors_tree'; - -export interface Handlers { - onMove: () => void; - onCancelMove: () => void; -} +import { Handlers } from './types'; export interface Props { processor: ProcessorInternal; + processorsDispatch: ProcessorsDispatch; + editor: ContextValueEditor; handlers: Handlers; selector: ProcessorSelector; description?: string; @@ -43,18 +41,16 @@ export interface Props { } export const PipelineProcessorsEditorItem: FunctionComponent = memo( - ({ + function PipelineProcessorsEditorItem({ processor, description, handlers: { onCancelMove, onMove }, selector, movingProcessor, renderOnFailureHandlers, - }) => { - const { - state: { editor, processors }, - } = usePipelineProcessorsContext(); - + editor, + processorsDispatch, + }) { const isDisabled = editor.mode.id !== 'idle'; const isInMoveMode = Boolean(movingProcessor); const isMovingThisProcessor = processor.id === movingProcessor?.id; @@ -78,9 +74,41 @@ export const PipelineProcessorsEditorItem: FunctionComponent = memo( 'pipelineProcessorsEditor__item--displayNone': isInMoveMode && !processor.options.description, }); - const cancelMoveButtonClasses = classNames('pipelineProcessorsEditor__item__cancelMoveButton', { - 'pipelineProcessorsEditor__item--displayNone': !isMovingThisProcessor, - }); + const renderMoveButton = () => { + const label = !isMovingThisProcessor + ? i18nTexts.moveButtonLabel + : i18nTexts.cancelMoveButtonLabel; + const dataTestSubj = !isMovingThisProcessor ? 'moveItemButton' : 'cancelMoveItemButton'; + const moveButtonClasses = classNames('pipelineProcessorsEditor__item__moveButton', { + 'pipelineProcessorsEditor__item__moveButton--cancel': isMovingThisProcessor, + }); + const icon = isMovingThisProcessor ? 'cross' : 'sortable'; + const moveButton = ( + (!isMovingThisProcessor ? onMove() : onCancelMove())} + /> + ); + // Remove the tooltip from the DOM to prevent it from lingering if the mouse leave event + // did not fire. + return ( +
+ {!isInMoveMode ? ( + {moveButton} + ) : ( + moveButton + )} +
+ ); + }; return ( @@ -93,6 +121,7 @@ export const PipelineProcessorsEditorItem: FunctionComponent = memo( > + {renderMoveButton()} = memo( description: nextDescription, }; } - processors.dispatch({ + processorsDispatch({ type: 'updateProcessor', payload: { processor: { @@ -149,25 +178,6 @@ export const PipelineProcessorsEditorItem: FunctionComponent = memo( )} - - {!isInMoveMode && ( - - - - )} - - - - {i18nTexts.cancelMoveButtonLabel} - - @@ -183,7 +193,7 @@ export const PipelineProcessorsEditorItem: FunctionComponent = memo( editor.setMode({ id: 'removingProcessor', arg: { selector } }); }} onDuplicate={() => { - processors.dispatch({ + processorsDispatch({ type: 'duplicateProcessor', payload: { source: selector, diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/types.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/types.ts new file mode 100644 index 000000000000000..893aee13fff8f91 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/types.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface Handlers { + onMove: () => void; + onCancelMove: () => void; +} diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.container.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.container.tsx index d76e9225c1a1375..2a537ba082eecdd 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.container.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.container.tsx @@ -10,6 +10,7 @@ import { useForm, OnFormUpdateArg, FormData } from '../../../../../shared_import import { ProcessorInternal } from '../../types'; import { ProcessorSettingsForm as ViewComponent } from './processor_settings_form'; +import { usePipelineProcessorsContext } from '../../context'; export type ProcessorSettingsFromOnSubmitArg = Omit; @@ -32,6 +33,10 @@ export const ProcessorSettingsForm: FunctionComponent = ({ onSubmit, ...rest }) => { + const { + links: { esDocsBasePath }, + } = usePipelineProcessorsContext(); + const handleSubmit = useCallback( async (data: FormData, isValid: boolean) => { if (isValid) { @@ -60,5 +65,7 @@ export const ProcessorSettingsForm: FunctionComponent = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [onFormUpdate]); - return ; + return ( + + ); }; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.tsx index 3eccda55fbb3a4a..015adae83e71ef3 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.tsx @@ -21,7 +21,6 @@ import { } from '@elastic/eui'; import { Form, FormDataProvider, FormHook } from '../../../../../shared_imports'; -import { usePipelineProcessorsContext } from '../../context'; import { ProcessorInternal } from '../../types'; import { DocumentationButton } from './documentation_button'; @@ -35,6 +34,7 @@ export interface Props { form: FormHook; onClose: () => void; onOpen: () => void; + esDocsBasePath: string; } const updateButtonLabel = i18n.translate( @@ -52,11 +52,7 @@ const cancelButtonLabel = i18n.translate( ); export const ProcessorSettingsForm: FunctionComponent = memo( - ({ processor, form, isOnFailure, onClose, onOpen }) => { - const { - links: { esDocsBasePath }, - } = usePipelineProcessorsContext(); - + ({ processor, form, isOnFailure, onClose, onOpen, esDocsBasePath }) => { const flyoutTitleContent = isOnFailure ? ( = ({ }; }, [onAction, stringSelector, processor]); // eslint-disable-line react-hooks/exhaustive-deps - const renderOnFailureHandlersTree = () => { + const renderOnFailureHandlersTree = useCallback(() => { if (!processor.onFailure?.length) { return; } @@ -79,7 +79,7 @@ export const TreeNode: FunctionComponent = ({ /> ); - }; + }, [processor.onFailure, stringSelector, onAction, movingProcessor, level]); // eslint-disable-line react-hooks/exhaustive-deps return ( ; - }; - }; -} - const PipelineProcessorsContext = createContext({} as any); export interface Props { @@ -81,7 +64,9 @@ export const PipelineProcessorsContextProvider: FunctionComponent = ({ children, }) => { const initRef = useRef(false); - const [mode, setMode] = useState({ id: 'idle' }); + const [mode, setMode] = useState(() => ({ + id: 'idle', + })); const deserializedResult = useMemo( () => deserialize({ @@ -199,15 +184,24 @@ export const PipelineProcessorsContextProvider: FunctionComponent = ({ [processorsDispatch, setMode] ); + // Memoize the state object to ensure we do not trigger unnecessary re-renders and so + // this object can be used safely further down the tree component tree. + const state = useMemo(() => { + return { + editor: { + mode, + setMode, + }, + processors: { state: processorsState, dispatch: processorsDispatch }, + }; + }, [mode, setMode, processorsState, processorsDispatch]); + return ( {children} diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/types.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/types.ts index aea8f0f0910f451..aaca4108bb58319 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/types.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/types.ts @@ -4,9 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import { OnFormUpdateArg } from '../../../../../../../src/plugins/es_ui_shared/static/forms/hook_form_lib'; +import { Dispatch } from 'react'; +import { OnFormUpdateArg } from '../../../shared_imports'; import { SerializeResult } from './serialize'; -import { ProcessorInfo } from './components/processors_tree'; +import { OnActionHandler, ProcessorInfo } from './components/processors_tree'; +import { ProcessorsDispatch, State as ProcessorsReducerState } from './processors_reducer'; + +export interface Links { + esDocsBasePath: string; +} /** * An array of keys that map to a value in an object @@ -51,3 +57,24 @@ export type EditorMode = | { id: 'editingProcessor'; arg: { processor: ProcessorInternal; selector: ProcessorSelector } } | { id: 'removingProcessor'; arg: { selector: ProcessorSelector } } | { id: 'idle' }; + +export interface ContextValueEditor { + mode: EditorMode; + setMode: Dispatch; +} + +export interface ContextValueProcessors { + state: ProcessorsReducerState; + dispatch: ProcessorsDispatch; +} + +export interface ContextValueState { + processors: ContextValueProcessors; + editor: ContextValueEditor; +} + +export interface ContextValue { + links: Links; + onTreeAction: OnActionHandler; + state: ContextValueState; +} From b1ec391d86ff887a9cbb454dc650dc1c43576bb5 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 7 Jul 2020 13:51:55 -0400 Subject: [PATCH 6/9] [Ingest Manager] Rate limit agent config update (#70871) --- .../ingest_manager/common/constants/agent.ts | 3 + .../ingest_manager/common/types/index.ts | 2 + .../ingest_manager/server/constants/index.ts | 2 + x-pack/plugins/ingest_manager/server/index.ts | 2 + .../services/agents/checkin/rxjs_utils.ts | 65 ++++++++++++++++++- .../agents/checkin/state_new_actions.ts | 29 ++++++--- 6 files changed, 92 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/constants/agent.ts b/x-pack/plugins/ingest_manager/common/constants/agent.ts index e9226fa68492531..7652c6ac87bced3 100644 --- a/x-pack/plugins/ingest_manager/common/constants/agent.ts +++ b/x-pack/plugins/ingest_manager/common/constants/agent.ts @@ -16,3 +16,6 @@ export const AGENT_POLLING_THRESHOLD_MS = 30000; export const AGENT_POLLING_INTERVAL = 1000; export const AGENT_UPDATE_LAST_CHECKIN_INTERVAL_MS = 30000; export const AGENT_UPDATE_ACTIONS_INTERVAL_MS = 5000; + +export const AGENT_CONFIG_ROLLUP_RATE_LIMIT_INTERVAL_MS = 5000; +export const AGENT_CONFIG_ROLLUP_RATE_LIMIT_REQUEST_PER_INTERVAL = 60; diff --git a/x-pack/plugins/ingest_manager/common/types/index.ts b/x-pack/plugins/ingest_manager/common/types/index.ts index 7f81b04f5e84a00..ff08b8a9252046f 100644 --- a/x-pack/plugins/ingest_manager/common/types/index.ts +++ b/x-pack/plugins/ingest_manager/common/types/index.ts @@ -24,6 +24,8 @@ export interface IngestManagerConfigType { host?: string; ca_sha256?: string; }; + agentConfigRollupRateLimitIntervalMs: number; + agentConfigRollupRateLimitRequestPerInterval: number; }; } diff --git a/x-pack/plugins/ingest_manager/server/constants/index.ts b/x-pack/plugins/ingest_manager/server/constants/index.ts index 650211ce9c1b2e5..d3c074ff2e8d0ea 100644 --- a/x-pack/plugins/ingest_manager/server/constants/index.ts +++ b/x-pack/plugins/ingest_manager/server/constants/index.ts @@ -10,6 +10,8 @@ export { AGENT_POLLING_THRESHOLD_MS, AGENT_POLLING_INTERVAL, AGENT_UPDATE_LAST_CHECKIN_INTERVAL_MS, + AGENT_CONFIG_ROLLUP_RATE_LIMIT_REQUEST_PER_INTERVAL, + AGENT_CONFIG_ROLLUP_RATE_LIMIT_INTERVAL_MS, AGENT_UPDATE_ACTIONS_INTERVAL_MS, INDEX_PATTERN_PLACEHOLDER_SUFFIX, // Routes diff --git a/x-pack/plugins/ingest_manager/server/index.ts b/x-pack/plugins/ingest_manager/server/index.ts index 5d6a1ad321b6dd4..811ec8a3d022240 100644 --- a/x-pack/plugins/ingest_manager/server/index.ts +++ b/x-pack/plugins/ingest_manager/server/index.ts @@ -37,6 +37,8 @@ export const config = { host: schema.maybe(schema.string()), ca_sha256: schema.maybe(schema.string()), }), + agentConfigRollupRateLimitIntervalMs: schema.number({ defaultValue: 5000 }), + agentConfigRollupRateLimitRequestPerInterval: schema.number({ defaultValue: 50 }), }), }), }; diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin/rxjs_utils.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin/rxjs_utils.ts index 1f9bba8b12be431..a806169019a1ed5 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin/rxjs_utils.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin/rxjs_utils.ts @@ -3,12 +3,13 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Observable } from 'rxjs'; + +import * as Rx from 'rxjs'; export class AbortError extends Error {} export const toPromiseAbortable = ( - observable: Observable, + observable: Rx.Observable, signal?: AbortSignal ): Promise => new Promise((resolve, reject) => { @@ -41,3 +42,63 @@ export const toPromiseAbortable = ( signal.addEventListener('abort', listener, { once: true }); } }); + +export function createLimiter(ratelimitIntervalMs: number, ratelimitRequestPerInterval: number) { + function createCurrentInterval() { + return { + startedAt: Rx.asyncScheduler.now(), + numRequests: 0, + }; + } + + let currentInterval: { startedAt: number; numRequests: number } = createCurrentInterval(); + let observers: Array<[Rx.Subscriber, any]> = []; + let timerSubscription: Rx.Subscription | undefined; + + function createTimeout() { + if (timerSubscription) { + return; + } + timerSubscription = Rx.asyncScheduler.schedule(() => { + timerSubscription = undefined; + currentInterval = createCurrentInterval(); + for (const [waitingObserver, value] of observers) { + if (currentInterval.numRequests >= ratelimitRequestPerInterval) { + createTimeout(); + continue; + } + currentInterval.numRequests++; + waitingObserver.next(value); + } + }, ratelimitIntervalMs); + } + + return function limit(): Rx.MonoTypeOperatorFunction { + return (observable) => + new Rx.Observable((observer) => { + const subscription = observable.subscribe({ + next(value) { + if (currentInterval.numRequests < ratelimitRequestPerInterval) { + currentInterval.numRequests++; + observer.next(value); + return; + } + + observers = [...observers, [observer, value]]; + createTimeout(); + }, + error(err) { + observer.error(err); + }, + complete() { + observer.complete(); + }, + }); + + return () => { + observers = observers.filter((o) => o[0] !== observer); + subscription.unsubscribe(); + }; + }); + }; +} diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_new_actions.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_new_actions.ts index 0f30ab409f38106..5ceb774a1946c2a 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_new_actions.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_new_actions.ts @@ -28,7 +28,7 @@ import * as APIKeysService from '../../api_keys'; import { AGENT_SAVED_OBJECT_TYPE, AGENT_UPDATE_ACTIONS_INTERVAL_MS } from '../../../constants'; import { createAgentAction, getNewActionsSince } from '../actions'; import { appContextService } from '../../app_context'; -import { toPromiseAbortable, AbortError } from './rxjs_utils'; +import { toPromiseAbortable, AbortError, createLimiter } from './rxjs_utils'; function getInternalUserSOClient() { const fakeRequest = ({ @@ -95,19 +95,23 @@ async function getOrCreateAgentDefaultOutputAPIKey( return outputAPIKey.key; } -async function createAgentActionFromConfigIfOutdated( - soClient: SavedObjectsClientContract, - agent: Agent, - config: FullAgentConfig | null -) { +function shouldCreateAgentConfigAction(agent: Agent, config: FullAgentConfig | null): boolean { if (!config || !config.revision) { - return; + return false; } const isAgentConfigOutdated = !agent.config_revision || agent.config_revision < config.revision; if (!isAgentConfigOutdated) { - return; + return false; } + return true; +} + +async function createAgentActionFromConfig( + soClient: SavedObjectsClientContract, + agent: Agent, + config: FullAgentConfig | null +) { // Deep clone !not supporting Date, and undefined value. const newConfig = JSON.parse(JSON.stringify(config)); @@ -129,6 +133,11 @@ export function agentCheckinStateNewActionsFactory() { // Shared Observables const agentConfigs$ = new Map>(); const newActions$ = createNewActionsSharedObservable(); + // Rx operators + const rateLimiter = createLimiter( + appContextService.getConfig()?.fleet.agentConfigRollupRateLimitIntervalMs || 5000, + appContextService.getConfig()?.fleet.agentConfigRollupRateLimitRequestPerInterval || 50 + ); async function subscribeToNewActions( soClient: SavedObjectsClientContract, @@ -148,7 +157,9 @@ export function agentCheckinStateNewActionsFactory() { } const stream$ = agentConfig$.pipe( timeout(appContextService.getConfig()?.fleet.pollingRequestTimeout || 0), - mergeMap((config) => createAgentActionFromConfigIfOutdated(soClient, agent, config)), + filter((config) => shouldCreateAgentConfigAction(agent, config)), + rateLimiter(), + mergeMap((config) => createAgentActionFromConfig(soClient, agent, config)), merge(newActions$), mergeMap(async (data) => { if (!data) { From 6e357988a2d52224094d4f1bf69fa53891a5ba43 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 7 Jul 2020 11:07:52 -0700 Subject: [PATCH 7/9] [Metrics UI] Performance improvements for Observability Homepage (#70869) --- .../public/metrics_overview_fetchers.test.ts | 12 +-- .../infra/public/metrics_overview_fetchers.ts | 7 +- .../infra/server/routes/source/index.ts | 52 +++++++++--- .../apis/metrics_ui/http_source.ts | 79 +++++++++++++++++++ .../api_integration/apis/metrics_ui/index.js | 1 + 5 files changed, 127 insertions(+), 24 deletions(-) create mode 100644 x-pack/test/api_integration/apis/metrics_ui/http_source.ts diff --git a/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts b/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts index 11348dab18b8262..24c51598ad25761 100644 --- a/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts +++ b/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts @@ -29,11 +29,7 @@ describe('Metrics UI Observability Homepage Functions', () => { it('should return true when true', async () => { const { core, mockedGetStartServices } = setup(); core.http.get.mockResolvedValue({ - status: { - indexFields: [], - logIndicesExist: false, - metricIndicesExist: true, - }, + hasData: true, }); const hasData = createMetricsHasData(mockedGetStartServices); const response = await hasData(); @@ -43,11 +39,7 @@ describe('Metrics UI Observability Homepage Functions', () => { it('should return false when false', async () => { const { core, mockedGetStartServices } = setup(); core.http.get.mockResolvedValue({ - status: { - indexFields: [], - logIndicesExist: false, - metricIndicesExist: false, - }, + hasData: false, }); const hasData = createMetricsHasData(mockedGetStartServices); const response = await hasData(); diff --git a/x-pack/plugins/infra/public/metrics_overview_fetchers.ts b/x-pack/plugins/infra/public/metrics_overview_fetchers.ts index 50b12696415100d..15751fab39abc0f 100644 --- a/x-pack/plugins/infra/public/metrics_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/metrics_overview_fetchers.ts @@ -16,15 +16,16 @@ import { } from '../common/http_api/snapshot_api'; import { SnapshotMetricType } from '../common/inventory_models/types'; import { InfraClientCoreSetup } from './types'; -import { SourceResponse } from '../common/http_api/source_api'; export const createMetricsHasData = ( getStartServices: InfraClientCoreSetup['getStartServices'] ) => async () => { const [coreServices] = await getStartServices(); const { http } = coreServices; - const results = await http.get('/api/metrics/source/default/metrics'); - return results.status.metricIndicesExist; + const results = await http.get<{ hasData: boolean }>( + '/api/metrics/source/default/metrics/hasData' + ); + return results.hasData; }; export const average = (values: number[]) => (values.length ? sum(values) / values.length : 0); diff --git a/x-pack/plugins/infra/server/routes/source/index.ts b/x-pack/plugins/infra/server/routes/source/index.ts index 62b7fd7ba902f58..2843897071e1914 100644 --- a/x-pack/plugins/infra/server/routes/source/index.ts +++ b/x-pack/plugins/infra/server/routes/source/index.ts @@ -37,22 +37,21 @@ export const initSourceRoute = (libs: InfraBackendLibs) => { try { const { type, sourceId } = request.params; - const source = await libs.sources.getSourceConfiguration( - requestContext.core.savedObjects.client, - sourceId - ); + const [source, logIndicesExist, metricIndicesExist, indexFields] = await Promise.all([ + libs.sources.getSourceConfiguration(requestContext.core.savedObjects.client, sourceId), + libs.sourceStatus.hasLogIndices(requestContext, sourceId), + libs.sourceStatus.hasMetricIndices(requestContext, sourceId), + libs.fields.getFields(requestContext, sourceId, typeToInfraIndexType(type)), + ]); + if (!source) { return response.notFound(); } const status = { - logIndicesExist: await libs.sourceStatus.hasLogIndices(requestContext, sourceId), - metricIndicesExist: await libs.sourceStatus.hasMetricIndices(requestContext, sourceId), - indexFields: await libs.fields.getFields( - requestContext, - sourceId, - typeToInfraIndexType(type) - ), + logIndicesExist, + metricIndicesExist, + indexFields, }; return response.ok({ @@ -65,4 +64,35 @@ export const initSourceRoute = (libs: InfraBackendLibs) => { } } ); + + framework.registerRoute( + { + method: 'get', + path: '/api/metrics/source/{sourceId}/{type}/hasData', + validate: { + params: schema.object({ + sourceId: schema.string(), + type: schema.string(), + }), + }, + }, + async (requestContext, request, response) => { + try { + const { type, sourceId } = request.params; + + const hasData = + type === 'metrics' + ? await libs.sourceStatus.hasMetricIndices(requestContext, sourceId) + : await libs.sourceStatus.hasLogIndices(requestContext, sourceId); + + return response.ok({ + body: { hasData }, + }); + } catch (error) { + return response.internalError({ + body: error.message, + }); + } + } + ); }; diff --git a/x-pack/test/api_integration/apis/metrics_ui/http_source.ts b/x-pack/test/api_integration/apis/metrics_ui/http_source.ts new file mode 100644 index 000000000000000..7e92caf0e37d785 --- /dev/null +++ b/x-pack/test/api_integration/apis/metrics_ui/http_source.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { SourceResponse } from '../../../../plugins/infra/server/lib/sources'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + const fetchSource = async (): Promise => { + const response = await supertest + .get('/api/metrics/source/default/metrics') + .set('kbn-xsrf', 'xxx') + .expect(200); + return response.body; + }; + const fetchHasData = async ( + type: 'logs' | 'metrics' + ): Promise<{ hasData: boolean } | undefined> => { + const response = await supertest + .get(`/api/metrics/source/default/${type}/hasData`) + .set('kbn-xsrf', 'xxx') + .expect(200); + return response.body; + }; + + describe('Source API via HTTP', () => { + describe('8.0.0', () => { + before(() => esArchiver.load('infra/8.0.0/logs_and_metrics')); + after(() => esArchiver.unload('infra/8.0.0/logs_and_metrics')); + describe('/api/metrics/source/default/metrics', () => { + it('should just work', () => { + const resp = fetchSource(); + return resp.then((data) => { + expect(data).to.have.property('source'); + expect(data?.source.configuration.metricAlias).to.equal('metrics-*,metricbeat-*'); + expect(data?.source.configuration.logAlias).to.equal( + 'logs-*,filebeat-*,kibana_sample_data_logs*' + ); + expect(data?.source.configuration.fields).to.eql({ + container: 'container.id', + host: 'host.name', + message: ['message', '@message'], + pod: 'kubernetes.pod.uid', + tiebreaker: '_doc', + timestamp: '@timestamp', + }); + expect(data).to.have.property('status'); + expect(data?.status.metricIndicesExist).to.equal(true); + expect(data?.status.logIndicesExist).to.equal(true); + }); + }); + }); + describe('/api/metrics/source/default/metrics/hasData', () => { + it('should just work', () => { + const resp = fetchHasData('metrics'); + return resp.then((data) => { + expect(data).to.have.property('hasData'); + expect(data?.hasData).to.be(true); + }); + }); + }); + describe('/api/metrics/source/default/logs/hasData', () => { + it('should just work', () => { + const resp = fetchHasData('logs'); + return resp.then((data) => { + expect(data).to.have.property('hasData'); + expect(data?.hasData).to.be(true); + }); + }); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/metrics_ui/index.js b/x-pack/test/api_integration/apis/metrics_ui/index.js index eb8ee77da582bea..fdd37fa4c335cbf 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/index.js +++ b/x-pack/test/api_integration/apis/metrics_ui/index.js @@ -21,5 +21,6 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./metrics_explorer')); loadTestFile(require.resolve('./feature_controls')); loadTestFile(require.resolve('./ip_to_hostname')); + loadTestFile(require.resolve('./http_source')); }); } From 465ed2119472621928ce5041d22c2dcfa95641f1 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 7 Jul 2020 14:14:30 -0400 Subject: [PATCH 8/9] [CI] Add pipeline task queue framework and merge workers into one (#64011) --- .ci/Dockerfile | 35 ++++ .ci/runbld_no_junit.yml | 2 +- .gitignore | 2 + Jenkinsfile | 45 +---- src/dev/ci_setup/checkout_sibling_es.sh | 12 +- src/dev/ci_setup/setup_env.sh | 4 +- src/dev/notice/generate_notice_from_source.ts | 2 + tasks/config/karma.js | 2 +- tasks/test_jest.js | 6 +- test/scripts/checks/doc_api_changes.sh | 5 + test/scripts/checks/file_casing.sh | 5 + test/scripts/checks/i18n.sh | 5 + test/scripts/checks/licenses.sh | 5 + test/scripts/checks/lock_file_symlinks.sh | 5 + test/scripts/checks/test_hardening.sh | 5 + test/scripts/checks/test_projects.sh | 5 + test/scripts/checks/ts_projects.sh | 5 + test/scripts/checks/type_check.sh | 5 + .../checks/verify_dependency_versions.sh | 5 + test/scripts/checks/verify_notice.sh | 5 + .../jenkins_build_kbn_sample_panel_action.sh | 0 test/scripts/jenkins_build_kibana.sh | 19 +- test/scripts/jenkins_build_plugins.sh | 19 ++ test/scripts/jenkins_ci_group.sh | 2 +- test/scripts/jenkins_plugin_functional.sh | 15 ++ .../jenkins_security_solution_cypress.sh | 8 +- .../jenkins_setup_parallel_workspace.sh | 32 ++++ test/scripts/jenkins_test_setup.sh | 4 + test/scripts/jenkins_test_setup_oss.sh | 15 +- test/scripts/jenkins_test_setup_xpack.sh | 15 +- test/scripts/jenkins_xpack_build_kibana.sh | 23 +-- test/scripts/jenkins_xpack_build_plugins.sh | 21 +++ .../jenkins_xpack_page_load_metrics.sh | 0 test/scripts/lint/eslint.sh | 5 + test/scripts/lint/sasslint.sh | 5 + test/scripts/test/api_integration.sh | 5 + test/scripts/test/jest_integration.sh | 5 + test/scripts/test/jest_unit.sh | 5 + test/scripts/test/karma_ci.sh | 5 + test/scripts/test/mocha.sh | 5 + test/scripts/test/xpack_jest_unit.sh | 6 + test/scripts/test/xpack_karma.sh | 6 + .../test/xpack_list_cyclic_dependency.sh | 6 + .../test/xpack_siem_cyclic_dependency.sh | 6 + vars/catchErrors.groovy | 11 +- vars/kibanaPipeline.groovy | 175 +++++++++++++++--- vars/task.groovy | 5 + vars/tasks.groovy | 118 ++++++++++++ vars/withTaskQueue.groovy | 154 +++++++++++++++ vars/workers.groovy | 12 +- .../canvas/.storybook/storyshots.test.js | 7 + 51 files changed, 749 insertions(+), 130 deletions(-) create mode 100644 .ci/Dockerfile create mode 100755 test/scripts/checks/doc_api_changes.sh create mode 100755 test/scripts/checks/file_casing.sh create mode 100755 test/scripts/checks/i18n.sh create mode 100755 test/scripts/checks/licenses.sh create mode 100755 test/scripts/checks/lock_file_symlinks.sh create mode 100755 test/scripts/checks/test_hardening.sh create mode 100755 test/scripts/checks/test_projects.sh create mode 100755 test/scripts/checks/ts_projects.sh create mode 100755 test/scripts/checks/type_check.sh create mode 100755 test/scripts/checks/verify_dependency_versions.sh create mode 100755 test/scripts/checks/verify_notice.sh mode change 100644 => 100755 test/scripts/jenkins_build_kbn_sample_panel_action.sh create mode 100755 test/scripts/jenkins_build_plugins.sh create mode 100755 test/scripts/jenkins_plugin_functional.sh mode change 100644 => 100755 test/scripts/jenkins_security_solution_cypress.sh create mode 100755 test/scripts/jenkins_setup_parallel_workspace.sh mode change 100644 => 100755 test/scripts/jenkins_test_setup.sh mode change 100644 => 100755 test/scripts/jenkins_test_setup_oss.sh mode change 100644 => 100755 test/scripts/jenkins_test_setup_xpack.sh create mode 100755 test/scripts/jenkins_xpack_build_plugins.sh mode change 100644 => 100755 test/scripts/jenkins_xpack_page_load_metrics.sh create mode 100755 test/scripts/lint/eslint.sh create mode 100755 test/scripts/lint/sasslint.sh create mode 100755 test/scripts/test/api_integration.sh create mode 100755 test/scripts/test/jest_integration.sh create mode 100755 test/scripts/test/jest_unit.sh create mode 100755 test/scripts/test/karma_ci.sh create mode 100755 test/scripts/test/mocha.sh create mode 100755 test/scripts/test/xpack_jest_unit.sh create mode 100755 test/scripts/test/xpack_karma.sh create mode 100755 test/scripts/test/xpack_list_cyclic_dependency.sh create mode 100755 test/scripts/test/xpack_siem_cyclic_dependency.sh create mode 100644 vars/task.groovy create mode 100644 vars/tasks.groovy create mode 100644 vars/withTaskQueue.groovy diff --git a/.ci/Dockerfile b/.ci/Dockerfile new file mode 100644 index 000000000000000..201e17b93c116be --- /dev/null +++ b/.ci/Dockerfile @@ -0,0 +1,35 @@ +ARG NODE_VERSION=10.21.0 + +FROM node:${NODE_VERSION} AS base + +RUN apt-get update && \ + apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \ + libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \ + libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \ + libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ + && apt-get update \ + && apt-get install -y rsync jq bsdtar google-chrome-stable \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN LATEST_VAULT_RELEASE=$(curl -s https://api.github.com/repos/hashicorp/vault/tags | jq --raw-output .[0].name[1:]) \ + && curl -L https://releases.hashicorp.com/vault/${LATEST_VAULT_RELEASE}/vault_${LATEST_VAULT_RELEASE}_linux_amd64.zip -o vault.zip \ + && unzip vault.zip \ + && rm vault.zip \ + && chmod +x vault \ + && mv vault /usr/local/bin/vault + +RUN groupadd -r kibana && useradd -r -g kibana kibana && mkdir /home/kibana && chown kibana:kibana /home/kibana + +COPY ./bash_standard_lib.sh /usr/local/bin/bash_standard_lib.sh +RUN chmod +x /usr/local/bin/bash_standard_lib.sh + +COPY ./runbld /usr/local/bin/runbld +RUN chmod +x /usr/local/bin/runbld + +USER kibana diff --git a/.ci/runbld_no_junit.yml b/.ci/runbld_no_junit.yml index 67b5002c1c43770..1bcb7e22a264807 100644 --- a/.ci/runbld_no_junit.yml +++ b/.ci/runbld_no_junit.yml @@ -3,4 +3,4 @@ profiles: - ".*": # Match any job tests: - junit-filename-pattern: "8d8bd494-d909-4e67-a052-7e8b5aaeb5e4" # A bogus path that should never exist + junit-filename-pattern: false diff --git a/.gitignore b/.gitignore index 32377ec0f1ffe80..25a8c369bb704d0 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,8 @@ npm-debug.log* .tern-project .nyc_output .ci/pipeline-library/build/ +.ci/runbld +.ci/bash_standard_lib.sh .gradle # apm plugin diff --git a/Jenkinsfile b/Jenkinsfile index f6f77ccae8427ad..491a1e386deb187 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,50 +8,7 @@ kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true) ciStats.trackBuild { catchError { retryable.enable() - parallel([ - 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), - 'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'), - 'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - 'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'), - 'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1), - 'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2), - 'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3), - 'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4), - 'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5), - 'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6), - 'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7), - 'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8), - 'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9), - 'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10), - 'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11), - 'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12), - 'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'), - // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'), - ]), - 'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - 'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'), - 'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1), - 'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2), - 'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3), - 'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4), - 'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5), - 'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6), - 'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7), - 'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8), - 'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9), - 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), - 'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'), - 'xpack-savedObjectsFieldMetrics': kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'), - // 'xpack-pageLoadMetrics': kibanaPipeline.functionalTestProcess('xpack-pageLoadMetrics', './test/scripts/jenkins_xpack_page_load_metrics.sh'), - 'xpack-securitySolutionCypress': { processNumber -> - whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/']) { - kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber) - } - }, - - // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'), - ]), - ]) + kibanaPipeline.allCiTasks() } } } diff --git a/src/dev/ci_setup/checkout_sibling_es.sh b/src/dev/ci_setup/checkout_sibling_es.sh index 915759d4214f9a8..3832ec9b4076a7b 100755 --- a/src/dev/ci_setup/checkout_sibling_es.sh +++ b/src/dev/ci_setup/checkout_sibling_es.sh @@ -7,10 +7,11 @@ function checkout_sibling { targetDir=$2 useExistingParamName=$3 useExisting="$(eval "echo "\$$useExistingParamName"")" + repoAddress="https://github.com/" if [ -z ${useExisting:+x} ]; then if [ -d "$targetDir" ]; then - echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$PARENT_DIR]!" + echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$WORKSPACE]!" echo echo "Either define '${useExistingParamName}' or remove the existing '${project}' sibling." exit 1 @@ -21,8 +22,9 @@ function checkout_sibling { cloneBranch="" function clone_target_is_valid { + echo " -> checking for '${cloneBranch}' branch at ${cloneAuthor}/${project}" - if [[ -n "$(git ls-remote --heads "git@github.com:${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then + if [[ -n "$(git ls-remote --heads "${repoAddress}${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then return 0 else return 1 @@ -71,7 +73,7 @@ function checkout_sibling { fi echo " -> checking out '${cloneBranch}' branch from ${cloneAuthor}/${project}..." - git clone -b "$cloneBranch" "git@github.com:${cloneAuthor}/${project}.git" "$targetDir" --depth=1 + git clone -b "$cloneBranch" "${repoAddress}${cloneAuthor}/${project}.git" "$targetDir" --depth=1 echo " -> checked out ${project} revision: $(git -C "${targetDir}" rev-parse HEAD)" echo } @@ -87,12 +89,12 @@ function checkout_sibling { fi } -checkout_sibling "elasticsearch" "${PARENT_DIR}/elasticsearch" "USE_EXISTING_ES" +checkout_sibling "elasticsearch" "${WORKSPACE}/elasticsearch" "USE_EXISTING_ES" export TEST_ES_FROM=${TEST_ES_FROM:-snapshot} # Set the JAVA_HOME based on the Java property file in the ES repo # This assumes the naming convention used on CI (ex: ~/.java/java10) -ES_DIR="$PARENT_DIR/elasticsearch" +ES_DIR="$WORKSPACE/elasticsearch" ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh index 343ff4719937546..f96a2240917e257 100644 --- a/src/dev/ci_setup/setup_env.sh +++ b/src/dev/ci_setup/setup_env.sh @@ -53,6 +53,8 @@ export PARENT_DIR="$parentDir" kbnBranch="$(jq -r .branch "$KIBANA_DIR/package.json")" export KIBANA_PKG_BRANCH="$kbnBranch" +export WORKSPACE="${WORKSPACE:-$PARENT_DIR}" + ### ### download node ### @@ -161,7 +163,7 @@ export -f checks-reporter-with-killswitch source "$KIBANA_DIR/src/dev/ci_setup/load_env_keys.sh" -ES_DIR="$PARENT_DIR/elasticsearch" +ES_DIR="$WORKSPACE/elasticsearch" ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then diff --git a/src/dev/notice/generate_notice_from_source.ts b/src/dev/notice/generate_notice_from_source.ts index fb74bed0f26f4b7..a2b05c6dc8a4eab 100644 --- a/src/dev/notice/generate_notice_from_source.ts +++ b/src/dev/notice/generate_notice_from_source.ts @@ -49,8 +49,10 @@ export async function generateNoticeFromSource({ productName, directory, log }: ignore: [ '{node_modules,build,target,dist,data,built_assets}/**', 'packages/*/{node_modules,build,target,dist}/**', + 'src/plugins/*/{node_modules,build,target,dist}/**', 'x-pack/{node_modules,build,target,dist,data}/**', 'x-pack/packages/*/{node_modules,build,target,dist}/**', + 'x-pack/plugins/*/{node_modules,build,target,dist}/**', ], }; diff --git a/tasks/config/karma.js b/tasks/config/karma.js index fa4bdc8ed226624..7c4f75bea8801cb 100644 --- a/tasks/config/karma.js +++ b/tasks/config/karma.js @@ -110,7 +110,7 @@ module.exports = function (grunt) { customLaunchers: { Chrome_Headless: { base: 'Chrome', - flags: ['--headless', '--disable-gpu', '--remote-debugging-port=9222'], + flags: ['--headless', '--disable-gpu', '--remote-debugging-port=9222', '--no-sandbox'], }, }, diff --git a/tasks/test_jest.js b/tasks/test_jest.js index d8f51806e8ddc8d..810ed423248400d 100644 --- a/tasks/test_jest.js +++ b/tasks/test_jest.js @@ -22,7 +22,7 @@ const { resolve } = require('path'); module.exports = function (grunt) { grunt.registerTask('test:jest', function () { const done = this.async(); - runJest(resolve(__dirname, '../scripts/jest.js')).then(done, done); + runJest(resolve(__dirname, '../scripts/jest.js'), ['--maxWorkers=10']).then(done, done); }); grunt.registerTask('test:jest_integration', function () { @@ -30,10 +30,10 @@ module.exports = function (grunt) { runJest(resolve(__dirname, '../scripts/jest_integration.js')).then(done, done); }); - function runJest(jestScript) { + function runJest(jestScript, args = []) { const serverCmd = { cmd: 'node', - args: [jestScript, '--ci'], + args: [jestScript, '--ci', ...args], opts: { stdio: 'inherit' }, }; diff --git a/test/scripts/checks/doc_api_changes.sh b/test/scripts/checks/doc_api_changes.sh new file mode 100755 index 000000000000000..503d12b2f6d73de --- /dev/null +++ b/test/scripts/checks/doc_api_changes.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:checkDocApiChanges diff --git a/test/scripts/checks/file_casing.sh b/test/scripts/checks/file_casing.sh new file mode 100755 index 000000000000000..513664263791b98 --- /dev/null +++ b/test/scripts/checks/file_casing.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:checkFileCasing diff --git a/test/scripts/checks/i18n.sh b/test/scripts/checks/i18n.sh new file mode 100755 index 000000000000000..7a6fd46c46c7697 --- /dev/null +++ b/test/scripts/checks/i18n.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:i18nCheck diff --git a/test/scripts/checks/licenses.sh b/test/scripts/checks/licenses.sh new file mode 100755 index 000000000000000..a08d7d07a24a138 --- /dev/null +++ b/test/scripts/checks/licenses.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:licenses diff --git a/test/scripts/checks/lock_file_symlinks.sh b/test/scripts/checks/lock_file_symlinks.sh new file mode 100755 index 000000000000000..1d43d32c9feb82d --- /dev/null +++ b/test/scripts/checks/lock_file_symlinks.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:checkLockfileSymlinks diff --git a/test/scripts/checks/test_hardening.sh b/test/scripts/checks/test_hardening.sh new file mode 100755 index 000000000000000..9184758577654a4 --- /dev/null +++ b/test/scripts/checks/test_hardening.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_hardening diff --git a/test/scripts/checks/test_projects.sh b/test/scripts/checks/test_projects.sh new file mode 100755 index 000000000000000..5f9aafe80e10e6d --- /dev/null +++ b/test/scripts/checks/test_projects.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_projects diff --git a/test/scripts/checks/ts_projects.sh b/test/scripts/checks/ts_projects.sh new file mode 100755 index 000000000000000..d667c753baec234 --- /dev/null +++ b/test/scripts/checks/ts_projects.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:checkTsProjects diff --git a/test/scripts/checks/type_check.sh b/test/scripts/checks/type_check.sh new file mode 100755 index 000000000000000..07c49638134be15 --- /dev/null +++ b/test/scripts/checks/type_check.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:typeCheck diff --git a/test/scripts/checks/verify_dependency_versions.sh b/test/scripts/checks/verify_dependency_versions.sh new file mode 100755 index 000000000000000..b73a71e7ff7fd54 --- /dev/null +++ b/test/scripts/checks/verify_dependency_versions.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:verifyDependencyVersions diff --git a/test/scripts/checks/verify_notice.sh b/test/scripts/checks/verify_notice.sh new file mode 100755 index 000000000000000..9f8343e5408615d --- /dev/null +++ b/test/scripts/checks/verify_notice.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:verifyNotice diff --git a/test/scripts/jenkins_build_kbn_sample_panel_action.sh b/test/scripts/jenkins_build_kbn_sample_panel_action.sh old mode 100644 new mode 100755 diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index 3e49edc8e6ae5f7..f449986713f97df 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -2,19 +2,9 @@ source src/dev/ci_setup/setup_env.sh -echo " -> building examples separate from test plugins" -node scripts/build_kibana_platform_plugins \ - --oss \ - --examples \ - --verbose; - -echo " -> building test plugins" -node scripts/build_kibana_platform_plugins \ - --oss \ - --no-examples \ - --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ - --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ - --verbose; +if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then + ./test/scripts/jenkins_build_plugins.sh +fi # doesn't persist, also set in kibanaPipeline.groovy export KBN_NP_PLUGINS_BUILT=true @@ -26,4 +16,7 @@ yarn run grunt functionalTests:ensureAllTestsInCiGroup; if [[ -z "$CODE_COVERAGE" ]] ; then echo " -> building and extracting OSS Kibana distributable for use in functional tests" node scripts/build --debug --oss + + mkdir -p "$WORKSPACE/kibana-build-oss" + cp -pR build/oss/kibana-*-SNAPSHOT-linux-x86_64/. $WORKSPACE/kibana-build-oss/ fi diff --git a/test/scripts/jenkins_build_plugins.sh b/test/scripts/jenkins_build_plugins.sh new file mode 100755 index 000000000000000..32b3942074b3468 --- /dev/null +++ b/test/scripts/jenkins_build_plugins.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +echo " -> building examples separate from test plugins" +node scripts/build_kibana_platform_plugins \ + --oss \ + --examples \ + --workers 6 \ + --verbose + +echo " -> building kibana platform plugins" +node scripts/build_kibana_platform_plugins \ + --oss \ + --no-examples \ + --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ + --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ + --workers 6 \ + --verbose diff --git a/test/scripts/jenkins_ci_group.sh b/test/scripts/jenkins_ci_group.sh index 60d7f0406f4c9f9..2542d7032e83bfd 100755 --- a/test/scripts/jenkins_ci_group.sh +++ b/test/scripts/jenkins_ci_group.sh @@ -5,7 +5,7 @@ source test/scripts/jenkins_test_setup_oss.sh if [[ -z "$CODE_COVERAGE" ]]; then checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}"; - if [ "$CI_GROUP" == "1" ]; then + if [[ ! "$TASK_QUEUE_PROCESS_ID" && "$CI_GROUP" == "1" ]]; then source test/scripts/jenkins_build_kbn_sample_panel_action.sh yarn run grunt run:pluginFunctionalTestsRelease --from=source; yarn run grunt run:exampleFunctionalTestsRelease --from=source; diff --git a/test/scripts/jenkins_plugin_functional.sh b/test/scripts/jenkins_plugin_functional.sh new file mode 100755 index 000000000000000..1d691d98982deac --- /dev/null +++ b/test/scripts/jenkins_plugin_functional.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +source test/scripts/jenkins_test_setup_oss.sh + +cd test/plugin_functional/plugins/kbn_sample_panel_action; +if [[ ! -d "target" ]]; then + yarn build; +fi +cd -; + +pwd + +yarn run grunt run:pluginFunctionalTestsRelease --from=source; +yarn run grunt run:exampleFunctionalTestsRelease --from=source; +yarn run grunt run:interpreterFunctionalTestsRelease; diff --git a/test/scripts/jenkins_security_solution_cypress.sh b/test/scripts/jenkins_security_solution_cypress.sh old mode 100644 new mode 100755 index 204911a3eedaa62..a5a1a2103801fad --- a/test/scripts/jenkins_security_solution_cypress.sh +++ b/test/scripts/jenkins_security_solution_cypress.sh @@ -1,12 +1,6 @@ #!/usr/bin/env bash -source test/scripts/jenkins_test_setup.sh - -installDir="$PARENT_DIR/install/kibana" -destDir="${installDir}-${CI_WORKER_NUMBER}" -cp -R "$installDir" "$destDir" - -export KIBANA_INSTALL_DIR="$destDir" +source test/scripts/jenkins_test_setup_xpack.sh echo " -> Running security solution cypress tests" cd "$XPACK_DIR" diff --git a/test/scripts/jenkins_setup_parallel_workspace.sh b/test/scripts/jenkins_setup_parallel_workspace.sh new file mode 100755 index 000000000000000..5274d05572e713d --- /dev/null +++ b/test/scripts/jenkins_setup_parallel_workspace.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -e + +CURRENT_DIR=$(pwd) + +# Copy everything except node_modules into the current workspace +rsync -a ${WORKSPACE}/kibana/* . --exclude node_modules +rsync -a ${WORKSPACE}/kibana/.??* . + +# Symlink all non-root, non-fixture node_modules into our new workspace +cd ${WORKSPACE}/kibana +find . -type d -name node_modules -not -path '*__fixtures__*' -not -path './node_modules*' -prune -print0 | xargs -0I % ln -s "${WORKSPACE}/kibana/%" "${CURRENT_DIR}/%" +find . -type d -wholename '*__fixtures__*node_modules' -not -path './node_modules*' -prune -print0 | xargs -0I % cp -R "${WORKSPACE}/kibana/%" "${CURRENT_DIR}/%" +cd "${CURRENT_DIR}" + +# Symlink all of the individual root-level node_modules into the node_modules/ directory +mkdir -p node_modules +ln -s ${WORKSPACE}/kibana/node_modules/* node_modules/ +ln -s ${WORKSPACE}/kibana/node_modules/.??* node_modules/ + +# Copy a few node_modules instead of symlinking them. They don't work correctly if symlinked +unlink node_modules/@kbn +unlink node_modules/css-loader +unlink node_modules/style-loader + +# packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts will fail if this is a symlink +unlink node_modules/val-loader + +cp -R ${WORKSPACE}/kibana/node_modules/@kbn node_modules/ +cp -R ${WORKSPACE}/kibana/node_modules/css-loader node_modules/ +cp -R ${WORKSPACE}/kibana/node_modules/style-loader node_modules/ +cp -R ${WORKSPACE}/kibana/node_modules/val-loader node_modules/ diff --git a/test/scripts/jenkins_test_setup.sh b/test/scripts/jenkins_test_setup.sh old mode 100644 new mode 100755 index 49ee8a6b526ca5e..7cced76eb650f55 --- a/test/scripts/jenkins_test_setup.sh +++ b/test/scripts/jenkins_test_setup.sh @@ -14,3 +14,7 @@ trap 'post_work' EXIT export TEST_BROWSER_HEADLESS=1 source src/dev/ci_setup/setup_env.sh + +if [[ ! -d .es && -d "$WORKSPACE/kibana/.es" ]]; then + cp -R $WORKSPACE/kibana/.es ./ +fi diff --git a/test/scripts/jenkins_test_setup_oss.sh b/test/scripts/jenkins_test_setup_oss.sh old mode 100644 new mode 100755 index 7bbb86752638431..b7eac33f3517681 --- a/test/scripts/jenkins_test_setup_oss.sh +++ b/test/scripts/jenkins_test_setup_oss.sh @@ -2,10 +2,17 @@ source test/scripts/jenkins_test_setup.sh -if [[ -z "$CODE_COVERAGE" ]] ; then - installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)" - destDir=${installDir}-${CI_PARALLEL_PROCESS_NUMBER} - cp -R "$installDir" "$destDir" +if [[ -z "$CODE_COVERAGE" ]]; then + + destDir="build/kibana-build-oss" + if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then + destDir="${destDir}-${CI_PARALLEL_PROCESS_NUMBER}" + fi + + if [[ ! -d $destDir ]]; then + mkdir -p $destDir + cp -pR "$WORKSPACE/kibana-build-oss/." $destDir/ + fi export KIBANA_INSTALL_DIR="$destDir" fi diff --git a/test/scripts/jenkins_test_setup_xpack.sh b/test/scripts/jenkins_test_setup_xpack.sh old mode 100644 new mode 100755 index a72e9749ebbd5bb..74a3de77e3a7609 --- a/test/scripts/jenkins_test_setup_xpack.sh +++ b/test/scripts/jenkins_test_setup_xpack.sh @@ -3,11 +3,18 @@ source test/scripts/jenkins_test_setup.sh if [[ -z "$CODE_COVERAGE" ]]; then - installDir="$PARENT_DIR/install/kibana" - destDir="${installDir}-${CI_PARALLEL_PROCESS_NUMBER}" - cp -R "$installDir" "$destDir" - export KIBANA_INSTALL_DIR="$destDir" + destDir="build/kibana-build-xpack" + if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then + destDir="${destDir}-${CI_PARALLEL_PROCESS_NUMBER}" + fi + + if [[ ! -d $destDir ]]; then + mkdir -p $destDir + cp -pR "$WORKSPACE/kibana-build-xpack/." $destDir/ + fi + + export KIBANA_INSTALL_DIR="$(realpath $destDir)" cd "$XPACK_DIR" fi diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh index 58ef6a42d3fe4cf..2452e2f5b8c58c1 100755 --- a/test/scripts/jenkins_xpack_build_kibana.sh +++ b/test/scripts/jenkins_xpack_build_kibana.sh @@ -3,21 +3,9 @@ cd "$KIBANA_DIR" source src/dev/ci_setup/setup_env.sh -echo " -> building examples separate from test plugins" -node scripts/build_kibana_platform_plugins \ - --examples \ - --verbose; - -echo " -> building test plugins" -node scripts/build_kibana_platform_plugins \ - --no-examples \ - --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ - --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ - --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ - --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \ - --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \ - --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \ - --verbose; +if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then + ./test/scripts/jenkins_xpack_build_plugins.sh +fi # doesn't persist, also set in kibanaPipeline.groovy export KBN_NP_PLUGINS_BUILT=true @@ -42,7 +30,10 @@ if [[ -z "$CODE_COVERAGE" ]] ; then cd "$KIBANA_DIR" node scripts/build --debug --no-oss linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" - installDir="$PARENT_DIR/install/kibana" + installDir="$KIBANA_DIR/install/kibana" mkdir -p "$installDir" tar -xzf "$linuxBuild" -C "$installDir" --strip=1 + + mkdir -p "$WORKSPACE/kibana-build-xpack" + cp -pR install/kibana/. $WORKSPACE/kibana-build-xpack/ fi diff --git a/test/scripts/jenkins_xpack_build_plugins.sh b/test/scripts/jenkins_xpack_build_plugins.sh new file mode 100755 index 000000000000000..fea30c547bd5fd3 --- /dev/null +++ b/test/scripts/jenkins_xpack_build_plugins.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +echo " -> building examples separate from test plugins" +node scripts/build_kibana_platform_plugins \ + --workers 12 \ + --examples \ + --verbose + +echo " -> building kibana platform plugins" +node scripts/build_kibana_platform_plugins \ + --no-examples \ + --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ + --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ + --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ + --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \ + --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \ + --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \ + --workers 12 \ + --verbose diff --git a/test/scripts/jenkins_xpack_page_load_metrics.sh b/test/scripts/jenkins_xpack_page_load_metrics.sh old mode 100644 new mode 100755 diff --git a/test/scripts/lint/eslint.sh b/test/scripts/lint/eslint.sh new file mode 100755 index 000000000000000..c3211300b96c54b --- /dev/null +++ b/test/scripts/lint/eslint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:eslint diff --git a/test/scripts/lint/sasslint.sh b/test/scripts/lint/sasslint.sh new file mode 100755 index 000000000000000..b9c683bcb049e12 --- /dev/null +++ b/test/scripts/lint/sasslint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:sasslint diff --git a/test/scripts/test/api_integration.sh b/test/scripts/test/api_integration.sh new file mode 100755 index 000000000000000..152c97a3ca7df76 --- /dev/null +++ b/test/scripts/test/api_integration.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:apiIntegrationTests diff --git a/test/scripts/test/jest_integration.sh b/test/scripts/test/jest_integration.sh new file mode 100755 index 000000000000000..73dbbddfb38f63d --- /dev/null +++ b/test/scripts/test/jest_integration.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_jest_integration diff --git a/test/scripts/test/jest_unit.sh b/test/scripts/test/jest_unit.sh new file mode 100755 index 000000000000000..e25452698cebc81 --- /dev/null +++ b/test/scripts/test/jest_unit.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_jest diff --git a/test/scripts/test/karma_ci.sh b/test/scripts/test/karma_ci.sh new file mode 100755 index 000000000000000..e9985300ba19d43 --- /dev/null +++ b/test/scripts/test/karma_ci.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_karma_ci diff --git a/test/scripts/test/mocha.sh b/test/scripts/test/mocha.sh new file mode 100755 index 000000000000000..43c00f0a09dcf7b --- /dev/null +++ b/test/scripts/test/mocha.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:mocha diff --git a/test/scripts/test/xpack_jest_unit.sh b/test/scripts/test/xpack_jest_unit.sh new file mode 100755 index 000000000000000..93d70ec35539102 --- /dev/null +++ b/test/scripts/test/xpack_jest_unit.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +cd x-pack +checks-reporter-with-killswitch "X-Pack Jest" node --max-old-space-size=6144 scripts/jest --ci --verbose --maxWorkers=10 diff --git a/test/scripts/test/xpack_karma.sh b/test/scripts/test/xpack_karma.sh new file mode 100755 index 000000000000000..9078f01f1b870fe --- /dev/null +++ b/test/scripts/test/xpack_karma.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +cd x-pack +checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:karma diff --git a/test/scripts/test/xpack_list_cyclic_dependency.sh b/test/scripts/test/xpack_list_cyclic_dependency.sh new file mode 100755 index 000000000000000..493fe9f58d322e7 --- /dev/null +++ b/test/scripts/test/xpack_list_cyclic_dependency.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +cd x-pack +checks-reporter-with-killswitch "X-Pack List cyclic dependency test" node plugins/lists/scripts/check_circular_deps diff --git a/test/scripts/test/xpack_siem_cyclic_dependency.sh b/test/scripts/test/xpack_siem_cyclic_dependency.sh new file mode 100755 index 000000000000000..b21301f25ad087d --- /dev/null +++ b/test/scripts/test/xpack_siem_cyclic_dependency.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +cd x-pack +checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node plugins/security_solution/scripts/check_circular_deps diff --git a/vars/catchErrors.groovy b/vars/catchErrors.groovy index 460a90b8ec0c04d..2a1b55d832606cf 100644 --- a/vars/catchErrors.groovy +++ b/vars/catchErrors.groovy @@ -1,8 +1,15 @@ // Basically, this is a shortcut for catchError(catchInterruptions: false) {} // By default, catchError will swallow aborts/timeouts, which we almost never want +// Also, by wrapping it in an additional try/catch, we cut down on spam in Pipeline Steps def call(Map params = [:], Closure closure) { - params.catchInterruptions = false - return catchError(params, closure) + try { + closure() + } catch (ex) { + params.catchInterruptions = false + catchError(params) { + throw ex + } + } } return this diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index f3fc5f84583c9cf..0f112043114511c 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -16,27 +16,34 @@ def withPostBuildReporting(Closure closure) { } } -def functionalTestProcess(String name, Closure closure) { - return { processNumber -> - def kibanaPort = "61${processNumber}1" - def esPort = "61${processNumber}2" - def esTransportPort = "61${processNumber}3" - def ingestManagementPackageRegistryPort = "61${processNumber}4" +def withFunctionalTestEnv(List additionalEnvs = [], Closure closure) { + // This can go away once everything that uses the deprecated workers.parallelProcesses() is moved to task queue + def parallelId = env.TASK_QUEUE_PROCESS_ID ?: env.CI_PARALLEL_PROCESS_NUMBER - withEnv([ - "CI_PARALLEL_PROCESS_NUMBER=${processNumber}", - "TEST_KIBANA_HOST=localhost", - "TEST_KIBANA_PORT=${kibanaPort}", - "TEST_KIBANA_URL=http://elastic:changeme@localhost:${kibanaPort}", - "TEST_ES_URL=http://elastic:changeme@localhost:${esPort}", - "TEST_ES_TRANSPORT_PORT=${esTransportPort}", - "INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=${ingestManagementPackageRegistryPort}", - "IS_PIPELINE_JOB=1", - "JOB=${name}", - "KBN_NP_PLUGINS_BUILT=true", - ]) { - closure() - } + def kibanaPort = "61${parallelId}1" + def esPort = "61${parallelId}2" + def esTransportPort = "61${parallelId}3" + def ingestManagementPackageRegistryPort = "61${parallelId}4" + + withEnv([ + "CI_GROUP=${parallelId}", + "REMOVE_KIBANA_INSTALL_DIR=1", + "CI_PARALLEL_PROCESS_NUMBER=${parallelId}", + "TEST_KIBANA_HOST=localhost", + "TEST_KIBANA_PORT=${kibanaPort}", + "TEST_KIBANA_URL=http://elastic:changeme@localhost:${kibanaPort}", + "TEST_ES_URL=http://elastic:changeme@localhost:${esPort}", + "TEST_ES_TRANSPORT_PORT=${esTransportPort}", + "KBN_NP_PLUGINS_BUILT=true", + "INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=${ingestManagementPackageRegistryPort}", + ] + additionalEnvs) { + closure() + } +} + +def functionalTestProcess(String name, Closure closure) { + return { + withFunctionalTestEnv(["JOB=${name}"], closure) } } @@ -100,11 +107,17 @@ def withGcsArtifactUpload(workerName, closure) { def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" def ARTIFACT_PATTERNS = [ 'target/kibana-*', + 'target/test-metrics/*', 'target/kibana-security-solution/**/*.png', 'target/junit/**/*', - 'test/**/screenshots/**/*.png', + 'target/test-suites-ci-plan.json', + 'test/**/screenshots/session/*.png', + 'test/**/screenshots/failure/*.png', + 'test/**/screenshots/diff/*.png', 'test/functional/failure_debug/html/*.html', - 'x-pack/test/**/screenshots/**/*.png', + 'x-pack/test/**/screenshots/session/*.png', + 'x-pack/test/**/screenshots/failure/*.png', + 'x-pack/test/**/screenshots/diff/*.png', 'x-pack/test/functional/failure_debug/html/*.html', 'x-pack/test/functional/apps/reporting/reports/session/*.pdf', ] @@ -119,6 +132,12 @@ def withGcsArtifactUpload(workerName, closure) { ARTIFACT_PATTERNS.each { pattern -> uploadGcsArtifact(uploadPrefix, pattern) } + + dir(env.WORKSPACE) { + ARTIFACT_PATTERNS.each { pattern -> + uploadGcsArtifact(uploadPrefix, "parallel/*/kibana/${pattern}") + } + } } } }) @@ -131,6 +150,11 @@ def withGcsArtifactUpload(workerName, closure) { def publishJunit() { junit(testResults: 'target/junit/**/*.xml', allowEmptyResults: true, keepLongStdio: true) + + // junit() is weird about paths for security reasons, so we need to actually change to an upper directory first + dir(env.WORKSPACE) { + junit(testResults: 'parallel/*/kibana/target/junit/**/*.xml', allowEmptyResults: true, keepLongStdio: true) + } } def sendMail() { @@ -194,12 +218,16 @@ def doSetup() { } } -def buildOss() { - runbld("./test/scripts/jenkins_build_kibana.sh", "Build OSS/Default Kibana") +def buildOss(maxWorkers = '') { + withEnv(["KBN_OPTIMIZER_MAX_WORKERS=${maxWorkers}"]) { + runbld("./test/scripts/jenkins_build_kibana.sh", "Build OSS/Default Kibana") + } } -def buildXpack() { - runbld("./test/scripts/jenkins_xpack_build_kibana.sh", "Build X-Pack Kibana") +def buildXpack(maxWorkers = '') { + withEnv(["KBN_OPTIMIZER_MAX_WORKERS=${maxWorkers}"]) { + runbld("./test/scripts/jenkins_xpack_build_kibana.sh", "Build X-Pack Kibana") + } } def runErrorReporter() { @@ -248,6 +276,100 @@ def call(Map params = [:], Closure closure) { } } +// Creates a task queue using withTaskQueue, and copies the bootstrapped kibana repo into each process's workspace +// Note that node_modules are mostly symlinked to save time/space. See test/scripts/jenkins_setup_parallel_workspace.sh +def withCiTaskQueue(Map options = [:], Closure closure) { + def setupClosure = { + // This can't use runbld, because it expects the source to be there, which isn't yet + bash("${env.WORKSPACE}/kibana/test/scripts/jenkins_setup_parallel_workspace.sh", "Set up duplicate workspace for parallel process") + } + + def config = [parallel: 24, setup: setupClosure] + options + + withTaskQueue(config) { + closure.call() + } +} + +def scriptTask(description, script) { + return { + withFunctionalTestEnv { + runbld(script, description) + } + } +} + +def scriptTaskDocker(description, script) { + return { + withDocker(scriptTask(description, script)) + } +} + +def buildDocker() { + sh( + script: """ + cp /usr/local/bin/runbld .ci/ + cp /usr/local/bin/bash_standard_lib.sh .ci/ + cd .ci + docker build -t kibana-ci -f ./Dockerfile . + """, + label: 'Build CI Docker image' + ) +} + +def withDocker(Closure closure) { + docker + .image('kibana-ci') + .inside( + "-v /etc/runbld:/etc/runbld:ro -v '${env.JENKINS_HOME}:${env.JENKINS_HOME}' -v '/dev/shm/workspace:/dev/shm/workspace' --shm-size 2GB --cpus 4", + closure + ) +} + +def buildOssPlugins() { + runbld('./test/scripts/jenkins_build_plugins.sh', 'Build OSS Plugins') +} + +def buildXpackPlugins() { + runbld('./test/scripts/jenkins_xpack_build_plugins.sh', 'Build X-Pack Plugins') +} + +def withTasks(Map params = [worker: [:]], Closure closure) { + catchErrors { + def config = [name: 'ci-worker', size: 'xxl', ramDisk: true] + (params.worker ?: [:]) + + workers.ci(config) { + withCiTaskQueue(parallel: 24) { + parallel([ + docker: { + retry(2) { + buildDocker() + } + }, + + // There are integration tests etc that require the plugins to be built first, so let's go ahead and build them before set up the parallel workspaces + ossPlugins: { buildOssPlugins() }, + xpackPlugins: { buildXpackPlugins() }, + ]) + + catchErrors { + closure() + } + } + } + } +} + +def allCiTasks() { + withTasks { + tasks.check() + tasks.lint() + tasks.test() + tasks.functionalOss() + tasks.functionalXpack() + } +} + def pipelineLibraryTests() { whenChanged(['vars/', '.ci/pipeline-library/']) { workers.base(size: 'flyweight', bootstrapped: false, ramDisk: false) { @@ -258,5 +380,4 @@ def pipelineLibraryTests() { } } - return this diff --git a/vars/task.groovy b/vars/task.groovy new file mode 100644 index 000000000000000..0c07b519b6fefca --- /dev/null +++ b/vars/task.groovy @@ -0,0 +1,5 @@ +def call(Closure closure) { + withTaskQueue.addTask(closure) +} + +return this diff --git a/vars/tasks.groovy b/vars/tasks.groovy new file mode 100644 index 000000000000000..4e8fcaad795cd23 --- /dev/null +++ b/vars/tasks.groovy @@ -0,0 +1,118 @@ +def call(List closures) { + withTaskQueue.addTasks(closures) +} + +def check() { + tasks([ + kibanaPipeline.scriptTask('Check TypeScript Projects', 'test/scripts/checks/ts_projects.sh'), + kibanaPipeline.scriptTask('Check Doc API Changes', 'test/scripts/checks/doc_api_changes.sh'), + kibanaPipeline.scriptTask('Check Types', 'test/scripts/checks/type_check.sh'), + kibanaPipeline.scriptTask('Check i18n', 'test/scripts/checks/i18n.sh'), + kibanaPipeline.scriptTask('Check File Casing', 'test/scripts/checks/file_casing.sh'), + kibanaPipeline.scriptTask('Check Lockfile Symlinks', 'test/scripts/checks/lock_file_symlinks.sh'), + kibanaPipeline.scriptTask('Check Licenses', 'test/scripts/checks/licenses.sh'), + kibanaPipeline.scriptTask('Verify Dependency Versions', 'test/scripts/checks/verify_dependency_versions.sh'), + kibanaPipeline.scriptTask('Verify NOTICE', 'test/scripts/checks/verify_notice.sh'), + kibanaPipeline.scriptTask('Test Projects', 'test/scripts/checks/test_projects.sh'), + kibanaPipeline.scriptTask('Test Hardening', 'test/scripts/checks/test_hardening.sh'), + ]) +} + +def lint() { + tasks([ + kibanaPipeline.scriptTask('Lint: eslint', 'test/scripts/lint/eslint.sh'), + kibanaPipeline.scriptTask('Lint: sasslint', 'test/scripts/lint/sasslint.sh'), + ]) +} + +def test() { + tasks([ + // These 4 tasks require isolation because of hard-coded, conflicting ports and such, so let's use Docker here + kibanaPipeline.scriptTaskDocker('Jest Integration Tests', 'test/scripts/test/jest_integration.sh'), + kibanaPipeline.scriptTaskDocker('Mocha Tests', 'test/scripts/test/mocha.sh'), + kibanaPipeline.scriptTaskDocker('Karma CI Tests', 'test/scripts/test/karma_ci.sh'), + kibanaPipeline.scriptTaskDocker('X-Pack Karma Tests', 'test/scripts/test/xpack_karma.sh'), + + kibanaPipeline.scriptTask('Jest Unit Tests', 'test/scripts/test/jest_unit.sh'), + kibanaPipeline.scriptTask('API Integration Tests', 'test/scripts/test/api_integration.sh'), + kibanaPipeline.scriptTask('X-Pack SIEM cyclic dependency', 'test/scripts/test/xpack_siem_cyclic_dependency.sh'), + kibanaPipeline.scriptTask('X-Pack List cyclic dependency', 'test/scripts/test/xpack_list_cyclic_dependency.sh'), + kibanaPipeline.scriptTask('X-Pack Jest Unit Tests', 'test/scripts/test/xpack_jest_unit.sh'), + ]) +} + +def functionalOss(Map params = [:]) { + def config = params ?: [ciGroups: true, firefox: true, accessibility: true, pluginFunctional: true, visualRegression: false] + + task { + kibanaPipeline.buildOss(6) + + if (config.ciGroups) { + def ciGroups = 1..12 + tasks(ciGroups.collect { kibanaPipeline.ossCiGroupProcess(it) }) + } + + if (config.firefox) { + task(kibanaPipeline.functionalTestProcess('oss-firefox', './test/scripts/jenkins_firefox_smoke.sh')) + } + + if (config.accessibility) { + task(kibanaPipeline.functionalTestProcess('oss-accessibility', './test/scripts/jenkins_accessibility.sh')) + } + + if (config.pluginFunctional) { + task(kibanaPipeline.functionalTestProcess('oss-pluginFunctional', './test/scripts/jenkins_plugin_functional.sh')) + } + + if (config.visualRegression) { + task(kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')) + } + } +} + +def functionalXpack(Map params = [:]) { + def config = params ?: [ + ciGroups: true, + firefox: true, + accessibility: true, + pluginFunctional: true, + savedObjectsFieldMetrics:true, + pageLoadMetrics: false, + visualRegression: false, + ] + + task { + kibanaPipeline.buildXpack(10) + + if (config.ciGroups) { + def ciGroups = 1..10 + tasks(ciGroups.collect { kibanaPipeline.xpackCiGroupProcess(it) }) + } + + if (config.firefox) { + task(kibanaPipeline.functionalTestProcess('xpack-firefox', './test/scripts/jenkins_xpack_firefox_smoke.sh')) + } + + if (config.accessibility) { + task(kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh')) + } + + if (config.visualRegression) { + task(kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')) + } + + if (config.pageLoadMetrics) { + task(kibanaPipeline.functionalTestProcess('xpack-pageLoadMetrics', './test/scripts/jenkins_xpack_page_load_metrics.sh')) + } + + if (config.savedObjectsFieldMetrics) { + task(kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh')) + } + + whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/']) { + task(kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')) + } + } +} + +return this diff --git a/vars/withTaskQueue.groovy b/vars/withTaskQueue.groovy new file mode 100644 index 000000000000000..8132d6264744f2c --- /dev/null +++ b/vars/withTaskQueue.groovy @@ -0,0 +1,154 @@ +import groovy.transform.Field + +public static @Field TASK_QUEUES = [:] +public static @Field TASK_QUEUES_COUNTER = 0 + +/** + withTaskQueue creates a queue of "tasks" (just plain closures to execute), and executes them with your desired level of concurrency. + This way, you can define, for example, 40 things that need to execute, then only allow 10 of them to execute at once. + + Each "process" will execute in a separate, unique, empty directory. + If you want each process to have a bootstrapped kibana repo, check out kibanaPipeline.withCiTaskQueue + + Using the queue currently requires an agent/worker. + + Usage: + + withTaskQueue(parallel: 10) { + task { print "This is a task" } + + // This is the same as calling task() multiple times + tasks([ { print "Another task" }, { print "And another task" } ]) + + // Tasks can queue up subsequent tasks + task { + buildThing() + task { print "I depend on buildThing()" } + } + } + + You can also define a setup task that each process should execute one time before executing tasks: + withTaskQueue(parallel: 10, setup: { sh "my-setup-scrupt.sh" }) { + ... + } + +*/ +def call(Map options = [:], Closure closure) { + def config = [ parallel: 10 ] + options + def counter = ++TASK_QUEUES_COUNTER + + // We're basically abusing withEnv() to create a "scope" for all steps inside of a withTaskQueue block + // This way, we could have multiple task queue instances in the same pipeline + withEnv(["TASK_QUEUE_ID=${counter}"]) { + withTaskQueue.TASK_QUEUES[env.TASK_QUEUE_ID] = [ + tasks: [], + tmpFile: sh(script: 'mktemp', returnStdout: true).trim() + ] + + closure.call() + + def processesExecuting = 0 + def processes = [:] + def iterationId = 0 + + for(def i = 1; i <= config.parallel; i++) { + def j = i + processes["task-queue-process-${j}"] = { + catchErrors { + withEnv([ + "TASK_QUEUE_PROCESS_ID=${j}", + "TASK_QUEUE_ITERATION_ID=${++iterationId}" + ]) { + dir("${WORKSPACE}/parallel/${j}/kibana") { + if (config.setup) { + config.setup.call(j) + } + + def isDone = false + while(!isDone) { // TODO some kind of timeout? + catchErrors { + if (!getTasks().isEmpty()) { + processesExecuting++ + catchErrors { + def task + try { + task = getTasks().pop() + } catch (java.util.NoSuchElementException ex) { + return + } + + task.call() + } + processesExecuting-- + // If a task finishes, and no new tasks were queued up, and nothing else is executing + // Then all of the processes should wake up and exit + if (processesExecuting < 1 && getTasks().isEmpty()) { + taskNotify() + } + return + } + + if (processesExecuting > 0) { + taskSleep() + return + } + + // Queue is empty, no processes are executing + isDone = true + } + } + } + } + } + } + } + parallel(processes) + } +} + +// If we sleep in a loop using Groovy code, Pipeline Steps is flooded with Sleep steps +// So, instead, we just watch a file and `touch` it whenever something happens that could modify the queue +// There's a 20 minute timeout just in case something goes wrong, +// in which case this method will get called again if the process is actually supposed to be waiting. +def taskSleep() { + sh(script: """#!/bin/bash + TIMESTAMP=\$(date '+%s' -d "0 seconds ago") + for (( i=1; i<=240; i++ )) + do + if [ "\$(stat -c %Y '${getTmpFile()}')" -ge "\$TIMESTAMP" ] + then + break + else + sleep 5 + if [[ \$i == 240 ]]; then + echo "Waited for new tasks for 20 minutes, exiting in case something went wrong" + fi + fi + done + """, label: "Waiting for new tasks...") +} + +// Used to let the task queue processes know that either a new task has been queued up, or work is complete +def taskNotify() { + sh "touch '${getTmpFile()}'" +} + +def getTasks() { + return withTaskQueue.TASK_QUEUES[env.TASK_QUEUE_ID].tasks +} + +def getTmpFile() { + return withTaskQueue.TASK_QUEUES[env.TASK_QUEUE_ID].tmpFile +} + +def addTask(Closure closure) { + getTasks() << closure + taskNotify() +} + +def addTasks(List closures) { + closures.reverse().each { + getTasks() << it + } + taskNotify() +} diff --git a/vars/workers.groovy b/vars/workers.groovy index 8b7e8525a7ce3b6..2e94ce12f34c072 100644 --- a/vars/workers.groovy +++ b/vars/workers.groovy @@ -13,6 +13,8 @@ def label(size) { return 'docker && tests-l' case 'xl': return 'docker && tests-xl' + case 'xl-highmem': + return 'docker && tests-xl-highmem' case 'xxl': return 'docker && tests-xxl' } @@ -55,6 +57,11 @@ def base(Map params, Closure closure) { } } + sh( + script: "mkdir -p ${env.WORKSPACE}/tmp", + label: "Create custom temp directory" + ) + def checkoutInfo = [:] if (config.scm) { @@ -89,6 +96,7 @@ def base(Map params, Closure closure) { "PR_AUTHOR=${env.ghprbPullAuthorLogin ?: ''}", "TEST_BROWSER_HEADLESS=1", "GIT_BRANCH=${checkoutInfo.branch}", + "TMPDIR=${env.WORKSPACE}/tmp", // For Chrome and anything else that respects it ]) { withCredentials([ string(credentialsId: 'vault-addr', variable: 'VAULT_ADDR'), @@ -167,7 +175,9 @@ def parallelProcesses(Map params) { sleep(delay) } - processClosure(processNumber) + withEnv(["CI_PARALLEL_PROCESS_NUMBER=${processNumber}"]) { + processClosure() + } } } diff --git a/x-pack/plugins/canvas/.storybook/storyshots.test.js b/x-pack/plugins/canvas/.storybook/storyshots.test.js index b9fe0914b369871..e3217ad4dbe58cc 100644 --- a/x-pack/plugins/canvas/.storybook/storyshots.test.js +++ b/x-pack/plugins/canvas/.storybook/storyshots.test.js @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import fs from 'fs'; import path from 'path'; import moment from 'moment'; import 'moment-timezone'; @@ -76,6 +77,12 @@ import { RenderedElement } from '../shareable_runtime/components/rendered_elemen jest.mock('../shareable_runtime/components/rendered_element'); RenderedElement.mockImplementation(() => 'RenderedElement'); +// Some of the code requires that this directory exists, but the tests don't actually require any css to be present +const cssDir = path.resolve(__dirname, '../../../../built_assets/css'); +if (!fs.existsSync(cssDir)) { + fs.mkdirSync(cssDir, { recursive: true }); +} + addSerializer(styleSheetSerializer); // Initialize Storyshots and build the Jest Snapshots From 06bc389189641d78c0c5280a88ed4f53f4f5ac32 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Tue, 7 Jul 2020 12:50:47 -0600 Subject: [PATCH 9/9] [Security Solution] [Timeline] Bugfix for timeline row actions disappear sometimes (#70958) --- .../alerts/components/alerts_table/index.tsx | 8 +++++--- .../components/alerts_viewer/alerts_table.tsx | 7 ++----- .../draggable_wrapper_hover_content.test.tsx | 10 +++------- .../components/events_viewer/events_viewer.tsx | 11 ----------- .../common/components/top_n/index.test.tsx | 8 +++----- .../pages/navigation/events_query_tab_body.tsx | 8 +++++++- .../components/manage_timeline/index.tsx | 16 ++++++++++------ .../timeline/body/events/event_column_view.tsx | 3 +-- .../data_providers/data_providers.test.tsx | 5 ++--- .../timeline/data_providers/providers.test.tsx | 5 ++--- .../timelines/components/timeline/timeline.tsx | 5 ++--- 11 files changed, 37 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/security_solution/public/alerts/components/alerts_table/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/alerts_table/index.tsx index 98bb6434ddafd7a..65f225c05459872 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/alerts_table/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/alerts_table/index.tsx @@ -48,6 +48,7 @@ import { displaySuccessToast, displayErrorToast, } from '../../../common/components/toasters'; +import { getInvestigateInResolverAction } from '../../../timelines/components/timeline/body/helpers'; interface OwnProps { timelineId: TimelineIdLiteral; @@ -331,13 +332,14 @@ export const AlertsTableComponent: React.FC = ({ useEffect(() => { initializeTimeline({ - id: timelineId, - documentType: i18n.ALERTS_DOCUMENT_TYPE, defaultModel: alertsDefaultModel, + documentType: i18n.ALERTS_DOCUMENT_TYPE, footerText: i18n.TOTAL_COUNT_OF_ALERTS, + id: timelineId, loadingText: i18n.LOADING_ALERTS, - title: i18n.ALERTS_TABLE_TITLE, selectAll: canUserCRUD ? selectAll : false, + timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId })], + title: i18n.ALERTS_TABLE_TITLE, }); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); diff --git a/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx b/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx index 6783fcbd17582bb..12b2853f8f7e1d8 100644 --- a/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx +++ b/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx @@ -69,7 +69,7 @@ const AlertsTableComponent: React.FC = ({ }) => { const dispatch = useDispatch(); const alertsFilter = useMemo(() => [...defaultAlertsFilters, ...pageFilters], [pageFilters]); - const { initializeTimeline, setTimelineRowActions } = useManageTimeline(); + const { initializeTimeline } = useManageTimeline(); useEffect(() => { initializeTimeline({ @@ -77,13 +77,10 @@ const AlertsTableComponent: React.FC = ({ documentType: i18n.ALERTS_DOCUMENT_TYPE, defaultModel: alertsDefaultModel, footerText: i18n.TOTAL_COUNT_OF_ALERTS, + timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId })], title: i18n.ALERTS_TABLE_TITLE, unit: i18n.UNIT, }); - setTimelineRowActions({ - id: timelineId, - timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId })], - }); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); return ( diff --git a/x-pack/plugins/security_solution/public/common/components/drag_and_drop/draggable_wrapper_hover_content.test.tsx b/x-pack/plugins/security_solution/public/common/components/drag_and_drop/draggable_wrapper_hover_content.test.tsx index 3507b0f8c447dd5..432e369cdd0f63d 100644 --- a/x-pack/plugins/security_solution/public/common/components/drag_and_drop/draggable_wrapper_hover_content.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/drag_and_drop/draggable_wrapper_hover_content.test.tsx @@ -18,7 +18,7 @@ import { useAddToTimeline } from '../../hooks/use_add_to_timeline'; import { DraggableWrapperHoverContent } from './draggable_wrapper_hover_content'; import { ManageGlobalTimeline, - timelineDefaults, + getTimelineDefaults, } from '../../../timelines/components/manage_timeline'; import { TimelineId } from '../../../../common/types/timeline'; @@ -152,10 +152,7 @@ describe('DraggableWrapperHoverContent', () => { beforeEach(() => { onFilterAdded = jest.fn(); const manageTimelineForTesting = { - [timelineId]: { - ...timelineDefaults, - id: timelineId, - }, + [timelineId]: getTimelineDefaults(timelineId), }; wrapper = mount( @@ -249,8 +246,7 @@ describe('DraggableWrapperHoverContent', () => { const manageTimelineForTesting = { [timelineId]: { - ...timelineDefaults, - id: timelineId, + ...getTimelineDefaults(timelineId), filterManager, }, }; diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx index 9e38b14c4334a58..910030d41a23e02 100644 --- a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx +++ b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx @@ -7,7 +7,6 @@ import { EuiPanel } from '@elastic/eui'; import { getOr, isEmpty, union } from 'lodash/fp'; import React, { useEffect, useMemo, useState } from 'react'; -import { useDispatch } from 'react-redux'; import styled from 'styled-components'; import deepEqual from 'fast-deep-equal'; @@ -35,7 +34,6 @@ import { } from '../../../../../../../src/plugins/data/public'; import { inputsModel } from '../../store'; import { useManageTimeline } from '../../../timelines/components/manage_timeline'; -import { getInvestigateInResolverAction } from '../../../timelines/components/timeline/body/helpers'; const DEFAULT_EVENTS_VIEWER_HEIGHT = 500; @@ -93,7 +91,6 @@ const EventsViewerComponent: React.FC = ({ toggleColumn, utilityBar, }) => { - const dispatch = useDispatch(); const columnsHeader = isEmpty(columns) ? defaultHeaders : columns; const kibana = useKibana(); const { filterManager } = useKibana().services.data.query; @@ -103,16 +100,8 @@ const EventsViewerComponent: React.FC = ({ getManageTimelineById, setIsTimelineLoading, setTimelineFilterManager, - setTimelineRowActions, } = useManageTimeline(); - useEffect(() => { - setTimelineRowActions({ - id, - timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId: id })], - }); - }, [setTimelineRowActions, id, dispatch]); - useEffect(() => { setIsTimelineLoading({ id, isLoading: isQueryLoading }); // eslint-disable-next-line react-hooks/exhaustive-deps diff --git a/x-pack/plugins/security_solution/public/common/components/top_n/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/top_n/index.test.tsx index 503e9983692f16f..da6ec784af6d44e 100644 --- a/x-pack/plugins/security_solution/public/common/components/top_n/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/top_n/index.test.tsx @@ -25,7 +25,7 @@ import { Props } from './top_n'; import { StatefulTopN } from '.'; import { ManageGlobalTimeline, - timelineDefaults, + getTimelineDefaults, } from '../../../timelines/components/manage_timeline'; import { TimelineId } from '../../../../common/types/timeline'; @@ -272,8 +272,7 @@ describe('StatefulTopN', () => { filterManager = new FilterManager(mockUiSettingsForFilterManager); const manageTimelineForTesting = { [TimelineId.active]: { - ...timelineDefaults, - id: TimelineId.active, + ...getTimelineDefaults(TimelineId.active), filterManager, }, }; @@ -351,8 +350,7 @@ describe('StatefulTopN', () => { const manageTimelineForTesting = { [TimelineId.active]: { - ...timelineDefaults, - id: TimelineId.active, + ...getTimelineDefaults(TimelineId.active), filterManager, documentType: 'alerts', }, diff --git a/x-pack/plugins/security_solution/public/hosts/pages/navigation/events_query_tab_body.tsx b/x-pack/plugins/security_solution/public/hosts/pages/navigation/events_query_tab_body.tsx index 64cfacaeaf6dc09..58026a28a04ce43 100644 --- a/x-pack/plugins/security_solution/public/hosts/pages/navigation/events_query_tab_body.tsx +++ b/x-pack/plugins/security_solution/public/hosts/pages/navigation/events_query_tab_body.tsx @@ -5,6 +5,7 @@ */ import React, { useEffect } from 'react'; +import { useDispatch } from 'react-redux'; import { TimelineId } from '../../../../common/types/timeline'; import { StatefulEventsViewer } from '../../../common/components/events_viewer'; import { HostsComponentsQueryProps } from './types'; @@ -18,6 +19,7 @@ import { MatrixHistogramContainer } from '../../../common/components/matrix_hist import * as i18n from '../translations'; import { HistogramType } from '../../../graphql/types'; import { useManageTimeline } from '../../../timelines/components/manage_timeline'; +import { getInvestigateInResolverAction } from '../../../timelines/components/timeline/body/helpers'; const EVENTS_HISTOGRAM_ID = 'eventsOverTimeQuery'; @@ -57,13 +59,17 @@ export const EventsQueryTabBody = ({ startDate, }: HostsComponentsQueryProps) => { const { initializeTimeline } = useManageTimeline(); + const dispatch = useDispatch(); useEffect(() => { initializeTimeline({ id: TimelineId.hostsPageEvents, defaultModel: eventsDefaultModel, + timelineRowActions: [ + getInvestigateInResolverAction({ dispatch, timelineId: TimelineId.hostsPageEvents }), + ], }); - }, [initializeTimeline]); + }, [dispatch, initializeTimeline]); useEffect(() => { return () => { diff --git a/x-pack/plugins/security_solution/public/timelines/components/manage_timeline/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/manage_timeline/index.tsx index 3b40c36fccd163f..c71087de4a07ee8 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/manage_timeline/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/manage_timeline/index.tsx @@ -6,6 +6,7 @@ import React, { createContext, useCallback, useContext, useReducer } from 'react'; import { noop } from 'lodash/fp'; + // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { FilterManager } from '../../../../../../../src/plugins/data/public/query/filter_manager'; import { TimelineRowAction } from '../timeline/body/actions'; @@ -22,6 +23,7 @@ interface ManageTimelineInit { indexToAdd?: string[] | null; loadingText?: string; selectAll?: boolean; + timelineRowActions: TimelineRowAction[]; title?: string; unit?: (totalCount: number) => string; } @@ -73,19 +75,20 @@ type ActionManageTimeline = payload: { filterManager: FilterManager }; }; -export const timelineDefaults = { +export const getTimelineDefaults = (id: string) => ({ indexToAdd: null, defaultModel: timelineDefaultModel, loadingText: i18n.LOADING_EVENTS, footerText: i18nF.TOTAL_COUNT_OF_EVENTS, documentType: i18nF.TOTAL_COUNT_OF_EVENTS, selectAll: false, + id, isLoading: false, queryFields: [], timelineRowActions: [], title: i18n.EVENTS, unit: (n: number) => i18n.UNIT(n), -}; +}); const reducerManageTimeline = ( state: ManageTimelineById, action: ActionManageTimeline @@ -95,7 +98,7 @@ const reducerManageTimeline = ( return { ...state, [action.id]: { - ...timelineDefaults, + ...getTimelineDefaults(action.id), ...state[action.id], ...action.payload, }, @@ -216,8 +219,8 @@ const useTimelineManager = (manageTimelineForTesting?: ManageTimelineById): UseT if (state[id] != null) { return state[id]; } - initializeTimeline({ id }); - return { ...timelineDefaults, id }; + initializeTimeline({ id, timelineRowActions: [] }); + return getTimelineDefaults(id); }, [initializeTimeline, state] ); @@ -236,7 +239,7 @@ const useTimelineManager = (manageTimelineForTesting?: ManageTimelineById): UseT }; const init = { - getManageTimelineById: (id: string) => ({ ...timelineDefaults, id }), + getManageTimelineById: (id: string) => getTimelineDefaults(id), getTimelineFilterManager: () => undefined, setIndexToAdd: () => undefined, isManagedTimeline: () => false, @@ -245,6 +248,7 @@ const init = { setTimelineRowActions: () => noop, setTimelineFilterManager: () => noop, }; + const ManageTimelineContext = createContext(init); export const useManageTimeline = () => useContext(ManageTimelineContext); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx index 8855cba7a4c890d..ae00edf5d1429bd 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx @@ -194,8 +194,7 @@ export const EventColumnView = React.memo( , ] : grouped.icon; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [button, ecsData, timelineActions, isPopoverOpen]); // , isPopoverOpen, closePopover, onButtonClick]); + }, [button, closePopover, id, onClickCb, ecsData, timelineActions, isPopoverOpen]); return ( diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_providers.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_providers.test.tsx index 1c85b6e6d72bf61..3a8c0d883121747 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_providers.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_providers.test.tsx @@ -13,7 +13,7 @@ import { useMountAppended } from '../../../../common/utils/use_mount_appended'; import { DataProviders } from '.'; import { DataProvider } from './data_provider'; import { mockDataProviders } from './mock/mock_data_providers'; -import { ManageGlobalTimeline, timelineDefaults } from '../../manage_timeline'; +import { ManageGlobalTimeline, getTimelineDefaults } from '../../manage_timeline'; import { FilterManager } from '../../../../../../../../src/plugins/data/public/query/filter_manager'; import { createKibanaCoreStartMock } from '../../../../common/mock/kibana_core'; const mockUiSettingsForFilterManager = createKibanaCoreStartMock().uiSettings; @@ -28,8 +28,7 @@ describe('DataProviders', () => { test('renders correctly against snapshot', () => { const manageTimelineForTesting = { foo: { - ...timelineDefaults, - id: 'foo', + ...getTimelineDefaults('foo'), filterManager, }, }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/providers.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/providers.test.tsx index 3ad83914c73b954..9dc0b762244582a 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/providers.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/providers.test.tsx @@ -16,7 +16,7 @@ import { mockDataProviders } from './mock/mock_data_providers'; import { Providers } from './providers'; import { DELETE_CLASS_NAME, ENABLE_CLASS_NAME, EXCLUDE_CLASS_NAME } from './provider_item_actions'; import { useMountAppended } from '../../../../common/utils/use_mount_appended'; -import { ManageGlobalTimeline, timelineDefaults } from '../../manage_timeline'; +import { ManageGlobalTimeline, getTimelineDefaults } from '../../manage_timeline'; const mockUiSettingsForFilterManager = createKibanaCoreStartMock().uiSettings; @@ -27,8 +27,7 @@ describe('Providers', () => { const manageTimelineForTesting = { foo: { - ...timelineDefaults, - id: 'foo', + ...getTimelineDefaults('foo'), filterManager, isLoading, }, diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/timeline.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/timeline.tsx index 18deaf01587239f..ed7f8a447c51d28 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/timeline.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/timeline.tsx @@ -177,12 +177,11 @@ export const TimelineComponent: React.FC = ({ setIndexToAdd, setIsTimelineLoading, setTimelineFilterManager, - setTimelineRowActions, } = useManageTimeline(); useEffect(() => { - initializeTimeline({ id, indexToAdd }); - setTimelineRowActions({ + initializeTimeline({ id, + indexToAdd, timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId: id })], }); // eslint-disable-next-line react-hooks/exhaustive-deps