diff --git a/.ci/Dockerfile b/.ci/Dockerfile new file mode 100644 index 000000000000000..201e17b93c116be --- /dev/null +++ b/.ci/Dockerfile @@ -0,0 +1,35 @@ +ARG NODE_VERSION=10.21.0 + +FROM node:${NODE_VERSION} AS base + +RUN apt-get update && \ + apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \ + libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \ + libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ + libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \ + libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ + && apt-get update \ + && apt-get install -y rsync jq bsdtar google-chrome-stable \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN LATEST_VAULT_RELEASE=$(curl -s https://api.github.com/repos/hashicorp/vault/tags | jq --raw-output .[0].name[1:]) \ + && curl -L https://releases.hashicorp.com/vault/${LATEST_VAULT_RELEASE}/vault_${LATEST_VAULT_RELEASE}_linux_amd64.zip -o vault.zip \ + && unzip vault.zip \ + && rm vault.zip \ + && chmod +x vault \ + && mv vault /usr/local/bin/vault + +RUN groupadd -r kibana && useradd -r -g kibana kibana && mkdir /home/kibana && chown kibana:kibana /home/kibana + +COPY ./bash_standard_lib.sh /usr/local/bin/bash_standard_lib.sh +RUN chmod +x /usr/local/bin/bash_standard_lib.sh + +COPY ./runbld /usr/local/bin/runbld +RUN chmod +x /usr/local/bin/runbld + +USER kibana diff --git a/.ci/runbld_no_junit.yml b/.ci/runbld_no_junit.yml index 67b5002c1c43770..1bcb7e22a264807 100644 --- a/.ci/runbld_no_junit.yml +++ b/.ci/runbld_no_junit.yml @@ -3,4 +3,4 @@ profiles: - ".*": # Match any job tests: - junit-filename-pattern: "8d8bd494-d909-4e67-a052-7e8b5aaeb5e4" # A bogus path that should never exist + junit-filename-pattern: false diff --git a/.gitignore b/.gitignore index 32377ec0f1ffe80..25a8c369bb704d0 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,8 @@ npm-debug.log* .tern-project .nyc_output .ci/pipeline-library/build/ +.ci/runbld +.ci/bash_standard_lib.sh .gradle # apm plugin diff --git a/Jenkinsfile b/Jenkinsfile index f6f77ccae8427ad..491a1e386deb187 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,50 +8,7 @@ kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true) ciStats.trackBuild { catchError { retryable.enable() - parallel([ - 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), - 'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'), - 'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - 'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'), - 'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1), - 'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2), - 'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3), - 'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4), - 'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5), - 'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6), - 'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7), - 'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8), - 'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9), - 'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10), - 'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11), - 'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12), - 'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'), - // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'), - ]), - 'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - 'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'), - 'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1), - 'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2), - 'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3), - 'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4), - 'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5), - 'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6), - 'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7), - 'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8), - 'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9), - 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), - 'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'), - 'xpack-savedObjectsFieldMetrics': kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'), - // 'xpack-pageLoadMetrics': kibanaPipeline.functionalTestProcess('xpack-pageLoadMetrics', './test/scripts/jenkins_xpack_page_load_metrics.sh'), - 'xpack-securitySolutionCypress': { processNumber -> - whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/']) { - kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber) - } - }, - - // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'), - ]), - ]) + kibanaPipeline.allCiTasks() } } } diff --git a/src/dev/ci_setup/checkout_sibling_es.sh b/src/dev/ci_setup/checkout_sibling_es.sh index 915759d4214f9a8..3832ec9b4076a7b 100755 --- a/src/dev/ci_setup/checkout_sibling_es.sh +++ b/src/dev/ci_setup/checkout_sibling_es.sh @@ -7,10 +7,11 @@ function checkout_sibling { targetDir=$2 useExistingParamName=$3 useExisting="$(eval "echo "\$$useExistingParamName"")" + repoAddress="https://github.com/" if [ -z ${useExisting:+x} ]; then if [ -d "$targetDir" ]; then - echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$PARENT_DIR]!" + echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$WORKSPACE]!" echo echo "Either define '${useExistingParamName}' or remove the existing '${project}' sibling." exit 1 @@ -21,8 +22,9 @@ function checkout_sibling { cloneBranch="" function clone_target_is_valid { + echo " -> checking for '${cloneBranch}' branch at ${cloneAuthor}/${project}" - if [[ -n "$(git ls-remote --heads "git@github.com:${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then + if [[ -n "$(git ls-remote --heads "${repoAddress}${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then return 0 else return 1 @@ -71,7 +73,7 @@ function checkout_sibling { fi echo " -> checking out '${cloneBranch}' branch from ${cloneAuthor}/${project}..." - git clone -b "$cloneBranch" "git@github.com:${cloneAuthor}/${project}.git" "$targetDir" --depth=1 + git clone -b "$cloneBranch" "${repoAddress}${cloneAuthor}/${project}.git" "$targetDir" --depth=1 echo " -> checked out ${project} revision: $(git -C "${targetDir}" rev-parse HEAD)" echo } @@ -87,12 +89,12 @@ function checkout_sibling { fi } -checkout_sibling "elasticsearch" "${PARENT_DIR}/elasticsearch" "USE_EXISTING_ES" +checkout_sibling "elasticsearch" "${WORKSPACE}/elasticsearch" "USE_EXISTING_ES" export TEST_ES_FROM=${TEST_ES_FROM:-snapshot} # Set the JAVA_HOME based on the Java property file in the ES repo # This assumes the naming convention used on CI (ex: ~/.java/java10) -ES_DIR="$PARENT_DIR/elasticsearch" +ES_DIR="$WORKSPACE/elasticsearch" ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh index 343ff4719937546..f96a2240917e257 100644 --- a/src/dev/ci_setup/setup_env.sh +++ b/src/dev/ci_setup/setup_env.sh @@ -53,6 +53,8 @@ export PARENT_DIR="$parentDir" kbnBranch="$(jq -r .branch "$KIBANA_DIR/package.json")" export KIBANA_PKG_BRANCH="$kbnBranch" +export WORKSPACE="${WORKSPACE:-$PARENT_DIR}" + ### ### download node ### @@ -161,7 +163,7 @@ export -f checks-reporter-with-killswitch source "$KIBANA_DIR/src/dev/ci_setup/load_env_keys.sh" -ES_DIR="$PARENT_DIR/elasticsearch" +ES_DIR="$WORKSPACE/elasticsearch" ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then diff --git a/src/dev/notice/generate_notice_from_source.ts b/src/dev/notice/generate_notice_from_source.ts index fb74bed0f26f4b7..a2b05c6dc8a4eab 100644 --- a/src/dev/notice/generate_notice_from_source.ts +++ b/src/dev/notice/generate_notice_from_source.ts @@ -49,8 +49,10 @@ export async function generateNoticeFromSource({ productName, directory, log }: ignore: [ '{node_modules,build,target,dist,data,built_assets}/**', 'packages/*/{node_modules,build,target,dist}/**', + 'src/plugins/*/{node_modules,build,target,dist}/**', 'x-pack/{node_modules,build,target,dist,data}/**', 'x-pack/packages/*/{node_modules,build,target,dist}/**', + 'x-pack/plugins/*/{node_modules,build,target,dist}/**', ], }; diff --git a/tasks/config/karma.js b/tasks/config/karma.js index fa4bdc8ed226624..7c4f75bea8801cb 100644 --- a/tasks/config/karma.js +++ b/tasks/config/karma.js @@ -110,7 +110,7 @@ module.exports = function (grunt) { customLaunchers: { Chrome_Headless: { base: 'Chrome', - flags: ['--headless', '--disable-gpu', '--remote-debugging-port=9222'], + flags: ['--headless', '--disable-gpu', '--remote-debugging-port=9222', '--no-sandbox'], }, }, diff --git a/tasks/test_jest.js b/tasks/test_jest.js index d8f51806e8ddc8d..810ed423248400d 100644 --- a/tasks/test_jest.js +++ b/tasks/test_jest.js @@ -22,7 +22,7 @@ const { resolve } = require('path'); module.exports = function (grunt) { grunt.registerTask('test:jest', function () { const done = this.async(); - runJest(resolve(__dirname, '../scripts/jest.js')).then(done, done); + runJest(resolve(__dirname, '../scripts/jest.js'), ['--maxWorkers=10']).then(done, done); }); grunt.registerTask('test:jest_integration', function () { @@ -30,10 +30,10 @@ module.exports = function (grunt) { runJest(resolve(__dirname, '../scripts/jest_integration.js')).then(done, done); }); - function runJest(jestScript) { + function runJest(jestScript, args = []) { const serverCmd = { cmd: 'node', - args: [jestScript, '--ci'], + args: [jestScript, '--ci', ...args], opts: { stdio: 'inherit' }, }; diff --git a/test/scripts/checks/doc_api_changes.sh b/test/scripts/checks/doc_api_changes.sh new file mode 100755 index 000000000000000..503d12b2f6d73de --- /dev/null +++ b/test/scripts/checks/doc_api_changes.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:checkDocApiChanges diff --git a/test/scripts/checks/file_casing.sh b/test/scripts/checks/file_casing.sh new file mode 100755 index 000000000000000..513664263791b98 --- /dev/null +++ b/test/scripts/checks/file_casing.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:checkFileCasing diff --git a/test/scripts/checks/i18n.sh b/test/scripts/checks/i18n.sh new file mode 100755 index 000000000000000..7a6fd46c46c7697 --- /dev/null +++ b/test/scripts/checks/i18n.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:i18nCheck diff --git a/test/scripts/checks/licenses.sh b/test/scripts/checks/licenses.sh new file mode 100755 index 000000000000000..a08d7d07a24a138 --- /dev/null +++ b/test/scripts/checks/licenses.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:licenses diff --git a/test/scripts/checks/lock_file_symlinks.sh b/test/scripts/checks/lock_file_symlinks.sh new file mode 100755 index 000000000000000..1d43d32c9feb82d --- /dev/null +++ b/test/scripts/checks/lock_file_symlinks.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:checkLockfileSymlinks diff --git a/test/scripts/checks/test_hardening.sh b/test/scripts/checks/test_hardening.sh new file mode 100755 index 000000000000000..9184758577654a4 --- /dev/null +++ b/test/scripts/checks/test_hardening.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_hardening diff --git a/test/scripts/checks/test_projects.sh b/test/scripts/checks/test_projects.sh new file mode 100755 index 000000000000000..5f9aafe80e10e6d --- /dev/null +++ b/test/scripts/checks/test_projects.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_projects diff --git a/test/scripts/checks/ts_projects.sh b/test/scripts/checks/ts_projects.sh new file mode 100755 index 000000000000000..d667c753baec234 --- /dev/null +++ b/test/scripts/checks/ts_projects.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:checkTsProjects diff --git a/test/scripts/checks/type_check.sh b/test/scripts/checks/type_check.sh new file mode 100755 index 000000000000000..07c49638134be15 --- /dev/null +++ b/test/scripts/checks/type_check.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:typeCheck diff --git a/test/scripts/checks/verify_dependency_versions.sh b/test/scripts/checks/verify_dependency_versions.sh new file mode 100755 index 000000000000000..b73a71e7ff7fd54 --- /dev/null +++ b/test/scripts/checks/verify_dependency_versions.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:verifyDependencyVersions diff --git a/test/scripts/checks/verify_notice.sh b/test/scripts/checks/verify_notice.sh new file mode 100755 index 000000000000000..9f8343e5408615d --- /dev/null +++ b/test/scripts/checks/verify_notice.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:verifyNotice diff --git a/test/scripts/jenkins_build_kbn_sample_panel_action.sh b/test/scripts/jenkins_build_kbn_sample_panel_action.sh old mode 100644 new mode 100755 diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index 3e49edc8e6ae5f7..f449986713f97df 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -2,19 +2,9 @@ source src/dev/ci_setup/setup_env.sh -echo " -> building examples separate from test plugins" -node scripts/build_kibana_platform_plugins \ - --oss \ - --examples \ - --verbose; - -echo " -> building test plugins" -node scripts/build_kibana_platform_plugins \ - --oss \ - --no-examples \ - --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ - --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ - --verbose; +if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then + ./test/scripts/jenkins_build_plugins.sh +fi # doesn't persist, also set in kibanaPipeline.groovy export KBN_NP_PLUGINS_BUILT=true @@ -26,4 +16,7 @@ yarn run grunt functionalTests:ensureAllTestsInCiGroup; if [[ -z "$CODE_COVERAGE" ]] ; then echo " -> building and extracting OSS Kibana distributable for use in functional tests" node scripts/build --debug --oss + + mkdir -p "$WORKSPACE/kibana-build-oss" + cp -pR build/oss/kibana-*-SNAPSHOT-linux-x86_64/. $WORKSPACE/kibana-build-oss/ fi diff --git a/test/scripts/jenkins_build_plugins.sh b/test/scripts/jenkins_build_plugins.sh new file mode 100755 index 000000000000000..32b3942074b3468 --- /dev/null +++ b/test/scripts/jenkins_build_plugins.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +echo " -> building examples separate from test plugins" +node scripts/build_kibana_platform_plugins \ + --oss \ + --examples \ + --workers 6 \ + --verbose + +echo " -> building kibana platform plugins" +node scripts/build_kibana_platform_plugins \ + --oss \ + --no-examples \ + --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ + --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ + --workers 6 \ + --verbose diff --git a/test/scripts/jenkins_ci_group.sh b/test/scripts/jenkins_ci_group.sh index 60d7f0406f4c9f9..2542d7032e83bfd 100755 --- a/test/scripts/jenkins_ci_group.sh +++ b/test/scripts/jenkins_ci_group.sh @@ -5,7 +5,7 @@ source test/scripts/jenkins_test_setup_oss.sh if [[ -z "$CODE_COVERAGE" ]]; then checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}"; - if [ "$CI_GROUP" == "1" ]; then + if [[ ! "$TASK_QUEUE_PROCESS_ID" && "$CI_GROUP" == "1" ]]; then source test/scripts/jenkins_build_kbn_sample_panel_action.sh yarn run grunt run:pluginFunctionalTestsRelease --from=source; yarn run grunt run:exampleFunctionalTestsRelease --from=source; diff --git a/test/scripts/jenkins_plugin_functional.sh b/test/scripts/jenkins_plugin_functional.sh new file mode 100755 index 000000000000000..1d691d98982deac --- /dev/null +++ b/test/scripts/jenkins_plugin_functional.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +source test/scripts/jenkins_test_setup_oss.sh + +cd test/plugin_functional/plugins/kbn_sample_panel_action; +if [[ ! -d "target" ]]; then + yarn build; +fi +cd -; + +pwd + +yarn run grunt run:pluginFunctionalTestsRelease --from=source; +yarn run grunt run:exampleFunctionalTestsRelease --from=source; +yarn run grunt run:interpreterFunctionalTestsRelease; diff --git a/test/scripts/jenkins_security_solution_cypress.sh b/test/scripts/jenkins_security_solution_cypress.sh old mode 100644 new mode 100755 index 204911a3eedaa62..a5a1a2103801fad --- a/test/scripts/jenkins_security_solution_cypress.sh +++ b/test/scripts/jenkins_security_solution_cypress.sh @@ -1,12 +1,6 @@ #!/usr/bin/env bash -source test/scripts/jenkins_test_setup.sh - -installDir="$PARENT_DIR/install/kibana" -destDir="${installDir}-${CI_WORKER_NUMBER}" -cp -R "$installDir" "$destDir" - -export KIBANA_INSTALL_DIR="$destDir" +source test/scripts/jenkins_test_setup_xpack.sh echo " -> Running security solution cypress tests" cd "$XPACK_DIR" diff --git a/test/scripts/jenkins_setup_parallel_workspace.sh b/test/scripts/jenkins_setup_parallel_workspace.sh new file mode 100755 index 000000000000000..5274d05572e713d --- /dev/null +++ b/test/scripts/jenkins_setup_parallel_workspace.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -e + +CURRENT_DIR=$(pwd) + +# Copy everything except node_modules into the current workspace +rsync -a ${WORKSPACE}/kibana/* . --exclude node_modules +rsync -a ${WORKSPACE}/kibana/.??* . + +# Symlink all non-root, non-fixture node_modules into our new workspace +cd ${WORKSPACE}/kibana +find . -type d -name node_modules -not -path '*__fixtures__*' -not -path './node_modules*' -prune -print0 | xargs -0I % ln -s "${WORKSPACE}/kibana/%" "${CURRENT_DIR}/%" +find . -type d -wholename '*__fixtures__*node_modules' -not -path './node_modules*' -prune -print0 | xargs -0I % cp -R "${WORKSPACE}/kibana/%" "${CURRENT_DIR}/%" +cd "${CURRENT_DIR}" + +# Symlink all of the individual root-level node_modules into the node_modules/ directory +mkdir -p node_modules +ln -s ${WORKSPACE}/kibana/node_modules/* node_modules/ +ln -s ${WORKSPACE}/kibana/node_modules/.??* node_modules/ + +# Copy a few node_modules instead of symlinking them. They don't work correctly if symlinked +unlink node_modules/@kbn +unlink node_modules/css-loader +unlink node_modules/style-loader + +# packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts will fail if this is a symlink +unlink node_modules/val-loader + +cp -R ${WORKSPACE}/kibana/node_modules/@kbn node_modules/ +cp -R ${WORKSPACE}/kibana/node_modules/css-loader node_modules/ +cp -R ${WORKSPACE}/kibana/node_modules/style-loader node_modules/ +cp -R ${WORKSPACE}/kibana/node_modules/val-loader node_modules/ diff --git a/test/scripts/jenkins_test_setup.sh b/test/scripts/jenkins_test_setup.sh old mode 100644 new mode 100755 index 49ee8a6b526ca5e..7cced76eb650f55 --- a/test/scripts/jenkins_test_setup.sh +++ b/test/scripts/jenkins_test_setup.sh @@ -14,3 +14,7 @@ trap 'post_work' EXIT export TEST_BROWSER_HEADLESS=1 source src/dev/ci_setup/setup_env.sh + +if [[ ! -d .es && -d "$WORKSPACE/kibana/.es" ]]; then + cp -R $WORKSPACE/kibana/.es ./ +fi diff --git a/test/scripts/jenkins_test_setup_oss.sh b/test/scripts/jenkins_test_setup_oss.sh old mode 100644 new mode 100755 index 7bbb86752638431..b7eac33f3517681 --- a/test/scripts/jenkins_test_setup_oss.sh +++ b/test/scripts/jenkins_test_setup_oss.sh @@ -2,10 +2,17 @@ source test/scripts/jenkins_test_setup.sh -if [[ -z "$CODE_COVERAGE" ]] ; then - installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)" - destDir=${installDir}-${CI_PARALLEL_PROCESS_NUMBER} - cp -R "$installDir" "$destDir" +if [[ -z "$CODE_COVERAGE" ]]; then + + destDir="build/kibana-build-oss" + if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then + destDir="${destDir}-${CI_PARALLEL_PROCESS_NUMBER}" + fi + + if [[ ! -d $destDir ]]; then + mkdir -p $destDir + cp -pR "$WORKSPACE/kibana-build-oss/." $destDir/ + fi export KIBANA_INSTALL_DIR="$destDir" fi diff --git a/test/scripts/jenkins_test_setup_xpack.sh b/test/scripts/jenkins_test_setup_xpack.sh old mode 100644 new mode 100755 index a72e9749ebbd5bb..74a3de77e3a7609 --- a/test/scripts/jenkins_test_setup_xpack.sh +++ b/test/scripts/jenkins_test_setup_xpack.sh @@ -3,11 +3,18 @@ source test/scripts/jenkins_test_setup.sh if [[ -z "$CODE_COVERAGE" ]]; then - installDir="$PARENT_DIR/install/kibana" - destDir="${installDir}-${CI_PARALLEL_PROCESS_NUMBER}" - cp -R "$installDir" "$destDir" - export KIBANA_INSTALL_DIR="$destDir" + destDir="build/kibana-build-xpack" + if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then + destDir="${destDir}-${CI_PARALLEL_PROCESS_NUMBER}" + fi + + if [[ ! -d $destDir ]]; then + mkdir -p $destDir + cp -pR "$WORKSPACE/kibana-build-xpack/." $destDir/ + fi + + export KIBANA_INSTALL_DIR="$(realpath $destDir)" cd "$XPACK_DIR" fi diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh index 58ef6a42d3fe4cf..2452e2f5b8c58c1 100755 --- a/test/scripts/jenkins_xpack_build_kibana.sh +++ b/test/scripts/jenkins_xpack_build_kibana.sh @@ -3,21 +3,9 @@ cd "$KIBANA_DIR" source src/dev/ci_setup/setup_env.sh -echo " -> building examples separate from test plugins" -node scripts/build_kibana_platform_plugins \ - --examples \ - --verbose; - -echo " -> building test plugins" -node scripts/build_kibana_platform_plugins \ - --no-examples \ - --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ - --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ - --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ - --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \ - --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \ - --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \ - --verbose; +if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then + ./test/scripts/jenkins_xpack_build_plugins.sh +fi # doesn't persist, also set in kibanaPipeline.groovy export KBN_NP_PLUGINS_BUILT=true @@ -42,7 +30,10 @@ if [[ -z "$CODE_COVERAGE" ]] ; then cd "$KIBANA_DIR" node scripts/build --debug --no-oss linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" - installDir="$PARENT_DIR/install/kibana" + installDir="$KIBANA_DIR/install/kibana" mkdir -p "$installDir" tar -xzf "$linuxBuild" -C "$installDir" --strip=1 + + mkdir -p "$WORKSPACE/kibana-build-xpack" + cp -pR install/kibana/. $WORKSPACE/kibana-build-xpack/ fi diff --git a/test/scripts/jenkins_xpack_build_plugins.sh b/test/scripts/jenkins_xpack_build_plugins.sh new file mode 100755 index 000000000000000..fea30c547bd5fd3 --- /dev/null +++ b/test/scripts/jenkins_xpack_build_plugins.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +echo " -> building examples separate from test plugins" +node scripts/build_kibana_platform_plugins \ + --workers 12 \ + --examples \ + --verbose + +echo " -> building kibana platform plugins" +node scripts/build_kibana_platform_plugins \ + --no-examples \ + --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ + --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ + --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ + --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \ + --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \ + --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \ + --workers 12 \ + --verbose diff --git a/test/scripts/jenkins_xpack_page_load_metrics.sh b/test/scripts/jenkins_xpack_page_load_metrics.sh old mode 100644 new mode 100755 diff --git a/test/scripts/lint/eslint.sh b/test/scripts/lint/eslint.sh new file mode 100755 index 000000000000000..c3211300b96c54b --- /dev/null +++ b/test/scripts/lint/eslint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:eslint diff --git a/test/scripts/lint/sasslint.sh b/test/scripts/lint/sasslint.sh new file mode 100755 index 000000000000000..b9c683bcb049e12 --- /dev/null +++ b/test/scripts/lint/sasslint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:sasslint diff --git a/test/scripts/test/api_integration.sh b/test/scripts/test/api_integration.sh new file mode 100755 index 000000000000000..152c97a3ca7df76 --- /dev/null +++ b/test/scripts/test/api_integration.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:apiIntegrationTests diff --git a/test/scripts/test/jest_integration.sh b/test/scripts/test/jest_integration.sh new file mode 100755 index 000000000000000..73dbbddfb38f63d --- /dev/null +++ b/test/scripts/test/jest_integration.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_jest_integration diff --git a/test/scripts/test/jest_unit.sh b/test/scripts/test/jest_unit.sh new file mode 100755 index 000000000000000..e25452698cebc81 --- /dev/null +++ b/test/scripts/test/jest_unit.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_jest diff --git a/test/scripts/test/karma_ci.sh b/test/scripts/test/karma_ci.sh new file mode 100755 index 000000000000000..e9985300ba19d43 --- /dev/null +++ b/test/scripts/test/karma_ci.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:test_karma_ci diff --git a/test/scripts/test/mocha.sh b/test/scripts/test/mocha.sh new file mode 100755 index 000000000000000..43c00f0a09dcf7b --- /dev/null +++ b/test/scripts/test/mocha.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +yarn run grunt run:mocha diff --git a/test/scripts/test/xpack_jest_unit.sh b/test/scripts/test/xpack_jest_unit.sh new file mode 100755 index 000000000000000..93d70ec35539102 --- /dev/null +++ b/test/scripts/test/xpack_jest_unit.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +cd x-pack +checks-reporter-with-killswitch "X-Pack Jest" node --max-old-space-size=6144 scripts/jest --ci --verbose --maxWorkers=10 diff --git a/test/scripts/test/xpack_karma.sh b/test/scripts/test/xpack_karma.sh new file mode 100755 index 000000000000000..9078f01f1b870fe --- /dev/null +++ b/test/scripts/test/xpack_karma.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +cd x-pack +checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:karma diff --git a/test/scripts/test/xpack_list_cyclic_dependency.sh b/test/scripts/test/xpack_list_cyclic_dependency.sh new file mode 100755 index 000000000000000..493fe9f58d322e7 --- /dev/null +++ b/test/scripts/test/xpack_list_cyclic_dependency.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +cd x-pack +checks-reporter-with-killswitch "X-Pack List cyclic dependency test" node plugins/lists/scripts/check_circular_deps diff --git a/test/scripts/test/xpack_siem_cyclic_dependency.sh b/test/scripts/test/xpack_siem_cyclic_dependency.sh new file mode 100755 index 000000000000000..b21301f25ad087d --- /dev/null +++ b/test/scripts/test/xpack_siem_cyclic_dependency.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +source src/dev/ci_setup/setup_env.sh + +cd x-pack +checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node plugins/security_solution/scripts/check_circular_deps diff --git a/vars/catchErrors.groovy b/vars/catchErrors.groovy index 460a90b8ec0c04d..2a1b55d832606cf 100644 --- a/vars/catchErrors.groovy +++ b/vars/catchErrors.groovy @@ -1,8 +1,15 @@ // Basically, this is a shortcut for catchError(catchInterruptions: false) {} // By default, catchError will swallow aborts/timeouts, which we almost never want +// Also, by wrapping it in an additional try/catch, we cut down on spam in Pipeline Steps def call(Map params = [:], Closure closure) { - params.catchInterruptions = false - return catchError(params, closure) + try { + closure() + } catch (ex) { + params.catchInterruptions = false + catchError(params) { + throw ex + } + } } return this diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index f3fc5f84583c9cf..0f112043114511c 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -16,27 +16,34 @@ def withPostBuildReporting(Closure closure) { } } -def functionalTestProcess(String name, Closure closure) { - return { processNumber -> - def kibanaPort = "61${processNumber}1" - def esPort = "61${processNumber}2" - def esTransportPort = "61${processNumber}3" - def ingestManagementPackageRegistryPort = "61${processNumber}4" +def withFunctionalTestEnv(List additionalEnvs = [], Closure closure) { + // This can go away once everything that uses the deprecated workers.parallelProcesses() is moved to task queue + def parallelId = env.TASK_QUEUE_PROCESS_ID ?: env.CI_PARALLEL_PROCESS_NUMBER - withEnv([ - "CI_PARALLEL_PROCESS_NUMBER=${processNumber}", - "TEST_KIBANA_HOST=localhost", - "TEST_KIBANA_PORT=${kibanaPort}", - "TEST_KIBANA_URL=http://elastic:changeme@localhost:${kibanaPort}", - "TEST_ES_URL=http://elastic:changeme@localhost:${esPort}", - "TEST_ES_TRANSPORT_PORT=${esTransportPort}", - "INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=${ingestManagementPackageRegistryPort}", - "IS_PIPELINE_JOB=1", - "JOB=${name}", - "KBN_NP_PLUGINS_BUILT=true", - ]) { - closure() - } + def kibanaPort = "61${parallelId}1" + def esPort = "61${parallelId}2" + def esTransportPort = "61${parallelId}3" + def ingestManagementPackageRegistryPort = "61${parallelId}4" + + withEnv([ + "CI_GROUP=${parallelId}", + "REMOVE_KIBANA_INSTALL_DIR=1", + "CI_PARALLEL_PROCESS_NUMBER=${parallelId}", + "TEST_KIBANA_HOST=localhost", + "TEST_KIBANA_PORT=${kibanaPort}", + "TEST_KIBANA_URL=http://elastic:changeme@localhost:${kibanaPort}", + "TEST_ES_URL=http://elastic:changeme@localhost:${esPort}", + "TEST_ES_TRANSPORT_PORT=${esTransportPort}", + "KBN_NP_PLUGINS_BUILT=true", + "INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=${ingestManagementPackageRegistryPort}", + ] + additionalEnvs) { + closure() + } +} + +def functionalTestProcess(String name, Closure closure) { + return { + withFunctionalTestEnv(["JOB=${name}"], closure) } } @@ -100,11 +107,17 @@ def withGcsArtifactUpload(workerName, closure) { def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" def ARTIFACT_PATTERNS = [ 'target/kibana-*', + 'target/test-metrics/*', 'target/kibana-security-solution/**/*.png', 'target/junit/**/*', - 'test/**/screenshots/**/*.png', + 'target/test-suites-ci-plan.json', + 'test/**/screenshots/session/*.png', + 'test/**/screenshots/failure/*.png', + 'test/**/screenshots/diff/*.png', 'test/functional/failure_debug/html/*.html', - 'x-pack/test/**/screenshots/**/*.png', + 'x-pack/test/**/screenshots/session/*.png', + 'x-pack/test/**/screenshots/failure/*.png', + 'x-pack/test/**/screenshots/diff/*.png', 'x-pack/test/functional/failure_debug/html/*.html', 'x-pack/test/functional/apps/reporting/reports/session/*.pdf', ] @@ -119,6 +132,12 @@ def withGcsArtifactUpload(workerName, closure) { ARTIFACT_PATTERNS.each { pattern -> uploadGcsArtifact(uploadPrefix, pattern) } + + dir(env.WORKSPACE) { + ARTIFACT_PATTERNS.each { pattern -> + uploadGcsArtifact(uploadPrefix, "parallel/*/kibana/${pattern}") + } + } } } }) @@ -131,6 +150,11 @@ def withGcsArtifactUpload(workerName, closure) { def publishJunit() { junit(testResults: 'target/junit/**/*.xml', allowEmptyResults: true, keepLongStdio: true) + + // junit() is weird about paths for security reasons, so we need to actually change to an upper directory first + dir(env.WORKSPACE) { + junit(testResults: 'parallel/*/kibana/target/junit/**/*.xml', allowEmptyResults: true, keepLongStdio: true) + } } def sendMail() { @@ -194,12 +218,16 @@ def doSetup() { } } -def buildOss() { - runbld("./test/scripts/jenkins_build_kibana.sh", "Build OSS/Default Kibana") +def buildOss(maxWorkers = '') { + withEnv(["KBN_OPTIMIZER_MAX_WORKERS=${maxWorkers}"]) { + runbld("./test/scripts/jenkins_build_kibana.sh", "Build OSS/Default Kibana") + } } -def buildXpack() { - runbld("./test/scripts/jenkins_xpack_build_kibana.sh", "Build X-Pack Kibana") +def buildXpack(maxWorkers = '') { + withEnv(["KBN_OPTIMIZER_MAX_WORKERS=${maxWorkers}"]) { + runbld("./test/scripts/jenkins_xpack_build_kibana.sh", "Build X-Pack Kibana") + } } def runErrorReporter() { @@ -248,6 +276,100 @@ def call(Map params = [:], Closure closure) { } } +// Creates a task queue using withTaskQueue, and copies the bootstrapped kibana repo into each process's workspace +// Note that node_modules are mostly symlinked to save time/space. See test/scripts/jenkins_setup_parallel_workspace.sh +def withCiTaskQueue(Map options = [:], Closure closure) { + def setupClosure = { + // This can't use runbld, because it expects the source to be there, which isn't yet + bash("${env.WORKSPACE}/kibana/test/scripts/jenkins_setup_parallel_workspace.sh", "Set up duplicate workspace for parallel process") + } + + def config = [parallel: 24, setup: setupClosure] + options + + withTaskQueue(config) { + closure.call() + } +} + +def scriptTask(description, script) { + return { + withFunctionalTestEnv { + runbld(script, description) + } + } +} + +def scriptTaskDocker(description, script) { + return { + withDocker(scriptTask(description, script)) + } +} + +def buildDocker() { + sh( + script: """ + cp /usr/local/bin/runbld .ci/ + cp /usr/local/bin/bash_standard_lib.sh .ci/ + cd .ci + docker build -t kibana-ci -f ./Dockerfile . + """, + label: 'Build CI Docker image' + ) +} + +def withDocker(Closure closure) { + docker + .image('kibana-ci') + .inside( + "-v /etc/runbld:/etc/runbld:ro -v '${env.JENKINS_HOME}:${env.JENKINS_HOME}' -v '/dev/shm/workspace:/dev/shm/workspace' --shm-size 2GB --cpus 4", + closure + ) +} + +def buildOssPlugins() { + runbld('./test/scripts/jenkins_build_plugins.sh', 'Build OSS Plugins') +} + +def buildXpackPlugins() { + runbld('./test/scripts/jenkins_xpack_build_plugins.sh', 'Build X-Pack Plugins') +} + +def withTasks(Map params = [worker: [:]], Closure closure) { + catchErrors { + def config = [name: 'ci-worker', size: 'xxl', ramDisk: true] + (params.worker ?: [:]) + + workers.ci(config) { + withCiTaskQueue(parallel: 24) { + parallel([ + docker: { + retry(2) { + buildDocker() + } + }, + + // There are integration tests etc that require the plugins to be built first, so let's go ahead and build them before set up the parallel workspaces + ossPlugins: { buildOssPlugins() }, + xpackPlugins: { buildXpackPlugins() }, + ]) + + catchErrors { + closure() + } + } + } + } +} + +def allCiTasks() { + withTasks { + tasks.check() + tasks.lint() + tasks.test() + tasks.functionalOss() + tasks.functionalXpack() + } +} + def pipelineLibraryTests() { whenChanged(['vars/', '.ci/pipeline-library/']) { workers.base(size: 'flyweight', bootstrapped: false, ramDisk: false) { @@ -258,5 +380,4 @@ def pipelineLibraryTests() { } } - return this diff --git a/vars/task.groovy b/vars/task.groovy new file mode 100644 index 000000000000000..0c07b519b6fefca --- /dev/null +++ b/vars/task.groovy @@ -0,0 +1,5 @@ +def call(Closure closure) { + withTaskQueue.addTask(closure) +} + +return this diff --git a/vars/tasks.groovy b/vars/tasks.groovy new file mode 100644 index 000000000000000..4e8fcaad795cd23 --- /dev/null +++ b/vars/tasks.groovy @@ -0,0 +1,118 @@ +def call(List closures) { + withTaskQueue.addTasks(closures) +} + +def check() { + tasks([ + kibanaPipeline.scriptTask('Check TypeScript Projects', 'test/scripts/checks/ts_projects.sh'), + kibanaPipeline.scriptTask('Check Doc API Changes', 'test/scripts/checks/doc_api_changes.sh'), + kibanaPipeline.scriptTask('Check Types', 'test/scripts/checks/type_check.sh'), + kibanaPipeline.scriptTask('Check i18n', 'test/scripts/checks/i18n.sh'), + kibanaPipeline.scriptTask('Check File Casing', 'test/scripts/checks/file_casing.sh'), + kibanaPipeline.scriptTask('Check Lockfile Symlinks', 'test/scripts/checks/lock_file_symlinks.sh'), + kibanaPipeline.scriptTask('Check Licenses', 'test/scripts/checks/licenses.sh'), + kibanaPipeline.scriptTask('Verify Dependency Versions', 'test/scripts/checks/verify_dependency_versions.sh'), + kibanaPipeline.scriptTask('Verify NOTICE', 'test/scripts/checks/verify_notice.sh'), + kibanaPipeline.scriptTask('Test Projects', 'test/scripts/checks/test_projects.sh'), + kibanaPipeline.scriptTask('Test Hardening', 'test/scripts/checks/test_hardening.sh'), + ]) +} + +def lint() { + tasks([ + kibanaPipeline.scriptTask('Lint: eslint', 'test/scripts/lint/eslint.sh'), + kibanaPipeline.scriptTask('Lint: sasslint', 'test/scripts/lint/sasslint.sh'), + ]) +} + +def test() { + tasks([ + // These 4 tasks require isolation because of hard-coded, conflicting ports and such, so let's use Docker here + kibanaPipeline.scriptTaskDocker('Jest Integration Tests', 'test/scripts/test/jest_integration.sh'), + kibanaPipeline.scriptTaskDocker('Mocha Tests', 'test/scripts/test/mocha.sh'), + kibanaPipeline.scriptTaskDocker('Karma CI Tests', 'test/scripts/test/karma_ci.sh'), + kibanaPipeline.scriptTaskDocker('X-Pack Karma Tests', 'test/scripts/test/xpack_karma.sh'), + + kibanaPipeline.scriptTask('Jest Unit Tests', 'test/scripts/test/jest_unit.sh'), + kibanaPipeline.scriptTask('API Integration Tests', 'test/scripts/test/api_integration.sh'), + kibanaPipeline.scriptTask('X-Pack SIEM cyclic dependency', 'test/scripts/test/xpack_siem_cyclic_dependency.sh'), + kibanaPipeline.scriptTask('X-Pack List cyclic dependency', 'test/scripts/test/xpack_list_cyclic_dependency.sh'), + kibanaPipeline.scriptTask('X-Pack Jest Unit Tests', 'test/scripts/test/xpack_jest_unit.sh'), + ]) +} + +def functionalOss(Map params = [:]) { + def config = params ?: [ciGroups: true, firefox: true, accessibility: true, pluginFunctional: true, visualRegression: false] + + task { + kibanaPipeline.buildOss(6) + + if (config.ciGroups) { + def ciGroups = 1..12 + tasks(ciGroups.collect { kibanaPipeline.ossCiGroupProcess(it) }) + } + + if (config.firefox) { + task(kibanaPipeline.functionalTestProcess('oss-firefox', './test/scripts/jenkins_firefox_smoke.sh')) + } + + if (config.accessibility) { + task(kibanaPipeline.functionalTestProcess('oss-accessibility', './test/scripts/jenkins_accessibility.sh')) + } + + if (config.pluginFunctional) { + task(kibanaPipeline.functionalTestProcess('oss-pluginFunctional', './test/scripts/jenkins_plugin_functional.sh')) + } + + if (config.visualRegression) { + task(kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')) + } + } +} + +def functionalXpack(Map params = [:]) { + def config = params ?: [ + ciGroups: true, + firefox: true, + accessibility: true, + pluginFunctional: true, + savedObjectsFieldMetrics:true, + pageLoadMetrics: false, + visualRegression: false, + ] + + task { + kibanaPipeline.buildXpack(10) + + if (config.ciGroups) { + def ciGroups = 1..10 + tasks(ciGroups.collect { kibanaPipeline.xpackCiGroupProcess(it) }) + } + + if (config.firefox) { + task(kibanaPipeline.functionalTestProcess('xpack-firefox', './test/scripts/jenkins_xpack_firefox_smoke.sh')) + } + + if (config.accessibility) { + task(kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh')) + } + + if (config.visualRegression) { + task(kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')) + } + + if (config.pageLoadMetrics) { + task(kibanaPipeline.functionalTestProcess('xpack-pageLoadMetrics', './test/scripts/jenkins_xpack_page_load_metrics.sh')) + } + + if (config.savedObjectsFieldMetrics) { + task(kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh')) + } + + whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/']) { + task(kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')) + } + } +} + +return this diff --git a/vars/withTaskQueue.groovy b/vars/withTaskQueue.groovy new file mode 100644 index 000000000000000..8132d6264744f2c --- /dev/null +++ b/vars/withTaskQueue.groovy @@ -0,0 +1,154 @@ +import groovy.transform.Field + +public static @Field TASK_QUEUES = [:] +public static @Field TASK_QUEUES_COUNTER = 0 + +/** + withTaskQueue creates a queue of "tasks" (just plain closures to execute), and executes them with your desired level of concurrency. + This way, you can define, for example, 40 things that need to execute, then only allow 10 of them to execute at once. + + Each "process" will execute in a separate, unique, empty directory. + If you want each process to have a bootstrapped kibana repo, check out kibanaPipeline.withCiTaskQueue + + Using the queue currently requires an agent/worker. + + Usage: + + withTaskQueue(parallel: 10) { + task { print "This is a task" } + + // This is the same as calling task() multiple times + tasks([ { print "Another task" }, { print "And another task" } ]) + + // Tasks can queue up subsequent tasks + task { + buildThing() + task { print "I depend on buildThing()" } + } + } + + You can also define a setup task that each process should execute one time before executing tasks: + withTaskQueue(parallel: 10, setup: { sh "my-setup-scrupt.sh" }) { + ... + } + +*/ +def call(Map options = [:], Closure closure) { + def config = [ parallel: 10 ] + options + def counter = ++TASK_QUEUES_COUNTER + + // We're basically abusing withEnv() to create a "scope" for all steps inside of a withTaskQueue block + // This way, we could have multiple task queue instances in the same pipeline + withEnv(["TASK_QUEUE_ID=${counter}"]) { + withTaskQueue.TASK_QUEUES[env.TASK_QUEUE_ID] = [ + tasks: [], + tmpFile: sh(script: 'mktemp', returnStdout: true).trim() + ] + + closure.call() + + def processesExecuting = 0 + def processes = [:] + def iterationId = 0 + + for(def i = 1; i <= config.parallel; i++) { + def j = i + processes["task-queue-process-${j}"] = { + catchErrors { + withEnv([ + "TASK_QUEUE_PROCESS_ID=${j}", + "TASK_QUEUE_ITERATION_ID=${++iterationId}" + ]) { + dir("${WORKSPACE}/parallel/${j}/kibana") { + if (config.setup) { + config.setup.call(j) + } + + def isDone = false + while(!isDone) { // TODO some kind of timeout? + catchErrors { + if (!getTasks().isEmpty()) { + processesExecuting++ + catchErrors { + def task + try { + task = getTasks().pop() + } catch (java.util.NoSuchElementException ex) { + return + } + + task.call() + } + processesExecuting-- + // If a task finishes, and no new tasks were queued up, and nothing else is executing + // Then all of the processes should wake up and exit + if (processesExecuting < 1 && getTasks().isEmpty()) { + taskNotify() + } + return + } + + if (processesExecuting > 0) { + taskSleep() + return + } + + // Queue is empty, no processes are executing + isDone = true + } + } + } + } + } + } + } + parallel(processes) + } +} + +// If we sleep in a loop using Groovy code, Pipeline Steps is flooded with Sleep steps +// So, instead, we just watch a file and `touch` it whenever something happens that could modify the queue +// There's a 20 minute timeout just in case something goes wrong, +// in which case this method will get called again if the process is actually supposed to be waiting. +def taskSleep() { + sh(script: """#!/bin/bash + TIMESTAMP=\$(date '+%s' -d "0 seconds ago") + for (( i=1; i<=240; i++ )) + do + if [ "\$(stat -c %Y '${getTmpFile()}')" -ge "\$TIMESTAMP" ] + then + break + else + sleep 5 + if [[ \$i == 240 ]]; then + echo "Waited for new tasks for 20 minutes, exiting in case something went wrong" + fi + fi + done + """, label: "Waiting for new tasks...") +} + +// Used to let the task queue processes know that either a new task has been queued up, or work is complete +def taskNotify() { + sh "touch '${getTmpFile()}'" +} + +def getTasks() { + return withTaskQueue.TASK_QUEUES[env.TASK_QUEUE_ID].tasks +} + +def getTmpFile() { + return withTaskQueue.TASK_QUEUES[env.TASK_QUEUE_ID].tmpFile +} + +def addTask(Closure closure) { + getTasks() << closure + taskNotify() +} + +def addTasks(List closures) { + closures.reverse().each { + getTasks() << it + } + taskNotify() +} diff --git a/vars/workers.groovy b/vars/workers.groovy index 8b7e8525a7ce3b6..2e94ce12f34c072 100644 --- a/vars/workers.groovy +++ b/vars/workers.groovy @@ -13,6 +13,8 @@ def label(size) { return 'docker && tests-l' case 'xl': return 'docker && tests-xl' + case 'xl-highmem': + return 'docker && tests-xl-highmem' case 'xxl': return 'docker && tests-xxl' } @@ -55,6 +57,11 @@ def base(Map params, Closure closure) { } } + sh( + script: "mkdir -p ${env.WORKSPACE}/tmp", + label: "Create custom temp directory" + ) + def checkoutInfo = [:] if (config.scm) { @@ -89,6 +96,7 @@ def base(Map params, Closure closure) { "PR_AUTHOR=${env.ghprbPullAuthorLogin ?: ''}", "TEST_BROWSER_HEADLESS=1", "GIT_BRANCH=${checkoutInfo.branch}", + "TMPDIR=${env.WORKSPACE}/tmp", // For Chrome and anything else that respects it ]) { withCredentials([ string(credentialsId: 'vault-addr', variable: 'VAULT_ADDR'), @@ -167,7 +175,9 @@ def parallelProcesses(Map params) { sleep(delay) } - processClosure(processNumber) + withEnv(["CI_PARALLEL_PROCESS_NUMBER=${processNumber}"]) { + processClosure() + } } } diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap new file mode 100644 index 000000000000000..3ac20a05639fbf0 --- /dev/null +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -0,0 +1,913 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the telemetry mapping 1`] = ` +Object { + "properties": Object { + "agents": Object { + "properties": Object { + "dotnet": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "go": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "java": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "js-base": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "nodejs": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "python": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "ruby": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "rum-js": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + }, + }, + "cardinality": Object { + "properties": Object { + "transaction": Object { + "properties": Object { + "name": Object { + "properties": Object { + "all_agents": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "rum": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + "user_agent": Object { + "properties": Object { + "original": Object { + "properties": Object { + "all_agents": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "rum": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + }, + }, + "counts": Object { + "properties": Object { + "agent_configuration": Object { + "properties": Object { + "all": Object { + "type": "long", + }, + }, + }, + "error": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "max_error_groups_per_service": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "max_transaction_groups_per_service": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "metric": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "onboarding": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "services": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "sourcemap": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "span": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "traces": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "transaction": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + }, + }, + "has_any_services": Object { + "type": "boolean", + }, + "indices": Object { + "properties": Object { + "all": Object { + "properties": Object { + "total": Object { + "properties": Object { + "docs": Object { + "properties": Object { + "count": Object { + "type": "long", + }, + }, + }, + "store": Object { + "properties": Object { + "size_in_bytes": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + "shards": Object { + "properties": Object { + "total": Object { + "type": "long", + }, + }, + }, + }, + }, + "integrations": Object { + "properties": Object { + "ml": Object { + "properties": Object { + "all_jobs_count": Object { + "type": "long", + }, + }, + }, + }, + }, + "retainment": Object { + "properties": Object { + "error": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "metric": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "onboarding": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "span": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "transaction": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "services_per_agent": Object { + "properties": Object { + "dotnet": Object { + "null_value": 0, + "type": "long", + }, + "go": Object { + "null_value": 0, + "type": "long", + }, + "java": Object { + "null_value": 0, + "type": "long", + }, + "js-base": Object { + "null_value": 0, + "type": "long", + }, + "nodejs": Object { + "null_value": 0, + "type": "long", + }, + "python": Object { + "null_value": 0, + "type": "long", + }, + "ruby": Object { + "null_value": 0, + "type": "long", + }, + "rum-js": Object { + "null_value": 0, + "type": "long", + }, + }, + }, + "tasks": Object { + "properties": Object { + "agent_configuration": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "agents": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "cardinality": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "groupings": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "indices_stats": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "integrations": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "processor_events": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "services": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "versions": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + "version": Object { + "properties": Object { + "apm_server": Object { + "properties": Object { + "major": Object { + "type": "long", + }, + "minor": Object { + "type": "long", + }, + "patch": Object { + "type": "long", + }, + }, + }, + }, + }, + }, +} +`; diff --git a/x-pack/plugins/apm/common/agent_name.ts b/x-pack/plugins/apm/common/agent_name.ts index 9d462dad87ec071..8b479d1d82fe7c3 100644 --- a/x-pack/plugins/apm/common/agent_name.ts +++ b/x-pack/plugins/apm/common/agent_name.ts @@ -15,15 +15,14 @@ import { AgentName } from '../typings/es_schemas/ui/fields/agent'; */ export const AGENT_NAMES: AgentName[] = [ - 'java', - 'js-base', - 'rum-js', 'dotnet', 'go', 'java', + 'js-base', 'nodejs', 'python', 'ruby', + 'rum-js', ]; export function isAgentName(agentName: string): agentName is AgentName { diff --git a/x-pack/plugins/apm/common/apm_telemetry.test.ts b/x-pack/plugins/apm/common/apm_telemetry.test.ts new file mode 100644 index 000000000000000..1612716142ce70e --- /dev/null +++ b/x-pack/plugins/apm/common/apm_telemetry.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + getApmTelemetryMapping, + mergeApmTelemetryMapping, +} from './apm_telemetry'; + +describe('APM telemetry helpers', () => { + describe('getApmTelemetry', () => { + it('generates a JSON object with the telemetry mapping', () => { + expect(getApmTelemetryMapping()).toMatchSnapshot(); + }); + }); + + describe('mergeApmTelemetryMapping', () => { + describe('with an invalid mapping', () => { + it('throws an error', () => { + expect(() => mergeApmTelemetryMapping({})).toThrowError(); + }); + }); + + describe('with a valid mapping', () => { + it('merges the mapping', () => { + // This is "valid" in the sense that it has all of the deep fields + // needed to merge. It's not a valid mapping opbject. + const validTelemetryMapping = { + mappings: { + properties: { + stack_stats: { + properties: { + kibana: { + properties: { plugins: { properties: { apm: {} } } }, + }, + }, + }, + }, + }, + }; + + expect( + mergeApmTelemetryMapping(validTelemetryMapping)?.mappings.properties + .stack_stats.properties.kibana.properties.plugins.properties.apm + ).toEqual(getApmTelemetryMapping()); + }); + }); + }); +}); diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts new file mode 100644 index 000000000000000..1532058adf64ff6 --- /dev/null +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -0,0 +1,229 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { produce } from 'immer'; +import { AGENT_NAMES } from './agent_name'; + +/** + * Generate an object containing the mapping used for APM telemetry. Can be used + * with the `upload-telemetry-data` script or to update the mapping in the + * telemetry repository. + * + * This function breaks things up to make the mapping easier to understand. + */ +export function getApmTelemetryMapping() { + const keyword = { + type: 'keyword', + ignore_above: 1024, + }; + + const long = { + type: 'long', + }; + + const allProperties = { + properties: { + all: long, + }, + }; + + const oneDayProperties = { + properties: { + '1d': long, + }, + }; + + const oneDayAllProperties = { + properties: { + '1d': long, + all: long, + }, + }; + + const msProperties = { + properties: { + ms: long, + }, + }; + + const tookProperties = { + properties: { + took: msProperties, + }, + }; + + const compositeNameVersionProperties = { + properties: { + composite: keyword, + name: keyword, + version: keyword, + }, + }; + + const agentProperties = { + properties: { version: keyword }, + }; + + const serviceProperties = { + properties: { + framework: compositeNameVersionProperties, + language: compositeNameVersionProperties, + runtime: compositeNameVersionProperties, + }, + }; + + return { + properties: { + agents: { + properties: AGENT_NAMES.reduce>( + (previousValue, currentValue) => { + previousValue[currentValue] = { + properties: { + agent: agentProperties, + service: serviceProperties, + }, + }; + + return previousValue; + }, + {} + ), + }, + counts: { + properties: { + agent_configuration: allProperties, + error: oneDayAllProperties, + max_error_groups_per_service: oneDayProperties, + max_transaction_groups_per_service: oneDayProperties, + metric: oneDayAllProperties, + onboarding: oneDayAllProperties, + services: oneDayProperties, + sourcemap: oneDayAllProperties, + span: oneDayAllProperties, + traces: oneDayProperties, + transaction: oneDayAllProperties, + }, + }, + cardinality: { + properties: { + user_agent: { + properties: { + original: { + properties: { + all_agents: oneDayProperties, + rum: oneDayProperties, + }, + }, + }, + }, + transaction: { + properties: { + name: { + properties: { + all_agents: oneDayProperties, + rum: oneDayProperties, + }, + }, + }, + }, + }, + }, + has_any_services: { + type: 'boolean', + }, + indices: { + properties: { + all: { + properties: { + total: { + properties: { + docs: { + properties: { + count: long, + }, + }, + store: { + properties: { + size_in_bytes: long, + }, + }, + }, + }, + }, + }, + shards: { + properties: { + total: long, + }, + }, + }, + }, + integrations: { + properties: { + ml: { + properties: { + all_jobs_count: long, + }, + }, + }, + }, + retainment: { + properties: { + error: msProperties, + metric: msProperties, + onboarding: msProperties, + span: msProperties, + transaction: msProperties, + }, + }, + services_per_agent: { + properties: AGENT_NAMES.reduce>( + (previousValue, currentValue) => { + previousValue[currentValue] = { ...long, null_value: 0 }; + return previousValue; + }, + {} + ), + }, + tasks: { + properties: { + agent_configuration: tookProperties, + agents: tookProperties, + cardinality: tookProperties, + groupings: tookProperties, + indices_stats: tookProperties, + integrations: tookProperties, + processor_events: tookProperties, + services: tookProperties, + versions: tookProperties, + }, + }, + version: { + properties: { + apm_server: { + properties: { + major: long, + minor: long, + patch: long, + }, + }, + }, + }, + }, + }; +} + +/** + * Merge a telemetry mapping object (from https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json) + * with the output from `getApmTelemetryMapping`. + */ +export function mergeApmTelemetryMapping( + xpackPhoneHomeMapping: Record +) { + return produce(xpackPhoneHomeMapping, (draft: Record) => { + draft.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = getApmTelemetryMapping(); + return draft; + }); +} diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md new file mode 100644 index 000000000000000..9674d39e57177b4 --- /dev/null +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -0,0 +1,69 @@ +# APM Telemetry + +In order to learn about our customers' usage and experience of APM, we collect +two types of telemetry, which we'll refer to here as "Data Telemetry" and +"Behavioral Telemetry." + +This document will explain how they are collected and how to make changes to +them. + +[The telemetry repository has information about accessing the clusters](https://github.com/elastic/telemetry#kibana-access). +Telemetry data is uploaded to the "xpack-phone-home" indices. + +## Data Telemetry + +Information that can be derived from a cluster's APM indices is queried and sent +to the telemetry cluster using the +[Usage Collection plugin](../../../../src/plugins/usage_collection/README.md). + +During the APM server-side plugin's setup phase a +[Saved Object](https://www.elastic.co/guide/en/kibana/master/managing-saved-objects.html) +for APM telemetry is registered and a +[task manager](../../task_manager/server/README.md) task is registered and started. +The task periodically queries the APM indices and saves the results in the Saved +Object, and the usage collector periodically gets the data from the saved object +and uploads it to the telemetry cluster. + +Once uploaded to the telemetry cluster, the data telemetry is stored in +`stack_stats.kibana.plugins.apm` in the xpack-phone-home index. + +### Generating sample data + +The script in `scripts/upload-telemetry-data` can generate sample telemetry data and upload it to a cluster of your choosing. + +You'll need to set the `GITHUB_TOKEN` environment variable to a token that has `repo` scope so it can read from the +[elastic/telemetry](https://github.com/elastic/telemetry) repository. (You probably have a token that works for this in +~/.backport/config.json.) + +The script will run as the `elastic` user using the elasticsearch hosts and password settings from the config/kibana.yml +and/or config/kibana.dev.yml files. + +Running the script with `--clear` will delete the index first. + +After running the script you should see sample telemetry data in the "xpack-phone-home" index. + +### Updating Data Telemetry Mappings + +In order for fields to be searchable on the telemetry cluster, they need to be +added to the cluster's mapping. The mapping is defined in +[the telemetry repository's xpack-phone-home template](https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json). + +The mapping for the telemetry data is here under `stack_stats.kibana.plugins.apm`. + +The mapping used there can be generated with the output of the [`getTelemetryMapping`](../common/apm_telemetry.ts) function. + +To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository. + +If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm: + +```bash +node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates/xpack-phone-home.json +``` + +this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. + +## Behavioral Telemetry + +Behavioral telemetry is recorded with the ui_metrics and application_usage methods from the Usage Collection plugin. + +Please fill this in with more details. diff --git a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/TransactionActionMenu.test.tsx b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/TransactionActionMenu.test.tsx index ec546b5c6280fc3..0a2cb90fdd5dafa 100644 --- a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/TransactionActionMenu.test.tsx +++ b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/TransactionActionMenu.test.tsx @@ -94,7 +94,7 @@ describe('TransactionActionMenu component', () => { expect(mock.core.application.navigateToApp).toHaveBeenCalledWith('logs', { path: - 'link-to/logs?time=1545092070952&filter=trace.id:%228b60bd32ecc6e1506735a8b6cfcf175c%22%20OR%208b60bd32ecc6e1506735a8b6cfcf175c', + 'link-to/logs?time=1545092070952&filter=trace.id:%228b60bd32ecc6e1506735a8b6cfcf175c%22%20OR%20%228b60bd32ecc6e1506735a8b6cfcf175c%22', }); }); diff --git a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/sections.test.ts b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/sections.test.ts index b2adc6cdac4a67d..50325e0b9d60441 100644 --- a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/sections.test.ts +++ b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/__test__/sections.test.ts @@ -42,7 +42,7 @@ describe('Transaction action menu', () => { key: 'traceLogs', label: 'Trace logs', href: - 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20123', + 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20%22123%22', condition: true, }, ], @@ -113,7 +113,7 @@ describe('Transaction action menu', () => { key: 'traceLogs', label: 'Trace logs', href: - 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20123', + 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20%22123%22', condition: true, }, ], @@ -183,7 +183,7 @@ describe('Transaction action menu', () => { key: 'traceLogs', label: 'Trace logs', href: - 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20123', + 'some-basepath/app/logs/link-to/logs?time=1580986800&filter=trace.id:%22123%22%20OR%20%22123%22', condition: true, }, ], diff --git a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/sections.ts b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/sections.ts index d3a9ade3925a129..5ca0285eb4eeb7b 100644 --- a/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/sections.ts +++ b/x-pack/plugins/apm/public/components/shared/TransactionActionMenu/sections.ts @@ -180,7 +180,7 @@ export const getSections = ({ path: `/link-to/logs`, query: { time, - filter: `trace.id:"${transaction.trace.id}" OR ${transaction.trace.id}`, + filter: `trace.id:"${transaction.trace.id}" OR "${transaction.trace.id}"`, }, }), condition: true, diff --git a/x-pack/plugins/apm/public/plugin.ts b/x-pack/plugins/apm/public/plugin.ts index d24cb29eaf24f95..f31ad83666a1760 100644 --- a/x-pack/plugins/apm/public/plugin.ts +++ b/x-pack/plugins/apm/public/plugin.ts @@ -66,8 +66,9 @@ export interface ApmPluginStartDeps { } export class ApmPlugin implements Plugin { - private readonly initializerContext: PluginInitializerContext; - constructor(initializerContext: PluginInitializerContext) { + constructor( + private readonly initializerContext: PluginInitializerContext + ) { this.initializerContext = initializerContext; } public setup(core: CoreSetup, plugins: ApmPluginSetupDeps) { diff --git a/x-pack/plugins/apm/readme.md b/x-pack/plugins/apm/readme.md index f460ff6ff9bf2c0..9b02972d353023f 100644 --- a/x-pack/plugins/apm/readme.md +++ b/x-pack/plugins/apm/readme.md @@ -162,3 +162,4 @@ You can access the development environment at http://localhost:9001. - [Cypress integration tests](./e2e/README.md) - [VSCode setup instructions](./dev_docs/vscode_setup.md) - [Github PR commands](./dev_docs/github_commands.md) +- [Telemetry](./dev_docs/telemetry.md) diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js b/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js new file mode 100644 index 000000000000000..741df981a9cb0a5 --- /dev/null +++ b/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// compile typescript on the fly +// eslint-disable-next-line import/no-extraneous-dependencies +require('@babel/register')({ + extensions: ['.ts'], + plugins: [ + '@babel/plugin-proposal-optional-chaining', + '@babel/plugin-proposal-nullish-coalescing-operator', + ], + presets: [ + '@babel/typescript', + ['@babel/preset-env', { targets: { node: 'current' } }], + ], +}); + +require('./merge-telemetry-mapping/index.ts'); diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts new file mode 100644 index 000000000000000..c06d4cec150dcfe --- /dev/null +++ b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { readFileSync, truncateSync, writeFileSync } from 'fs'; +import { resolve } from 'path'; +import { argv } from 'yargs'; +import { mergeApmTelemetryMapping } from '../../common/apm_telemetry'; + +function errorExit(error?: Error) { + console.error(`usage: ${argv.$0} /path/to/xpack-phone-home.json`); // eslint-disable-line no-console + if (error) { + throw error; + } + process.exit(1); +} + +try { + const filename = resolve(argv._[0]); + const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8')); + + const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping); + + truncateSync(filename); + writeFileSync(filename, JSON.stringify(newMapping, null, 2)); +} catch (error) { + errorExit(error); +} diff --git a/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts b/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts index 3f88b73f5598437..6d44e12fb00a2bd 100644 --- a/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts +++ b/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts @@ -30,6 +30,11 @@ export async function createOrUpdateIndex({ } } + // Some settings are non-updateable and need to be removed. + const settings = { ...template.settings }; + delete settings?.index?.number_of_shards; + delete settings?.index?.sort; + const indexExists = ( await client.indices.exists({ index: indexName, @@ -42,6 +47,7 @@ export async function createOrUpdateIndex({ body: template, }); } else { + await client.indices.close({ index: indexName }); await Promise.all([ template.mappings ? client.indices.putMapping({ @@ -49,12 +55,13 @@ export async function createOrUpdateIndex({ body: template.mappings, }) : Promise.resolve(undefined as any), - template.settings + settings ? client.indices.putSettings({ index: indexName, - body: template.settings, + body: settings, }) : Promise.resolve(undefined as any), ]); + await client.indices.open({ index: indexName }); } } diff --git a/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts b/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts index 5f9c72810fc91c4..a44fad82f20e648 100644 --- a/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts +++ b/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts @@ -11,7 +11,7 @@ // - Easier testing of the telemetry tasks // - Validate whether we can run the queries we want to on the telemetry data -import { merge, chunk, flatten } from 'lodash'; +import { merge, chunk, flatten, omit } from 'lodash'; import { Client } from '@elastic/elasticsearch'; import { argv } from 'yargs'; import { Logger } from 'kibana/server'; @@ -20,7 +20,7 @@ import { stampLogger } from '../shared/stamp-logger'; import { CollectTelemetryParams } from '../../server/lib/apm_telemetry/collect_data_telemetry'; import { downloadTelemetryTemplate } from '../shared/download-telemetry-template'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { apmTelemetry } from '../../server/saved_objects/apm_telemetry'; +import { mergeApmTelemetryMapping } from '../../common/apm_telemetry'; import { generateSampleDocuments } from './generate-sample-documents'; import { readKibanaConfig } from '../shared/read-kibana-config'; import { getHttpAuth } from '../shared/get-http-auth'; @@ -40,8 +40,6 @@ async function uploadData() { githubToken, }); - const kibanaMapping = apmTelemetry.mappings; - const config = readKibanaConfig(); const httpAuth = getHttpAuth(config); @@ -50,19 +48,25 @@ async function uploadData() { nodes: [config['elasticsearch.hosts']], ...(httpAuth ? { - auth: httpAuth, + auth: { ...httpAuth, username: 'elastic' }, } : {}), }); - const newTemplate = merge(telemetryTemplate, { - settings: { - index: { mapping: { total_fields: { limit: 10000 } } }, - }, - }); - - // override apm mapping instead of merging - newTemplate.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = kibanaMapping; + // The new template is the template downloaded from the telemetry repo, with + // our current telemetry mapping merged in, with the "index_patterns" key + // (which cannot be used when creating an index) removed. + const newTemplate = omit( + mergeApmTelemetryMapping( + merge(telemetryTemplate, { + index_patterns: undefined, + settings: { + index: { mapping: { total_fields: { limit: 10000 } } }, + }, + }) + ), + 'index_patterns' + ); await createOrUpdateIndex({ indexName: xpackTelemetryIndexName, diff --git a/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts b/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts index f741b61fc7a0404..411f453042b9351 100644 --- a/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts +++ b/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts @@ -4,918 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ import { SavedObjectsType } from 'src/core/server'; +import { APM_TELEMETRY_SAVED_OBJECT_ID } from '../../common/apm_saved_object_constants'; export const apmTelemetry: SavedObjectsType = { - name: 'apm-telemetry', + name: APM_TELEMETRY_SAVED_OBJECT_ID, hidden: false, namespaceType: 'agnostic', mappings: { - properties: { - agents: { - properties: { - dotnet: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - go: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - java: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - 'js-base': { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - nodejs: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - python: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - ruby: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - 'rum-js': { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - }, - }, - counts: { - properties: { - agent_configuration: { - properties: { - all: { - type: 'long', - }, - }, - }, - error: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - max_error_groups_per_service: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - max_transaction_groups_per_service: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - metric: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - onboarding: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - services: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - sourcemap: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - span: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - traces: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - transaction: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - }, - }, - cardinality: { - properties: { - user_agent: { - properties: { - original: { - properties: { - all_agents: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - rum: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - transaction: { - properties: { - name: { - properties: { - all_agents: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - rum: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - }, - }, - has_any_services: { - type: 'boolean', - }, - indices: { - properties: { - all: { - properties: { - total: { - properties: { - docs: { - properties: { - count: { - type: 'long', - }, - }, - }, - store: { - properties: { - size_in_bytes: { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - shards: { - properties: { - total: { - type: 'long', - }, - }, - }, - }, - }, - integrations: { - properties: { - ml: { - properties: { - all_jobs_count: { - type: 'long', - }, - }, - }, - }, - }, - retainment: { - properties: { - error: { - properties: { - ms: { - type: 'long', - }, - }, - }, - metric: { - properties: { - ms: { - type: 'long', - }, - }, - }, - onboarding: { - properties: { - ms: { - type: 'long', - }, - }, - }, - span: { - properties: { - ms: { - type: 'long', - }, - }, - }, - transaction: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - services_per_agent: { - properties: { - dotnet: { - type: 'long', - null_value: 0, - }, - go: { - type: 'long', - null_value: 0, - }, - java: { - type: 'long', - null_value: 0, - }, - 'js-base': { - type: 'long', - null_value: 0, - }, - nodejs: { - type: 'long', - null_value: 0, - }, - python: { - type: 'long', - null_value: 0, - }, - ruby: { - type: 'long', - null_value: 0, - }, - 'rum-js': { - type: 'long', - null_value: 0, - }, - }, - }, - tasks: { - properties: { - agent_configuration: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - agents: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - cardinality: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - groupings: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - indices_stats: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - integrations: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - processor_events: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - services: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - versions: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - version: { - properties: { - apm_server: { - properties: { - major: { - type: 'long', - }, - minor: { - type: 'long', - }, - patch: { - type: 'long', - }, - }, - }, - }, - }, - } as SavedObjectsType['mappings']['properties'], + dynamic: false, + properties: {}, }, }; diff --git a/x-pack/plugins/canvas/.storybook/storyshots.test.js b/x-pack/plugins/canvas/.storybook/storyshots.test.js index b9fe0914b369871..e3217ad4dbe58cc 100644 --- a/x-pack/plugins/canvas/.storybook/storyshots.test.js +++ b/x-pack/plugins/canvas/.storybook/storyshots.test.js @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import fs from 'fs'; import path from 'path'; import moment from 'moment'; import 'moment-timezone'; @@ -76,6 +77,12 @@ import { RenderedElement } from '../shareable_runtime/components/rendered_elemen jest.mock('../shareable_runtime/components/rendered_element'); RenderedElement.mockImplementation(() => 'RenderedElement'); +// Some of the code requires that this directory exists, but the tests don't actually require any css to be present +const cssDir = path.resolve(__dirname, '../../../../built_assets/css'); +if (!fs.existsSync(cssDir)) { + fs.mkdirSync(cssDir, { recursive: true }); +} + addSerializer(styleSheetSerializer); // Initialize Storyshots and build the Jest Snapshots diff --git a/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts b/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts index 21946c7c5653ab5..24c51598ad25761 100644 --- a/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts +++ b/x-pack/plugins/infra/public/metrics_overview_fetchers.test.ts @@ -29,11 +29,7 @@ describe('Metrics UI Observability Homepage Functions', () => { it('should return true when true', async () => { const { core, mockedGetStartServices } = setup(); core.http.get.mockResolvedValue({ - status: { - indexFields: [], - logIndicesExist: false, - metricIndicesExist: true, - }, + hasData: true, }); const hasData = createMetricsHasData(mockedGetStartServices); const response = await hasData(); @@ -43,11 +39,7 @@ describe('Metrics UI Observability Homepage Functions', () => { it('should return false when false', async () => { const { core, mockedGetStartServices } = setup(); core.http.get.mockResolvedValue({ - status: { - indexFields: [], - logIndicesExist: false, - metricIndicesExist: false, - }, + hasData: false, }); const hasData = createMetricsHasData(mockedGetStartServices); const response = await hasData(); @@ -76,6 +68,7 @@ describe('Metrics UI Observability Homepage Functions', () => { metrics: [{ type: 'cpu' }, { type: 'memory' }, { type: 'rx' }, { type: 'tx' }], groupBy: [], nodeType: 'host', + includeTimeseries: true, timerange: { from: startTime.valueOf(), to: endTime.valueOf(), diff --git a/x-pack/plugins/infra/public/metrics_overview_fetchers.ts b/x-pack/plugins/infra/public/metrics_overview_fetchers.ts index d10ad5dda53204d..15751fab39abc0f 100644 --- a/x-pack/plugins/infra/public/metrics_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/metrics_overview_fetchers.ts @@ -16,15 +16,16 @@ import { } from '../common/http_api/snapshot_api'; import { SnapshotMetricType } from '../common/inventory_models/types'; import { InfraClientCoreSetup } from './types'; -import { SourceResponse } from '../common/http_api/source_api'; export const createMetricsHasData = ( getStartServices: InfraClientCoreSetup['getStartServices'] ) => async () => { const [coreServices] = await getStartServices(); const { http } = coreServices; - const results = await http.get('/api/metrics/source/default/metrics'); - return results.status.metricIndicesExist; + const results = await http.get<{ hasData: boolean }>( + '/api/metrics/source/default/metrics/hasData' + ); + return results.hasData; }; export const average = (values: number[]) => (values.length ? sum(values) / values.length : 0); @@ -88,6 +89,7 @@ export const createMetricsFetchData = ( metrics: ['cpu', 'memory', 'rx', 'tx'].map((type) => ({ type })) as SnapshotMetricInput[], groupBy: [], nodeType: 'host', + includeTimeseries: true, timerange: { from: moment(startTime).valueOf(), to: moment(endTime).valueOf(), diff --git a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts index 65ea53a8465bbfc..5a0a996287959cd 100644 --- a/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts +++ b/x-pack/plugins/infra/public/utils/logs_overview_fetchers.ts @@ -6,6 +6,7 @@ import { encode } from 'rison-node'; import { i18n } from '@kbn/i18n'; +import { SearchResponse } from 'src/plugins/data/public'; import { DEFAULT_SOURCE_ID } from '../../common/constants'; import { InfraClientCoreSetup, InfraClientStartDeps } from '../types'; import { @@ -89,9 +90,10 @@ async function fetchLogsOverview( params: FetchDataParams, dataPlugin: InfraClientStartDeps['data'] ): Promise { - const esSearcher = dataPlugin.search.getSearchStrategy('es'); return new Promise((resolve, reject) => { - esSearcher + let esResponse: SearchResponse = {}; + + dataPlugin.search .search({ params: { index: logParams.index, @@ -103,14 +105,15 @@ async function fetchLogsOverview( }, }) .subscribe( - (response) => { - if (response.rawResponse.aggregations) { - resolve(processLogsOverviewAggregations(response.rawResponse.aggregations)); + (response) => (esResponse = response.rawResponse), + (error) => reject(error), + () => { + if (esResponse.aggregations) { + resolve(processLogsOverviewAggregations(esResponse.aggregations)); } else { resolve({ stats: {}, series: {} }); } - }, - (error) => reject(error) + } ); }); } diff --git a/x-pack/plugins/infra/server/routes/source/index.ts b/x-pack/plugins/infra/server/routes/source/index.ts index 62b7fd7ba902f58..2843897071e1914 100644 --- a/x-pack/plugins/infra/server/routes/source/index.ts +++ b/x-pack/plugins/infra/server/routes/source/index.ts @@ -37,22 +37,21 @@ export const initSourceRoute = (libs: InfraBackendLibs) => { try { const { type, sourceId } = request.params; - const source = await libs.sources.getSourceConfiguration( - requestContext.core.savedObjects.client, - sourceId - ); + const [source, logIndicesExist, metricIndicesExist, indexFields] = await Promise.all([ + libs.sources.getSourceConfiguration(requestContext.core.savedObjects.client, sourceId), + libs.sourceStatus.hasLogIndices(requestContext, sourceId), + libs.sourceStatus.hasMetricIndices(requestContext, sourceId), + libs.fields.getFields(requestContext, sourceId, typeToInfraIndexType(type)), + ]); + if (!source) { return response.notFound(); } const status = { - logIndicesExist: await libs.sourceStatus.hasLogIndices(requestContext, sourceId), - metricIndicesExist: await libs.sourceStatus.hasMetricIndices(requestContext, sourceId), - indexFields: await libs.fields.getFields( - requestContext, - sourceId, - typeToInfraIndexType(type) - ), + logIndicesExist, + metricIndicesExist, + indexFields, }; return response.ok({ @@ -65,4 +64,35 @@ export const initSourceRoute = (libs: InfraBackendLibs) => { } } ); + + framework.registerRoute( + { + method: 'get', + path: '/api/metrics/source/{sourceId}/{type}/hasData', + validate: { + params: schema.object({ + sourceId: schema.string(), + type: schema.string(), + }), + }, + }, + async (requestContext, request, response) => { + try { + const { type, sourceId } = request.params; + + const hasData = + type === 'metrics' + ? await libs.sourceStatus.hasMetricIndices(requestContext, sourceId) + : await libs.sourceStatus.hasLogIndices(requestContext, sourceId); + + return response.ok({ + body: { hasData }, + }); + } catch (error) { + return response.internalError({ + body: error.message, + }); + } + } + ); }; diff --git a/x-pack/plugins/ingest_manager/common/constants/agent.ts b/x-pack/plugins/ingest_manager/common/constants/agent.ts index e9226fa68492531..7652c6ac87bced3 100644 --- a/x-pack/plugins/ingest_manager/common/constants/agent.ts +++ b/x-pack/plugins/ingest_manager/common/constants/agent.ts @@ -16,3 +16,6 @@ export const AGENT_POLLING_THRESHOLD_MS = 30000; export const AGENT_POLLING_INTERVAL = 1000; export const AGENT_UPDATE_LAST_CHECKIN_INTERVAL_MS = 30000; export const AGENT_UPDATE_ACTIONS_INTERVAL_MS = 5000; + +export const AGENT_CONFIG_ROLLUP_RATE_LIMIT_INTERVAL_MS = 5000; +export const AGENT_CONFIG_ROLLUP_RATE_LIMIT_REQUEST_PER_INTERVAL = 60; diff --git a/x-pack/plugins/ingest_manager/common/types/index.ts b/x-pack/plugins/ingest_manager/common/types/index.ts index 7f81b04f5e84a00..ff08b8a9252046f 100644 --- a/x-pack/plugins/ingest_manager/common/types/index.ts +++ b/x-pack/plugins/ingest_manager/common/types/index.ts @@ -24,6 +24,8 @@ export interface IngestManagerConfigType { host?: string; ca_sha256?: string; }; + agentConfigRollupRateLimitIntervalMs: number; + agentConfigRollupRateLimitRequestPerInterval: number; }; } diff --git a/x-pack/plugins/ingest_manager/server/constants/index.ts b/x-pack/plugins/ingest_manager/server/constants/index.ts index 650211ce9c1b2e5..d3c074ff2e8d0ea 100644 --- a/x-pack/plugins/ingest_manager/server/constants/index.ts +++ b/x-pack/plugins/ingest_manager/server/constants/index.ts @@ -10,6 +10,8 @@ export { AGENT_POLLING_THRESHOLD_MS, AGENT_POLLING_INTERVAL, AGENT_UPDATE_LAST_CHECKIN_INTERVAL_MS, + AGENT_CONFIG_ROLLUP_RATE_LIMIT_REQUEST_PER_INTERVAL, + AGENT_CONFIG_ROLLUP_RATE_LIMIT_INTERVAL_MS, AGENT_UPDATE_ACTIONS_INTERVAL_MS, INDEX_PATTERN_PLACEHOLDER_SUFFIX, // Routes diff --git a/x-pack/plugins/ingest_manager/server/index.ts b/x-pack/plugins/ingest_manager/server/index.ts index 5d6a1ad321b6dd4..811ec8a3d022240 100644 --- a/x-pack/plugins/ingest_manager/server/index.ts +++ b/x-pack/plugins/ingest_manager/server/index.ts @@ -37,6 +37,8 @@ export const config = { host: schema.maybe(schema.string()), ca_sha256: schema.maybe(schema.string()), }), + agentConfigRollupRateLimitIntervalMs: schema.number({ defaultValue: 5000 }), + agentConfigRollupRateLimitRequestPerInterval: schema.number({ defaultValue: 50 }), }), }), }; diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin/rxjs_utils.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin/rxjs_utils.ts index 1f9bba8b12be431..a806169019a1ed5 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin/rxjs_utils.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin/rxjs_utils.ts @@ -3,12 +3,13 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Observable } from 'rxjs'; + +import * as Rx from 'rxjs'; export class AbortError extends Error {} export const toPromiseAbortable = ( - observable: Observable, + observable: Rx.Observable, signal?: AbortSignal ): Promise => new Promise((resolve, reject) => { @@ -41,3 +42,63 @@ export const toPromiseAbortable = ( signal.addEventListener('abort', listener, { once: true }); } }); + +export function createLimiter(ratelimitIntervalMs: number, ratelimitRequestPerInterval: number) { + function createCurrentInterval() { + return { + startedAt: Rx.asyncScheduler.now(), + numRequests: 0, + }; + } + + let currentInterval: { startedAt: number; numRequests: number } = createCurrentInterval(); + let observers: Array<[Rx.Subscriber, any]> = []; + let timerSubscription: Rx.Subscription | undefined; + + function createTimeout() { + if (timerSubscription) { + return; + } + timerSubscription = Rx.asyncScheduler.schedule(() => { + timerSubscription = undefined; + currentInterval = createCurrentInterval(); + for (const [waitingObserver, value] of observers) { + if (currentInterval.numRequests >= ratelimitRequestPerInterval) { + createTimeout(); + continue; + } + currentInterval.numRequests++; + waitingObserver.next(value); + } + }, ratelimitIntervalMs); + } + + return function limit(): Rx.MonoTypeOperatorFunction { + return (observable) => + new Rx.Observable((observer) => { + const subscription = observable.subscribe({ + next(value) { + if (currentInterval.numRequests < ratelimitRequestPerInterval) { + currentInterval.numRequests++; + observer.next(value); + return; + } + + observers = [...observers, [observer, value]]; + createTimeout(); + }, + error(err) { + observer.error(err); + }, + complete() { + observer.complete(); + }, + }); + + return () => { + observers = observers.filter((o) => o[0] !== observer); + subscription.unsubscribe(); + }; + }); + }; +} diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_new_actions.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_new_actions.ts index 0f30ab409f38106..5ceb774a1946c2a 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_new_actions.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin/state_new_actions.ts @@ -28,7 +28,7 @@ import * as APIKeysService from '../../api_keys'; import { AGENT_SAVED_OBJECT_TYPE, AGENT_UPDATE_ACTIONS_INTERVAL_MS } from '../../../constants'; import { createAgentAction, getNewActionsSince } from '../actions'; import { appContextService } from '../../app_context'; -import { toPromiseAbortable, AbortError } from './rxjs_utils'; +import { toPromiseAbortable, AbortError, createLimiter } from './rxjs_utils'; function getInternalUserSOClient() { const fakeRequest = ({ @@ -95,19 +95,23 @@ async function getOrCreateAgentDefaultOutputAPIKey( return outputAPIKey.key; } -async function createAgentActionFromConfigIfOutdated( - soClient: SavedObjectsClientContract, - agent: Agent, - config: FullAgentConfig | null -) { +function shouldCreateAgentConfigAction(agent: Agent, config: FullAgentConfig | null): boolean { if (!config || !config.revision) { - return; + return false; } const isAgentConfigOutdated = !agent.config_revision || agent.config_revision < config.revision; if (!isAgentConfigOutdated) { - return; + return false; } + return true; +} + +async function createAgentActionFromConfig( + soClient: SavedObjectsClientContract, + agent: Agent, + config: FullAgentConfig | null +) { // Deep clone !not supporting Date, and undefined value. const newConfig = JSON.parse(JSON.stringify(config)); @@ -129,6 +133,11 @@ export function agentCheckinStateNewActionsFactory() { // Shared Observables const agentConfigs$ = new Map>(); const newActions$ = createNewActionsSharedObservable(); + // Rx operators + const rateLimiter = createLimiter( + appContextService.getConfig()?.fleet.agentConfigRollupRateLimitIntervalMs || 5000, + appContextService.getConfig()?.fleet.agentConfigRollupRateLimitRequestPerInterval || 50 + ); async function subscribeToNewActions( soClient: SavedObjectsClientContract, @@ -148,7 +157,9 @@ export function agentCheckinStateNewActionsFactory() { } const stream$ = agentConfig$.pipe( timeout(appContextService.getConfig()?.fleet.pollingRequestTimeout || 0), - mergeMap((config) => createAgentActionFromConfigIfOutdated(soClient, agent, config)), + filter((config) => shouldCreateAgentConfigAction(agent, config)), + rateLimiter(), + mergeMap((config) => createAgentActionFromConfig(soClient, agent, config)), merge(newActions$), mergeMap(async (data) => { if (!data) { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.helpers.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.helpers.tsx index cc3817d92d5e3c0..e7258a74f473269 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.helpers.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.helpers.tsx @@ -112,7 +112,7 @@ const createActions = (testBed: TestBed) => { moveProcessor(processorSelector: string, dropZoneSelector: string) { act(() => { - find(`${processorSelector}.moveItemButton`).simulate('click'); + find(`${processorSelector}.moveItemButton`).simulate('change'); }); component.update(); act(() => { @@ -144,12 +144,13 @@ const createActions = (testBed: TestBed) => { startAndCancelMove(processorSelector: string) { act(() => { - find(`${processorSelector}.moveItemButton`).simulate('click'); + find(`${processorSelector}.moveItemButton`).simulate('change'); }); component.update(); act(() => { - find(`${processorSelector}.cancelMoveItemButton`).simulate('click'); + find(`${processorSelector}.cancelMoveItemButton`).simulate('change'); }); + component.update(); }, duplicateProcessor(processorSelector: string) { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.test.tsx index a4bbf840dff7101..acfa012990b21c4 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.test.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/pipeline_processors_editor.test.tsx @@ -153,7 +153,7 @@ describe('Pipeline Editor', () => { const processorSelector = 'processors>0'; actions.startAndCancelMove(processorSelector); // Assert that we have exited move mode for this processor - expect(exists(`moveItemButton-${processorSelector}`)); + expect(exists(`${processorSelector}.moveItemButton`)).toBe(true); const [onUpdateResult] = onUpdate.mock.calls[onUpdate.mock.calls.length - 1]; const { processors } = onUpdateResult.getData(); // Assert that nothing has changed diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/_shared.scss b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/_shared.scss index 8d17a3970d94f85..c7c49c00bb5cfc8 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/_shared.scss +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/_shared.scss @@ -1,2 +1,2 @@ -$dropZoneZIndex: 1; /* Prevent the next item down from obscuring the button */ -$cancelButtonZIndex: 2; +$dropZoneZIndex: $euiZLevel1; /* Prevent the next item down from obscuring the button */ +$cancelButtonZIndex: $euiZLevel2; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/index.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/index.ts index 02bafdb32602444..fb3f513300c6a01 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/index.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/index.ts @@ -4,4 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -export { PipelineProcessorsEditorItem, Handlers } from './pipeline_processors_editor_item'; +export { PipelineProcessorsEditorItem } from './pipeline_processors_editor_item.container'; + +export { Handlers } from './types'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.container.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.container.tsx new file mode 100644 index 000000000000000..5201320e97d3a10 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.container.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent } from 'react'; + +import { usePipelineProcessorsContext } from '../../context'; + +import { + PipelineProcessorsEditorItem as ViewComponent, + Props as ViewComponentProps, +} from './pipeline_processors_editor_item'; + +type Props = Omit; + +export const PipelineProcessorsEditorItem: FunctionComponent = (props) => { + const { state } = usePipelineProcessorsContext(); + + return ( + + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.scss b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.scss index 6b5e11808460632..85a123b421975fb 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.scss +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.scss @@ -1,10 +1,12 @@ @import '../shared'; .pipelineProcessorsEditor__item { - transition: border-color 1s; + transition: border-color $euiAnimSpeedExtraSlow $euiAnimSlightResistance; min-height: 50px; + &--selected { - border: 1px solid $euiColorPrimary; + border: $euiBorderThin; + border-color: $euiColorPrimary; } &--displayNone { @@ -25,15 +27,14 @@ } &__textContainer { - padding: 4px; - border-radius: 2px; - - transition: border-color 0.3s; - border: 2px solid transparent; + cursor: text; + border-bottom: 1px dashed transparent; &--notEditing { + border-bottom: $euiBorderEditable; + border-width: $euiBorderWidthThin; &:hover { - border: 2px solid $euiColorLightShade; + border-color: $euiColorMediumShade; } } } @@ -46,12 +47,17 @@ } &__textInput { - height: 21px; - min-width: 150px; + height: $euiSizeL; + min-width: 200px; } - &__cancelMoveButton { - // Ensure that the cancel button is above the drop zones - z-index: $cancelButtonZIndex; + &__moveButton { + &:hover { + transform: none !important; + } + &--cancel { + // Ensure that the cancel button is above the drop zones + z-index: $cancelButtonZIndex; + } } } diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.tsx index 09c047d1d51b764..97b57a971ff7d6f 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/pipeline_processors_editor_item.tsx @@ -8,7 +8,7 @@ import classNames from 'classnames'; import React, { FunctionComponent, memo } from 'react'; import { EuiButtonIcon, - EuiButton, + EuiButtonToggle, EuiFlexGroup, EuiFlexItem, EuiPanel, @@ -16,25 +16,23 @@ import { EuiToolTip, } from '@elastic/eui'; -import { ProcessorInternal, ProcessorSelector } from '../../types'; +import { ProcessorInternal, ProcessorSelector, ContextValueEditor } from '../../types'; import { selectorToDataTestSubject } from '../../utils'; +import { ProcessorsDispatch } from '../../processors_reducer'; -import { usePipelineProcessorsContext } from '../../context'; +import { ProcessorInfo } from '../processors_tree'; import './pipeline_processors_editor_item.scss'; import { InlineTextInput } from './inline_text_input'; import { ContextMenu } from './context_menu'; import { i18nTexts } from './i18n_texts'; -import { ProcessorInfo } from '../processors_tree'; - -export interface Handlers { - onMove: () => void; - onCancelMove: () => void; -} +import { Handlers } from './types'; export interface Props { processor: ProcessorInternal; + processorsDispatch: ProcessorsDispatch; + editor: ContextValueEditor; handlers: Handlers; selector: ProcessorSelector; description?: string; @@ -43,18 +41,16 @@ export interface Props { } export const PipelineProcessorsEditorItem: FunctionComponent = memo( - ({ + function PipelineProcessorsEditorItem({ processor, description, handlers: { onCancelMove, onMove }, selector, movingProcessor, renderOnFailureHandlers, - }) => { - const { - state: { editor, processors }, - } = usePipelineProcessorsContext(); - + editor, + processorsDispatch, + }) { const isDisabled = editor.mode.id !== 'idle'; const isInMoveMode = Boolean(movingProcessor); const isMovingThisProcessor = processor.id === movingProcessor?.id; @@ -78,9 +74,41 @@ export const PipelineProcessorsEditorItem: FunctionComponent = memo( 'pipelineProcessorsEditor__item--displayNone': isInMoveMode && !processor.options.description, }); - const cancelMoveButtonClasses = classNames('pipelineProcessorsEditor__item__cancelMoveButton', { - 'pipelineProcessorsEditor__item--displayNone': !isMovingThisProcessor, - }); + const renderMoveButton = () => { + const label = !isMovingThisProcessor + ? i18nTexts.moveButtonLabel + : i18nTexts.cancelMoveButtonLabel; + const dataTestSubj = !isMovingThisProcessor ? 'moveItemButton' : 'cancelMoveItemButton'; + const moveButtonClasses = classNames('pipelineProcessorsEditor__item__moveButton', { + 'pipelineProcessorsEditor__item__moveButton--cancel': isMovingThisProcessor, + }); + const icon = isMovingThisProcessor ? 'cross' : 'sortable'; + const moveButton = ( + (!isMovingThisProcessor ? onMove() : onCancelMove())} + /> + ); + // Remove the tooltip from the DOM to prevent it from lingering if the mouse leave event + // did not fire. + return ( +
+ {!isInMoveMode ? ( + {moveButton} + ) : ( + moveButton + )} +
+ ); + }; return ( @@ -93,6 +121,7 @@ export const PipelineProcessorsEditorItem: FunctionComponent = memo( > + {renderMoveButton()} = memo( description: nextDescription, }; } - processors.dispatch({ + processorsDispatch({ type: 'updateProcessor', payload: { processor: { @@ -149,25 +178,6 @@ export const PipelineProcessorsEditorItem: FunctionComponent = memo( )} - - {!isInMoveMode && ( - - - - )} - - - - {i18nTexts.cancelMoveButtonLabel} - - @@ -183,7 +193,7 @@ export const PipelineProcessorsEditorItem: FunctionComponent = memo( editor.setMode({ id: 'removingProcessor', arg: { selector } }); }} onDuplicate={() => { - processors.dispatch({ + processorsDispatch({ type: 'duplicateProcessor', payload: { source: selector, diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/types.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/types.ts new file mode 100644 index 000000000000000..893aee13fff8f91 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/pipeline_processors_editor_item/types.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface Handlers { + onMove: () => void; + onCancelMove: () => void; +} diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.container.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.container.tsx index d76e9225c1a1375..2a537ba082eecdd 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.container.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.container.tsx @@ -10,6 +10,7 @@ import { useForm, OnFormUpdateArg, FormData } from '../../../../../shared_import import { ProcessorInternal } from '../../types'; import { ProcessorSettingsForm as ViewComponent } from './processor_settings_form'; +import { usePipelineProcessorsContext } from '../../context'; export type ProcessorSettingsFromOnSubmitArg = Omit; @@ -32,6 +33,10 @@ export const ProcessorSettingsForm: FunctionComponent = ({ onSubmit, ...rest }) => { + const { + links: { esDocsBasePath }, + } = usePipelineProcessorsContext(); + const handleSubmit = useCallback( async (data: FormData, isValid: boolean) => { if (isValid) { @@ -60,5 +65,7 @@ export const ProcessorSettingsForm: FunctionComponent = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [onFormUpdate]); - return ; + return ( + + ); }; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.tsx index 3eccda55fbb3a4a..015adae83e71ef3 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/processor_settings_form/processor_settings_form.tsx @@ -21,7 +21,6 @@ import { } from '@elastic/eui'; import { Form, FormDataProvider, FormHook } from '../../../../../shared_imports'; -import { usePipelineProcessorsContext } from '../../context'; import { ProcessorInternal } from '../../types'; import { DocumentationButton } from './documentation_button'; @@ -35,6 +34,7 @@ export interface Props { form: FormHook; onClose: () => void; onOpen: () => void; + esDocsBasePath: string; } const updateButtonLabel = i18n.translate( @@ -52,11 +52,7 @@ const cancelButtonLabel = i18n.translate( ); export const ProcessorSettingsForm: FunctionComponent = memo( - ({ processor, form, isOnFailure, onClose, onOpen }) => { - const { - links: { esDocsBasePath }, - } = usePipelineProcessorsContext(); - + ({ processor, form, isOnFailure, onClose, onOpen, esDocsBasePath }) => { const flyoutTitleContent = isOnFailure ? ( = ({ }; }, [onAction, stringSelector, processor]); // eslint-disable-line react-hooks/exhaustive-deps - const renderOnFailureHandlersTree = () => { + const renderOnFailureHandlersTree = useCallback(() => { if (!processor.onFailure?.length) { return; } @@ -79,7 +79,7 @@ export const TreeNode: FunctionComponent = ({ /> ); - }; + }, [processor.onFailure, stringSelector, onAction, movingProcessor, level]); // eslint-disable-line react-hooks/exhaustive-deps return ( ; - }; - }; -} - const PipelineProcessorsContext = createContext({} as any); export interface Props { @@ -81,7 +64,9 @@ export const PipelineProcessorsContextProvider: FunctionComponent = ({ children, }) => { const initRef = useRef(false); - const [mode, setMode] = useState({ id: 'idle' }); + const [mode, setMode] = useState(() => ({ + id: 'idle', + })); const deserializedResult = useMemo( () => deserialize({ @@ -199,15 +184,24 @@ export const PipelineProcessorsContextProvider: FunctionComponent = ({ [processorsDispatch, setMode] ); + // Memoize the state object to ensure we do not trigger unnecessary re-renders and so + // this object can be used safely further down the tree component tree. + const state = useMemo(() => { + return { + editor: { + mode, + setMode, + }, + processors: { state: processorsState, dispatch: processorsDispatch }, + }; + }, [mode, setMode, processorsState, processorsDispatch]); + return ( {children} diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/types.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/types.ts index aea8f0f0910f451..aaca4108bb58319 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/types.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/types.ts @@ -4,9 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import { OnFormUpdateArg } from '../../../../../../../src/plugins/es_ui_shared/static/forms/hook_form_lib'; +import { Dispatch } from 'react'; +import { OnFormUpdateArg } from '../../../shared_imports'; import { SerializeResult } from './serialize'; -import { ProcessorInfo } from './components/processors_tree'; +import { OnActionHandler, ProcessorInfo } from './components/processors_tree'; +import { ProcessorsDispatch, State as ProcessorsReducerState } from './processors_reducer'; + +export interface Links { + esDocsBasePath: string; +} /** * An array of keys that map to a value in an object @@ -51,3 +57,24 @@ export type EditorMode = | { id: 'editingProcessor'; arg: { processor: ProcessorInternal; selector: ProcessorSelector } } | { id: 'removingProcessor'; arg: { selector: ProcessorSelector } } | { id: 'idle' }; + +export interface ContextValueEditor { + mode: EditorMode; + setMode: Dispatch; +} + +export interface ContextValueProcessors { + state: ProcessorsReducerState; + dispatch: ProcessorsDispatch; +} + +export interface ContextValueState { + processors: ContextValueProcessors; + editor: ContextValueEditor; +} + +export interface ContextValue { + links: Links; + onTreeAction: OnActionHandler; + state: ContextValueState; +} diff --git a/x-pack/plugins/security_solution/public/alerts/components/alerts_table/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/alerts_table/index.tsx index 98bb6434ddafd7a..65f225c05459872 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/alerts_table/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/alerts_table/index.tsx @@ -48,6 +48,7 @@ import { displaySuccessToast, displayErrorToast, } from '../../../common/components/toasters'; +import { getInvestigateInResolverAction } from '../../../timelines/components/timeline/body/helpers'; interface OwnProps { timelineId: TimelineIdLiteral; @@ -331,13 +332,14 @@ export const AlertsTableComponent: React.FC = ({ useEffect(() => { initializeTimeline({ - id: timelineId, - documentType: i18n.ALERTS_DOCUMENT_TYPE, defaultModel: alertsDefaultModel, + documentType: i18n.ALERTS_DOCUMENT_TYPE, footerText: i18n.TOTAL_COUNT_OF_ALERTS, + id: timelineId, loadingText: i18n.LOADING_ALERTS, - title: i18n.ALERTS_TABLE_TITLE, selectAll: canUserCRUD ? selectAll : false, + timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId })], + title: i18n.ALERTS_TABLE_TITLE, }); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); diff --git a/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx b/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx index 6783fcbd17582bb..12b2853f8f7e1d8 100644 --- a/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx +++ b/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx @@ -69,7 +69,7 @@ const AlertsTableComponent: React.FC = ({ }) => { const dispatch = useDispatch(); const alertsFilter = useMemo(() => [...defaultAlertsFilters, ...pageFilters], [pageFilters]); - const { initializeTimeline, setTimelineRowActions } = useManageTimeline(); + const { initializeTimeline } = useManageTimeline(); useEffect(() => { initializeTimeline({ @@ -77,13 +77,10 @@ const AlertsTableComponent: React.FC = ({ documentType: i18n.ALERTS_DOCUMENT_TYPE, defaultModel: alertsDefaultModel, footerText: i18n.TOTAL_COUNT_OF_ALERTS, + timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId })], title: i18n.ALERTS_TABLE_TITLE, unit: i18n.UNIT, }); - setTimelineRowActions({ - id: timelineId, - timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId })], - }); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); return ( diff --git a/x-pack/plugins/security_solution/public/common/components/drag_and_drop/draggable_wrapper_hover_content.test.tsx b/x-pack/plugins/security_solution/public/common/components/drag_and_drop/draggable_wrapper_hover_content.test.tsx index 3507b0f8c447dd5..432e369cdd0f63d 100644 --- a/x-pack/plugins/security_solution/public/common/components/drag_and_drop/draggable_wrapper_hover_content.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/drag_and_drop/draggable_wrapper_hover_content.test.tsx @@ -18,7 +18,7 @@ import { useAddToTimeline } from '../../hooks/use_add_to_timeline'; import { DraggableWrapperHoverContent } from './draggable_wrapper_hover_content'; import { ManageGlobalTimeline, - timelineDefaults, + getTimelineDefaults, } from '../../../timelines/components/manage_timeline'; import { TimelineId } from '../../../../common/types/timeline'; @@ -152,10 +152,7 @@ describe('DraggableWrapperHoverContent', () => { beforeEach(() => { onFilterAdded = jest.fn(); const manageTimelineForTesting = { - [timelineId]: { - ...timelineDefaults, - id: timelineId, - }, + [timelineId]: getTimelineDefaults(timelineId), }; wrapper = mount( @@ -249,8 +246,7 @@ describe('DraggableWrapperHoverContent', () => { const manageTimelineForTesting = { [timelineId]: { - ...timelineDefaults, - id: timelineId, + ...getTimelineDefaults(timelineId), filterManager, }, }; diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx index 9e38b14c4334a58..910030d41a23e02 100644 --- a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx +++ b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx @@ -7,7 +7,6 @@ import { EuiPanel } from '@elastic/eui'; import { getOr, isEmpty, union } from 'lodash/fp'; import React, { useEffect, useMemo, useState } from 'react'; -import { useDispatch } from 'react-redux'; import styled from 'styled-components'; import deepEqual from 'fast-deep-equal'; @@ -35,7 +34,6 @@ import { } from '../../../../../../../src/plugins/data/public'; import { inputsModel } from '../../store'; import { useManageTimeline } from '../../../timelines/components/manage_timeline'; -import { getInvestigateInResolverAction } from '../../../timelines/components/timeline/body/helpers'; const DEFAULT_EVENTS_VIEWER_HEIGHT = 500; @@ -93,7 +91,6 @@ const EventsViewerComponent: React.FC = ({ toggleColumn, utilityBar, }) => { - const dispatch = useDispatch(); const columnsHeader = isEmpty(columns) ? defaultHeaders : columns; const kibana = useKibana(); const { filterManager } = useKibana().services.data.query; @@ -103,16 +100,8 @@ const EventsViewerComponent: React.FC = ({ getManageTimelineById, setIsTimelineLoading, setTimelineFilterManager, - setTimelineRowActions, } = useManageTimeline(); - useEffect(() => { - setTimelineRowActions({ - id, - timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId: id })], - }); - }, [setTimelineRowActions, id, dispatch]); - useEffect(() => { setIsTimelineLoading({ id, isLoading: isQueryLoading }); // eslint-disable-next-line react-hooks/exhaustive-deps diff --git a/x-pack/plugins/security_solution/public/common/components/top_n/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/top_n/index.test.tsx index 503e9983692f16f..da6ec784af6d44e 100644 --- a/x-pack/plugins/security_solution/public/common/components/top_n/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/top_n/index.test.tsx @@ -25,7 +25,7 @@ import { Props } from './top_n'; import { StatefulTopN } from '.'; import { ManageGlobalTimeline, - timelineDefaults, + getTimelineDefaults, } from '../../../timelines/components/manage_timeline'; import { TimelineId } from '../../../../common/types/timeline'; @@ -272,8 +272,7 @@ describe('StatefulTopN', () => { filterManager = new FilterManager(mockUiSettingsForFilterManager); const manageTimelineForTesting = { [TimelineId.active]: { - ...timelineDefaults, - id: TimelineId.active, + ...getTimelineDefaults(TimelineId.active), filterManager, }, }; @@ -351,8 +350,7 @@ describe('StatefulTopN', () => { const manageTimelineForTesting = { [TimelineId.active]: { - ...timelineDefaults, - id: TimelineId.active, + ...getTimelineDefaults(TimelineId.active), filterManager, documentType: 'alerts', }, diff --git a/x-pack/plugins/security_solution/public/hosts/pages/navigation/events_query_tab_body.tsx b/x-pack/plugins/security_solution/public/hosts/pages/navigation/events_query_tab_body.tsx index 64cfacaeaf6dc09..58026a28a04ce43 100644 --- a/x-pack/plugins/security_solution/public/hosts/pages/navigation/events_query_tab_body.tsx +++ b/x-pack/plugins/security_solution/public/hosts/pages/navigation/events_query_tab_body.tsx @@ -5,6 +5,7 @@ */ import React, { useEffect } from 'react'; +import { useDispatch } from 'react-redux'; import { TimelineId } from '../../../../common/types/timeline'; import { StatefulEventsViewer } from '../../../common/components/events_viewer'; import { HostsComponentsQueryProps } from './types'; @@ -18,6 +19,7 @@ import { MatrixHistogramContainer } from '../../../common/components/matrix_hist import * as i18n from '../translations'; import { HistogramType } from '../../../graphql/types'; import { useManageTimeline } from '../../../timelines/components/manage_timeline'; +import { getInvestigateInResolverAction } from '../../../timelines/components/timeline/body/helpers'; const EVENTS_HISTOGRAM_ID = 'eventsOverTimeQuery'; @@ -57,13 +59,17 @@ export const EventsQueryTabBody = ({ startDate, }: HostsComponentsQueryProps) => { const { initializeTimeline } = useManageTimeline(); + const dispatch = useDispatch(); useEffect(() => { initializeTimeline({ id: TimelineId.hostsPageEvents, defaultModel: eventsDefaultModel, + timelineRowActions: [ + getInvestigateInResolverAction({ dispatch, timelineId: TimelineId.hostsPageEvents }), + ], }); - }, [initializeTimeline]); + }, [dispatch, initializeTimeline]); useEffect(() => { return () => { diff --git a/x-pack/plugins/security_solution/public/timelines/components/manage_timeline/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/manage_timeline/index.tsx index 3b40c36fccd163f..c71087de4a07ee8 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/manage_timeline/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/manage_timeline/index.tsx @@ -6,6 +6,7 @@ import React, { createContext, useCallback, useContext, useReducer } from 'react'; import { noop } from 'lodash/fp'; + // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { FilterManager } from '../../../../../../../src/plugins/data/public/query/filter_manager'; import { TimelineRowAction } from '../timeline/body/actions'; @@ -22,6 +23,7 @@ interface ManageTimelineInit { indexToAdd?: string[] | null; loadingText?: string; selectAll?: boolean; + timelineRowActions: TimelineRowAction[]; title?: string; unit?: (totalCount: number) => string; } @@ -73,19 +75,20 @@ type ActionManageTimeline = payload: { filterManager: FilterManager }; }; -export const timelineDefaults = { +export const getTimelineDefaults = (id: string) => ({ indexToAdd: null, defaultModel: timelineDefaultModel, loadingText: i18n.LOADING_EVENTS, footerText: i18nF.TOTAL_COUNT_OF_EVENTS, documentType: i18nF.TOTAL_COUNT_OF_EVENTS, selectAll: false, + id, isLoading: false, queryFields: [], timelineRowActions: [], title: i18n.EVENTS, unit: (n: number) => i18n.UNIT(n), -}; +}); const reducerManageTimeline = ( state: ManageTimelineById, action: ActionManageTimeline @@ -95,7 +98,7 @@ const reducerManageTimeline = ( return { ...state, [action.id]: { - ...timelineDefaults, + ...getTimelineDefaults(action.id), ...state[action.id], ...action.payload, }, @@ -216,8 +219,8 @@ const useTimelineManager = (manageTimelineForTesting?: ManageTimelineById): UseT if (state[id] != null) { return state[id]; } - initializeTimeline({ id }); - return { ...timelineDefaults, id }; + initializeTimeline({ id, timelineRowActions: [] }); + return getTimelineDefaults(id); }, [initializeTimeline, state] ); @@ -236,7 +239,7 @@ const useTimelineManager = (manageTimelineForTesting?: ManageTimelineById): UseT }; const init = { - getManageTimelineById: (id: string) => ({ ...timelineDefaults, id }), + getManageTimelineById: (id: string) => getTimelineDefaults(id), getTimelineFilterManager: () => undefined, setIndexToAdd: () => undefined, isManagedTimeline: () => false, @@ -245,6 +248,7 @@ const init = { setTimelineRowActions: () => noop, setTimelineFilterManager: () => noop, }; + const ManageTimelineContext = createContext(init); export const useManageTimeline = () => useContext(ManageTimelineContext); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx index 8855cba7a4c890d..ae00edf5d1429bd 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx @@ -194,8 +194,7 @@ export const EventColumnView = React.memo( , ] : grouped.icon; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [button, ecsData, timelineActions, isPopoverOpen]); // , isPopoverOpen, closePopover, onButtonClick]); + }, [button, closePopover, id, onClickCb, ecsData, timelineActions, isPopoverOpen]); return ( diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_providers.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_providers.test.tsx index 1c85b6e6d72bf61..3a8c0d883121747 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_providers.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_providers.test.tsx @@ -13,7 +13,7 @@ import { useMountAppended } from '../../../../common/utils/use_mount_appended'; import { DataProviders } from '.'; import { DataProvider } from './data_provider'; import { mockDataProviders } from './mock/mock_data_providers'; -import { ManageGlobalTimeline, timelineDefaults } from '../../manage_timeline'; +import { ManageGlobalTimeline, getTimelineDefaults } from '../../manage_timeline'; import { FilterManager } from '../../../../../../../../src/plugins/data/public/query/filter_manager'; import { createKibanaCoreStartMock } from '../../../../common/mock/kibana_core'; const mockUiSettingsForFilterManager = createKibanaCoreStartMock().uiSettings; @@ -28,8 +28,7 @@ describe('DataProviders', () => { test('renders correctly against snapshot', () => { const manageTimelineForTesting = { foo: { - ...timelineDefaults, - id: 'foo', + ...getTimelineDefaults('foo'), filterManager, }, }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/providers.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/providers.test.tsx index 3ad83914c73b954..9dc0b762244582a 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/providers.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/providers.test.tsx @@ -16,7 +16,7 @@ import { mockDataProviders } from './mock/mock_data_providers'; import { Providers } from './providers'; import { DELETE_CLASS_NAME, ENABLE_CLASS_NAME, EXCLUDE_CLASS_NAME } from './provider_item_actions'; import { useMountAppended } from '../../../../common/utils/use_mount_appended'; -import { ManageGlobalTimeline, timelineDefaults } from '../../manage_timeline'; +import { ManageGlobalTimeline, getTimelineDefaults } from '../../manage_timeline'; const mockUiSettingsForFilterManager = createKibanaCoreStartMock().uiSettings; @@ -27,8 +27,7 @@ describe('Providers', () => { const manageTimelineForTesting = { foo: { - ...timelineDefaults, - id: 'foo', + ...getTimelineDefaults('foo'), filterManager, isLoading, }, diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/timeline.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/timeline.tsx index 18deaf01587239f..ed7f8a447c51d28 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/timeline.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/timeline.tsx @@ -177,12 +177,11 @@ export const TimelineComponent: React.FC = ({ setIndexToAdd, setIsTimelineLoading, setTimelineFilterManager, - setTimelineRowActions, } = useManageTimeline(); useEffect(() => { - initializeTimeline({ id, indexToAdd }); - setTimelineRowActions({ + initializeTimeline({ id, + indexToAdd, timelineRowActions: [getInvestigateInResolverAction({ dispatch, timelineId: id })], }); // eslint-disable-next-line react-hooks/exhaustive-deps diff --git a/x-pack/test/api_integration/apis/metrics_ui/http_source.ts b/x-pack/test/api_integration/apis/metrics_ui/http_source.ts new file mode 100644 index 000000000000000..7e92caf0e37d785 --- /dev/null +++ b/x-pack/test/api_integration/apis/metrics_ui/http_source.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { SourceResponse } from '../../../../plugins/infra/server/lib/sources'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + const fetchSource = async (): Promise => { + const response = await supertest + .get('/api/metrics/source/default/metrics') + .set('kbn-xsrf', 'xxx') + .expect(200); + return response.body; + }; + const fetchHasData = async ( + type: 'logs' | 'metrics' + ): Promise<{ hasData: boolean } | undefined> => { + const response = await supertest + .get(`/api/metrics/source/default/${type}/hasData`) + .set('kbn-xsrf', 'xxx') + .expect(200); + return response.body; + }; + + describe('Source API via HTTP', () => { + describe('8.0.0', () => { + before(() => esArchiver.load('infra/8.0.0/logs_and_metrics')); + after(() => esArchiver.unload('infra/8.0.0/logs_and_metrics')); + describe('/api/metrics/source/default/metrics', () => { + it('should just work', () => { + const resp = fetchSource(); + return resp.then((data) => { + expect(data).to.have.property('source'); + expect(data?.source.configuration.metricAlias).to.equal('metrics-*,metricbeat-*'); + expect(data?.source.configuration.logAlias).to.equal( + 'logs-*,filebeat-*,kibana_sample_data_logs*' + ); + expect(data?.source.configuration.fields).to.eql({ + container: 'container.id', + host: 'host.name', + message: ['message', '@message'], + pod: 'kubernetes.pod.uid', + tiebreaker: '_doc', + timestamp: '@timestamp', + }); + expect(data).to.have.property('status'); + expect(data?.status.metricIndicesExist).to.equal(true); + expect(data?.status.logIndicesExist).to.equal(true); + }); + }); + }); + describe('/api/metrics/source/default/metrics/hasData', () => { + it('should just work', () => { + const resp = fetchHasData('metrics'); + return resp.then((data) => { + expect(data).to.have.property('hasData'); + expect(data?.hasData).to.be(true); + }); + }); + }); + describe('/api/metrics/source/default/logs/hasData', () => { + it('should just work', () => { + const resp = fetchHasData('logs'); + return resp.then((data) => { + expect(data).to.have.property('hasData'); + expect(data?.hasData).to.be(true); + }); + }); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/metrics_ui/index.js b/x-pack/test/api_integration/apis/metrics_ui/index.js index eb8ee77da582bea..fdd37fa4c335cbf 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/index.js +++ b/x-pack/test/api_integration/apis/metrics_ui/index.js @@ -21,5 +21,6 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./metrics_explorer')); loadTestFile(require.resolve('./feature_controls')); loadTestFile(require.resolve('./ip_to_hostname')); + loadTestFile(require.resolve('./http_source')); }); }