Merge branch 'main' of github.com:elastic/kibana into adds-alerts-det…

…ected-created-metrics

.bazelrc

@@ -2,14 +2,6 @@
 # Import shared settings first so we can override below
 import %workspace%/.bazelrc.common
 
-# Remote cache settings for local env
-# build --remote_cache=grpcs://cloud.buildbuddy.io
-# build --incompatible_remote_results_ignore_disk=true
-# build --noremote_upload_local_results
-# build --remote_timeout=30
-# build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze
-# build --remote_accept_cached=true
-
 # Enable this in case you want to share your build info
 # build --build_metadata=VISIBILITY=PUBLIC
 build --build_metadata=TEST_GROUPS=//packages

.buildkite/pipelines/es_snapshots/verify.yml

@@ -29,7 +29,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 30
     agents:
       queue: n2-4
     depends_on: build
@@ -40,18 +40,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11

.buildkite/pipelines/flaky_tests/groups.json

@@ -32,11 +32,7 @@
     {
       "key": "xpack/cigroup",
       "name": "Default CI Group",
-      "ciGroups": 27
-    },
-    {
-      "key": "xpack/cigroup/Docker",
-      "name": "Default CI Group Docker"
+      "ciGroups": 30
     },
     {
       "key": "xpack/firefox",
@@ -47,4 +43,4 @@
       "name": "Default Accessibility"
     }
   ]
-}
+}

.buildkite/pipelines/on_merge.yml

@@ -51,7 +51,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 30
     agents:
       queue: n2-4-spot
     depends_on: build
@@ -64,20 +64,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11

.buildkite/pipelines/pull_request/base.yml

@@ -17,26 +17,16 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 30
     agents:
-      queue: n2-4
+      queue: n2-4-spot-2
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
     retry:
       automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 

.buildkite/scripts/build_kibana_plugins.sh

@@ -3,18 +3,4 @@
 set -euo pipefail
 
 echo "--- Build Platform Plugins"
-node scripts/build_kibana_platform_plugins \
-  --scan-dir "$KIBANA_DIR/test/analytics/__fixtures__/plugins" \
-  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
-  --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
-  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
-  --scan-dir "$KIBANA_DIR/examples" \
-  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
-  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
-  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
-  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
-  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
-  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
-  --scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
-  --scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
-  --scan-dir "$XPACK_DIR/examples"
+node scripts/build_kibana_platform_plugins --examples --test-plugins

.buildkite/scripts/common/setup_bazel.sh

@@ -15,8 +15,8 @@ if [[ "${BAZEL_CACHE_MODE:-none}" == read* ]]; then
   echo "[bazel] enabling caching"
 cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --bes_results_url=https://app.buildbuddy.io/invocation/
-  build --bes_backend=grpcs://cloud.buildbuddy.io
-  build --remote_cache=grpcs://cloud.buildbuddy.io
+  build --bes_backend=grpcs://remote.buildbuddy.io
+  build --remote_cache=grpcs://remote.buildbuddy.io
   build --remote_timeout=3600
   build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY
 EOF

.buildkite/scripts/lifecycle/pre_build.sh

@@ -12,3 +12,11 @@ export CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-iss
 export CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
 
 node "$(dirname "${0}")/ci_stats_start.js"
+
+# We resolve the latest manifest URL at the beginning of the build to ensure that all steps in the build will use the same manifest
+# Otherwise, the manifest could change if a step is running around the time that a new one is promoted
+if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+  BUCKET=$(curl -s "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(cat package.json | jq -r .version)/manifest-latest-verified.json" | jq -r .bucket)
+  ES_SNAPSHOT_MANIFEST_DEFAULT="https://storage.googleapis.com/$BUCKET/manifest.json"
+  buildkite-agent meta-data set ES_SNAPSHOT_MANIFEST_DEFAULT "$ES_SNAPSHOT_MANIFEST_DEFAULT"
+fi

.buildkite/scripts/lifecycle/pre_command.sh

@@ -65,6 +65,15 @@ EOF
   fi
 }
 
+# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build
+{
+  if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+    ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')}
+    export ES_SNAPSHOT_MANIFEST
+    echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST"
+  fi
+}
+
 # Setup CI Stats
 {
   CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"

.buildkite/scripts/steps/artifacts/build.sh

@@ -9,9 +9,18 @@ source .buildkite/scripts/steps/artifacts/env.sh
 echo "--- Build Kibana artifacts"
 node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS")
 
-echo "--- Build and upload dependencies report"
+echo "--- Extract default i18n messages"
+mkdir -p target/i18n
+node scripts/i18n_extract --output-dir=target/i18n
+
+echo "--- Build dependencies report"
 node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv"
+(cd target; sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt")
+
+echo "--- Upload Kibana Artifacts"
 cd target
-sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt"
-buildkite-agent artifact upload "*"
+buildkite-agent artifact upload 'kibana-*'
+buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv"
+buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv.sha512.txt"
+buildkite-agent artifact upload 'i18n/*.json'
 cd -

.buildkite/scripts/steps/artifacts/publish.sh

@@ -51,30 +51,35 @@ trap 'docker logout docker.elastic.co' EXIT
 docker pull docker.elastic.co/infra/release-manager:latest
 
 echo "--- Publish artifacts"
-export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
-export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
-export VAULT_ADDR="https://secrets.elastic.co:8200"
-docker run --rm \
-  --name release-manager \
-  -e VAULT_ADDR \
-  -e VAULT_ROLE_ID \
-  -e VAULT_SECRET_ID \
-  --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
-  docker.elastic.co/infra/release-manager:latest \
-    cli collect \
-      --project kibana \
-      --branch "$KIBANA_BASE_BRANCH" \
-      --commit "$GIT_COMMIT" \
-      --workflow "$WORKFLOW" \
-      --version "$BASE_VERSION" \
-      --qualifier "$VERSION_QUALIFIER" \
-      --artifact-set main
-
-ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
-ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url')
-
-cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
+if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
+  export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
+  export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+  export VAULT_ADDR="https://secrets.elastic.co:8200"
+  docker run --rm \
+    --name release-manager \
+    -e VAULT_ADDR \
+    -e VAULT_ROLE_ID \
+    -e VAULT_SECRET_ID \
+    --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
+    docker.elastic.co/infra/release-manager:latest \
+      cli collect \
+        --project kibana \
+        --branch "$KIBANA_BASE_BRANCH" \
+        --commit "$GIT_COMMIT" \
+        --workflow "$WORKFLOW" \
+        --version "$BASE_VERSION" \
+        --qualifier "$VERSION_QUALIFIER" \
+        --artifact-set main
+
+  ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
+  ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url')
+
+  cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
   ### Artifacts Summary
 
   $ARTIFACTS_SUMMARY
 EOF
+
+else
+  echo "Skipping publish for untracked branch $BUILDKITE_BRANCH"
+fi

.buildkite/scripts/steps/es_snapshots/create_manifest.js

@@ -8,7 +8,7 @@
 
 const fs = require('fs');
 const { execSync } = require('child_process');
-const { BASE_BUCKET_DAILY } = require('./bucket_config.js');
+const { BASE_BUCKET_DAILY } = require('./bucket_config');
 
 (async () => {
   console.log('--- Create ES Snapshot Manifest');

.buildkite/scripts/steps/es_snapshots/promote_manifest.js

@@ -8,7 +8,7 @@
 
 const fs = require('fs');
 const { execSync } = require('child_process');
-const { BASE_BUCKET_DAILY, BASE_BUCKET_PERMANENT } = require('./bucket_config.js');
+const { BASE_BUCKET_DAILY, BASE_BUCKET_PERMANENT } = require('./bucket_config');
 
 (async () => {
   try {

.buildkite/scripts/steps/functional/performance_playwright.sh

@@ -20,15 +20,15 @@ sleep 120
 
 cd "$XPACK_DIR"
 
-journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard")
+journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover")
 
 for i in "${journeys[@]}"; do
     echo "JOURNEY[${i}] is running"
 
     export TEST_PERFORMANCE_PHASE=WARMUP
     export ELASTIC_APM_ACTIVE=false
     export JOURNEY_NAME="${i}"
-    
+
     checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: WARMUP)" \
       node scripts/functional_tests \
       --config test/performance/config.playwright.ts \

.buildkite/scripts/steps/package_testing/test.sh

@@ -20,7 +20,7 @@ elif [[ "$TEST_PACKAGE" == "docker" ]]; then
 fi
 cd ..
 
-export VAGRANT_CWD=test/package
+export VAGRANT_CWD=$PWD/test/package
 vagrant up "$TEST_PACKAGE" --no-provision
 
 node scripts/es snapshot \
@@ -29,6 +29,16 @@ node scripts/es snapshot \
   --license=trial &
 while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
 
+function echoKibanaLogs {
+  echo '--- Kibana logs'
+  if [[ "$TEST_PACKAGE" == "deb" ]] || [[ "$TEST_PACKAGE" == "rpm" ]]; then
+    vagrant ssh $TEST_PACKAGE -t -c 'sudo cat /var/log/kibana/kibana.log'
+  elif [[ "$TEST_PACKAGE" == "docker" ]]; then
+    vagrant ssh $TEST_PACKAGE -t -c 'sudo docker logs kibana'
+  fi
+}
+trap "echoKibanaLogs" EXIT
+
 vagrant provision "$TEST_PACKAGE"
 
 export TEST_BROWSER_HEADLESS=1

.ci/.storybook/main.js

@@ -7,7 +7,7 @@
  */
 
 const config = require('@kbn/storybook').defaultConfig;
-const aliases = require('../../src/dev/storybook/aliases.ts').storybookAliases;
+const aliases = require('../../src/dev/storybook/aliases').storybookAliases;
 
 config.refs = {};
 

.ci/ci_groups.yml

@@ -39,4 +39,6 @@ xpack:
   - ciGroup25
   - ciGroup26
   - ciGroup27
-  - ciGroupDocker
+  - ciGroup28
+  - ciGroup29
+  - ciGroup30