Merge remote-tracking branch 'upstream/main' into json-diff-for-rule-…

…upgrade

.backportrc.json

@@ -3,6 +3,7 @@
   "repoName": "kibana",
   "targetBranchChoices": [
     "main",
+    "8.12",
     "8.11",
     "8.10",
     "8.9",
@@ -48,7 +49,7 @@
     "backport"
   ],
   "branchLabelMapping": {
-    "^v8.12.0$": "main",
+    "^v8.13.0$": "main",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/ftr_configs.yml

@@ -124,6 +124,7 @@ enabled:
   - test/functional/apps/kibana_overview/config.ts
   - test/functional/apps/management/config.ts
   - test/functional/apps/saved_objects_management/config.ts
+  - test/functional/apps/sharing/config.ts
   - test/functional/apps/status_page/config.ts
   - test/functional/apps/visualize/group1/config.ts
   - test/functional/apps/visualize/group2/config.ts
@@ -306,6 +307,7 @@ enabled:
   - x-pack/test/functional/apps/ml/stack_management_jobs/config.ts
   - x-pack/test/functional/apps/monitoring/config.ts
   - x-pack/test/functional/apps/observability_log_explorer/config.ts
+  - x-pack/test/functional/apps/painless_lab/config.ts
   - x-pack/test/functional/apps/remote_clusters/config.ts
   - x-pack/test/functional/apps/reporting_management/config.ts
   - x-pack/test/functional/apps/rollup_job/config.ts

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -147,6 +147,19 @@ steps:
             - exit_status: '*'
               limit: 1
 
+      - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
+        label: 'Serverless AI Assistant - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
+        parallelism: 1
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
       - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
         label: 'Defend Workflows Cypress Tests on Serverless'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"

.buildkite/pipelines/flaky_tests/groups.json

@@ -49,7 +49,6 @@
       "key": "cypress/security_serverless_detection_engine",
       "name": "[Serverless] Security Solution Detection Engine - Cypress"
     },
-
     {
       "key": "cypress/security_solution_detection_engine_exceptions",
       "name": "Security Solution Detection Engine - Exceptions - Cypress"
@@ -58,6 +57,14 @@
       "key": "cypress/security_serverless_detection_engine_exceptions",
       "name": "[Serverless] Security Solution Detection Engine - Exceptions - Cypress"
     },
+    {
+      "key": "cypress/security_solution_ai_assistant",
+      "name": "Security Solution AI Assistant - Cypress"
+    },
+    {
+      "key": "cypress/security_serverless_ai_assistant",
+      "name": "[Serverless] Security Solution AI Assistant - Cypress"
+    },
     {
       "key": "cypress/defend_workflows",
       "name": "Security Solution Defend Workflows - Cypress"

.buildkite/pipelines/on_merge.yml

@@ -211,6 +211,30 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
+    label: 'Serverless AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh
+    label: 'AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/base.yml

@@ -141,6 +141,18 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
+    label: 'Serverless AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1        
+
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Cypress Tests'
     agents:
@@ -213,6 +225,18 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh
+    label: 'AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1        
+
   - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
     label: 'Investigations - Security Solution Cypress Tests'
     agents:
@@ -231,7 +255,7 @@ steps:
       queue: n2-4-virt
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 10
+    parallelism: 16
     retry:
       automatic:
         - exit_status: '*'

.buildkite/pipelines/pull_request/cypress_burn.yml

@@ -1,6 +1,6 @@
 steps:
   - command: .buildkite/scripts/steps/functional/defend_workflows_burn.sh
-    label: 'Defend Workflows Cypress Tests, burning changed specs'
+    label: '[Soft fail] Defend Workflows Cypress Tests, burning changed specs'
     agents:
       queue: n2-4-virt
     depends_on: build
@@ -11,7 +11,7 @@ steps:
       automatic: false
 
   - command: .buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh
-    label: 'Defend Workflows Cypress Tests on Serverless, burning changed specs'
+    label: '[Soft fail] Defend Workflows Cypress Tests on Serverless, burning changed specs'
     agents:
       queue: n2-4-virt
     depends_on: build
@@ -22,7 +22,7 @@ steps:
       automatic: false
 
   - command: .buildkite/scripts/steps/functional/security_solution_burn.sh
-    label: 'Security Solution Cypress tests, burning changed specs'
+    label: '[Soft fail] Security Solution Cypress tests, burning changed specs'
     agents:
       queue: n2-4-spot
     depends_on: build
@@ -33,7 +33,7 @@ steps:
     soft_fail: true
 
   - command: .buildkite/scripts/steps/functional/osquery_cypress_burn.sh
-    label: 'Osquery Cypress Tests, burning changed specs'
+    label: '[Soft fail] Osquery Cypress Tests, burning changed specs'
     agents:
       queue: n2-4-spot
     depends_on: build

.buildkite/pipelines/security_solution/security_solution_cypress.yml

@@ -81,4 +81,16 @@ steps:
     retry:
       automatic:
         - exit_status: '*'
-          limit: 1
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:ai_assistant
+    label: 'Serverless MKI QA AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1          

.buildkite/pipelines/sonarqube.yml

@@ -0,0 +1,4 @@
+
+steps:
+  - label: Placeholder
+    command: echo "Hello!"

.buildkite/pull_requests.json

@@ -13,8 +13,9 @@
       "commit_status_context": "kibana-ci",
       "build_on_commit": true,
       "build_on_comment": true,
-      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
-      "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
+      "build_drafts": false,
+      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^\\/ci$",
+      "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^\\/ci$",
       "skip_ci_labels": ["skip-ci", "jenkins-ci"],
       "skip_target_branches": ["6.8", "7.11", "7.12"],
       "enable_skippable_commits": true,
@@ -84,7 +85,8 @@
         "^\\.backportrc\\.json$",
         "^nav-kibana-dev\\.docnav\\.json$",
         "^src/dev/prs/kibana_qa_pr_list\\.json$",
-        "^\\.buildkite/pull_requests\\.json$"
+        "^\\.buildkite/pull_requests\\.json$",
+        "^\\.catalog-info\\.yaml$"
       ],
       "always_require_ci_on_changed": [
         "^docs/developer/plugin-list.asciidoc$",

.buildkite/scripts/common/util.sh

@@ -170,3 +170,34 @@ npm_install_global() {
 download_artifact() {
   retry 3 1 timeout 3m buildkite-agent artifact download "$@"
 }
+
+
+vault_get() {
+  path=$1
+  field=$2
+
+  fullPath="secret/ci/elastic-kibana/$path"
+  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
+    fullPath="secret/kibana-issues/dev/$path"
+  fi
+
+  if [[ -z "${2:-}" ]]; then
+    retry 5 5 vault read "$fullPath"
+  else
+    retry 5 5 vault read -field="$field" "$fullPath"
+  fi
+}
+
+vault_set() {
+  path=$1
+  shift
+  fields=("$@")
+
+  fullPath="secret/ci/elastic-kibana/$path"
+  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
+    fullPath="secret/kibana-issues/dev/$path"
+  fi
+
+  # shellcheck disable=SC2068
+  retry 5 5 vault write "$fullPath" ${fields[@]}
+}

.buildkite/scripts/lifecycle/pre_command.sh

@@ -8,7 +8,7 @@ echo '--- Setup environment vars'
 source .buildkite/scripts/common/env.sh
 source .buildkite/scripts/common/setup_node.sh
 
-BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
+BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)"
 export BUILDKITE_TOKEN
 
 echo '--- Install/build buildkite dependencies'
@@ -77,11 +77,11 @@ EOF
 {
   CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"
   export CI_STATS_BUILD_ID
-  
-  CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-issues/dev/kibana_ci_stats)"
+
+  CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)"
   export CI_STATS_TOKEN
-  
-  CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
+
+  CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)"
   export CI_STATS_HOST
 
   if [[ "$CI_STATS_BUILD_ID" ]]; then
@@ -97,66 +97,66 @@ EOF
   fi
 }
 
-GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)
+GITHUB_TOKEN=$(vault_get kibanamachine github_token)
 export GITHUB_TOKEN
 
-KIBANA_CI_GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibana-ci-github)
+KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token)
 export KIBANA_CI_GITHUB_TOKEN
 
-KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
+KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value)
 export KIBANA_CI_REPORTER_KEY
 
-KIBANA_DOCKER_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/container-registry)"
+KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
 export KIBANA_DOCKER_USERNAME
 
-KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
+KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
 export KIBANA_DOCKER_PASSWORD
 
-EC_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-cloud-deploy)"
+EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)"
 export EC_API_KEY
 
-PROJECT_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-project-deploy)"
+PROJECT_API_KEY="$(vault_get kibana-ci-project-deploy pr_deploy_api_key)"
 export PROJECT_API_KEY
 
-PROJECT_API_DOMAIN="$(retry 5 5 vault read -field=pr_deploy_domain secret/kibana-issues/dev/kibana-ci-project-deploy)"
+PROJECT_API_DOMAIN="$(vault_get kibana-ci-project-deploy pr_deploy_domain)"
 export PROJECT_API_DOMAIN
 
-SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)"
 export SYNTHETICS_SERVICE_USERNAME
 
-SYNTHETICS_SERVICE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)"
 export SYNTHETICS_SERVICE_PASSWORD
 
-SYNTHETICS_SERVICE_MANIFEST="$(retry 5 5 vault read -field=manifest secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)"
 export SYNTHETICS_SERVICE_MANIFEST
 
-SYNTHETICS_REMOTE_KIBANA_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
+SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)"
 export SYNTHETICS_REMOTE_KIBANA_USERNAME
 
-SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
+SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)"
 export SYNTHETICS_REMOTE_KIBANA_PASSWORD
 
-SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(retry 5 5 vault read -field=url secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"}
+SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"}
 export SYNTHETICS_REMOTE_KIBANA_URL
 
-DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(retry 5 5 vault read -field=DEPLOY_TAGGER_SLACK_WEBHOOK_URL secret/kibana-issues/dev/kibana-serverless-release-tools)"}
+DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"}
 export DEPLOY_TAGGER_SLACK_WEBHOOK_URL
 
 # Setup Failed Test Reporter Elasticsearch credentials
 {
-  TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)
+  TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id)
   export TEST_FAILURES_ES_CLOUD_ID
 
-  TEST_FAILURES_ES_USERNAME=$(retry 5 5 vault read -field=username secret/kibana-issues/dev/failed_tests_reporter_es)
+  TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username)
   export TEST_FAILURES_ES_USERNAME
 
-  TEST_FAILURES_ES_PASSWORD=$(retry 5 5 vault read -field=password secret/kibana-issues/dev/failed_tests_reporter_es)
+  TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password)
   export TEST_FAILURES_ES_PASSWORD
 }
 
 BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
 export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
-retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
+vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
 
 PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
 if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then