Merge branch 'main' into rylnd/prebuilt_rule_import

.buildkite/pipeline-resource-definitions/kibana-pr.yml

@@ -20,8 +20,8 @@ spec:
     spec:
       env:
         ELASTIC_PR_COMMENTS_ENABLED: 'true'
-        ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
-        ELASTIC_GITHUB_STEP_COMMIT_STATUS_ENABLED: 'true'
+        GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+        GITHUB_STEP_COMMIT_STATUS_ENABLED: 'true'
         GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-ci
       allow_rebuilds: true
       branch_configuration: ''

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-production.yaml

@@ -10,7 +10,7 @@ steps:
       env:
         TARGET_ENV: production
         CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
+        CHECK_SLO_TAG: kbn-quality-gate
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
         DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml

@@ -10,7 +10,7 @@ steps:
       env:
         TARGET_ENV: production
         CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
+        CHECK_SLO_TAG: kbn-quality-gate
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
         DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

config/serverless.es.yml

@@ -34,6 +34,7 @@ xpack.cloud.serverless.project_type: search
 
 ## Enable the Serverless Search plugin
 xpack.serverless.search.enabled: true
+xpack.searchIndices.enabled: true
 
 ## Set the home route
 uiSettings.overrides.defaultRoute: /app/elasticsearch

config/serverless.security.yml

@@ -105,3 +105,6 @@ xpack.ml.compatibleModuleType: 'security'
 
 # Disable the embedded Dev Console
 console.ui.embeddedEnabled: false
+
+# Enable project level rentention checks in DSL form from Index Management UI
+xpack.index_management.enableProjectLevelRetentionChecks: true

config/serverless.yml

@@ -110,6 +110,8 @@ xpack.index_management.editableIndexSettings: limited
 xpack.index_management.enableMappingsSourceFieldSection: false
 # Disable toggle for enabling data retention in DSL form from Index Management UI
 xpack.index_management.enableTogglingDataRetention: false
+# Disable project level rentention checks in DSL form from Index Management UI
+xpack.index_management.enableProjectLevelRetentionChecks: false
 
 # Keep deeplinks visible so that they are shown in the sidenav
 dev_tools.deeplinks.navLinkStatus: visible

dev_docs/tutorials/versioning_http_apis.mdx

@@ -15,6 +15,13 @@ All Kibana HTTP API developers and maintainers must ensure that past versions of
   The exact number of past APIs and the length of time they are kept available will vary per use case. Generally, the length of time will be shorter for internal HTTP APIs than for public HTTP APIs.
 </DocCallOut>
 
+<DocAccordion buttonContent="FAQ: What is the difference between an internal and a public HTTP API?">
+<DocCallOut >
+
+We only declare HTTP APIs that are stable and reliable as public and keep all development changes behind internal HTTP APIs. Public HTTP APIs are intended for external consumption and are typically garanteed not to change within a major. Internal HTTP APIs may change more frequently, as long as they adhere to the versioning principles outlined in the rest of this doc.
+
+</DocCallOut></DocAccordion>
+
 Versioned HTTP APIs should hold to the following set of properties. **Note:** how you meet these is properties is up to you. Use the examples provided as a guide.
 
 ### 1. Do not directly expose persistence schemas on your HTTP API endpoints

package.json

@@ -1493,7 +1493,7 @@
     "@octokit/rest": "^17.11.2",
     "@parcel/watcher": "^2.1.0",
     "@playwright/test": "=1.46.0",
-    "@redocly/cli": "^1.25.4",
+    "@redocly/cli": "^1.25.5",
     "@statoscope/webpack-plugin": "^5.28.2",
     "@storybook/addon-a11y": "^6.5.16",
     "@storybook/addon-actions": "^6.5.16",

packages/core/feature-flags/README.mdx

@@ -18,6 +18,10 @@ For a code example, refer to the [Feature Flags Example plugin](../../../example
 
 ## Registering a feature flag
 
+> [!IMPORTANT]
+> At the moment, we follow a manual process to manage our feature flags. Refer to [this repo](https://github.com/elastic/kibana-feature-flags) to learn more about our current internal process.
+> Our goal is to achieve the _gitops_ approach detailed below. But, at the moment, it's not available, and you can skip it if you want.
+
 Kibana follows a _gitops_ approach when managing feature flags. To declare a feature flag, add your flags definitions in
 your plugin's `server/index.ts` file:
 

packages/core/http/core-http-server-internal/src/lifecycle_handlers.test.ts

@@ -19,6 +19,7 @@ import type {
 } from '@kbn/core-http-server';
 import { mockRouter } from '@kbn/core-http-router-server-mocks';
 import {
+  INTERNAL_API_RESTRICTED_LOGGER_NAME,
   createBuildNrMismatchLoggerPreResponseHandler,
   createCustomHeadersPreResponseHandler,
   createRestrictInternalRoutesPostAuthHandler,
@@ -267,11 +268,19 @@ describe('versionCheck post-auth handler', () => {
 describe('restrictInternal post-auth handler', () => {
   let toolkit: ToolkitMock;
   let responseFactory: ReturnType<typeof mockRouter.createResponseFactory>;
+  let logger: jest.Mocked<Logger>;
+  let config: HttpConfig;
 
   beforeEach(() => {
     toolkit = createToolkit();
     responseFactory = mockRouter.createResponseFactory();
+    logger = loggerMock.create();
+    config = createConfig({
+      name: 'my-server-name',
+      restrictInternalApis: true,
+    });
   });
+
   const createForgeRequest = (
     access: 'internal' | 'public',
     headers: Record<string, string> | undefined = {},
@@ -298,92 +307,115 @@ describe('restrictInternal post-auth handler', () => {
     expect(result).toBe('next');
   };
 
-  describe('when restriction is enabled', () => {
-    const config = createConfig({
-      name: 'my-server-name',
-      restrictInternalApis: true,
-    });
+  it('injects a logger prefix', () => {
+    createRestrictInternalRoutesPostAuthHandler(config, logger);
+    expect(logger.get).toHaveBeenCalledTimes(1);
+    expect(logger.get).toHaveBeenCalledWith(`server`, INTERNAL_API_RESTRICTED_LOGGER_NAME);
+  });
 
-    it('returns a bad request if called without internal origin header for internal API', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('internal');
+  it('when enabled, does not log deprecation warning for internal API access restriction', () => {
+    createRestrictInternalRoutesPostAuthHandler(config, logger);
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
 
-      responseFactory.badRequest.mockReturnValue('badRequest' as any);
+  it('when enabled, returns a bad request if called without internal origin header for internal API', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(config, logger);
+    const request = createForgeRequest('internal');
 
-      const result = handler(request, responseFactory, toolkit);
+    responseFactory.badRequest.mockReturnValue('badRequest' as any);
 
-      expect(toolkit.next).not.toHaveBeenCalled();
-      expect(responseFactory.badRequest.mock.calls[0][0]?.body).toMatchInlineSnapshot(
-        `"uri [/internal/some-path] with method [get] exists but is not available with the current configuration"`
-      );
-      expect(result).toBe('badRequest');
-    });
+    const result = handler(request, responseFactory, toolkit);
 
-    it('forward the request to the next interceptor if called with internal origin header for internal API', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('internal', { 'x-elastic-internal-origin': 'Kibana' });
-      createForwardSuccess(handler, request);
-    });
+    expect(toolkit.next).not.toHaveBeenCalled();
+    expect(responseFactory.badRequest.mock.calls[0][0]?.body).toMatchInlineSnapshot(
+      `"uri [/internal/some-path] with method [get] exists but is not available with the current configuration"`
+    );
+    expect(result).toBe('badRequest');
+  });
 
-    it('forward the request to the next interceptor if called with internal origin header for public APIs', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('public', { 'x-elastic-internal-origin': 'Kibana' });
-      createForwardSuccess(handler, request);
-    });
+  it('when enabled, forward the request to the next interceptor if called with internal origin header for internal API', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(config, logger);
+    const request = createForgeRequest('internal', { 'x-elastic-internal-origin': 'Kibana' });
+    createForwardSuccess(handler, request);
+  });
 
-    it('forward the request to the next interceptor if called without internal origin header for public APIs', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('public');
-      createForwardSuccess(handler, request);
-    });
+  it('when enabled, forward the request to the next interceptor if called with internal origin header for public APIs', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(config, logger);
+    const request = createForgeRequest('public', { 'x-elastic-internal-origin': 'Kibana' });
+    createForwardSuccess(handler, request);
+  });
 
-    it('forward the request to the next interceptor if called with internal origin query param for internal API', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('internal', undefined, { elasticInternalOrigin: 'true' });
-      createForwardSuccess(handler, request);
-    });
+  it('when enabled, forward the request to the next interceptor if called without internal origin header for public APIs', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(config, logger);
+    const request = createForgeRequest('public');
+    createForwardSuccess(handler, request);
+  });
 
-    it('forward the request to the next interceptor if called with internal origin query param for public APIs', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('internal', undefined, { elasticInternalOrigin: 'true' });
-      createForwardSuccess(handler, request);
-    });
+  it('when enabled, forward the request to the next interceptor if called with internal origin query param for internal API', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(config, logger);
+    const request = createForgeRequest('internal', undefined, { elasticInternalOrigin: 'true' });
+    createForwardSuccess(handler, request);
+  });
 
-    it('forward the request to the next interceptor if called without internal origin query param for public APIs', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('public');
-      createForwardSuccess(handler, request);
-    });
+  it('when enabled, forward the request to the next interceptor if called with internal origin query param for public APIs', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(config, logger);
+    const request = createForgeRequest('internal', undefined, { elasticInternalOrigin: 'true' });
+    createForwardSuccess(handler, request);
   });
 
-  describe('when restriction is not enabled', () => {
-    const config = createConfig({
-      name: 'my-server-name',
-      restrictInternalApis: false,
-    });
-    it('forward the request to the next interceptor if called without internal origin header for internal APIs', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('internal');
-      createForwardSuccess(handler, request);
-    });
+  it('when enabled, forward the request to the next interceptor if called without internal origin query param for public APIs', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(config, logger);
+    const request = createForgeRequest('public');
+    createForwardSuccess(handler, request);
+  });
 
-    it('forward the request to the next interceptor if called with internal origin header for internal API', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('internal', { 'x-elastic-internal-origin': 'Kibana' });
-      createForwardSuccess(handler, request);
-    });
+  it('when not enabled, logs deprecation warning for internal API access restriction', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(
+      { ...config, restrictInternalApis: false },
+      logger
+    );
+    const request = createForgeRequest('internal');
+    createForwardSuccess(handler, request);
+    expect(logger.warn).toHaveBeenCalledTimes(1);
+    expect(logger.warn).toHaveBeenCalledWith(
+      `Access to uri [/internal/some-path] with method [get] is deprecated`
+    );
+  });
 
-    it('forward the request to the next interceptor if called without internal origin header for public APIs', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('public');
-      createForwardSuccess(handler, request);
-    });
+  it('when not enabled, forward the request to the next interceptor if called without internal origin header for internal APIs', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(
+      { ...config, restrictInternalApis: false },
+      logger
+    );
+    const request = createForgeRequest('internal');
+    createForwardSuccess(handler, request);
+  });
 
-    it('forward the request to the next interceptor if called with internal origin header for public APIs', () => {
-      const handler = createRestrictInternalRoutesPostAuthHandler(config as HttpConfig);
-      const request = createForgeRequest('public', { 'x-elastic-internal-origin': 'Kibana' });
-      createForwardSuccess(handler, request);
-    });
+  it('when not enabled, forward the request to the next interceptor if called with internal origin header for internal API', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(
+      { ...config, restrictInternalApis: false },
+      logger
+    );
+    const request = createForgeRequest('internal', { 'x-elastic-internal-origin': 'Kibana' });
+    createForwardSuccess(handler, request);
+  });
+
+  it('when not enabled, forward the request to the next interceptor if called without internal origin header for public APIs', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(
+      { ...config, restrictInternalApis: false },
+      logger
+    );
+    const request = createForgeRequest('public');
+    createForwardSuccess(handler, request);
+  });
+
+  it('when not enabled, forward the request to the next interceptor if called with internal origin header for public APIs', () => {
+    const handler = createRestrictInternalRoutesPostAuthHandler(
+      { ...config, restrictInternalApis: false },
+      logger
+    );
+    const request = createForgeRequest('public', { 'x-elastic-internal-origin': 'Kibana' });
+    createForwardSuccess(handler, request);
   });
 });
 

packages/core/http/core-http-server-internal/src/lifecycle_handlers.ts

@@ -45,18 +45,35 @@ export const createXsrfPostAuthHandler = (config: HttpConfig): OnPostAuthHandler
   };
 };
 
+/**
+ * This should remain part of the logger prefix so that we can notify/track
+ * when we see this logged for observability purposes.
+ */
+export const INTERNAL_API_RESTRICTED_LOGGER_NAME = 'kbn-internal-api-restricted';
 export const createRestrictInternalRoutesPostAuthHandler = (
-  config: HttpConfig
+  config: HttpConfig,
+  log: Logger
 ): OnPostAuthHandler => {
   const isRestrictionEnabled = config.restrictInternalApis;
+  log = log.get('server', `${INTERNAL_API_RESTRICTED_LOGGER_NAME}`);
 
   return (request, response, toolkit) => {
     const isInternalRoute = request.route.options.access === 'internal';
-    if (isRestrictionEnabled && isInternalRoute && !request.isInternalApiRequest) {
-      // throw 400
-      return response.badRequest({
-        body: `uri [${request.url.pathname}] with method [${request.route.method}] exists but is not available with the current configuration`,
-      });
+    if (isInternalRoute && !request.isInternalApiRequest) {
+      if (!isRestrictionEnabled) {
+        // warn if the restriction is not enforced
+        log.warn(
+          `Access to uri [${request.url.pathname}] with method [${request.route.method}] is deprecated`
+        );
+      } else {
+        log.error(
+          `Access to uri [${request.url.pathname}] with method [${request.route.method}] is not available with the current configuration`
+        );
+        // throw 400
+        return response.badRequest({
+          body: `uri [${request.url.pathname}] with method [${request.route.method}] exists but is not available with the current configuration`,
+        });
+      }
     }
     return toolkit.next();
   };

packages/core/http/core-http-server-internal/src/register_lifecycle_handlers.ts

@@ -38,5 +38,5 @@ export const registerCoreHandlers = (
     );
   }
   // add check on header if the route is internal
-  registrar.registerOnPostAuth(createRestrictInternalRoutesPostAuthHandler(config)); // strictly speaking, we should have access to route.options.access from the request on postAuth
+  registrar.registerOnPostAuth(createRestrictInternalRoutesPostAuthHandler(config, log)); // strictly speaking, we should have access to route.options.access from the request on postAuth
 };

packages/kbn-ftr-common-functional-services/index.ts

@@ -30,6 +30,7 @@ export type {
   InternalRequestHeader,
   RoleCredentials,
   CookieCredentials,
+  KibanaRoleDescriptors,
 } from './services/saml_auth';
 
 import { SamlAuthProvider } from './services/saml_auth/saml_auth_provider';

packages/kbn-ftr-common-functional-services/services/saml_auth/get_auth_provider.ts

@@ -12,8 +12,10 @@ import { ServerlessAuthProvider } from './serverless/auth_provider';
 import { StatefulAuthProvider } from './stateful/auth_provider';
 
 export interface AuthProvider {
-  getSupportedRoleDescriptors(): Record<string, unknown>;
+  getSupportedRoleDescriptors(): Map<string, any>;
   getDefaultRole(): string;
+  isCustomRoleEnabled(): boolean;
+  getCustomRole(): string;
   getRolesDefinitionPath(): string;
   getCommonRequestHeader(): { [key: string]: string };
   getInternalRequestHeader(): { [key: string]: string };

packages/kbn-ftr-common-functional-services/services/saml_auth/index.ts

@@ -8,5 +8,9 @@
  */
 
 export { SamlAuthProvider } from './saml_auth_provider';
-export type { RoleCredentials, CookieCredentials } from './saml_auth_provider';
+export type {
+  RoleCredentials,
+  CookieCredentials,
+  KibanaRoleDescriptors,
+} from './saml_auth_provider';
 export type { InternalRequestHeader } from './default_request_headers';