Merge branch 'master' into remove-injected-vars-from-maps-legacy

x-pack/plugins/index_management/public/application/components/shared/components/index.ts

@@ -6,4 +6,9 @@
 
 export { TabAliases, TabMappings, TabSettings } from './details_panel';
 
-export { StepAliases, StepMappings, StepSettings } from './wizard_steps';
+export {
+  StepAliasesContainer,
+  StepMappingsContainer,
+  StepSettingsContainer,
+  CommonWizardSteps,
+} from './wizard_steps';

x-pack/plugins/index_management/public/application/components/shared/components/wizard_steps/index.ts

@@ -4,6 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export { StepAliases } from './step_aliases';
-export { StepMappings } from './step_mappings';
-export { StepSettings } from './step_settings';
+export { StepAliasesContainer } from './step_aliases_container';
+export { StepMappingsContainer } from './step_mappings_container';
+export { StepSettingsContainer } from './step_settings_container';
+
+export { CommonWizardSteps } from './types';

x-pack/plugins/index_management/public/application/components/shared/components/wizard_steps/step_aliases_container.tsx

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React from 'react';
+
+import { Forms } from '../../../../../shared_imports';
+import { CommonWizardSteps } from './types';
+import { StepAliases } from './step_aliases';
+
+interface Props {
+  esDocsBase: string;
+}
+
+export const StepAliasesContainer: React.FunctionComponent<Props> = ({ esDocsBase }) => {
+  const { defaultValue, updateContent } = Forms.useContent<CommonWizardSteps>('aliases');
+
+  return (
+    <StepAliases defaultValue={defaultValue} onChange={updateContent} esDocsBase={esDocsBase} />
+  );
+};

x-pack/plugins/index_management/public/application/components/template_form/steps/step_mappings_container.tsx → x-pack/plugins/index_management/public/application/components/shared/components/wizard_steps/step_mappings_container.tsx

@@ -5,20 +5,23 @@
  */
 import React from 'react';
 
-import { Forms } from '../../../../shared_imports';
-import { documentationService } from '../../../services/documentation';
-import { StepMappings } from '../../shared';
-import { WizardContent } from '../template_form';
+import { Forms } from '../../../../../shared_imports';
+import { CommonWizardSteps } from './types';
+import { StepMappings } from './step_mappings';
 
-export const StepMappingsContainer = () => {
-  const { defaultValue, updateContent, getData } = Forms.useContent<WizardContent>('mappings');
+interface Props {
+  esDocsBase: string;
+}
+
+export const StepMappingsContainer: React.FunctionComponent<Props> = ({ esDocsBase }) => {
+  const { defaultValue, updateContent, getData } = Forms.useContent<CommonWizardSteps>('mappings');
 
   return (
     <StepMappings
       defaultValue={defaultValue}
       onChange={updateContent}
       indexSettings={getData().settings}
-      esDocsBase={documentationService.getEsDocsBase()}
+      esDocsBase={esDocsBase}
     />
   );
 };

x-pack/plugins/index_management/public/application/components/shared/components/wizard_steps/step_settings_container.tsx

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React from 'react';
+
+import { Forms } from '../../../../../shared_imports';
+import { CommonWizardSteps } from './types';
+import { StepSettings } from './step_settings';
+
+interface Props {
+  esDocsBase: string;
+}
+
+export const StepSettingsContainer = React.memo(({ esDocsBase }: Props) => {
+  const { defaultValue, updateContent } = Forms.useContent<CommonWizardSteps>('settings');
+
+  return (
+    <StepSettings defaultValue={defaultValue} onChange={updateContent} esDocsBase={esDocsBase} />
+  );
+});

x-pack/plugins/index_management/public/application/components/shared/components/wizard_steps/types.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Mappings, IndexSettings, Aliases } from '../../../../../../common';
+
+export interface CommonWizardSteps {
+  settings?: IndexSettings;
+  mappings?: Mappings;
+  aliases?: Aliases;
+}

x-pack/plugins/index_management/public/application/components/shared/index.ts

@@ -8,7 +8,8 @@ export {
   TabAliases,
   TabMappings,
   TabSettings,
-  StepAliases,
-  StepMappings,
-  StepSettings,
+  StepAliasesContainer,
+  StepMappingsContainer,
+  StepSettingsContainer,
+  CommonWizardSteps,
 } from './components';

x-pack/plugins/index_management/public/application/components/template_form/steps/index.ts

@@ -5,7 +5,4 @@
  */
 
 export { StepLogisticsContainer } from './step_logistics_container';
-export { StepAliasesContainer } from './step_aliases_container';
-export { StepMappingsContainer } from './step_mappings_container';
-export { StepSettingsContainer } from './step_settings_container';
 export { StepReviewContainer } from './step_review_container';

x-pack/plugins/index_management/public/application/components/template_form/template_form.tsx

@@ -11,13 +11,14 @@ import { EuiSpacer } from '@elastic/eui';
 import { TemplateDeserialized, CREATE_LEGACY_TEMPLATE_BY_DEFAULT } from '../../../../common';
 import { serializers, Forms } from '../../../shared_imports';
 import { SectionError } from '../section_error';
+import { StepLogisticsContainer, StepReviewContainer } from './steps';
 import {
-  StepLogisticsContainer,
+  CommonWizardSteps,
   StepSettingsContainer,
   StepMappingsContainer,
   StepAliasesContainer,
-  StepReviewContainer,
-} from './steps';
+} from '../shared';
+import { documentationService } from '../../services/documentation';
 
 const { stripEmptyFields } = serializers;
 const { FormWizard, FormWizardStep } = Forms;
@@ -31,11 +32,8 @@ interface Props {
   isEditing?: boolean;
 }
 
-export interface WizardContent {
+export interface WizardContent extends CommonWizardSteps {
   logistics: Omit<TemplateDeserialized, '_kbnMeta' | 'template'>;
-  settings: TemplateDeserialized['template']['settings'];
-  mappings: TemplateDeserialized['template']['mappings'];
-  aliases: TemplateDeserialized['template']['aliases'];
 }
 
 export type WizardSection = keyof WizardContent | 'review';
@@ -183,15 +181,15 @@ export const TemplateForm = ({
       </FormWizardStep>
 
       <FormWizardStep id={wizardSections.settings.id} label={wizardSections.settings.label}>
-        <StepSettingsContainer />
+        <StepSettingsContainer esDocsBase={documentationService.getEsDocsBase()} />
       </FormWizardStep>
 
       <FormWizardStep id={wizardSections.mappings.id} label={wizardSections.mappings.label}>
-        <StepMappingsContainer />
+        <StepMappingsContainer esDocsBase={documentationService.getEsDocsBase()} />
       </FormWizardStep>
 
       <FormWizardStep id={wizardSections.aliases.id} label={wizardSections.aliases.label}>
-        <StepAliasesContainer />
+        <StepAliasesContainer esDocsBase={documentationService.getEsDocsBase()} />
       </FormWizardStep>
 
       <FormWizardStep id={wizardSections.review.id} label={wizardSections.review.label}>

x-pack/plugins/ingest_manager/server/index.ts

@@ -11,6 +11,7 @@ export {
   IngestManagerSetupContract,
   IngestManagerSetupDeps,
   IngestManagerStartContract,
+  ExternalCallback,
 } from './plugin';
 
 export const config = {
@@ -42,6 +43,8 @@ export const config = {
 
 export type IngestManagerConfigType = TypeOf<typeof config.schema>;
 
+export { DatasourceServiceInterface } from './services/datasource';
+
 export const plugin = (initializerContext: PluginInitializerContext) => {
   return new IngestManagerPlugin(initializerContext);
 };

x-pack/plugins/ingest_manager/server/mocks.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks';
+import { IngestManagerAppContext } from './plugin';
+import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks';
+import { securityMock } from '../../security/server/mocks';
+import { DatasourceServiceInterface } from './services/datasource';
+
+export const createAppContextStartContractMock = (): IngestManagerAppContext => {
+  return {
+    encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(),
+    savedObjects: savedObjectsServiceMock.createStartContract(),
+    security: securityMock.createSetup(),
+    logger: loggingSystemMock.create().get(),
+    isProductionMode: true,
+    kibanaVersion: '8.0.0',
+  };
+};
+
+export const createDatasourceServiceMock = () => {
+  return {
+    assignPackageStream: jest.fn(),
+    buildDatasourceFromPackage: jest.fn(),
+    bulkCreate: jest.fn(),
+    create: jest.fn(),
+    delete: jest.fn(),
+    get: jest.fn(),
+    getByIDs: jest.fn(),
+    list: jest.fn(),
+    update: jest.fn(),
+  } as jest.Mocked<DatasourceServiceInterface>;
+};

x-pack/plugins/ingest_manager/server/plugin.ts

@@ -45,15 +45,16 @@ import {
   registerSettingsRoutes,
   registerAppRoutes,
 } from './routes';
-import { IngestManagerConfigType } from '../common';
+import { IngestManagerConfigType, NewDatasource } from '../common';
 import {
   appContextService,
   licenseService,
   ESIndexPatternSavedObjectService,
   ESIndexPatternService,
   AgentService,
+  datasourceService,
 } from './services';
-import { getAgentStatusById } from './services/agents';
+import { getAgentStatusById, authenticateAgentWithAccessToken } from './services/agents';
 import { CloudSetup } from '../../cloud/server';
 import { agentCheckinState } from './services/agents/checkin/state';
 
@@ -92,12 +93,31 @@ const allSavedObjectTypes = [
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
 ];
 
+/**
+ * Callbacks supported by the Ingest plugin
+ */
+export type ExternalCallback = [
+  'datasourceCreate',
+  (newDatasource: NewDatasource) => Promise<NewDatasource>
+];
+
+export type ExternalCallbacksStorage = Map<ExternalCallback[0], Set<ExternalCallback[1]>>;
+
 /**
  * Describes public IngestManager plugin contract returned at the `startup` stage.
  */
 export interface IngestManagerStartContract {
   esIndexPatternService: ESIndexPatternService;
   agentService: AgentService;
+  /**
+   * Services for Ingest's Datasources
+   */
+  datasourceService: typeof datasourceService;
+  /**
+   * Register callbacks for inclusion in ingest API processing
+   * @param args
+   */
+  registerExternalCallback: (...args: ExternalCallback) => void;
 }
 
 export class IngestManagerPlugin
@@ -236,6 +256,11 @@ export class IngestManagerPlugin
       esIndexPatternService: new ESIndexPatternSavedObjectService(),
       agentService: {
         getAgentStatusById,
+        authenticateAgentWithAccessToken,
+      },
+      datasourceService,
+      registerExternalCallback: (...args: ExternalCallback) => {
+        return appContextService.addExternalCallback(...args);
       },
     };
   }

x-pack/plugins/ingest_manager/server/routes/agent/acks_handlers.test.ts

@@ -77,7 +77,7 @@ describe('test acks handlers', () => {
           id: 'action1',
         },
       ]),
-      getAgentByAccessAPIKeyId: jest.fn().mockReturnValueOnce({
+      authenticateAgentWithAccessToken: jest.fn().mockReturnValueOnce({
         id: 'agent',
       }),
       getSavedObjectsClientContract: jest.fn().mockReturnValueOnce(mockSavedObjectsClient),

x-pack/plugins/ingest_manager/server/routes/agent/acks_handlers.ts

@@ -9,7 +9,6 @@
 import { RequestHandler } from 'kibana/server';
 import { TypeOf } from '@kbn/config-schema';
 import { PostAgentAcksRequestSchema } from '../../types/rest_spec';
-import * as APIKeyService from '../../services/api_keys';
 import { AcksService } from '../../services/agents';
 import { AgentEvent } from '../../../common/types/models';
 import { PostAgentAcksResponse } from '../../../common/types/rest_spec';
@@ -24,8 +23,7 @@ export const postAgentAcksHandlerBuilder = function (
   return async (context, request, response) => {
     try {
       const soClient = ackService.getSavedObjectsClientContract(request);
-      const res = APIKeyService.parseApiKeyFromHeaders(request.headers);
-      const agent = await ackService.getAgentByAccessAPIKeyId(soClient, res.apiKeyId as string);
+      const agent = await ackService.authenticateAgentWithAccessToken(soClient, request);
       const agentEvents = request.body.events as AgentEvent[];
 
       // validate that all events are for the authorized agent obtained from the api key

x-pack/plugins/ingest_manager/server/routes/agent/handlers.ts

@@ -171,8 +171,7 @@ export const postAgentCheckinHandler: RequestHandler<
 > = async (context, request, response) => {
   try {
     const soClient = appContextService.getInternalUserSOClient(request);
-    const res = APIKeyService.parseApiKeyFromHeaders(request.headers);
-    const agent = await AgentService.getAgentByAccessAPIKeyId(soClient, res.apiKeyId);
+    const agent = await AgentService.authenticateAgentWithAccessToken(soClient, request);
     const abortController = new AbortController();
     request.events.aborted$.subscribe(() => {
       abortController.abort();

x-pack/plugins/ingest_manager/server/routes/agent/index.ts

@@ -109,7 +109,7 @@ export const registerRoutes = (router: IRouter) => {
     },
     postAgentAcksHandlerBuilder({
       acknowledgeAgentActions: AgentService.acknowledgeAgentActions,
-      getAgentByAccessAPIKeyId: AgentService.getAgentByAccessAPIKeyId,
+      authenticateAgentWithAccessToken: AgentService.authenticateAgentWithAccessToken,
       getSavedObjectsClientContract: appContextService.getInternalUserSOClient.bind(
         appContextService
       ),