[7.x] [Telemetry] collector set to np (#51618) (#51787)

* merge conflicts

* fix upgrade_assistant code

* fix upgrade_assistant code

* checkout upgrade_assistant from master

* checkout upgrade_assistant from master

* redo upgrade_assistant work on 7.x branch

src/core/CONVENTIONS.md

@@ -0,0 +1,249 @@
+# Kibana Conventions
+
+- [Plugin Structure](#plugin-structure)
+
+## Plugin Structure
+
+All Kibana plugins built at Elastic should follow the same structure.
+
+```
+my_plugin/
+├── kibana.json
+├── public
+│   ├── applications
+│   │   ├── my_app
+│   │   │   └── index.ts
+│   │   └── index.ts
+│   ├── services
+│   │   ├── my_service
+│   │   │   └── index.ts
+│   │   └── index.ts
+│   ├── index.ts
+│   └── plugin.ts
+└── server
+    ├── routes
+    │   └── index.ts
+    ├── services
+    │   ├── my_service
+    │   │   └── index.ts
+    │   └── index.ts
+    ├── index.ts
+    └── plugin.ts
+```
+
+- Both `server` and `public` should have an `index.ts` and a `plugin.ts` file:
+  - `index.ts` should only contain:
+    - The `plugin` export
+    - The `config` export (server only, optional)
+    - Type exports for your Setup and Start interfaces, and any relevant sub-types
+    - Static, pure exports to be consumed by other plugins
+  - `plugin.ts` should only export a single class that implements the `Plugin` interface (more details below)
+- UI applications should live as modules inside the `applications` directory.
+  - Applications should export a single `renderApp` function.
+  - Applications should be loaded using dynamic async imports (more details below)
+  - If there is only a single application, this directory can be called `application` that exports the `renderApp` function.
+- Services provided to other plugins as APIs should live inside the `services` subdirectory.
+  - Services should model the plugin lifecycle (more details below).
+- HTTP routes should be contained inside the `routes` directory.
+  - More should be fleshed out here...
+
+### The PluginInitializer
+
+```ts
+// my_plugin/public/index.ts
+
+import { PluginInitializer } from '../../src/core/public';
+import { MyPlugin, MyPluginSetup, MyPluginStart } from './plugin';
+
+export const plugin: PluginInitializer<MyPluginSetup, MyPluginStart> = () => new MyPlugin();
+export {
+  MyPluginSetup,
+  MyPluginStart
+}
+```
+
+### The Plugin class
+
+```ts
+// my_plugin/public/plugin.ts
+
+import { CoreSetup, CoreStart, Plugin } from '../../src/core/public';
+import { OtherPluginSetup, OtherPluginStart } from '../other_plugin';
+import { ThirdPluginSetup, ThirdPluginStart } from '../third_plugin';
+
+export interface MyPluginSetup {
+  registerThing(...);
+}
+
+export interface MyPluginStart {
+  getThing(): Thing;
+}
+
+export interface MyPluginSetupDeps {
+  otherPlugin: OtherPluginSetup;
+  thirdPlugin?: ThirdPluginSetup;  // Optional dependency
+}
+
+export interface MyPluginStartDeps {
+  otherPlugin: OtherPluginStart;
+  thirdPlugin?: ThirdPluginStart;  // Optional dependency
+}
+
+export class MyPlugin implements Plugin<
+  // All of these types are optional. If your plugin does not expose anything
+  // or depend on other plugins, these can be omitted.
+  MyPluginSetup,
+  MyPluginStart,
+  MyPluginSetupDeps,
+  MyPluginStartDeps,
+> {
+
+  public setup(core: CoreSetup, plugins: MyPluginSetupDeps) {
+    // should return a value that matches `MyPluginSetup`
+  }
+
+  public start(core: CoreStart, plugins: MyPluginStartDeps) {
+    // should return a value that matches `MyPluginStart`
+  }
+
+  public stop() { ... }
+}
+```
+
+Difference between `setup` and `start`:
+- `setup` is reserved for "registration" work
+- `start` is where any "running" logic for your plugin would live. This only applies if you need to start listening for
+  outside events (polling for work, listening on a port, etc.)
+
+The bulk of your plugin logic will most likely live inside _handlers_ registered during `setup`.
+
+### Applications
+
+It's important that UI code is not included in the main bundle for your plugin. Our webpack configuration supports
+dynamic async imports to split out imports into a separate bundle. Every app's rendering logic and UI code should
+leverage this pattern.
+
+```tsx
+// my_plugin/public/applications/my_app.ts
+
+import React from 'react';
+import ReactDOM from 'react-dom';
+import { ApplicationMountContext } from '../../src/core/public';
+
+import { MyAppRoot } from './components/app.ts';
+
+/**
+ * This module will be loaded asynchronously to reduce the bundle size of your plugin's main bundle.
+ */
+export const renderApp = (context: ApplicationMountContext, domElement: HTMLDivElement) => {
+  ReactDOM.render(<MyAppRoot context={context} />, domElement);
+  return () => ReactDOM.unmountComponentAtNode(domElement);
+}
+```
+
+```ts
+// my_plugin/public/plugin.ts
+
+import { Plugin } from '../../src/core/public';
+
+export class MyPlugin implements Plugin {
+  public setup(core) {
+    core.application.register({
+      id: 'my-app',
+      async mount(context, domElement) {
+        // Load application bundle
+        const { renderApp } = await import('./application/my_app');
+        return renderApp(context, domElement);
+      }
+    });
+  }
+}
+```
+
+### Services
+
+Service structure should mirror the plugin lifecycle to make reasoning about how the service is executed more clear.
+
+```ts
+// my_plugin/public/services/my_service.ts
+
+export class MyService {
+  private readonly strings$ = new BehaviorSubject<string[]>();
+
+  public setup() {
+    return {
+      registerStrings: (newString: string) =>
+        this.strings$.next([...this.strings$.value, newString]);
+    }
+  }
+
+  public start() {
+    this.strings$.complete();
+
+    return {
+      strings: this.strings$.value
+    };
+  }
+}
+```
+
+Constructing and interacting with this service becomes very simple from the top-level Plugin class:
+
+```ts
+// my_plugin/public/plugin.ts
+
+import { MyService } from './services';
+
+export class Plugin {
+  private readonly myService = new MyService();
+
+  public setup() {
+    return {
+      myService: myService.setup();
+    }
+  }
+
+  public start() {
+    return {
+      myService: myService.start();
+    }
+  }
+}
+```
+
+### Usage Collection
+
+For creating and registering a Usage Collector. Collectors would be defined in a separate directory `server/collectors/register.ts`. You can read more about usage collectors on `src/plugins/usage_collection/README.md`.
+
+```ts
+// server/collectors/register.ts
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
+import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
+
+export function registerMyPluginUsageCollector(usageCollection?: UsageCollectionSetup): void {
+  // usageCollection is an optional dependency, so make sure to return if it is not registered.
+  if (!usageCollection) {
+    return;
+  }
+
+  // create usage collector
+  const myCollector = usageCollection.makeUsageCollector({
+    type: MY_USAGE_TYPE,
+    fetch: async (callCluster: CallCluster) => {
+
+    // query ES and get some data
+    // summarize the data into a model
+    // return the modeled object that includes whatever you want to track
+
+      return {
+        my_objects: {
+          total: SOME_NUMBER
+        }
+      };
+    },
+  });
+
+  // register usage collector
+  usageCollection.registerCollector(myCollector);
+}
+```

src/legacy/core_plugins/kibana/index.js

@@ -324,6 +324,7 @@ export default function (kibana) {
     },
 
     init: async function (server) {
+      const { usageCollection } = server.newPlatform.setup.plugins;
       // uuid
       await manageUuid(server);
       // routes
@@ -338,8 +339,8 @@ export default function (kibana) {
       registerKqlTelemetryApi(server);
       registerFieldFormats(server);
       registerTutorials(server);
-      makeKQLUsageCollector(server);
-      registerCspCollector(server);
+      makeKQLUsageCollector(usageCollection, server);
+      registerCspCollector(usageCollection, server);
       server.expose('systemApi', systemApi);
       server.injectUiAppVars('kibana', () => injectVars(server));
     },

src/legacy/core_plugins/kibana/server/lib/csp_usage_collector/csp_collector.ts

@@ -19,6 +19,7 @@
 
 import { Server } from 'hapi';
 import { createCSPRuleString, DEFAULT_CSP_RULES } from '../../../../../server/csp';
+import { UsageCollectionSetup } from '../../../../../../plugins/usage_collection/server';
 
 export function createCspCollector(server: Server) {
   return {
@@ -42,8 +43,7 @@ export function createCspCollector(server: Server) {
   };
 }
 
-export function registerCspCollector(server: Server): void {
-  const { collectorSet } = server.usage;
-  const collector = collectorSet.makeUsageCollector(createCspCollector(server));
-  collectorSet.register(collector);
+export function registerCspCollector(usageCollection: UsageCollectionSetup, server: Server): void {
+  const collector = usageCollection.makeUsageCollector(createCspCollector(server));
+  usageCollection.registerCollector(collector);
 }

src/legacy/core_plugins/kibana/server/lib/kql_usage_collector/make_kql_usage_collector.js

@@ -19,14 +19,14 @@
 
 import { fetchProvider } from './fetch';
 
-export function makeKQLUsageCollector(server) {
+export function makeKQLUsageCollector(usageCollection, server) {
   const index = server.config().get('kibana.index');
   const fetch = fetchProvider(index);
-  const kqlUsageCollector = server.usage.collectorSet.makeUsageCollector({
+  const kqlUsageCollector = usageCollection.makeUsageCollector({
     type: 'kql',
     fetch,
     isReady: () => true,
   });
 
-  server.usage.collectorSet.register(kqlUsageCollector);
+  usageCollection.registerCollector(kqlUsageCollector);
 }

src/legacy/core_plugins/kibana/server/lib/kql_usage_collector/make_kql_usage_collector.test.js

@@ -20,29 +20,30 @@
 import { makeKQLUsageCollector } from './make_kql_usage_collector';
 
 describe('makeKQLUsageCollector', () => {
-
   let server;
   let makeUsageCollectorStub;
   let registerStub;
+  let usageCollection;
 
   beforeEach(() => {
     makeUsageCollectorStub = jest.fn();
     registerStub = jest.fn();
+    usageCollection = {
+      makeUsageCollector: makeUsageCollectorStub,
+      registerCollector: registerStub,
+    };
     server = {
-      usage: {
-        collectorSet: { makeUsageCollector: makeUsageCollectorStub, register: registerStub },
-      },
       config: () => ({ get: () => '.kibana' })
     };
   });
 
-  it('should call collectorSet.register', () => {
-    makeKQLUsageCollector(server);
+  it('should call registerCollector', () => {
+    makeKQLUsageCollector(usageCollection, server);
     expect(registerStub).toHaveBeenCalledTimes(1);
   });
 
   it('should call makeUsageCollector with type = kql', () => {
-    makeKQLUsageCollector(server);
+    makeKQLUsageCollector(usageCollection, server);
     expect(makeUsageCollectorStub).toHaveBeenCalledTimes(1);
     expect(makeUsageCollectorStub.mock.calls[0][0].type).toBe('kql');
   });

src/legacy/core_plugins/telemetry/README.md

@@ -0,0 +1,9 @@
+# Kibana Telemetry Service
+
+Telemetry allows Kibana features to have usage tracked in the wild. The general term "telemetry" refers to multiple things:
+
+1. Integrating with the telemetry service to express how to collect usage data (Collecting).
+2. Sending a payload of usage data up to Elastic's telemetry cluster.
+3. Viewing usage data in the Kibana instance of the telemetry cluster (Viewing).
+
+This plugin is responsible for sending usage data to the telemetry cluster. For collecting usage data, use 

src/legacy/core_plugins/telemetry/index.ts

@@ -27,14 +27,7 @@ import { i18n } from '@kbn/i18n';
 import mappings from './mappings.json';
 import { CONFIG_TELEMETRY, getConfigTelemetryDesc } from './common/constants';
 import { getXpackConfigWithDeprecated } from './common/get_xpack_config_with_deprecated';
-import { telemetryPlugin, replaceTelemetryInjectedVars, FetcherTask } from './server';
-
-import {
-  createLocalizationUsageCollector,
-  createTelemetryUsageCollector,
-  createUiMetricUsageCollector,
-  createTelemetryPluginUsageCollector,
-} from './server/collectors';
+import { telemetryPlugin, replaceTelemetryInjectedVars, FetcherTask, PluginsSetup } from './server';
 
 const ENDPOINT_VERSION = 'v2';
 
@@ -123,6 +116,7 @@ const telemetry = (kibana: any) => {
       fetcherTask.start();
     },
     init(server: Server) {
+      const { usageCollection } = server.newPlatform.setup.plugins;
       const initializerContext = {
         env: {
           packageInfo: {
@@ -149,12 +143,11 @@ const telemetry = (kibana: any) => {
         log: server.log,
       } as any) as CoreSetup;
 
-      telemetryPlugin(initializerContext).setup(coreSetup);
-      // register collectors
-      server.usage.collectorSet.register(createTelemetryPluginUsageCollector(server));
-      server.usage.collectorSet.register(createLocalizationUsageCollector(server));
-      server.usage.collectorSet.register(createTelemetryUsageCollector(server));
-      server.usage.collectorSet.register(createUiMetricUsageCollector(server));
+      const pluginsSetup: PluginsSetup = {
+        usageCollection,
+      };
+
+      telemetryPlugin(initializerContext).setup(coreSetup, pluginsSetup, server);
     },
   });
 };