[APM] Additional data telemetry changes

x-pack/plugins/apm/common/apm_telemetry.ts

@@ -199,6 +199,7 @@ export function getApmTelemetryMapping() {
           agent_configuration: tookProperties,
           agents: tookProperties,
           cardinality: tookProperties,
+          cloud: tookProperties,
           groupings: tookProperties,
           indices_stats: tookProperties,
           integrations: tookProperties,
@@ -234,3 +235,26 @@ export function mergeApmTelemetryMapping(
     return draft;
   });
 }
+
+/**
+ * Create the just the mapping at its full path
+ */
+export function getApmTelemetryMappingFullPath() {
+  return {
+    properties: {
+      stack_stats: {
+        properties: {
+          kibana: {
+            properties: {
+              plugins: {
+                properties: {
+                  apm: getApmTelemetryMapping(),
+                },
+              },
+            },
+          },
+        },
+      },
+    },
+  };
+}

x-pack/plugins/apm/dev_docs/telemetry.md

@@ -57,13 +57,19 @@ The mapping used there can be generated with the output of the [`getTelemetryMap
 
 To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository.
 
+When adding a task, the key of the task and the `took` properties need to be added under the `tasks` properties in the mapping, as when tasks run they report the time they took.
+
 If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm:
 
 ```bash
-node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates/xpack-phone-home.json
+node ./scripts/merge-telemetry-mapping.js ../../../../telemetry slug-for-my-change 7.9
 ```
 
-this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes.
+this will replace the contents of the mapping in the repository checkout with the updated mapping.
+
+It will also create a file in the mapping_migrations directory named "00XX-slug-for-my-change". You'll need to rename the file to replace the "XX" with the next sequential migration number based on what's there already and in open pull requests.
+
+You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes.
 
 The queries for the stats are in the [collect data telemetry tasks](../server/lib/apm_telemetry/collect_data_telemetry/tasks.ts).
 

x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts

@@ -5,26 +5,59 @@
  */
 
 import { readFileSync, truncateSync, writeFileSync } from 'fs';
-import { resolve } from 'path';
+import { join, resolve } from 'path';
 import { argv } from 'yargs';
-import { mergeApmTelemetryMapping } from '../../common/apm_telemetry';
+import {
+  getApmTelemetryMappingFullPath,
+  mergeApmTelemetryMapping,
+} from '../../common/apm_telemetry';
 
 function errorExit(error?: Error) {
-  console.error(`usage: ${argv.$0} /path/to/xpack-phone-home.json`); // eslint-disable-line no-console
+  // eslint-disable-next-line no-console
+  console.error(
+    `\n    usage: ${argv.$0} /path/to/telemetry/repo migration-slug stack-minor-version\n`
+  );
   if (error) {
     throw error;
   }
   process.exit(1);
 }
 
 try {
-  const filename = resolve(argv._[0]);
-  const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8'));
+  const telemetryRepoPath = argv._[0];
+  const filename = resolve(
+    join(telemetryRepoPath, 'config/templates/xpack-phone-home.json')
+  );
+  const slug = argv._[1];
+  const stackMinorVersion = argv._[2];
 
-  const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping);
+  if (!slug || !telemetryRepoPath || !stackMinorVersion) {
+    errorExit();
+  }
 
+  const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8'));
+  const migrationFilename = resolve(
+    join(telemetryRepoPath, 'mapping_migrations', `00XX-${slug}`)
+  );
+  const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping);
+  // The suffix for the all-xpack-phone-home index. Will be "202*" for the 2020s
+  const allSuffix = new Date().getFullYear().toString().replace(/.$/, '*');
+  const versionQuery = {
+    query: { range: { 'version.minor': { gte: String(stackMinorVersion) } } },
+  };
   truncateSync(filename);
   writeFileSync(filename, JSON.stringify(newMapping, null, 2));
+
+  writeFileSync(
+    migrationFilename,
+    [
+      `PUT xpack-phone-home,all-xpack-phone-home-${allSuffix}/_mapping`,
+      JSON.stringify(getApmTelemetryMappingFullPath(), null, 2),
+      '',
+      `POST xpack-phone-home,all-xpack-phone-home-${allSuffix}/_update_by_query?wait_for_completion=false&conflicts=proceed`,
+      JSON.stringify(versionQuery, null, 2),
+    ].join('\n')
+  );
 } catch (error) {
   errorExit(error);
 }

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts

@@ -65,4 +65,63 @@ describe('data telemetry collection tasks', () => {
       });
     });
   });
+
+  describe('indices_stats', () => {
+    const indicesStatsTask = tasks.find(
+      (task) => task.name === 'indices_stats'
+    );
+
+    it('returns a map of index stats', async () => {
+      const indicesStats = jest.fn().mockResolvedValueOnce({
+        _all: { total: { docs: { count: 1 }, store: { size_in_bytes: 1 } } },
+        _shards: { total: 1 },
+      });
+
+      expect(
+        await indicesStatsTask?.executor({ indices, indicesStats } as any)
+      ).toEqual({
+        indices: {
+          shards: {
+            total: 1,
+          },
+          all: {
+            total: {
+              docs: {
+                count: 1,
+              },
+              store: {
+                size_in_bytes: 1,
+              },
+            },
+          },
+        },
+      });
+    });
+
+    describe('with no results', () => {
+      it('returns zero values', async () => {
+        const indicesStats = jest.fn().mockResolvedValueOnce({});
+
+        expect(
+          await indicesStatsTask?.executor({ indices, indicesStats } as any)
+        ).toEqual({
+          indices: {
+            shards: {
+              total: 0,
+            },
+            all: {
+              total: {
+                docs: {
+                  count: 0,
+                },
+                store: {
+                  size_in_bytes: 0,
+                },
+              },
+            },
+          },
+        });
+      });
+    });
+  });
 });

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

@@ -33,6 +33,8 @@ import { APMTelemetry } from '../types';
 
 const TIME_RANGES = ['1d', 'all'] as const;
 type TimeRange = typeof TIME_RANGES[number];
+const range1d = { range: { '@timestamp': { gte: 'now-1d' } } };
+const timeout = '5m';
 
 export const tasks: TelemetryTask[] = [
   {
@@ -62,6 +64,7 @@ export const tasks: TelemetryTask[] = [
         ],
         body: {
           size: 0,
+          timeout,
           aggs: {
             [az]: {
               terms: {
@@ -128,21 +131,12 @@ export const tasks: TelemetryTask[] = [
             index: indicesByProcessorEvent[processorEvent],
             body: {
               size: 0,
+              timeout,
               query: {
                 bool: {
                   filter: [
                     { term: { [PROCESSOR_EVENT]: processorEvent } },
-                    ...(timeRange !== 'all'
-                      ? [
-                          {
-                            range: {
-                              '@timestamp': {
-                                gte: `now-${timeRange}`,
-                              },
-                            },
-                          },
-                        ]
-                      : []),
+                    ...(timeRange !== 'all' ? [range1d] : []),
                   ],
                 },
               },
@@ -155,6 +149,7 @@ export const tasks: TelemetryTask[] = [
               ? await search({
                   index: indicesByProcessorEvent[processorEvent],
                   body: {
+                    timeout,
                     query: {
                       bool: {
                         filter: [
@@ -208,6 +203,7 @@ export const tasks: TelemetryTask[] = [
           index: indices.apmAgentConfigurationIndex,
           body: {
             size: 0,
+            timeout,
             track_total_hits: true,
           },
         })
@@ -237,6 +233,7 @@ export const tasks: TelemetryTask[] = [
               ],
               body: {
                 size: 0,
+                timeout,
                 query: {
                   bool: {
                     filter: [
@@ -245,13 +242,7 @@ export const tasks: TelemetryTask[] = [
                           [AGENT_NAME]: agentName,
                         },
                       },
-                      {
-                        range: {
-                          '@timestamp': {
-                            gte: 'now-1d',
-                          },
-                        },
-                      },
+                      range1d,
                     ],
                   },
                 },
@@ -297,6 +288,7 @@ export const tasks: TelemetryTask[] = [
             },
           },
           size: 1,
+          timeout,
           sort: {
             '@timestamp': 'desc',
           },
@@ -330,12 +322,12 @@ export const tasks: TelemetryTask[] = [
   {
     name: 'groupings',
     executor: async ({ search, indices }) => {
-      const range1d = { range: { '@timestamp': { gte: 'now-1d' } } };
       const errorGroupsCount = (
         await search({
           index: indices['apm_oss.errorIndices'],
           body: {
             size: 0,
+            timeout,
             query: {
               bool: {
                 filter: [{ term: { [PROCESSOR_EVENT]: 'error' } }, range1d],
@@ -368,6 +360,7 @@ export const tasks: TelemetryTask[] = [
           index: indices['apm_oss.transactionIndices'],
           body: {
             size: 0,
+            timeout,
             query: {
               bool: {
                 filter: [
@@ -415,6 +408,7 @@ export const tasks: TelemetryTask[] = [
             },
             track_total_hits: true,
             size: 0,
+            timeout,
           },
         })
       ).hits.total.value;
@@ -428,6 +422,7 @@ export const tasks: TelemetryTask[] = [
           ],
           body: {
             size: 0,
+            timeout,
             query: {
               bool: {
                 filter: [range1d],
@@ -497,12 +492,10 @@ export const tasks: TelemetryTask[] = [
           ],
           body: {
             size: 0,
+            timeout,
             query: {
               bool: {
-                filter: [
-                  { term: { [AGENT_NAME]: agentName } },
-                  { range: { '@timestamp': { gte: 'now-1d' } } },
-                ],
+                filter: [{ term: { [AGENT_NAME]: agentName } }, range1d],
               },
             },
             sort: {
@@ -699,15 +692,15 @@ export const tasks: TelemetryTask[] = [
       return {
         indices: {
           shards: {
-            total: response._shards.total,
+            total: response._shards?.total ?? 0,
           },
           all: {
             total: {
               docs: {
-                count: response._all.total.docs.count,
+                count: response._all?.total?.docs?.count ?? 0,
               },
               store: {
-                size_in_bytes: response._all.total.store.size_in_bytes,
+                size_in_bytes: response._all?.total?.store?.size_in_bytes ?? 0,
               },
             },
           },
@@ -721,9 +714,10 @@ export const tasks: TelemetryTask[] = [
       const allAgentsCardinalityResponse = await search({
         body: {
           size: 0,
+          timeout,
           query: {
             bool: {
-              filter: [{ range: { '@timestamp': { gte: 'now-1d' } } }],
+              filter: [range1d],
             },
           },
           aggs: {
@@ -744,10 +738,11 @@ export const tasks: TelemetryTask[] = [
       const rumAgentCardinalityResponse = await search({
         body: {
           size: 0,
+          timeout,
           query: {
             bool: {
               filter: [
-                { range: { '@timestamp': { gte: 'now-1d' } } },
+                range1d,
                 { terms: { [AGENT_NAME]: ['rum-js', 'js-base'] } },
               ],
             },