[APM] Additional data telemetry changes

x-pack/plugins/apm/common/apm_telemetry.test.ts

@@ -4,48 +4,43 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import {
-  getApmTelemetryMapping,
-  mergeApmTelemetryMapping,
-} from './apm_telemetry';
+import { getApmTelemetryMapping } from './apm_telemetry';
+
+// Add this snapshot serializer for this test. The default snapshot serializer
+// prints "Object" next to objects in the JSON output, but we want to be able to
+// Use the output from this JSON snapshot to share with the telemetry team. When
+// new fields are added to the mapping, we'll have a diff in the snapshot.
+expect.addSnapshotSerializer({
+  print: (contents) => {
+    return JSON.stringify(contents, null, 2);
+  },
+  test: () => true,
+});
 
 describe('APM telemetry helpers', () => {
-  describe('getApmTelemetry', () => {
+  describe('getApmTelemetryFullPath', () => {
+    // This test creates a snapshot with the JSON of our full telemetry mapping
+    // that can be PUT in a query to the index on the telemetry cluster. Sharing
+    // the contents of the snapshot with the telemetry team can provide them with
+    // useful information about changes to our telmetry.
     it('generates a JSON object with the telemetry mapping', () => {
-      expect(getApmTelemetryMapping()).toMatchSnapshot();
-    });
-  });
-
-  describe('mergeApmTelemetryMapping', () => {
-    describe('with an invalid mapping', () => {
-      it('throws an error', () => {
-        expect(() => mergeApmTelemetryMapping({})).toThrowError();
-      });
-    });
-
-    describe('with a valid mapping', () => {
-      it('merges the mapping', () => {
-        // This is "valid" in the sense that it has all of the deep fields
-        // needed to merge. It's not a valid mapping opbject.
-        const validTelemetryMapping = {
-          mappings: {
+      expect({
+        properties: {
+          stack_stats: {
             properties: {
-              stack_stats: {
+              kibana: {
                 properties: {
-                  kibana: {
-                    properties: { plugins: { properties: { apm: {} } } },
+                  plugins: {
+                    properties: {
+                      apm: getApmTelemetryMapping(),
+                    },
                   },
                 },
               },
             },
           },
-        };
-
-        expect(
-          mergeApmTelemetryMapping(validTelemetryMapping)?.mappings.properties
-            .stack_stats.properties.kibana.properties.plugins.properties.apm
-        ).toEqual(getApmTelemetryMapping());
-      });
+        },
+      }).toMatchSnapshot();
     });
   });
 });

x-pack/plugins/apm/common/apm_telemetry.ts

@@ -3,7 +3,6 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { produce } from 'immer';
 import { AGENT_NAMES } from './agent_name';
 
 /**
@@ -199,6 +198,7 @@ export function getApmTelemetryMapping() {
           agent_configuration: tookProperties,
           agents: tookProperties,
           cardinality: tookProperties,
+          cloud: tookProperties,
           groupings: tookProperties,
           indices_stats: tookProperties,
           integrations: tookProperties,
@@ -221,16 +221,3 @@ export function getApmTelemetryMapping() {
     },
   };
 }
-
-/**
- * Merge a telemetry mapping object (from https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json)
- * with the output from `getApmTelemetryMapping`.
- */
-export function mergeApmTelemetryMapping(
-  xpackPhoneHomeMapping: Record<string, any>
-) {
-  return produce(xpackPhoneHomeMapping, (draft: Record<string, any>) => {
-    draft.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = getApmTelemetryMapping();
-    return draft;
-  });
-}

x-pack/plugins/apm/dev_docs/telemetry.md

@@ -55,15 +55,9 @@ The mapping for the telemetry data is here under `stack_stats.kibana.plugins.apm
 
 The mapping used there can be generated with the output of the [`getTelemetryMapping`](../common/apm_telemetry.ts) function.
 
-To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository.
+The `schema` property of the `makeUsageCollector` call in the [`createApmTelemetry` function](../server/lib/apm_telemetry/index.ts) contains the output of `getTelemetryMapping`. Pull requests with changes to the schema should automatically notify the Telemetry team so they can update the mapping in the telemetry clusters.
 
-If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm:
-
-```bash
-node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates/xpack-phone-home.json
-```
-
-this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes.
+When adding a task, the key of the task and the `took` properties need to be added under the `tasks` properties in the mapping, as when tasks run they report the time they took.
 
 The queries for the stats are in the [collect data telemetry tasks](../server/lib/apm_telemetry/collect_data_telemetry/tasks.ts).
 

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts

@@ -16,7 +16,7 @@ describe('data telemetry collection tasks', () => {
   } as ApmIndicesConfig;
 
   describe('cloud', () => {
-    const cloudTask = tasks.find((task) => task.name === 'cloud');
+    const task = tasks.find((t) => t.name === 'cloud');
 
     it('returns a map of cloud provider data', async () => {
       const search = jest.fn().mockResolvedValueOnce({
@@ -42,7 +42,7 @@ describe('data telemetry collection tasks', () => {
         },
       });
 
-      expect(await cloudTask?.executor({ indices, search } as any)).toEqual({
+      expect(await task?.executor({ indices, search } as any)).toEqual({
         cloud: {
           availability_zone: ['us-west-1', 'europe-west1-c'],
           provider: ['aws', 'gcp'],
@@ -55,7 +55,7 @@ describe('data telemetry collection tasks', () => {
       it('returns an empty map', async () => {
         const search = jest.fn().mockResolvedValueOnce({});
 
-        expect(await cloudTask?.executor({ indices, search } as any)).toEqual({
+        expect(await task?.executor({ indices, search } as any)).toEqual({
           cloud: {
             availability_zone: [],
             provider: [],
@@ -66,16 +66,91 @@ describe('data telemetry collection tasks', () => {
     });
   });
 
+  describe('processor_events', () => {
+    const task = tasks.find((t) => t.name === 'processor_events');
+
+    it('returns a map of processor events', async () => {
+      const getTime = jest
+        .spyOn(Date.prototype, 'getTime')
+        .mockReturnValue(1594330792957);
+
+      const search = jest.fn().mockImplementation((params: any) => {
+        const isTotalHitsQuery = params?.body?.track_total_hits;
+
+        return Promise.resolve(
+          isTotalHitsQuery
+            ? { hits: { total: { value: 1 } } }
+            : {
+                hits: {
+                  hits: [{ _source: { '@timestamp': 1 } }],
+                },
+              }
+        );
+      });
+
+      expect(await task?.executor({ indices, search } as any)).toEqual({
+        counts: {
+          error: {
+            '1d': 1,
+            all: 1,
+          },
+          metric: {
+            '1d': 1,
+            all: 1,
+          },
+          onboarding: {
+            '1d': 1,
+            all: 1,
+          },
+          sourcemap: {
+            '1d': 1,
+            all: 1,
+          },
+          span: {
+            '1d': 1,
+            all: 1,
+          },
+          transaction: {
+            '1d': 1,
+            all: 1,
+          },
+        },
+        retainment: {
+          error: {
+            ms: 0,
+          },
+          metric: {
+            ms: 0,
+          },
+          onboarding: {
+            ms: 0,
+          },
+          sourcemap: {
+            ms: 0,
+          },
+          span: {
+            ms: 0,
+          },
+          transaction: {
+            ms: 0,
+          },
+        },
+      });
+
+      getTime.mockRestore();
+    });
+  });
+
   describe('integrations', () => {
-    const integrationsTask = tasks.find((task) => task.name === 'integrations');
+    const task = tasks.find((t) => t.name === 'integrations');
 
     it('returns the count of ML jobs', async () => {
       const transportRequest = jest
         .fn()
         .mockResolvedValueOnce({ body: { count: 1 } });
 
       expect(
-        await integrationsTask?.executor({ indices, transportRequest } as any)
+        await task?.executor({ indices, transportRequest } as any)
       ).toEqual({
         integrations: {
           ml: {
@@ -90,7 +165,7 @@ describe('data telemetry collection tasks', () => {
         const transportRequest = jest.fn().mockResolvedValueOnce({});
 
         expect(
-          await integrationsTask?.executor({ indices, transportRequest } as any)
+          await task?.executor({ indices, transportRequest } as any)
         ).toEqual({
           integrations: {
             ml: {
@@ -101,4 +176,57 @@ describe('data telemetry collection tasks', () => {
       });
     });
   });
+
+  describe('indices_stats', () => {
+    const task = tasks.find((t) => t.name === 'indices_stats');
+
+    it('returns a map of index stats', async () => {
+      const indicesStats = jest.fn().mockResolvedValueOnce({
+        _all: { total: { docs: { count: 1 }, store: { size_in_bytes: 1 } } },
+        _shards: { total: 1 },
+      });
+
+      expect(await task?.executor({ indices, indicesStats } as any)).toEqual({
+        indices: {
+          shards: {
+            total: 1,
+          },
+          all: {
+            total: {
+              docs: {
+                count: 1,
+              },
+              store: {
+                size_in_bytes: 1,
+              },
+            },
+          },
+        },
+      });
+    });
+
+    describe('with no results', () => {
+      it('returns zero values', async () => {
+        const indicesStats = jest.fn().mockResolvedValueOnce({});
+
+        expect(await task?.executor({ indices, indicesStats } as any)).toEqual({
+          indices: {
+            shards: {
+              total: 0,
+            },
+            all: {
+              total: {
+                docs: {
+                  count: 0,
+                },
+                store: {
+                  size_in_bytes: 0,
+                },
+              },
+            },
+          },
+        });
+      });
+    });
+  });
 });