From 043c722399ff132542bb9db5e3f42fdfcf24dd71 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Wed, 8 Jul 2020 10:54:07 -0500 Subject: [PATCH 01/11] [APM] Add more bounds to telemetry tasks * Add a date range of `now-1d` to the cloud query * Add a timeout of 5m to all queries (we'll investigate using async queries to improve this in the future.) * Factor out the date range filter into a variable * Fix a bug with the `indices_stats` tasks when it doesn't return data * Update the merge mapping script to create a migration file --- .../__snapshots__/apm_telemetry.test.ts.snap | 11 ++++ x-pack/plugins/apm/common/apm_telemetry.ts | 24 ++++++++ x-pack/plugins/apm/dev_docs/telemetry.md | 10 +++- .../scripts/merge-telemetry-mapping/index.ts | 45 ++++++++++++-- .../collect_data_telemetry/tasks.test.ts | 59 +++++++++++++++++++ .../collect_data_telemetry/tasks.ts | 51 ++++++++-------- 6 files changed, 164 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index 4ee7692222d689..b13e90e94ad47d 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -839,6 +839,17 @@ Object { }, }, }, + "cloud": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, "groupings": Object { "properties": Object { "took": Object { diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index 5837648f3e5054..0a56923d0c14fb 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -199,6 +199,7 @@ export function getApmTelemetryMapping() { agent_configuration: tookProperties, agents: tookProperties, cardinality: tookProperties, + cloud: tookProperties, groupings: tookProperties, indices_stats: tookProperties, integrations: tookProperties, @@ -234,3 +235,26 @@ export function mergeApmTelemetryMapping( return draft; }); } + +/** + * Create the just the mapping at its full path + */ +export function getApmTelemetryMappingFullPath() { + return { + properties: { + stack_stats: { + properties: { + kibana: { + properties: { + plugins: { + properties: { + apm: getApmTelemetryMapping(), + }, + }, + }, + }, + }, + }, + }, + }; +} diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md index fa8e057a595954..f2815f1d1262ec 100644 --- a/x-pack/plugins/apm/dev_docs/telemetry.md +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -57,13 +57,19 @@ The mapping used there can be generated with the output of the [`getTelemetryMap To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository. +When adding a task, the key of the task and the `took` properties need to be added under the `tasks` properties in the mapping, as when tasks run they report the time they took. + If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm: ```bash -node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates/xpack-phone-home.json +node ./scripts/merge-telemetry-mapping.js ../../../../telemetry slug-for-my-change 7.9 ``` -this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. +this will replace the contents of the mapping in the repository checkout with the updated mapping. + +It will also create a file in the mapping_migrations directory named "00XX-slug-for-my-change". You'll need to rename the file to replace the "XX" with the next sequential migration number based on what's there already and in open pull requests. + +You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. The queries for the stats are in the [collect data telemetry tasks](../server/lib/apm_telemetry/collect_data_telemetry/tasks.ts). diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts index c06d4cec150dcf..b4d1817681cde8 100644 --- a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts +++ b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts @@ -5,12 +5,18 @@ */ import { readFileSync, truncateSync, writeFileSync } from 'fs'; -import { resolve } from 'path'; +import { join, resolve } from 'path'; import { argv } from 'yargs'; -import { mergeApmTelemetryMapping } from '../../common/apm_telemetry'; +import { + getApmTelemetryMappingFullPath, + mergeApmTelemetryMapping, +} from '../../common/apm_telemetry'; function errorExit(error?: Error) { - console.error(`usage: ${argv.$0} /path/to/xpack-phone-home.json`); // eslint-disable-line no-console + // eslint-disable-next-line no-console + console.error( + `\n usage: ${argv.$0} /path/to/telemetry/repo migration-slug stack-minor-version\n` + ); if (error) { throw error; } @@ -18,13 +24,40 @@ function errorExit(error?: Error) { } try { - const filename = resolve(argv._[0]); - const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8')); + const telemetryRepoPath = argv._[0]; + const filename = resolve( + join(telemetryRepoPath, 'config/templates/xpack-phone-home.json') + ); + const slug = argv._[1]; + const stackMinorVersion = argv._[2]; - const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping); + if (!slug || !telemetryRepoPath || !stackMinorVersion) { + errorExit(); + } + const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8')); + const migrationFilename = resolve( + join(telemetryRepoPath, 'mapping_migrations', `00XX-${slug}`) + ); + const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping); + // The suffix for the all-xpack-phone-home index. Will be "202*" for the 2020s + const allSuffix = new Date().getFullYear().toString().replace(/.$/, '*'); + const versionQuery = { + query: { range: { 'version.minor': { gte: String(stackMinorVersion) } } }, + }; truncateSync(filename); writeFileSync(filename, JSON.stringify(newMapping, null, 2)); + + writeFileSync( + migrationFilename, + [ + `PUT xpack-phone-home,all-xpack-phone-home-${allSuffix}/_mapping`, + JSON.stringify(getApmTelemetryMappingFullPath(), null, 2), + '', + `POST xpack-phone-home,all-xpack-phone-home-${allSuffix}/_update_by_query?wait_for_completion=false&conflicts=proceed`, + JSON.stringify(versionQuery, null, 2), + ].join('\n') + ); } catch (error) { errorExit(error); } diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index c648cf4cc116a6..fd95eab5f64b4b 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -65,4 +65,63 @@ describe('data telemetry collection tasks', () => { }); }); }); + + describe('indices_stats', () => { + const indicesStatsTask = tasks.find( + (task) => task.name === 'indices_stats' + ); + + it('returns a map of index stats', async () => { + const indicesStats = jest.fn().mockResolvedValueOnce({ + _all: { total: { docs: { count: 1 }, store: { size_in_bytes: 1 } } }, + _shards: { total: 1 }, + }); + + expect( + await indicesStatsTask?.executor({ indices, indicesStats } as any) + ).toEqual({ + indices: { + shards: { + total: 1, + }, + all: { + total: { + docs: { + count: 1, + }, + store: { + size_in_bytes: 1, + }, + }, + }, + }, + }); + }); + + describe('with no results', () => { + it('returns zero values', async () => { + const indicesStats = jest.fn().mockResolvedValueOnce({}); + + expect( + await indicesStatsTask?.executor({ indices, indicesStats } as any) + ).toEqual({ + indices: { + shards: { + total: 0, + }, + all: { + total: { + docs: { + count: 0, + }, + store: { + size_in_bytes: 0, + }, + }, + }, + }, + }); + }); + }); + }); }); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index f27af9a2cc5165..43056dfc83704b 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -33,6 +33,8 @@ import { APMTelemetry } from '../types'; const TIME_RANGES = ['1d', 'all'] as const; type TimeRange = typeof TIME_RANGES[number]; +const range1d = { range: { '@timestamp': { gte: 'now-1d' } } }; +const timeout = '5m'; export const tasks: TelemetryTask[] = [ { @@ -62,6 +64,7 @@ export const tasks: TelemetryTask[] = [ ], body: { size: 0, + timeout, aggs: { [az]: { terms: { @@ -128,21 +131,12 @@ export const tasks: TelemetryTask[] = [ index: indicesByProcessorEvent[processorEvent], body: { size: 0, + timeout, query: { bool: { filter: [ { term: { [PROCESSOR_EVENT]: processorEvent } }, - ...(timeRange !== 'all' - ? [ - { - range: { - '@timestamp': { - gte: `now-${timeRange}`, - }, - }, - }, - ] - : []), + ...(timeRange !== 'all' ? [range1d] : []), ], }, }, @@ -155,6 +149,7 @@ export const tasks: TelemetryTask[] = [ ? await search({ index: indicesByProcessorEvent[processorEvent], body: { + timeout, query: { bool: { filter: [ @@ -208,6 +203,7 @@ export const tasks: TelemetryTask[] = [ index: indices.apmAgentConfigurationIndex, body: { size: 0, + timeout, track_total_hits: true, }, }) @@ -237,6 +233,7 @@ export const tasks: TelemetryTask[] = [ ], body: { size: 0, + timeout, query: { bool: { filter: [ @@ -245,13 +242,7 @@ export const tasks: TelemetryTask[] = [ [AGENT_NAME]: agentName, }, }, - { - range: { - '@timestamp': { - gte: 'now-1d', - }, - }, - }, + range1d, ], }, }, @@ -297,6 +288,7 @@ export const tasks: TelemetryTask[] = [ }, }, size: 1, + timeout, sort: { '@timestamp': 'desc', }, @@ -330,12 +322,12 @@ export const tasks: TelemetryTask[] = [ { name: 'groupings', executor: async ({ search, indices }) => { - const range1d = { range: { '@timestamp': { gte: 'now-1d' } } }; const errorGroupsCount = ( await search({ index: indices['apm_oss.errorIndices'], body: { size: 0, + timeout, query: { bool: { filter: [{ term: { [PROCESSOR_EVENT]: 'error' } }, range1d], @@ -368,6 +360,7 @@ export const tasks: TelemetryTask[] = [ index: indices['apm_oss.transactionIndices'], body: { size: 0, + timeout, query: { bool: { filter: [ @@ -415,6 +408,7 @@ export const tasks: TelemetryTask[] = [ }, track_total_hits: true, size: 0, + timeout, }, }) ).hits.total.value; @@ -428,6 +422,7 @@ export const tasks: TelemetryTask[] = [ ], body: { size: 0, + timeout, query: { bool: { filter: [range1d], @@ -497,12 +492,10 @@ export const tasks: TelemetryTask[] = [ ], body: { size: 0, + timeout, query: { bool: { - filter: [ - { term: { [AGENT_NAME]: agentName } }, - { range: { '@timestamp': { gte: 'now-1d' } } }, - ], + filter: [{ term: { [AGENT_NAME]: agentName } }, range1d], }, }, sort: { @@ -699,15 +692,15 @@ export const tasks: TelemetryTask[] = [ return { indices: { shards: { - total: response._shards.total, + total: response._shards?.total ?? 0, }, all: { total: { docs: { - count: response._all.total.docs.count, + count: response._all?.total?.docs?.count ?? 0, }, store: { - size_in_bytes: response._all.total.store.size_in_bytes, + size_in_bytes: response._all?.total?.store?.size_in_bytes ?? 0, }, }, }, @@ -721,9 +714,10 @@ export const tasks: TelemetryTask[] = [ const allAgentsCardinalityResponse = await search({ body: { size: 0, + timeout, query: { bool: { - filter: [{ range: { '@timestamp': { gte: 'now-1d' } } }], + filter: [range1d], }, }, aggs: { @@ -744,10 +738,11 @@ export const tasks: TelemetryTask[] = [ const rumAgentCardinalityResponse = await search({ body: { size: 0, + timeout, query: { bool: { filter: [ - { range: { '@timestamp': { gte: 'now-1d' } } }, + range1d, { terms: { [AGENT_NAME]: ['rum-js', 'js-base'] } }, ], }, From 99c869107141a6aac0db0e2de06c810ee7e904a6 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Thu, 9 Jul 2020 14:43:43 -0500 Subject: [PATCH 02/11] update serializer --- .../__snapshots__/apm_telemetry.test.ts.snap | 1497 ++++++++--------- .../plugins/apm/common/apm_telemetry.test.ts | 11 + x-pack/plugins/apm/common/apm_telemetry.ts | 2 +- 3 files changed, 755 insertions(+), 755 deletions(-) diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index b13e90e94ad47d..0f791c3fccf2e2 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -1,940 +1,929 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the telemetry mapping 1`] = ` -Object { - "properties": Object { - "agents": Object { - "properties": Object { - "dotnet": Object { - "properties": Object { - "agent": Object { - "properties": Object { - "version": Object { - "ignore_above": 1024, +{ + "properties": { + "agents": { + "properties": { + "dotnet": { + "properties": { + "agent": { + "properties": { + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "service": Object { - "properties": Object { - "framework": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "language": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "language": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "runtime": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "runtime": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, - }, - }, - }, - }, - }, - "go": Object { - "properties": Object { - "agent": Object { - "properties": Object { - "version": Object { - "ignore_above": 1024, + "ignore_above": 1024 + } + } + } + } + } + } + }, + "go": { + "properties": { + "agent": { + "properties": { + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "service": Object { - "properties": Object { - "framework": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "language": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "language": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "runtime": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "runtime": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, - }, - }, - }, - }, - }, - "java": Object { - "properties": Object { - "agent": Object { - "properties": Object { - "version": Object { - "ignore_above": 1024, + "ignore_above": 1024 + } + } + } + } + } + } + }, + "java": { + "properties": { + "agent": { + "properties": { + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "service": Object { - "properties": Object { - "framework": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "language": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "language": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "runtime": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "runtime": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, - }, - }, - }, - }, - }, - "js-base": Object { - "properties": Object { - "agent": Object { - "properties": Object { - "version": Object { - "ignore_above": 1024, + "ignore_above": 1024 + } + } + } + } + } + } + }, + "js-base": { + "properties": { + "agent": { + "properties": { + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "service": Object { - "properties": Object { - "framework": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "language": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "language": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "runtime": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "runtime": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, - }, - }, - }, - }, - }, - "nodejs": Object { - "properties": Object { - "agent": Object { - "properties": Object { - "version": Object { - "ignore_above": 1024, + "ignore_above": 1024 + } + } + } + } + } + } + }, + "nodejs": { + "properties": { + "agent": { + "properties": { + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "service": Object { - "properties": Object { - "framework": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "language": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "language": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "runtime": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "runtime": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, - }, - }, - }, - }, - }, - "python": Object { - "properties": Object { - "agent": Object { - "properties": Object { - "version": Object { - "ignore_above": 1024, + "ignore_above": 1024 + } + } + } + } + } + } + }, + "python": { + "properties": { + "agent": { + "properties": { + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "service": Object { - "properties": Object { - "framework": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "language": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "language": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "runtime": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "runtime": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, - }, - }, - }, - }, - }, - "ruby": Object { - "properties": Object { - "agent": Object { - "properties": Object { - "version": Object { - "ignore_above": 1024, + "ignore_above": 1024 + } + } + } + } + } + } + }, + "ruby": { + "properties": { + "agent": { + "properties": { + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "service": Object { - "properties": Object { - "framework": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "language": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "language": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "runtime": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "runtime": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, - }, - }, - }, - }, - }, - "rum-js": Object { - "properties": Object { - "agent": Object { - "properties": Object { - "version": Object { - "ignore_above": 1024, + "ignore_above": 1024 + } + } + } + } + } + } + }, + "rum-js": { + "properties": { + "agent": { + "properties": { + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "service": Object { - "properties": Object { - "framework": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "language": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "language": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "runtime": Object { - "properties": Object { - "composite": Object { - "ignore_above": 1024, + "runtime": { + "properties": { + "composite": { "type": "keyword", + "ignore_above": 1024 }, - "name": Object { - "ignore_above": 1024, + "name": { "type": "keyword", + "ignore_above": 1024 }, - "version": Object { - "ignore_above": 1024, + "version": { "type": "keyword", - }, - }, - }, - }, - }, - }, - }, - }, + "ignore_above": 1024 + } + } + } + } + } + } + } + } }, - "cardinality": Object { - "properties": Object { - "transaction": Object { - "properties": Object { - "name": Object { - "properties": Object { - "all_agents": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - }, - }, - "rum": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - }, - }, - }, - }, - }, - }, - "user_agent": Object { - "properties": Object { - "original": Object { - "properties": Object { - "all_agents": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - }, - }, - "rum": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - }, - }, - }, - }, - }, - }, - }, - }, - "cloud": Object { - "properties": Object { - "availability_zone": Object { - "ignore_above": 1024, + "cloud": { + "properties": { + "availability_zone": { "type": "keyword", + "ignore_above": 1024 }, - "provider": Object { - "ignore_above": 1024, + "provider": { "type": "keyword", + "ignore_above": 1024 }, - "region": Object { - "ignore_above": 1024, + "region": { "type": "keyword", - }, - }, + "ignore_above": 1024 + } + } }, - "counts": Object { - "properties": Object { - "agent_configuration": Object { - "properties": Object { - "all": Object { - "type": "long", - }, - }, - }, - "error": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - "all": Object { - "type": "long", - }, - }, - }, - "max_error_groups_per_service": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - }, - }, - "max_transaction_groups_per_service": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - }, - }, - "metric": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - "all": Object { - "type": "long", - }, - }, - }, - "onboarding": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - "all": Object { - "type": "long", - }, - }, - }, - "services": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - }, - }, - "sourcemap": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - "all": Object { - "type": "long", - }, - }, - }, - "span": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - "all": Object { - "type": "long", - }, - }, - }, - "traces": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - }, - }, - "transaction": Object { - "properties": Object { - "1d": Object { - "type": "long", - }, - "all": Object { - "type": "long", - }, - }, - }, - }, + "counts": { + "properties": { + "agent_configuration": { + "properties": { + "all": { + "type": "long" + } + } + }, + "error": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "max_error_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "max_transaction_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "services": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "sourcemap": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "span": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "traces": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + } + } }, - "has_any_services": Object { - "type": "boolean", + "cardinality": { + "properties": { + "user_agent": { + "properties": { + "original": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + }, + "transaction": { + "properties": { + "name": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + } + } }, - "indices": Object { - "properties": Object { - "all": Object { - "properties": Object { - "total": Object { - "properties": Object { - "docs": Object { - "properties": Object { - "count": Object { - "type": "long", - }, - }, - }, - "store": Object { - "properties": Object { - "size_in_bytes": Object { - "type": "long", - }, - }, - }, - }, - }, - }, - }, - "shards": Object { - "properties": Object { - "total": Object { - "type": "long", - }, - }, - }, - }, + "has_any_services": { + "type": "boolean" }, - "integrations": Object { - "properties": Object { - "ml": Object { - "properties": Object { - "all_jobs_count": Object { - "type": "long", - }, - }, - }, - }, + "indices": { + "properties": { + "all": { + "properties": { + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + } + } + } + } + } + } + }, + "shards": { + "properties": { + "total": { + "type": "long" + } + } + } + } }, - "retainment": Object { - "properties": Object { - "error": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - "metric": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - "onboarding": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - "span": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - "transaction": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, + "integrations": { + "properties": { + "ml": { + "properties": { + "all_jobs_count": { + "type": "long" + } + } + } + } }, - "services_per_agent": Object { - "properties": Object { - "dotnet": Object { - "null_value": 0, + "retainment": { + "properties": { + "error": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "span": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "services_per_agent": { + "properties": { + "dotnet": { "type": "long", + "null_value": 0 }, - "go": Object { - "null_value": 0, + "go": { "type": "long", + "null_value": 0 }, - "java": Object { - "null_value": 0, + "java": { "type": "long", + "null_value": 0 }, - "js-base": Object { - "null_value": 0, + "js-base": { "type": "long", + "null_value": 0 }, - "nodejs": Object { - "null_value": 0, + "nodejs": { "type": "long", + "null_value": 0 }, - "python": Object { - "null_value": 0, + "python": { "type": "long", + "null_value": 0 }, - "ruby": Object { - "null_value": 0, + "ruby": { "type": "long", + "null_value": 0 }, - "rum-js": Object { - "null_value": 0, + "rum-js": { "type": "long", - }, - }, + "null_value": 0 + } + } }, - "tasks": Object { - "properties": Object { - "agent_configuration": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - "agents": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - "cardinality": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - "cloud": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - "groupings": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - "indices_stats": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - "integrations": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - "processor_events": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - "services": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - "versions": Object { - "properties": Object { - "took": Object { - "properties": Object { - "ms": Object { - "type": "long", - }, - }, - }, - }, - }, - }, - }, - "version": Object { - "properties": Object { - "apm_server": Object { - "properties": Object { - "major": Object { - "type": "long", - }, - "minor": Object { - "type": "long", - }, - "patch": Object { - "type": "long", - }, - }, - }, - }, + "tasks": { + "properties": { + "agent_configuration": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "agents": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "cardinality": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "groupings": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "indices_stats": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "integrations": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "processor_events": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "services": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "versions": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } }, - }, + "version": { + "properties": { + "apm_server": { + "properties": { + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "patch": { + "type": "long" + } + } + } + } + } + } } `; diff --git a/x-pack/plugins/apm/common/apm_telemetry.test.ts b/x-pack/plugins/apm/common/apm_telemetry.test.ts index 1612716142ce70..565536e6b83407 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.test.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.test.ts @@ -9,6 +9,17 @@ import { mergeApmTelemetryMapping, } from './apm_telemetry'; +// Add this snapshot serializer for this test. The default snapshot serializer +// prints "Object" next to objects in the JSON output, but we want to be able to +// Use the output from this JSON snapshot to share with the telemetry team. When +// new fields are added to the mapping, we'll have a diff in the snapshot. +expect.addSnapshotSerializer({ + print: (contents) => { + return JSON.stringify(contents, null, 2); + }, + test: () => true, +}); + describe('APM telemetry helpers', () => { describe('getApmTelemetry', () => { it('generates a JSON object with the telemetry mapping', () => { diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index 0a56923d0c14fb..02bb07f95569d5 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -199,7 +199,7 @@ export function getApmTelemetryMapping() { agent_configuration: tookProperties, agents: tookProperties, cardinality: tookProperties, - cloud: tookProperties, + // cloud: tookProperties, groupings: tookProperties, indices_stats: tookProperties, integrations: tookProperties, From 5e403b968aa2d323486144e93d7dde84764f903c Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Thu, 9 Jul 2020 14:53:14 -0500 Subject: [PATCH 03/11] Use full path for mapping snapshot --- .../__snapshots__/apm_telemetry.test.ts.snap | 1830 +++++++++-------- .../plugins/apm/common/apm_telemetry.test.ts | 7 +- 2 files changed, 927 insertions(+), 910 deletions(-) diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index 0f791c3fccf2e2..2186176c995224 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -1,926 +1,942 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the telemetry mapping 1`] = ` +exports[`APM telemetry helpers getApmTelemetryFullPath generates a JSON object with the full path telemetry mapping 1`] = ` { "properties": { - "agents": { + "stack_stats": { "properties": { - "dotnet": { + "kibana": { "properties": { - "agent": { + "plugins": { "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - } - } - }, - "go": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - } - } - }, - "java": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - } - } - }, - "js-base": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - } - } - }, - "nodejs": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "runtime": { + "apm": { "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 + "agents": { + "properties": { + "dotnet": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + } + } + }, + "go": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + } + } + }, + "java": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + } + } + }, + "js-base": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + } + } + }, + "nodejs": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + } + } + }, + "python": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + } + } + }, + "ruby": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + } + } + }, + "rum-js": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + } + } + } + } + }, + "cloud": { + "properties": { + "availability_zone": { + "type": "keyword", + "ignore_above": 1024 + }, + "provider": { + "type": "keyword", + "ignore_above": 1024 + }, + "region": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "counts": { + "properties": { + "agent_configuration": { + "properties": { + "all": { + "type": "long" + } + } + }, + "error": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "max_error_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "max_transaction_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "services": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "sourcemap": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "span": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "traces": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + } + } + }, + "cardinality": { + "properties": { + "user_agent": { + "properties": { + "original": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + }, + "transaction": { + "properties": { + "name": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "has_any_services": { + "type": "boolean" + }, + "indices": { + "properties": { + "all": { + "properties": { + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + } + } + } + } + } + } + }, + "shards": { + "properties": { + "total": { + "type": "long" + } + } + } + } + }, + "integrations": { + "properties": { + "ml": { + "properties": { + "all_jobs_count": { + "type": "long" + } + } + } + } + }, + "retainment": { + "properties": { + "error": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "span": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "services_per_agent": { + "properties": { + "dotnet": { + "type": "long", + "null_value": 0 + }, + "go": { + "type": "long", + "null_value": 0 + }, + "java": { + "type": "long", + "null_value": 0 + }, + "js-base": { + "type": "long", + "null_value": 0 + }, + "nodejs": { + "type": "long", + "null_value": 0 + }, + "python": { + "type": "long", + "null_value": 0 + }, + "ruby": { + "type": "long", + "null_value": 0 + }, + "rum-js": { + "type": "long", + "null_value": 0 + } + } + }, + "tasks": { + "properties": { + "agent_configuration": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "agents": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "cardinality": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "groupings": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "indices_stats": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "integrations": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "processor_events": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "services": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "versions": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } }, "version": { - "type": "keyword", - "ignore_above": 1024 + "properties": { + "apm_server": { + "properties": { + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "patch": { + "type": "long" + } + } + } + } } } } } } } - }, - "python": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - } - } - }, - "ruby": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - } - } - }, - "rum-js": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - } - } - } - } - }, - "cloud": { - "properties": { - "availability_zone": { - "type": "keyword", - "ignore_above": 1024 - }, - "provider": { - "type": "keyword", - "ignore_above": 1024 - }, - "region": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "counts": { - "properties": { - "agent_configuration": { - "properties": { - "all": { - "type": "long" - } - } - }, - "error": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "max_error_groups_per_service": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "max_transaction_groups_per_service": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "metric": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "onboarding": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "services": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "sourcemap": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "span": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "traces": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "transaction": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - } - } - }, - "cardinality": { - "properties": { - "user_agent": { - "properties": { - "original": { - "properties": { - "all_agents": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "rum": { - "properties": { - "1d": { - "type": "long" - } - } - } - } - } - } - }, - "transaction": { - "properties": { - "name": { - "properties": { - "all_agents": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "rum": { - "properties": { - "1d": { - "type": "long" - } - } - } - } - } - } - } - } - }, - "has_any_services": { - "type": "boolean" - }, - "indices": { - "properties": { - "all": { - "properties": { - "total": { - "properties": { - "docs": { - "properties": { - "count": { - "type": "long" - } - } - }, - "store": { - "properties": { - "size_in_bytes": { - "type": "long" - } - } - } - } - } - } - }, - "shards": { - "properties": { - "total": { - "type": "long" - } - } - } - } - }, - "integrations": { - "properties": { - "ml": { - "properties": { - "all_jobs_count": { - "type": "long" - } - } - } - } - }, - "retainment": { - "properties": { - "error": { - "properties": { - "ms": { - "type": "long" - } - } - }, - "metric": { - "properties": { - "ms": { - "type": "long" - } - } - }, - "onboarding": { - "properties": { - "ms": { - "type": "long" - } - } - }, - "span": { - "properties": { - "ms": { - "type": "long" - } - } - }, - "transaction": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "services_per_agent": { - "properties": { - "dotnet": { - "type": "long", - "null_value": 0 - }, - "go": { - "type": "long", - "null_value": 0 - }, - "java": { - "type": "long", - "null_value": 0 - }, - "js-base": { - "type": "long", - "null_value": 0 - }, - "nodejs": { - "type": "long", - "null_value": 0 - }, - "python": { - "type": "long", - "null_value": 0 - }, - "ruby": { - "type": "long", - "null_value": 0 - }, - "rum-js": { - "type": "long", - "null_value": 0 - } - } - }, - "tasks": { - "properties": { - "agent_configuration": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "agents": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "cardinality": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "groupings": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "indices_stats": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "integrations": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "processor_events": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "services": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "versions": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - } - } - }, - "version": { - "properties": { - "apm_server": { - "properties": { - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "patch": { - "type": "long" - } - } } } } diff --git a/x-pack/plugins/apm/common/apm_telemetry.test.ts b/x-pack/plugins/apm/common/apm_telemetry.test.ts index 565536e6b83407..ae76221c289fe7 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.test.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.test.ts @@ -6,6 +6,7 @@ import { getApmTelemetryMapping, + getApmTelemetryMappingFullPath, mergeApmTelemetryMapping, } from './apm_telemetry'; @@ -21,9 +22,9 @@ expect.addSnapshotSerializer({ }); describe('APM telemetry helpers', () => { - describe('getApmTelemetry', () => { - it('generates a JSON object with the telemetry mapping', () => { - expect(getApmTelemetryMapping()).toMatchSnapshot(); + describe('getApmTelemetryFullPath', () => { + it('generates a JSON object with the full path telemetry mapping', () => { + expect(getApmTelemetryMappingFullPath()).toMatchSnapshot(); }); }); From 374d6a3c1e86e03c3a4c60d83e68e9bb4a825a2f Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Thu, 9 Jul 2020 14:55:08 -0500 Subject: [PATCH 04/11] Cloud properties --- .../common/__snapshots__/apm_telemetry.test.ts.snap | 11 +++++++++++ x-pack/plugins/apm/common/apm_telemetry.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index 2186176c995224..d08c16455f03f5 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -847,6 +847,17 @@ exports[`APM telemetry helpers getApmTelemetryFullPath generates a JSON object w } } }, + "cloud": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, "groupings": { "properties": { "took": { diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index 02bb07f95569d5..0a56923d0c14fb 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -199,7 +199,7 @@ export function getApmTelemetryMapping() { agent_configuration: tookProperties, agents: tookProperties, cardinality: tookProperties, - // cloud: tookProperties, + cloud: tookProperties, groupings: tookProperties, indices_stats: tookProperties, integrations: tookProperties, From 078619049e9be645891f18f7372ee7348baff547 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Thu, 9 Jul 2020 15:54:02 -0500 Subject: [PATCH 05/11] remove script; update docs; add schema --- .../__snapshots__/apm_telemetry.test.ts.snap | 2 +- .../plugins/apm/common/apm_telemetry.test.ts | 49 +++++---------- x-pack/plugins/apm/common/apm_telemetry.ts | 37 ----------- x-pack/plugins/apm/dev_docs/telemetry.md | 14 +---- .../apm/scripts/merge-telemetry-mapping.js | 21 ------- .../scripts/merge-telemetry-mapping/index.ts | 63 ------------------- .../apm/server/lib/apm_telemetry/index.ts | 12 ++-- 7 files changed, 25 insertions(+), 173 deletions(-) delete mode 100644 x-pack/plugins/apm/scripts/merge-telemetry-mapping.js delete mode 100644 x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index d08c16455f03f5..59c0d3459008dc 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`APM telemetry helpers getApmTelemetryFullPath generates a JSON object with the full path telemetry mapping 1`] = ` +exports[`APM telemetry helpers getApmTelemetryFullPath generates a JSON object with the telemetry mapping 1`] = ` { "properties": { "stack_stats": { diff --git a/x-pack/plugins/apm/common/apm_telemetry.test.ts b/x-pack/plugins/apm/common/apm_telemetry.test.ts index ae76221c289fe7..92c71f33f0d99c 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.test.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.test.ts @@ -4,11 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - getApmTelemetryMapping, - getApmTelemetryMappingFullPath, - mergeApmTelemetryMapping, -} from './apm_telemetry'; +import { getApmTelemetryMapping } from './apm_telemetry'; // Add this snapshot serializer for this test. The default snapshot serializer // prints "Object" next to objects in the JSON output, but we want to be able to @@ -23,41 +19,28 @@ expect.addSnapshotSerializer({ describe('APM telemetry helpers', () => { describe('getApmTelemetryFullPath', () => { - it('generates a JSON object with the full path telemetry mapping', () => { - expect(getApmTelemetryMappingFullPath()).toMatchSnapshot(); - }); - }); - - describe('mergeApmTelemetryMapping', () => { - describe('with an invalid mapping', () => { - it('throws an error', () => { - expect(() => mergeApmTelemetryMapping({})).toThrowError(); - }); - }); - - describe('with a valid mapping', () => { - it('merges the mapping', () => { - // This is "valid" in the sense that it has all of the deep fields - // needed to merge. It's not a valid mapping opbject. - const validTelemetryMapping = { - mappings: { + // This test creates a snapshot with the JSON of our full telemetry mapping + // that can be PUT in a query to the index on the telemetry cluster. Sharing + // the contents of the snapshot with the telemetry team can provide them with + // useful information about changes to our telmetry. + it('generates a JSON object with the telemetry mapping', () => { + expect({ + properties: { + stack_stats: { properties: { - stack_stats: { + kibana: { properties: { - kibana: { - properties: { plugins: { properties: { apm: {} } } }, + plugins: { + properties: { + apm: getApmTelemetryMapping(), + }, }, }, }, }, }, - }; - - expect( - mergeApmTelemetryMapping(validTelemetryMapping)?.mappings.properties - .stack_stats.properties.kibana.properties.plugins.properties.apm - ).toEqual(getApmTelemetryMapping()); - }); + }, + }).toMatchSnapshot(); }); }); }); diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index 0a56923d0c14fb..cfc3a75dc89841 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { produce } from 'immer'; import { AGENT_NAMES } from './agent_name'; /** @@ -222,39 +221,3 @@ export function getApmTelemetryMapping() { }, }; } - -/** - * Merge a telemetry mapping object (from https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json) - * with the output from `getApmTelemetryMapping`. - */ -export function mergeApmTelemetryMapping( - xpackPhoneHomeMapping: Record -) { - return produce(xpackPhoneHomeMapping, (draft: Record) => { - draft.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = getApmTelemetryMapping(); - return draft; - }); -} - -/** - * Create the just the mapping at its full path - */ -export function getApmTelemetryMappingFullPath() { - return { - properties: { - stack_stats: { - properties: { - kibana: { - properties: { - plugins: { - properties: { - apm: getApmTelemetryMapping(), - }, - }, - }, - }, - }, - }, - }, - }; -} diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md index f2815f1d1262ec..92614f80f3b6c5 100644 --- a/x-pack/plugins/apm/dev_docs/telemetry.md +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -55,22 +55,10 @@ The mapping for the telemetry data is here under `stack_stats.kibana.plugins.apm The mapping used there can be generated with the output of the [`getTelemetryMapping`](../common/apm_telemetry.ts) function. -To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository. +The `schema` property of the `makeUsageCollector` call in the [`createApmTelemetry` function](../server/lib/apm_telemetry/index.ts) contains the output of `getTelemetryMapping`. Pull requests with changes to the schema should automatically notify the Telemetry team so they can update the mapping in the telemetry clusters. When adding a task, the key of the task and the `took` properties need to be added under the `tasks` properties in the mapping, as when tasks run they report the time they took. -If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm: - -```bash -node ./scripts/merge-telemetry-mapping.js ../../../../telemetry slug-for-my-change 7.9 -``` - -this will replace the contents of the mapping in the repository checkout with the updated mapping. - -It will also create a file in the mapping_migrations directory named "00XX-slug-for-my-change". You'll need to rename the file to replace the "XX" with the next sequential migration number based on what's there already and in open pull requests. - -You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. - The queries for the stats are in the [collect data telemetry tasks](../server/lib/apm_telemetry/collect_data_telemetry/tasks.ts). The collection tasks also use the [`APMDataTelemetry` type](../server/lib/apm_telemetry/types.ts) which also needs to be updated with any changes to the fields. diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js b/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js deleted file mode 100644 index 741df981a9cb0a..00000000000000 --- a/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -// compile typescript on the fly -// eslint-disable-next-line import/no-extraneous-dependencies -require('@babel/register')({ - extensions: ['.ts'], - plugins: [ - '@babel/plugin-proposal-optional-chaining', - '@babel/plugin-proposal-nullish-coalescing-operator', - ], - presets: [ - '@babel/typescript', - ['@babel/preset-env', { targets: { node: 'current' } }], - ], -}); - -require('./merge-telemetry-mapping/index.ts'); diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts deleted file mode 100644 index b4d1817681cde8..00000000000000 --- a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { readFileSync, truncateSync, writeFileSync } from 'fs'; -import { join, resolve } from 'path'; -import { argv } from 'yargs'; -import { - getApmTelemetryMappingFullPath, - mergeApmTelemetryMapping, -} from '../../common/apm_telemetry'; - -function errorExit(error?: Error) { - // eslint-disable-next-line no-console - console.error( - `\n usage: ${argv.$0} /path/to/telemetry/repo migration-slug stack-minor-version\n` - ); - if (error) { - throw error; - } - process.exit(1); -} - -try { - const telemetryRepoPath = argv._[0]; - const filename = resolve( - join(telemetryRepoPath, 'config/templates/xpack-phone-home.json') - ); - const slug = argv._[1]; - const stackMinorVersion = argv._[2]; - - if (!slug || !telemetryRepoPath || !stackMinorVersion) { - errorExit(); - } - - const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8')); - const migrationFilename = resolve( - join(telemetryRepoPath, 'mapping_migrations', `00XX-${slug}`) - ); - const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping); - // The suffix for the all-xpack-phone-home index. Will be "202*" for the 2020s - const allSuffix = new Date().getFullYear().toString().replace(/.$/, '*'); - const versionQuery = { - query: { range: { 'version.minor': { gte: String(stackMinorVersion) } } }, - }; - truncateSync(filename); - writeFileSync(filename, JSON.stringify(newMapping, null, 2)); - - writeFileSync( - migrationFilename, - [ - `PUT xpack-phone-home,all-xpack-phone-home-${allSuffix}/_mapping`, - JSON.stringify(getApmTelemetryMappingFullPath(), null, 2), - '', - `POST xpack-phone-home,all-xpack-phone-home-${allSuffix}/_update_by_query?wait_for_completion=false&conflicts=proceed`, - JSON.stringify(versionQuery, null, 2), - ].join('\n') - ); -} catch (error) { - errorExit(error); -} diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts index 632e653a2f6e94..d04d8f106b217f 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts @@ -3,25 +3,26 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { CoreSetup, Logger } from 'src/core/server'; import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; +import { CoreSetup, Logger } from 'src/core/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { getApmTelemetryMapping } from '../../../common/apm_telemetry'; +import { APMConfig } from '../..'; import { - TaskManagerStartContract, TaskManagerSetupContract, + TaskManagerStartContract, } from '../../../../task_manager/server'; -import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { APM_TELEMETRY_SAVED_OBJECT_ID, APM_TELEMETRY_SAVED_OBJECT_TYPE, } from '../../../common/apm_saved_object_constants'; +import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; +import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { collectDataTelemetry, CollectTelemetryParams, } from './collect_data_telemetry'; -import { APMConfig } from '../..'; -import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task'; @@ -97,6 +98,7 @@ export async function createApmTelemetry({ const collector = usageCollector.makeUsageCollector({ type: 'apm', + schema: getApmTelemetryMapping(), fetch: async () => { try { const data = ( From 81bce44f7a1f7c20c5660b5428970760fc292635 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Thu, 9 Jul 2020 17:01:09 -0500 Subject: [PATCH 06/11] =?UTF-8?q?Updates=20based=20on=20S=C3=B8ren's=20fee?= =?UTF-8?q?dback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../collect_data_telemetry/tasks.test.ts | 99 ++++++++++++++++--- .../collect_data_telemetry/tasks.ts | 16 +-- .../apm/server/lib/apm_telemetry/index.ts | 2 +- 3 files changed, 93 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index c9e4a4c7dd70dc..b697abfe9bdeb9 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -16,7 +16,7 @@ describe('data telemetry collection tasks', () => { } as ApmIndicesConfig; describe('cloud', () => { - const cloudTask = tasks.find((task) => task.name === 'cloud'); + const task = tasks.find((t) => t.name === 'cloud'); it('returns a map of cloud provider data', async () => { const search = jest.fn().mockResolvedValueOnce({ @@ -42,7 +42,7 @@ describe('data telemetry collection tasks', () => { }, }); - expect(await cloudTask?.executor({ indices, search } as any)).toEqual({ + expect(await task?.executor({ indices, search } as any)).toEqual({ cloud: { availability_zone: ['us-west-1', 'europe-west1-c'], provider: ['aws', 'gcp'], @@ -55,7 +55,7 @@ describe('data telemetry collection tasks', () => { it('returns an empty map', async () => { const search = jest.fn().mockResolvedValueOnce({}); - expect(await cloudTask?.executor({ indices, search } as any)).toEqual({ + expect(await task?.executor({ indices, search } as any)).toEqual({ cloud: { availability_zone: [], provider: [], @@ -66,8 +66,83 @@ describe('data telemetry collection tasks', () => { }); }); + describe('processor_events', () => { + const task = tasks.find((t) => t.name === 'processor_events'); + + it('returns a map of processor events', async () => { + const getTime = jest + .spyOn(Date.prototype, 'getTime') + .mockReturnValue(1594330792957); + + const search = jest.fn().mockImplementation((params: any) => { + const isTotalHitsQuery = params?.body?.track_total_hits; + + return Promise.resolve( + isTotalHitsQuery + ? { hits: { total: { value: 1 } } } + : { + hits: { + hits: [{ _source: { '@timestamp': 1 } }], + }, + } + ); + }); + + expect(await task?.executor({ indices, search } as any)).toEqual({ + counts: { + error: { + '1d': 1, + all: 1, + }, + metric: { + '1d': 1, + all: 1, + }, + onboarding: { + '1d': 1, + all: 1, + }, + sourcemap: { + '1d': 1, + all: 1, + }, + span: { + '1d': 1, + all: 1, + }, + transaction: { + '1d': 1, + all: 1, + }, + }, + retainment: { + error: { + ms: 0, + }, + metric: { + ms: 0, + }, + onboarding: { + ms: 0, + }, + sourcemap: { + ms: 0, + }, + span: { + ms: 0, + }, + transaction: { + ms: 0, + }, + }, + }); + + getTime.mockRestore(); + }); + }); + describe('integrations', () => { - const integrationsTask = tasks.find((task) => task.name === 'integrations'); + const task = tasks.find((t) => t.name === 'integrations'); it('returns the count of ML jobs', async () => { const transportRequest = jest @@ -75,7 +150,7 @@ describe('data telemetry collection tasks', () => { .mockResolvedValueOnce({ body: { count: 1 } }); expect( - await integrationsTask?.executor({ indices, transportRequest } as any) + await task?.executor({ indices, transportRequest } as any) ).toEqual({ integrations: { ml: { @@ -90,7 +165,7 @@ describe('data telemetry collection tasks', () => { const transportRequest = jest.fn().mockResolvedValueOnce({}); expect( - await integrationsTask?.executor({ indices, transportRequest } as any) + await task?.executor({ indices, transportRequest } as any) ).toEqual({ integrations: { ml: { @@ -103,9 +178,7 @@ describe('data telemetry collection tasks', () => { }); describe('indices_stats', () => { - const indicesStatsTask = tasks.find( - (task) => task.name === 'indices_stats' - ); + const task = tasks.find((t) => t.name === 'indices_stats'); it('returns a map of index stats', async () => { const indicesStats = jest.fn().mockResolvedValueOnce({ @@ -113,9 +186,7 @@ describe('data telemetry collection tasks', () => { _shards: { total: 1 }, }); - expect( - await indicesStatsTask?.executor({ indices, indicesStats } as any) - ).toEqual({ + expect(await task?.executor({ indices, indicesStats } as any)).toEqual({ indices: { shards: { total: 1, @@ -138,9 +209,7 @@ describe('data telemetry collection tasks', () => { it('returns zero values', async () => { const indicesStats = jest.fn().mockResolvedValueOnce({}); - expect( - await indicesStatsTask?.executor({ indices, indicesStats } as any) - ).toEqual({ + expect(await task?.executor({ indices, indicesStats } as any)).toEqual({ indices: { shards: { total: 0, diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index 6243316274fd9a..bd2c09c5a2d6b7 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -33,6 +33,7 @@ import { APMTelemetry } from '../types'; const TIME_RANGES = ['1d', 'all'] as const; type TimeRange = typeof TIME_RANGES[number]; + const range1d = { range: { '@timestamp': { gte: 'now-1d' } } }; const timeout = '5m'; @@ -112,15 +113,14 @@ export const tasks: TelemetryTask[] = [ type ProcessorEvent = keyof typeof indicesByProcessorEvent; - const jobs: Array<{ + interface Job { processorEvent: ProcessorEvent; timeRange: TimeRange; - }> = flatten( - (Object.keys( - indicesByProcessorEvent - ) as ProcessorEvent[]).map((processorEvent) => - TIME_RANGES.map((timeRange) => ({ processorEvent, timeRange })) - ) + } + + const events = Object.keys(indicesByProcessorEvent) as ProcessorEvent[]; + const jobs: Job[] = events.flatMap((processorEvent) => + TIME_RANGES.map((timeRange) => ({ processorEvent, timeRange })) ); const allData = await jobs.reduce((prevJob, current) => { @@ -136,7 +136,7 @@ export const tasks: TelemetryTask[] = [ bool: { filter: [ { term: { [PROCESSOR_EVENT]: processorEvent } }, - ...(timeRange !== 'all' ? [range1d] : []), + ...(timeRange === '1d' ? [range1d] : []), ], }, }, diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts index d04d8f106b217f..2836cf100a4324 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts @@ -7,7 +7,6 @@ import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; import { CoreSetup, Logger } from 'src/core/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { getApmTelemetryMapping } from '../../../common/apm_telemetry'; import { APMConfig } from '../..'; import { TaskManagerSetupContract, @@ -17,6 +16,7 @@ import { APM_TELEMETRY_SAVED_OBJECT_ID, APM_TELEMETRY_SAVED_OBJECT_TYPE, } from '../../../common/apm_saved_object_constants'; +import { getApmTelemetryMapping } from '../../../common/apm_telemetry'; import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { From a092d45294d9e07e60edac37f0eec892ecca8993 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Mon, 13 Jul 2020 13:46:39 -0500 Subject: [PATCH 07/11] Fix test name; add country code cardinality --- .../__snapshots__/apm_telemetry.test.ts.snap | 17 ++++++++- .../plugins/apm/common/apm_telemetry.test.ts | 2 +- x-pack/plugins/apm/common/apm_telemetry.ts | 9 +++++ .../collect_data_telemetry/tasks.test.ts | 36 +++++++++++++++++++ .../collect_data_telemetry/tasks.ts | 16 +++++++++ 5 files changed, 78 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index 59c0d3459008dc..440e8130adba5d 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`APM telemetry helpers getApmTelemetryFullPath generates a JSON object with the telemetry mapping 1`] = ` +exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the telemetry mapping 1`] = ` { "properties": { "stack_stats": { @@ -644,6 +644,21 @@ exports[`APM telemetry helpers getApmTelemetryFullPath generates a JSON object w }, "cardinality": { "properties": { + "client": { + "properties": { + "geo": { + "properites": { + "country_iso_code": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + }, "user_agent": { "properties": { "original": { diff --git a/x-pack/plugins/apm/common/apm_telemetry.test.ts b/x-pack/plugins/apm/common/apm_telemetry.test.ts index 92c71f33f0d99c..035c546a5b49a2 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.test.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.test.ts @@ -18,7 +18,7 @@ expect.addSnapshotSerializer({ }); describe('APM telemetry helpers', () => { - describe('getApmTelemetryFullPath', () => { + describe('getApmTelemetry', () => { // This test creates a snapshot with the JSON of our full telemetry mapping // that can be PUT in a query to the index on the telemetry cluster. Sharing // the contents of the snapshot with the telemetry team can provide them with diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index cfc3a75dc89841..4369c9775187b9 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -114,6 +114,15 @@ export function getApmTelemetryMapping() { }, cardinality: { properties: { + client: { + properties: { + geo: { + properites: { + country_iso_code: oneDayProperties, + }, + }, + }, + }, user_agent: { properties: { original: { diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index b697abfe9bdeb9..ea2b57c01acff7 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -229,4 +229,40 @@ describe('data telemetry collection tasks', () => { }); }); }); + + describe('cardinality', () => { + const task = tasks.find((t) => t.name === 'cardinality'); + + it('returns cardinalities', async () => { + const search = jest.fn().mockImplementation((params: any) => { + const isRumQuery = params.body.query.bool.filter.length === 2; + if (isRumQuery) { + return Promise.resolve({ + aggregations: { + 'client.geo.country_iso_code': { value: 5 }, + 'transaction.name': { value: 1 }, + 'user_agent.original': { value: 2 }, + }, + }); + } else { + return Promise.resolve({ + aggregations: { + 'transaction.name': { value: 3 }, + 'user_agent.original': { value: 4 }, + }, + }); + } + }); + + expect(await task?.executor({ search } as any)).toEqual({ + cardinality: { + client: { geo: { country_iso_code: { rum: { '1d': 5 } } } }, + transaction: { name: { all_agents: { '1d': 3 }, rum: { '1d': 1 } } }, + user_agent: { + original: { all_agents: { '1d': 4 }, rum: { '1d': 2 } }, + }, + }, + }); + }); + }); }); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index bd2c09c5a2d6b7..2ecb5a935893f4 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -9,6 +9,7 @@ import { AGENT_NAMES } from '../../../../common/agent_name'; import { AGENT_NAME, AGENT_VERSION, + CLIENT_GEO_COUNTRY_ISO_CODE, CLOUD_AVAILABILITY_ZONE, CLOUD_PROVIDER, CLOUD_REGION, @@ -748,6 +749,9 @@ export const tasks: TelemetryTask[] = [ }, }, aggs: { + [CLIENT_GEO_COUNTRY_ISO_CODE]: { + cardinality: { field: CLIENT_GEO_COUNTRY_ISO_CODE }, + }, [TRANSACTION_NAME]: { cardinality: { field: TRANSACTION_NAME, @@ -764,6 +768,18 @@ export const tasks: TelemetryTask[] = [ return { cardinality: { + client: { + geo: { + country_iso_code: { + rum: { + '1d': + rumAgentCardinalityResponse.aggregations?.[ + CLIENT_GEO_COUNTRY_ISO_CODE + ].value, + }, + }, + }, + }, transaction: { name: { all_agents: { From 8fde7b6f26b7f3f23e05d42cfa297c2a1a00a6fb Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Mon, 13 Jul 2020 16:18:21 -0500 Subject: [PATCH 08/11] Docs update --- x-pack/.telemetryrc.json | 1 + x-pack/plugins/apm/dev_docs/telemetry.md | 4 +- .../__test__/__snapshots__/List.test.tsx.snap | 48 +++++++------------ 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/x-pack/.telemetryrc.json b/x-pack/.telemetryrc.json index 2c16491c1096bf..4da44667e167fe 100644 --- a/x-pack/.telemetryrc.json +++ b/x-pack/.telemetryrc.json @@ -7,6 +7,7 @@ "plugins/apm/server/lib/apm_telemetry/index.ts", "plugins/canvas/server/collectors/collector.ts", "plugins/infra/server/usage/usage_collector.ts", + "plugins/ingest_manager/server/collectors/register.ts", "plugins/lens/server/usage/collectors.ts", "plugins/reporting/server/usage/reporting_usage_collector.ts", "plugins/maps/server/maps_telemetry/collectors/register.ts" diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md index 92614f80f3b6c5..d61afbe07522f1 100644 --- a/x-pack/plugins/apm/dev_docs/telemetry.md +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -55,7 +55,7 @@ The mapping for the telemetry data is here under `stack_stats.kibana.plugins.apm The mapping used there can be generated with the output of the [`getTelemetryMapping`](../common/apm_telemetry.ts) function. -The `schema` property of the `makeUsageCollector` call in the [`createApmTelemetry` function](../server/lib/apm_telemetry/index.ts) contains the output of `getTelemetryMapping`. Pull requests with changes to the schema should automatically notify the Telemetry team so they can update the mapping in the telemetry clusters. +The `schema` property of the `makeUsageCollector` call in the [`createApmTelemetry` function](../server/lib/apm_telemetry/index.ts) contains the output of `getTelemetryMapping`. When adding a task, the key of the task and the `took` properties need to be added under the `tasks` properties in the mapping, as when tasks run they report the time they took. @@ -63,6 +63,8 @@ The queries for the stats are in the [collect data telemetry tasks](../server/li The collection tasks also use the [`APMDataTelemetry` type](../server/lib/apm_telemetry/types.ts) which also needs to be updated with any changes to the fields. +Running `node scripts/telemetry_check --fix` from the root Kibana directory will update the schemas which schema should automatically notify the Telemetry team when a pull request is opened so they can update the mapping in the telemetry clusters. (At the time of this writing the APM schema is excluded. #70180 is open to remove these exclusions so at this time any pull requests with mapping changes will have to manually request the Telemetry team as a reviewer.) + ## Behavioral Telemetry Behavioral telemetry is recorded with the ui_metrics and application_usage methods from the Usage Collection plugin. diff --git a/x-pack/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap b/x-pack/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap index a86f7fdf41f4fb..6a20e3c103709d 100644 --- a/x-pack/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap +++ b/x-pack/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap @@ -133,8 +133,6 @@ exports[`ErrorGroupOverview -> List should render empty state 1`] = `
List should render with data 1`] = `
List should render with data 1`] = `
- +
From ec33c3b32d2b25eb5ffbac935223f7543748df0b Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Mon, 13 Jul 2020 16:23:42 -0500 Subject: [PATCH 09/11] More stuff --- x-pack/.telemetryrc.json | 1 - x-pack/plugins/apm/common/apm_telemetry.ts | 2 +- x-pack/plugins/apm/server/lib/apm_telemetry/types.ts | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/.telemetryrc.json b/x-pack/.telemetryrc.json index 4da44667e167fe..2c16491c1096bf 100644 --- a/x-pack/.telemetryrc.json +++ b/x-pack/.telemetryrc.json @@ -7,7 +7,6 @@ "plugins/apm/server/lib/apm_telemetry/index.ts", "plugins/canvas/server/collectors/collector.ts", "plugins/infra/server/usage/usage_collector.ts", - "plugins/ingest_manager/server/collectors/register.ts", "plugins/lens/server/usage/collectors.ts", "plugins/reporting/server/usage/reporting_usage_collector.ts", "plugins/maps/server/maps_telemetry/collectors/register.ts" diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index 4369c9775187b9..5fb6414674d1cb 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -118,7 +118,7 @@ export function getApmTelemetryMapping() { properties: { geo: { properites: { - country_iso_code: oneDayProperties, + country_iso_code: { rum: oneDayProperties }, }, }, }, diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts index a1d94333b1a08f..4c376aac52f5b5 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts @@ -44,6 +44,7 @@ export type APMDataTelemetry = DeepPartial<{ services: TimeframeMap; }; cardinality: { + client: { geo: { country_iso_code: { rum: TimeframeMap1d } } }; user_agent: { original: { all_agents: TimeframeMap1d; From 6477c068ca07d90b4d3a52e9a95c9e2cc15298fc Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Mon, 13 Jul 2020 16:28:45 -0500 Subject: [PATCH 10/11] More stuff --- .../apm/common/__snapshots__/apm_telemetry.test.ts.snap | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index 440e8130adba5d..1d8cfa28aea75e 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -649,9 +649,11 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the "geo": { "properites": { "country_iso_code": { - "properties": { - "1d": { - "type": "long" + "rum": { + "properties": { + "1d": { + "type": "long" + } } } } From 331ea8a2c6198997ba02f802e1460ada78c3b6a5 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Tue, 14 Jul 2020 00:09:15 -0500 Subject: [PATCH 11/11] snapshot update --- .../__test__/__snapshots__/List.test.tsx.snap | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap b/x-pack/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap index 6a20e3c103709d..a86f7fdf41f4fb 100644 --- a/x-pack/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap +++ b/x-pack/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap @@ -133,6 +133,8 @@ exports[`ErrorGroupOverview -> List should render empty state 1`] = `
List should render with data 1`] = `
List should render with data 1`] = `
-
- + + + 1 + + + + + -
+