diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index 4ee7692222d689..b13e90e94ad47d 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -839,6 +839,17 @@ Object { }, }, }, + "cloud": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, "groupings": Object { "properties": Object { "took": Object { diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index 5837648f3e5054..0a56923d0c14fb 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -199,6 +199,7 @@ export function getApmTelemetryMapping() { agent_configuration: tookProperties, agents: tookProperties, cardinality: tookProperties, + cloud: tookProperties, groupings: tookProperties, indices_stats: tookProperties, integrations: tookProperties, @@ -234,3 +235,26 @@ export function mergeApmTelemetryMapping( return draft; }); } + +/** + * Create the just the mapping at its full path + */ +export function getApmTelemetryMappingFullPath() { + return { + properties: { + stack_stats: { + properties: { + kibana: { + properties: { + plugins: { + properties: { + apm: getApmTelemetryMapping(), + }, + }, + }, + }, + }, + }, + }, + }; +} diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md index fa8e057a595954..f2815f1d1262ec 100644 --- a/x-pack/plugins/apm/dev_docs/telemetry.md +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -57,13 +57,19 @@ The mapping used there can be generated with the output of the [`getTelemetryMap To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository. +When adding a task, the key of the task and the `took` properties need to be added under the `tasks` properties in the mapping, as when tasks run they report the time they took. + If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm: ```bash -node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates/xpack-phone-home.json +node ./scripts/merge-telemetry-mapping.js ../../../../telemetry slug-for-my-change 7.9 ``` -this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. +this will replace the contents of the mapping in the repository checkout with the updated mapping. + +It will also create a file in the mapping_migrations directory named "00XX-slug-for-my-change". You'll need to rename the file to replace the "XX" with the next sequential migration number based on what's there already and in open pull requests. + +You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. The queries for the stats are in the [collect data telemetry tasks](../server/lib/apm_telemetry/collect_data_telemetry/tasks.ts). diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts index c06d4cec150dcf..b4d1817681cde8 100644 --- a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts +++ b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts @@ -5,12 +5,18 @@ */ import { readFileSync, truncateSync, writeFileSync } from 'fs'; -import { resolve } from 'path'; +import { join, resolve } from 'path'; import { argv } from 'yargs'; -import { mergeApmTelemetryMapping } from '../../common/apm_telemetry'; +import { + getApmTelemetryMappingFullPath, + mergeApmTelemetryMapping, +} from '../../common/apm_telemetry'; function errorExit(error?: Error) { - console.error(`usage: ${argv.$0} /path/to/xpack-phone-home.json`); // eslint-disable-line no-console + // eslint-disable-next-line no-console + console.error( + `\n usage: ${argv.$0} /path/to/telemetry/repo migration-slug stack-minor-version\n` + ); if (error) { throw error; } @@ -18,13 +24,40 @@ function errorExit(error?: Error) { } try { - const filename = resolve(argv._[0]); - const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8')); + const telemetryRepoPath = argv._[0]; + const filename = resolve( + join(telemetryRepoPath, 'config/templates/xpack-phone-home.json') + ); + const slug = argv._[1]; + const stackMinorVersion = argv._[2]; - const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping); + if (!slug || !telemetryRepoPath || !stackMinorVersion) { + errorExit(); + } + const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8')); + const migrationFilename = resolve( + join(telemetryRepoPath, 'mapping_migrations', `00XX-${slug}`) + ); + const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping); + // The suffix for the all-xpack-phone-home index. Will be "202*" for the 2020s + const allSuffix = new Date().getFullYear().toString().replace(/.$/, '*'); + const versionQuery = { + query: { range: { 'version.minor': { gte: String(stackMinorVersion) } } }, + }; truncateSync(filename); writeFileSync(filename, JSON.stringify(newMapping, null, 2)); + + writeFileSync( + migrationFilename, + [ + `PUT xpack-phone-home,all-xpack-phone-home-${allSuffix}/_mapping`, + JSON.stringify(getApmTelemetryMappingFullPath(), null, 2), + '', + `POST xpack-phone-home,all-xpack-phone-home-${allSuffix}/_update_by_query?wait_for_completion=false&conflicts=proceed`, + JSON.stringify(versionQuery, null, 2), + ].join('\n') + ); } catch (error) { errorExit(error); } diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index c648cf4cc116a6..fd95eab5f64b4b 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -65,4 +65,63 @@ describe('data telemetry collection tasks', () => { }); }); }); + + describe('indices_stats', () => { + const indicesStatsTask = tasks.find( + (task) => task.name === 'indices_stats' + ); + + it('returns a map of index stats', async () => { + const indicesStats = jest.fn().mockResolvedValueOnce({ + _all: { total: { docs: { count: 1 }, store: { size_in_bytes: 1 } } }, + _shards: { total: 1 }, + }); + + expect( + await indicesStatsTask?.executor({ indices, indicesStats } as any) + ).toEqual({ + indices: { + shards: { + total: 1, + }, + all: { + total: { + docs: { + count: 1, + }, + store: { + size_in_bytes: 1, + }, + }, + }, + }, + }); + }); + + describe('with no results', () => { + it('returns zero values', async () => { + const indicesStats = jest.fn().mockResolvedValueOnce({}); + + expect( + await indicesStatsTask?.executor({ indices, indicesStats } as any) + ).toEqual({ + indices: { + shards: { + total: 0, + }, + all: { + total: { + docs: { + count: 0, + }, + store: { + size_in_bytes: 0, + }, + }, + }, + }, + }); + }); + }); + }); }); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index f27af9a2cc5165..43056dfc83704b 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -33,6 +33,8 @@ import { APMTelemetry } from '../types'; const TIME_RANGES = ['1d', 'all'] as const; type TimeRange = typeof TIME_RANGES[number]; +const range1d = { range: { '@timestamp': { gte: 'now-1d' } } }; +const timeout = '5m'; export const tasks: TelemetryTask[] = [ { @@ -62,6 +64,7 @@ export const tasks: TelemetryTask[] = [ ], body: { size: 0, + timeout, aggs: { [az]: { terms: { @@ -128,21 +131,12 @@ export const tasks: TelemetryTask[] = [ index: indicesByProcessorEvent[processorEvent], body: { size: 0, + timeout, query: { bool: { filter: [ { term: { [PROCESSOR_EVENT]: processorEvent } }, - ...(timeRange !== 'all' - ? [ - { - range: { - '@timestamp': { - gte: `now-${timeRange}`, - }, - }, - }, - ] - : []), + ...(timeRange !== 'all' ? [range1d] : []), ], }, }, @@ -155,6 +149,7 @@ export const tasks: TelemetryTask[] = [ ? await search({ index: indicesByProcessorEvent[processorEvent], body: { + timeout, query: { bool: { filter: [ @@ -208,6 +203,7 @@ export const tasks: TelemetryTask[] = [ index: indices.apmAgentConfigurationIndex, body: { size: 0, + timeout, track_total_hits: true, }, }) @@ -237,6 +233,7 @@ export const tasks: TelemetryTask[] = [ ], body: { size: 0, + timeout, query: { bool: { filter: [ @@ -245,13 +242,7 @@ export const tasks: TelemetryTask[] = [ [AGENT_NAME]: agentName, }, }, - { - range: { - '@timestamp': { - gte: 'now-1d', - }, - }, - }, + range1d, ], }, }, @@ -297,6 +288,7 @@ export const tasks: TelemetryTask[] = [ }, }, size: 1, + timeout, sort: { '@timestamp': 'desc', }, @@ -330,12 +322,12 @@ export const tasks: TelemetryTask[] = [ { name: 'groupings', executor: async ({ search, indices }) => { - const range1d = { range: { '@timestamp': { gte: 'now-1d' } } }; const errorGroupsCount = ( await search({ index: indices['apm_oss.errorIndices'], body: { size: 0, + timeout, query: { bool: { filter: [{ term: { [PROCESSOR_EVENT]: 'error' } }, range1d], @@ -368,6 +360,7 @@ export const tasks: TelemetryTask[] = [ index: indices['apm_oss.transactionIndices'], body: { size: 0, + timeout, query: { bool: { filter: [ @@ -415,6 +408,7 @@ export const tasks: TelemetryTask[] = [ }, track_total_hits: true, size: 0, + timeout, }, }) ).hits.total.value; @@ -428,6 +422,7 @@ export const tasks: TelemetryTask[] = [ ], body: { size: 0, + timeout, query: { bool: { filter: [range1d], @@ -497,12 +492,10 @@ export const tasks: TelemetryTask[] = [ ], body: { size: 0, + timeout, query: { bool: { - filter: [ - { term: { [AGENT_NAME]: agentName } }, - { range: { '@timestamp': { gte: 'now-1d' } } }, - ], + filter: [{ term: { [AGENT_NAME]: agentName } }, range1d], }, }, sort: { @@ -699,15 +692,15 @@ export const tasks: TelemetryTask[] = [ return { indices: { shards: { - total: response._shards.total, + total: response._shards?.total ?? 0, }, all: { total: { docs: { - count: response._all.total.docs.count, + count: response._all?.total?.docs?.count ?? 0, }, store: { - size_in_bytes: response._all.total.store.size_in_bytes, + size_in_bytes: response._all?.total?.store?.size_in_bytes ?? 0, }, }, }, @@ -721,9 +714,10 @@ export const tasks: TelemetryTask[] = [ const allAgentsCardinalityResponse = await search({ body: { size: 0, + timeout, query: { bool: { - filter: [{ range: { '@timestamp': { gte: 'now-1d' } } }], + filter: [range1d], }, }, aggs: { @@ -744,10 +738,11 @@ export const tasks: TelemetryTask[] = [ const rumAgentCardinalityResponse = await search({ body: { size: 0, + timeout, query: { bool: { filter: [ - { range: { '@timestamp': { gte: 'now-1d' } } }, + range1d, { terms: { [AGENT_NAME]: ['rum-js', 'js-base'] } }, ], },