From fbf68ee3aca89358eb48827e3fa61fe80789855e Mon Sep 17 00:00:00 2001 From: restrry Date: Wed, 28 Jul 2021 09:50:52 +0300 Subject: [PATCH 1/2] do not import types from the legacy es package --- .../server/es_client/instantiate_client.ts | 4 +-- .../telemetry_collection/get_beats_stats.ts | 33 +++++++++---------- .../telemetry_collection/get_es_stats.ts | 9 +++-- .../get_high_level_stats.ts | 13 ++++---- .../telemetry_collection/get_kibana_stats.ts | 8 ++--- .../telemetry_collection/get_licenses.ts | 9 +++-- .../get_logstash_stats.ts | 21 ++++++------ 7 files changed, 45 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/monitoring/server/es_client/instantiate_client.ts b/x-pack/plugins/monitoring/server/es_client/instantiate_client.ts index 58da336a5447dc..096c82a9c456ca 100644 --- a/x-pack/plugins/monitoring/server/es_client/instantiate_client.ts +++ b/x-pack/plugins/monitoring/server/es_client/instantiate_client.ts @@ -4,8 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { ConfigOptions } from 'elasticsearch'; import { Logger, ICustomClusterClient, ElasticsearchClientConfig } from 'kibana/server'; // @ts-ignore import { monitoringBulk } from '../kibana_monitoring/lib/monitoring_bulk'; @@ -18,7 +16,7 @@ import { MonitoringElasticsearchConfig } from '../config'; * Kibana itself is connected to a production cluster. */ -type ESClusterConfig = MonitoringElasticsearchConfig & Pick; +type ESClusterConfig = MonitoringElasticsearchConfig; export function instantiateClient( elasticsearchConfig: MonitoringElasticsearchConfig, diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.ts index c62fbf4e8a1cfb..00dba8b7278838 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.ts @@ -6,7 +6,6 @@ */ import { get } from 'lodash'; -import { SearchResponse } from 'elasticsearch'; import { ElasticsearchClient } from 'kibana/server'; import { estypes } from '@elastic/elasticsearch'; import { createQuery } from './create_query'; @@ -156,7 +155,7 @@ export interface BeatsArchitecture { * @param {Object} clusterModuleSets - the object keyed by cluster UUIDs to count the unique modules */ export function processResults( - results: SearchResponse, + results: estypes.SearchResponse, { clusters, clusterHostSets, @@ -167,7 +166,7 @@ export function processResults( ) { const currHits = results?.hits?.hits || []; currHits.forEach((hit) => { - const clusterUuid = hit._source.cluster_uuid; + const clusterUuid = hit._source!.cluster_uuid; if (clusters[clusterUuid] === undefined) { clusters[clusterUuid] = getBaseStats(); clusterHostSets[clusterUuid] = new Set(); @@ -178,30 +177,30 @@ export function processResults( const processBeatsStatsResults = () => { const { versions, types, outputs } = clusters[clusterUuid]; - const thisVersion = hit._source.beats_stats?.beat?.version; + const thisVersion = hit._source?.beats_stats?.beat?.version; if (thisVersion !== undefined) { const thisVersionAccum = versions[thisVersion] || 0; versions[thisVersion] = thisVersionAccum + 1; } - const thisType = hit._source.beats_stats?.beat?.type; + const thisType = hit._source?.beats_stats?.beat?.type; if (thisType !== undefined) { const thisTypeAccum = types[thisType] || 0; types[thisType] = thisTypeAccum + 1; } - const thisOutput = hit._source.beats_stats?.metrics?.libbeat?.output?.type; + const thisOutput = hit._source?.beats_stats?.metrics?.libbeat?.output?.type; if (thisOutput !== undefined) { const thisOutputAccum = outputs[thisOutput] || 0; outputs[thisOutput] = thisOutputAccum + 1; } - const thisEvents = hit._source.beats_stats?.metrics?.libbeat?.pipeline?.events?.published; + const thisEvents = hit._source?.beats_stats?.metrics?.libbeat?.pipeline?.events?.published; if (thisEvents !== undefined) { clusters[clusterUuid].eventsPublished += thisEvents; } - const thisHost = hit._source.beats_stats?.beat?.host; + const thisHost = hit._source?.beats_stats?.beat?.host; if (thisHost !== undefined) { const hostsMap = clusterHostSets[clusterUuid]; hostsMap.add(thisHost); @@ -210,7 +209,7 @@ export function processResults( }; const processBeatsStateResults = () => { - const stateInput = hit._source.beats_state?.state?.input; + const stateInput = hit._source?.beats_state?.state?.input; if (stateInput !== undefined) { const inputSet = clusterInputSets[clusterUuid]; stateInput.names.forEach((name) => inputSet.add(name)); @@ -218,8 +217,8 @@ export function processResults( clusters[clusterUuid].input.count += stateInput.count; } - const stateModule = hit._source.beats_state?.state?.module; - const statsType = hit._source.beats_state?.beat?.type; + const stateModule = hit._source?.beats_state?.state?.module; + const statsType = hit._source?.beats_state?.beat?.type; if (stateModule !== undefined) { const moduleSet = clusterModuleSets[clusterUuid]; stateModule.names.forEach((name) => moduleSet.add(statsType + '.' + name)); @@ -227,12 +226,12 @@ export function processResults( clusters[clusterUuid].module.count += stateModule.count; } - const stateQueue = hit._source.beats_state?.state?.queue?.name; + const stateQueue = hit._source?.beats_state?.state?.queue?.name; if (stateQueue !== undefined) { clusters[clusterUuid].queue[stateQueue] += 1; } - const heartbeatState = hit._source.beats_state?.state?.heartbeat; + const heartbeatState = hit._source?.beats_state?.state?.heartbeat; if (heartbeatState !== undefined) { if (!clusters[clusterUuid].hasOwnProperty('heartbeat')) { clusters[clusterUuid].heartbeat = { @@ -264,7 +263,7 @@ export function processResults( } } - const functionbeatState = hit._source.beats_state?.state?.functionbeat; + const functionbeatState = hit._source?.beats_state?.state?.functionbeat; if (functionbeatState !== undefined) { if (!clusters[clusterUuid].hasOwnProperty('functionbeat')) { clusters[clusterUuid].functionbeat = { @@ -278,7 +277,7 @@ export function processResults( functionbeatState.functions?.count || 0; } - const stateHost = hit._source.beats_state?.state?.host; + const stateHost = hit._source?.beats_state?.state?.host; if (stateHost !== undefined) { const hostMap = clusterArchitectureMaps[clusterUuid]; const hostKey = `${stateHost.architecture}/${stateHost.os.platform}`; @@ -362,11 +361,11 @@ async function fetchBeatsByType( }, }; - const { body: results } = await callCluster.search(params); + const { body: results } = await callCluster.search(params); const hitsLength = results?.hits?.hits.length || 0; if (hitsLength > 0) { // further augment the clusters object with more stats - processResults(results as SearchResponse, options); + processResults(results, options); if (hitsLength === HITS_SIZE) { // call recursively diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_es_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_es_stats.ts index 217ab70ab0ecaa..92aa48cbe90efd 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_es_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_es_stats.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { SearchResponse } from 'elasticsearch'; import { ElasticsearchClient } from 'kibana/server'; import { estypes } from '@elastic/elasticsearch'; import { INDEX_PATTERN_ELASTICSEARCH } from '../../common/constants'; @@ -70,8 +69,8 @@ export async function fetchElasticsearchStats( }, }; - const { body: response } = await callCluster.search(params); - return response as SearchResponse; + const { body: response } = await callCluster.search(params); + return response; } export interface ESClusterStats { @@ -86,8 +85,8 @@ export interface ESClusterStats { /** * Extract the cluster stats for each cluster. */ -export function handleElasticsearchStats(response: SearchResponse) { +export function handleElasticsearchStats(response: estypes.SearchResponse) { const clusters = response.hits?.hits || []; - return clusters.map((cluster) => cluster._source); + return clusters.map((cluster) => cluster._source!); } diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.ts index 2fb3814ac40224..77aa6478e96fe1 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.ts @@ -6,7 +6,6 @@ */ import { get } from 'lodash'; -import { SearchResponse } from 'elasticsearch'; import { ElasticsearchClient } from 'kibana/server'; import { estypes } from '@elastic/elasticsearch'; import { createQuery } from './create_query'; @@ -149,14 +148,14 @@ interface OSData { * Returns a map of the Cluster UUID to an {@link Object} containing the {@code count} and {@code versions} {@link Map} */ function groupInstancesByCluster( - instances: Array<{ _source: T }>, + instances: Array<{ _source?: T }>, product: string ) { const clusterMap = new Map(); // hits are sorted arbitrarily by product UUID instances.map((instance) => { - const clusterUuid = instance._source.cluster_uuid; + const clusterUuid = instance._source!.cluster_uuid; const version: string | undefined = get( instance, `_source.${product}_stats.${product}.version` @@ -276,7 +275,7 @@ export async function fetchHighLevelStats< end: string, product: string, maxBucketSize: number -): Promise> { +): Promise> { const isKibanaIndex = product === KIBANA_SYSTEM_ID; const filters: object[] = [{ terms: { cluster_uuid: clusterUuids } }]; @@ -332,12 +331,12 @@ export async function fetchHighLevelStats< }, }; - const { body: response } = await callCluster.search(params, { + const { body: response } = await callCluster.search(params, { headers: { 'X-QUERY-SOURCE': TELEMETRY_QUERY_SOURCE, }, }); - return response as SearchResponse; + return response; } /** @@ -349,7 +348,7 @@ export async function fetchHighLevelStats< * Returns an object keyed by the cluster UUIDs to make grouping easier. */ export function handleHighLevelStatsResponse( - response: SearchResponse<{ cluster_uuid?: string }>, + response: estypes.SearchResponse<{ cluster_uuid?: string }>, product: string ): ClustersHighLevelStats { const instances = response.hits?.hits || []; diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts index 4be5e02dff3a66..45ef1fbd00339e 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts @@ -7,7 +7,7 @@ import moment from 'moment'; import { isEmpty } from 'lodash'; -import { SearchResponse } from 'elasticsearch'; +import type { estypes } from '@elastic/elasticsearch'; import { ElasticsearchClient } from 'kibana/server'; import { KIBANA_SYSTEM_ID, TELEMETRY_COLLECTION_INTERVAL } from '../../common/constants'; import { @@ -70,14 +70,14 @@ export interface KibanaStats { /* * @param {Object} rawStats */ -export function getUsageStats(rawStats: SearchResponse) { +export function getUsageStats(rawStats: estypes.SearchResponse) { const clusterIndexCache = new Set(); const rawStatsHits = rawStats.hits?.hits || []; // get usage stats per cluster / .kibana index return rawStatsHits.reduce((accum, currInstance) => { - const clusterUuid = currInstance._source.cluster_uuid; - const currUsage = currInstance._source.kibana_stats?.usage || {}; + const clusterUuid = currInstance._source!.cluster_uuid; + const currUsage = currInstance._source!.kibana_stats?.usage || {}; const clusterIndexCombination = clusterUuid + currUsage.index; // return early if usage data is empty or if this cluster/index has already been processed diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.ts index 633e37a2049bdd..514b04bf1a0e77 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { SearchResponse } from 'elasticsearch'; import { ElasticsearchClient } from 'kibana/server'; import { estypes } from '@elastic/elasticsearch'; import { ESLicense } from '../../../telemetry_collection_xpack/server'; @@ -60,8 +59,8 @@ export async function fetchLicenses( }, }; - const { body: response } = await callCluster.search(params); - return response as SearchResponse; + const { body: response } = await callCluster.search(params); + return response; } export interface ESClusterStatsWithLicense { @@ -73,13 +72,13 @@ export interface ESClusterStatsWithLicense { /** * Extract the cluster stats for each cluster. */ -export function handleLicenses(response: SearchResponse) { +export function handleLicenses(response: estypes.SearchResponse) { const clusters = response.hits?.hits || []; return clusters.reduce( (acc, { _source }) => ({ ...acc, - [_source.cluster_uuid]: _source.license, + [_source!.cluster_uuid]: _source!.license, }), {} ); diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts index ac3f5fceb47ade..306c9b62015576 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { SearchResponse } from 'elasticsearch'; import { ElasticsearchClient } from 'kibana/server'; import { estypes } from '@elastic/elasticsearch'; import { createQuery } from './create_query'; @@ -116,18 +115,18 @@ export interface LogstashProcessOptions { * @param {Object} plugins - plugin information keyed by cluster UUIDs to count the unique plugins */ export function processStatsResults( - results: SearchResponse, + results: estypes.SearchResponse, { clusters, allEphemeralIds, versions, plugins }: LogstashProcessOptions ) { const currHits = results?.hits?.hits || []; currHits.forEach((hit) => { - const clusterUuid = hit._source.cluster_uuid; + const clusterUuid = hit._source!.cluster_uuid; if (clusters[clusterUuid] === undefined) { clusters[clusterUuid] = getLogstashBaseStats(); versions[clusterUuid] = new Map(); plugins[clusterUuid] = new Map(); } - const logstashStats = hit._source.logstash_stats; + const logstashStats = hit._source?.logstash_stats; const clusterStats = clusters[clusterUuid].cluster_stats; if (clusterStats !== undefined && logstashStats !== undefined) { @@ -139,7 +138,7 @@ export function processStatsResults( clusters[clusterUuid].versions = mapToList(a, 'version'); // Internal Collection has no agent field, so default to 'internal_collection' - let thisCollectionType = hit._source.agent?.type; + let thisCollectionType = hit._source?.agent?.type; if (thisCollectionType === undefined) { thisCollectionType = 'internal_collection'; } @@ -176,7 +175,7 @@ export function processStatsResults( * @param {Object} plugins - plugin information keyed by cluster UUIDs to count the unique plugins */ export function processLogstashStateResults( - results: SearchResponse, + results: estypes.SearchResponse, clusterUuid: string, { clusters, plugins }: LogstashProcessOptions ) { @@ -185,7 +184,7 @@ export function processLogstashStateResults( const pipelineStats = clusters[clusterUuid].cluster_stats?.pipelines; currHits.forEach((hit) => { - const thisLogstashStatePipeline = hit._source.logstash_state?.pipeline; + const thisLogstashStatePipeline = hit._source?.logstash_state?.pipeline; if (pipelineStats !== undefined && thisLogstashStatePipeline !== undefined) { pipelineStats.count = (pipelineStats.count || 0) + 1; @@ -301,7 +300,7 @@ export async function fetchLogstashStats( }, }; - const { body: results } = await callCluster.search(params, { + const { body: results } = await callCluster.search(params, { headers: { 'X-QUERY-SOURCE': TELEMETRY_QUERY_SOURCE, }, @@ -310,7 +309,7 @@ export async function fetchLogstashStats( if (hitsLength > 0) { // further augment the clusters object with more stats - processStatsResults(results as SearchResponse, options); + processStatsResults(results, options); if (hitsLength === HITS_SIZE) { // call recursively @@ -362,7 +361,7 @@ export async function fetchLogstashState( }, }; - const { body: results } = await callCluster.search>(params, { + const { body: results } = await callCluster.search(params, { headers: { 'X-QUERY-SOURCE': TELEMETRY_QUERY_SOURCE, }, @@ -371,7 +370,7 @@ export async function fetchLogstashState( const hitsLength = results?.hits?.hits.length || 0; if (hitsLength > 0) { // further augment the clusters object with more stats - processLogstashStateResults(results as SearchResponse, clusterUuid, options); + processLogstashStateResults(results, clusterUuid, options); if (hitsLength === HITS_SIZE) { // call recursively From 695ea8e1e7d10d5f9ac5fcb7284ba8a47e0f833f Mon Sep 17 00:00:00 2001 From: restrry Date: Wed, 28 Jul 2021 10:09:00 +0300 Subject: [PATCH 2/2] adjust tests --- .../server/lib/errors/auth_errors.test.js | 3 +-- .../server/lib/errors/known_errors.test.js | 16 ++++++++-------- .../get_kibana_stats.test.ts | 5 ++--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/monitoring/server/lib/errors/auth_errors.test.js b/x-pack/plugins/monitoring/server/lib/errors/auth_errors.test.js index 34feb16d3a2a54..8ab9581aa7d0a9 100644 --- a/x-pack/plugins/monitoring/server/lib/errors/auth_errors.test.js +++ b/x-pack/plugins/monitoring/server/lib/errors/auth_errors.test.js @@ -5,13 +5,12 @@ * 2.0. */ -import { errors } from 'elasticsearch'; import { forbidden, unauthorized } from '@hapi/boom'; import { isAuthError, handleAuthError } from './auth_errors'; describe('Error handling for 401/403 errors', () => { it('ignores an unknown type', () => { - const err = new errors.Generic(); + const err = new Error(); expect(isAuthError(err)).toBe(false); }); diff --git a/x-pack/plugins/monitoring/server/lib/errors/known_errors.test.js b/x-pack/plugins/monitoring/server/lib/errors/known_errors.test.js index 97e01dffe6fd21..37faf5a941bfab 100644 --- a/x-pack/plugins/monitoring/server/lib/errors/known_errors.test.js +++ b/x-pack/plugins/monitoring/server/lib/errors/known_errors.test.js @@ -5,19 +5,19 @@ * 2.0. */ -import { errors } from 'elasticsearch'; +import { errors } from '@elastic/elasticsearch'; import { isKnownError, handleKnownError } from './known_errors'; import { MonitoringLicenseError } from './custom_errors'; // TODO: tests were not running and are not up to date describe.skip('Error handling for 503 errors', () => { it('ignores an unknown type', () => { - const err = new errors.Generic(); + const err = new Error(); expect(isKnownError(err)).toBe(false); }); - it('handles ConnectionFault', () => { - const err = new errors.ConnectionFault(); + it('handles ConnectionError', () => { + const err = new errors.ConnectionError(); expect(isKnownError(err)).toBe(true); const wrappedErr = handleKnownError(err); @@ -41,8 +41,8 @@ describe.skip('Error handling for 503 errors', () => { }); }); - it('handles NoConnections', () => { - const err = new errors.NoConnections(); + it('handles NoLivingConnectionsError', () => { + const err = new errors.NoLivingConnectionsError(); expect(isKnownError(err)).toBe(true); const wrappedErr = handleKnownError(err); @@ -66,8 +66,8 @@ describe.skip('Error handling for 503 errors', () => { }); }); - it('handles RequestTimeout', () => { - const err = new errors.RequestTimeout(); + it('handles TimeoutError', () => { + const err = new errors.TimeoutError(); expect(isKnownError(err)).toBe(true); const wrappedErr = handleKnownError(err); diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.test.ts index 4464e87362d216..08997a73d26bb7 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.test.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import type { estypes } from '@elastic/elasticsearch'; import { getUsageStats, combineStats, @@ -12,12 +12,11 @@ import { ensureTimeSpan, KibanaUsageStats, } from './get_kibana_stats'; -import { SearchResponse } from 'elasticsearch'; describe('Get Kibana Stats', () => { describe('Make a map of usage stats for each cluster', () => { test('passes through if there are no kibana instances', () => { - const rawStats = {} as SearchResponse; + const rawStats = {} as estypes.SearchResponse; expect(getUsageStats(rawStats)).toStrictEqual({}); });