From a2469da81dcef6527d56fe41b9d9a3b5f26af3d3 Mon Sep 17 00:00:00 2001 From: Corey Robertson Date: Fri, 11 Sep 2020 11:35:18 -0400 Subject: [PATCH 01/11] Properly handle kibana assets with underscores in their path --- .../ingest_manager/common/types/models/epm.ts | 9 +- .../ingest_manager/sections/epm/constants.tsx | 4 +- .../services/epm/kibana/assets/install.ts | 84 +++++++++++++++---- .../server/services/epm/packages/install.ts | 8 +- .../server/services/epm/packages/remove.ts | 42 +++++++--- .../server/services/epm/registry/index.ts | 4 +- .../apis/epm/index.js | 2 +- .../apis/epm/install_remove_assets.ts | 37 +++++++- .../0.1.0/kibana/index_pattern/invalid.json | 11 +++ .../0.1.0/kibana/index_pattern/test-*.json | 11 +++ 10 files changed, 173 insertions(+), 39 deletions(-) create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/index_pattern/invalid.json create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/index_pattern/test-*.json diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 8bc5d9f7210b25..3ba174f176d6ff 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -27,11 +27,14 @@ export type DetailViewPanelName = 'overview' | 'usages' | 'settings'; export type ServiceName = 'kibana' | 'elasticsearch'; export type AssetType = KibanaAssetType | ElasticsearchAssetType | AgentAssetType; +/* + Enum mapping of a saved object asset type to how it would appear in a package file path (snake cased) +*/ export enum KibanaAssetType { dashboard = 'dashboard', visualization = 'visualization', search = 'search', - indexPattern = 'index-pattern', + indexPattern = 'index_pattern', map = 'map', } @@ -255,9 +258,7 @@ export type NotInstalled = T & { export type AssetReference = KibanaAssetReference | EsAssetReference; -export type KibanaAssetReference = Pick & { - type: KibanaAssetType; -}; +export type KibanaAssetReference = Pick; export type EsAssetReference = Pick & { type: ElasticsearchAssetType; }; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx index da3cab1a4b8a3d..1dad25e9cf0595 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx @@ -20,7 +20,7 @@ export const AssetTitleMap: Record = { ilm_policy: 'ILM Policy', ingest_pipeline: 'Ingest Pipeline', transform: 'Transform', - 'index-pattern': 'Index Pattern', + index_pattern: 'Index Pattern', index_template: 'Index Template', component_template: 'Component Template', search: 'Saved Search', @@ -36,7 +36,7 @@ export const ServiceTitleMap: Record = { export const AssetIcons: Record = { dashboard: 'dashboardApp', - 'index-pattern': 'indexPatternApp', + index_pattern: 'indexPatternApp', search: 'searchProfilerApp', visualization: 'visualizeApp', map: 'mapApp', diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts index 201003629e5ea3..1e8679ee112b97 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts @@ -11,7 +11,7 @@ import { } from 'src/core/server'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common'; import * as Registry from '../../registry'; -import { AssetType, KibanaAssetType, AssetReference } from '../../../../types'; +import { AssetType, KibanaAssetType, AssetReference, AssetParts } from '../../../../types'; import { savedObjectTypes } from '../../packages'; type SavedObjectToBe = Required> & { @@ -24,6 +24,31 @@ export type ArchiveAsset = Pick< type: AssetType; }; +// KibanaSavedObjectTypes are used to ensure saved objects being created for a given +// KibanaAssetType have the correct type +const KibanaSavedObjectTypes: Record = { + [KibanaAssetType.dashboard]: 'dashboard', + [KibanaAssetType.indexPattern]: 'index-pattern', + [KibanaAssetType.map]: 'map', + [KibanaAssetType.search]: 'search', + [KibanaAssetType.visualization]: 'visualization', +}; + +// Define how each asset type will be installed +const AssetInstallers: Record< + KibanaAssetType, + (args: { + savedObjectsClient: SavedObjectsClientContract; + kibanaAssets: ArchiveAsset[]; + }) => Promise>> +> = { + [KibanaAssetType.dashboard]: installKibanaSavedObjects, + [KibanaAssetType.indexPattern]: installKibanaSavedObjects, + [KibanaAssetType.map]: installKibanaSavedObjects, + [KibanaAssetType.search]: installKibanaSavedObjects, + [KibanaAssetType.visualization]: installKibanaSavedObjects, +}; + export async function getKibanaAsset(key: string): Promise { const buffer = Registry.getAsset(key); @@ -47,16 +72,22 @@ export function createSavedObjectKibanaAsset(asset: ArchiveAsset): SavedObjectTo export async function installKibanaAssets(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; - kibanaAssets: ArchiveAsset[]; + kibanaAssets: Record; }): Promise { const { savedObjectsClient, kibanaAssets } = options; // install the assets const kibanaAssetTypes = Object.values(KibanaAssetType); const installedAssets = await Promise.all( - kibanaAssetTypes.map((assetType) => - installKibanaSavedObjects({ savedObjectsClient, assetType, kibanaAssets }) - ) + kibanaAssetTypes.map((assetType) => { + if (kibanaAssets[assetType]) { + return AssetInstallers[assetType]({ + savedObjectsClient, + kibanaAssets: kibanaAssets[assetType], + }); + } + return []; + }) ); return installedAssets.flat(); } @@ -75,24 +106,47 @@ export const deleteKibanaInstalledRefs = async ( }); }; export async function getKibanaAssets(paths: string[]) { - const isKibanaAssetType = (path: string) => Registry.pathParts(path).type in KibanaAssetType; - const filteredPaths = paths.filter(isKibanaAssetType); - const kibanaAssets = await Promise.all(filteredPaths.map((path) => getKibanaAsset(path))); - return kibanaAssets; + const kibanaAssetTypes = Object.values(KibanaAssetType); + const isKibanaAssetType = (path: string) => { + const parts = Registry.pathParts(path); + + return parts.service === 'kibana' && (kibanaAssetTypes as string[]).includes(parts.type); + }; + + const filteredPaths = paths + .filter(isKibanaAssetType) + .map<[string, AssetParts]>((path) => [path, Registry.pathParts(path)]); + + const assetArrays: Array> = []; + for (const assetType of kibanaAssetTypes) { + const matching = filteredPaths.filter(([path, parts]) => parts.type === assetType); + + assetArrays.push(Promise.all(matching.map(([path]) => path).map(getKibanaAsset))); + } + + const resolvedAssets = await Promise.all(assetArrays); + + const result = {} as Record; + + for (const [index, assetType] of kibanaAssetTypes.entries()) { + const expectedType = KibanaSavedObjectTypes[assetType]; + const properlyTypedAssets = resolvedAssets[index].filter(({ type }) => type === expectedType); + + result[assetType] = properlyTypedAssets; + } + + return result; } + async function installKibanaSavedObjects({ savedObjectsClient, - assetType, kibanaAssets, }: { savedObjectsClient: SavedObjectsClientContract; - assetType: KibanaAssetType; kibanaAssets: ArchiveAsset[]; }) { - const isSameType = (asset: ArchiveAsset) => assetType === asset.type; - const filteredKibanaAssets = kibanaAssets.filter((asset) => isSameType(asset)); const toBeSavedObjects = await Promise.all( - filteredKibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset)) + kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset)) ); if (toBeSavedObjects.length === 0) { @@ -106,7 +160,7 @@ async function installKibanaSavedObjects({ } export function toAssetReference({ id, type }: SavedObject) { - const reference: AssetReference = { id, type: type as KibanaAssetType }; + const reference: AssetReference = { id, type }; return reference; } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 54b9c4d3fbb172..1e63e755ec5a64 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -17,6 +17,7 @@ import { EsAssetReference, ElasticsearchAssetType, InstallType, + KibanaAssetType, } from '../../../types'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import * as Registry from '../registry'; @@ -150,12 +151,13 @@ export async function installPackage({ } const installIndexPatternPromise = installIndexPatterns(savedObjectsClient, pkgName, pkgVersion); const kibanaAssets = await getKibanaAssets(paths); + if (installedPkg) await deleteKibanaSavedObjectsAssets( savedObjectsClient, installedPkg.attributes.installed_kibana ); - // save new kibana refs before installing the assets + const installedKibanaAssetsRefs = await saveKibanaAssetsRefs( savedObjectsClient, pkgName, @@ -288,9 +290,9 @@ export async function createInstallation(options: { export const saveKibanaAssetsRefs = async ( savedObjectsClient: SavedObjectsClientContract, pkgName: string, - kibanaAssets: ArchiveAsset[] + kibanaAssets: Record ) => { - const assetRefs = kibanaAssets.map(toAssetReference); + const assetRefs = Object.values(kibanaAssets).flat().map(toAssetReference); await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { installed_kibana: assetRefs, }); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts index 2434ebf27aa5db..19d603f9e432bc 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts @@ -12,6 +12,9 @@ import { AssetType, CallESAsCurrentUser, ElasticsearchAssetType, + EsAssetReference, + KibanaAssetReference, + Installation, } from '../../../types'; import { getInstallation, savedObjectTypes } from './index'; import { deletePipeline } from '../elasticsearch/ingest_pipeline/'; @@ -49,7 +52,7 @@ export async function removeInstallation(options: { // Delete the installed assets const installedAssets = [...installation.installed_kibana, ...installation.installed_es]; - await deleteAssets(installedAssets, savedObjectsClient, callCluster); + await deleteAssets(installation, savedObjectsClient, callCluster); // Delete the manager saved object with references to the asset objects // could also update with [] or some other state @@ -63,17 +66,20 @@ export async function removeInstallation(options: { // successful delete's in SO client return {}. return something more useful return installedAssets; } -async function deleteAssets( - installedObjects: AssetReference[], - savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser + +function deleteKibanaAssets( + installedObjects: KibanaAssetReference[], + savedObjectsClient: SavedObjectsClientContract ) { - const logger = appContextService.getLogger(); - const deletePromises = installedObjects.map(async ({ id, type }) => { + return installedObjects.map(async ({ id, type }) => { + return savedObjectsClient.delete(type, id); + }); +} + +function deleteESAssets(installedObjects: EsAssetReference[], callCluster: CallESAsCurrentUser) { + return installedObjects.map(async ({ id, type }) => { const assetType = type as AssetType; - if (savedObjectTypes.includes(assetType)) { - return savedObjectsClient.delete(assetType, id); - } else if (assetType === ElasticsearchAssetType.ingestPipeline) { + if (assetType === ElasticsearchAssetType.ingestPipeline) { return deletePipeline(callCluster, id); } else if (assetType === ElasticsearchAssetType.indexTemplate) { return deleteTemplate(callCluster, id); @@ -81,8 +87,22 @@ async function deleteAssets( return deleteTransforms(callCluster, [id]); } }); +} + +async function deleteAssets( + { installed_es: installedEs, installed_kibana: installedKibana }: Installation, + savedObjectsClient: SavedObjectsClientContract, + callCluster: CallESAsCurrentUser +) { + const logger = appContextService.getLogger(); + + const deletePromises: Array> = [ + ...deleteESAssets(installedEs, callCluster), + ...deleteKibanaAssets(installedKibana, savedObjectsClient), + ]; + try { - await Promise.all([...deletePromises]); + await Promise.all(deletePromises); } catch (err) { logger.error(err); } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts index 96f75306413905..00ed593388e08d 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts @@ -232,10 +232,12 @@ export function getAsset(key: string) { } export function groupPathsByService(paths: string[]): AssetsGroupedByServiceByType { + const kibanaAssetTypes = Object.values(KibanaAssetType); + // ASK: best way, if any, to avoid `any`? const assets = paths.reduce((map: any, path) => { const parts = pathParts(path.replace(/^\/package\//, '')); - if (parts.type in KibanaAssetType) { + if (parts.service === 'kibana' && kibanaAssetTypes.includes(parts.type)) { if (!map[parts.service]) map[parts.service] = {}; if (!map[parts.service][parts.type]) map[parts.service][parts.type] = []; map[parts.service][parts.type].push(parts); diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/index.js b/x-pack/test/ingest_manager_api_integration/apis/epm/index.js index 85558142459099..7e46cc1985276c 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/index.js @@ -9,7 +9,7 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./list')); loadTestFile(require.resolve('./get')); loadTestFile(require.resolve('./file')); - //loadTestFile(require.resolve('./template')); + loadTestFile(require.resolve('./template')); loadTestFile(require.resolve('./ilm')); loadTestFile(require.resolve('./install_overrides')); loadTestFile(require.resolve('./install_prerelease')); diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts index 198c129b7482fa..e2f59cb0aa7c52 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts @@ -92,6 +92,7 @@ export default function (providerContext: FtrProviderContext) { expect(res.statusCode).equal(200); }); it('should have installed the kibana assets', async function () { + // Kibana Assets installed as part of Fleet, not from a particular package const resIndexPatternLogs = await kibanaServer.savedObjects.get({ type: 'index-pattern', id: 'logs-*', @@ -106,6 +107,8 @@ export default function (providerContext: FtrProviderContext) { type: 'dashboard', id: 'sample_dashboard', }); + + // Kibana Assets installed from the package expect(resDashboard.id).equal('sample_dashboard'); const resDashboard2 = await kibanaServer.savedObjects.get({ type: 'dashboard', @@ -122,6 +125,22 @@ export default function (providerContext: FtrProviderContext) { id: 'sample_search', }); expect(resSearch.id).equal('sample_search'); + const resIndexPattern = await kibanaServer.savedObjects.get({ + type: 'index-pattern', + id: 'test-*', + }); + expect(resIndexPattern.id).equal('test-*'); + + let resInvalidTypeIndexPattern; + try { + resInvalidTypeIndexPattern = await kibanaServer.savedObjects.get({ + type: 'invalid-type', + id: 'invalid', + }); + } catch (err) { + resInvalidTypeIndexPattern = err; + } + expect(resInvalidTypeIndexPattern.response.data.statusCode).equal(404); }); it('should have created the correct saved object', async function () { const res = await kibanaServer.savedObjects.get({ @@ -138,13 +157,17 @@ export default function (providerContext: FtrProviderContext) { id: 'sample_dashboard2', type: 'dashboard', }, + { + id: 'sample_visualization', + type: 'visualization', + }, { id: 'sample_search', type: 'search', }, { - id: 'sample_visualization', - type: 'visualization', + id: 'test-*', + type: 'index-pattern', }, ], installed_es: [ @@ -301,6 +324,16 @@ export default function (providerContext: FtrProviderContext) { resSearch = err; } expect(resSearch.response.data.statusCode).equal(404); + let resIndexPattern; + try { + resIndexPattern = await kibanaServer.savedObjects.get({ + type: 'index-pattern', + id: 'test-*', + }); + } catch (err) { + resIndexPattern = err; + } + expect(resIndexPattern.response.data.statusCode).equal(404); }); it('should have removed the saved object', async function () { let res; diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/index_pattern/invalid.json b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/index_pattern/invalid.json new file mode 100644 index 00000000000000..bffc52ded73d65 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/index_pattern/invalid.json @@ -0,0 +1,11 @@ +{ + "attributes": { + "fieldFormatMap": "{}", + "fields": "[]", + "timeFieldName": "@timestamp", + "title": "invalid" + }, + "id": "invalid", + "references": [], + "type": "invalid-type" +} diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/index_pattern/test-*.json b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/index_pattern/test-*.json new file mode 100644 index 00000000000000..48ba36a1167093 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/kibana/index_pattern/test-*.json @@ -0,0 +1,11 @@ +{ + "attributes": { + "fieldFormatMap": "{}", + "fields": "[]", + "timeFieldName": "@timestamp", + "title": "test-*" + }, + "id": "test-*", + "references": [], + "type": "index-pattern" +} From a3d768bd896fad6dcfc88db5b4e5897aef20ebea Mon Sep 17 00:00:00 2001 From: Corey Robertson Date: Mon, 14 Sep 2020 11:13:56 -0400 Subject: [PATCH 02/11] Recomment test --- x-pack/test/ingest_manager_api_integration/apis/epm/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/index.js b/x-pack/test/ingest_manager_api_integration/apis/epm/index.js index 7e46cc1985276c..85558142459099 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/index.js @@ -9,7 +9,7 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./list')); loadTestFile(require.resolve('./get')); loadTestFile(require.resolve('./file')); - loadTestFile(require.resolve('./template')); + //loadTestFile(require.resolve('./template')); loadTestFile(require.resolve('./ilm')); loadTestFile(require.resolve('./install_overrides')); loadTestFile(require.resolve('./install_prerelease')); From 06094fa4eb8f93269aa54270bac55e5bf3ed9813 Mon Sep 17 00:00:00 2001 From: Corey Robertson Date: Mon, 14 Sep 2020 17:08:46 -0400 Subject: [PATCH 03/11] Fix type check --- .../common/services/package_to_package_policy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts index 6c3559d7cc5a03..939a2ae7af35a0 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts @@ -24,7 +24,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => { dashboard: [], visualization: [], search: [], - 'index-pattern': [], + index_pattern: [], map: [], }, }, From 639f90dd0377a30d2cae5309683d7b412a72ea06 Mon Sep 17 00:00:00 2001 From: Corey Robertson Date: Wed, 16 Sep 2020 12:09:46 -0400 Subject: [PATCH 04/11] Don't install index patterns that are reserved --- .../services/epm/kibana/assets/install.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts index 1e8679ee112b97..7c7d3e386f81dc 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts @@ -13,6 +13,7 @@ import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common'; import * as Registry from '../../registry'; import { AssetType, KibanaAssetType, AssetReference, AssetParts } from '../../../../types'; import { savedObjectTypes } from '../../packages'; +import { IndexPatternType } from '../index_pattern/install'; type SavedObjectToBe = Required> & { type: AssetType; @@ -43,7 +44,7 @@ const AssetInstallers: Record< }) => Promise>> > = { [KibanaAssetType.dashboard]: installKibanaSavedObjects, - [KibanaAssetType.indexPattern]: installKibanaSavedObjects, + [KibanaAssetType.indexPattern]: installKibanaIndexPatterns, [KibanaAssetType.map]: installKibanaSavedObjects, [KibanaAssetType.search]: installKibanaSavedObjects, [KibanaAssetType.visualization]: installKibanaSavedObjects, @@ -159,6 +160,21 @@ async function installKibanaSavedObjects({ } } +async function installKibanaIndexPatterns({ + savedObjectsClient, + kibanaAssets, +}: { + savedObjectsClient: SavedObjectsClientContract; + kibanaAssets: ArchiveAsset[]; +}) { + // Filter out any reserved index patterns + const reservedPatterns = Object.values(IndexPatternType).map((pattern) => `${pattern}-*`); + + const nonReservedPatterns = kibanaAssets.filter((asset) => !reservedPatterns.includes(asset.id)); + + return installKibanaSavedObjects({ savedObjectsClient, kibanaAssets: nonReservedPatterns }); +} + export function toAssetReference({ id, type }: SavedObject) { const reference: AssetReference = { id, type }; From a9fb9dab7989a020859e6ab08c673f21dbae7210 Mon Sep 17 00:00:00 2001 From: Corey Robertson Date: Mon, 21 Sep 2020 13:36:51 -0400 Subject: [PATCH 05/11] Introduce SavedObjectType to use on AssetReference --- .../ingest_manager/common/types/models/epm.ts | 15 ++++++++- .../server/routes/data_streams/handlers.ts | 4 +-- .../services/epm/kibana/assets/install.ts | 32 ++++++++++++------- .../services/epm/packages/install.test.ts | 6 ++-- .../ingest_manager/server/types/index.tsx | 1 + 5 files changed, 40 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 3ba174f176d6ff..47925c4c58bd6a 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -38,6 +38,17 @@ export enum KibanaAssetType { map = 'map', } +/* + Enum of saved object types that are allowed to be installed +*/ +export enum KibanaSavedObjectType { + dashboard = 'dashboard', + visualization = 'visualization', + search = 'search', + indexPattern = 'index-pattern', + map = 'map', +} + export enum ElasticsearchAssetType { componentTemplate = 'component_template', ingestPipeline = 'ingest_pipeline', @@ -258,7 +269,9 @@ export type NotInstalled = T & { export type AssetReference = KibanaAssetReference | EsAssetReference; -export type KibanaAssetReference = Pick; +export type KibanaAssetReference = Pick & { + type: KibanaSavedObjectType; +}; export type EsAssetReference = Pick & { type: ElasticsearchAssetType; }; diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts index 43ae2b72f6077c..1928368411105c 100644 --- a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts @@ -5,7 +5,7 @@ */ import { RequestHandler, SavedObjectsClientContract } from 'src/core/server'; import { DataStream } from '../../types'; -import { GetDataStreamsResponse, KibanaAssetType } from '../../../common'; +import { GetDataStreamsResponse, KibanaAssetType, KibanaSavedObjectType } from '../../../common'; import { getPackageSavedObjects, getKibanaSavedObject } from '../../services/epm/packages/get'; const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*'; @@ -123,7 +123,7 @@ export const getListHandler: RequestHandler = async (context, request, response) // then pick the dashboards from the package saved object const dashboards = pkgSavedObject[0].attributes?.installed_kibana?.filter( - (o) => o.type === KibanaAssetType.dashboard + (o) => o.type === KibanaSavedObjectType.dashboard ) || []; // and then pick the human-readable titles from the dashboard saved objects const enhancedDashboards = await getEnhancedDashboards( diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts index 7c7d3e386f81dc..e7e94aa9d45c8b 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts @@ -11,28 +11,34 @@ import { } from 'src/core/server'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common'; import * as Registry from '../../registry'; -import { AssetType, KibanaAssetType, AssetReference, AssetParts } from '../../../../types'; +import { + AssetType, + KibanaAssetType, + AssetReference, + AssetParts, + KibanaSavedObjectType, +} from '../../../../types'; import { savedObjectTypes } from '../../packages'; import { IndexPatternType } from '../index_pattern/install'; type SavedObjectToBe = Required> & { - type: AssetType; + type: KibanaSavedObjectType; }; export type ArchiveAsset = Pick< SavedObject, 'id' | 'attributes' | 'migrationVersion' | 'references' > & { - type: AssetType; + type: KibanaSavedObjectType; }; // KibanaSavedObjectTypes are used to ensure saved objects being created for a given // KibanaAssetType have the correct type -const KibanaSavedObjectTypes: Record = { - [KibanaAssetType.dashboard]: 'dashboard', - [KibanaAssetType.indexPattern]: 'index-pattern', - [KibanaAssetType.map]: 'map', - [KibanaAssetType.search]: 'search', - [KibanaAssetType.visualization]: 'visualization', +const KibanaSavedObjectTypeMapping: Record = { + [KibanaAssetType.dashboard]: KibanaSavedObjectType.dashboard, + [KibanaAssetType.indexPattern]: KibanaSavedObjectType.indexPattern, + [KibanaAssetType.map]: KibanaSavedObjectType.map, + [KibanaAssetType.search]: KibanaSavedObjectType.search, + [KibanaAssetType.visualization]: KibanaSavedObjectType.visualization, }; // Define how each asset type will be installed @@ -106,7 +112,9 @@ export const deleteKibanaInstalledRefs = async ( installed_kibana: installedAssetsToSave, }); }; -export async function getKibanaAssets(paths: string[]) { +export async function getKibanaAssets( + paths: string[] +): Promise> { const kibanaAssetTypes = Object.values(KibanaAssetType); const isKibanaAssetType = (path: string) => { const parts = Registry.pathParts(path); @@ -130,7 +138,7 @@ export async function getKibanaAssets(paths: string[]) { const result = {} as Record; for (const [index, assetType] of kibanaAssetTypes.entries()) { - const expectedType = KibanaSavedObjectTypes[assetType]; + const expectedType = KibanaSavedObjectTypeMapping[assetType]; const properlyTypedAssets = resolvedAssets[index].filter(({ type }) => type === expectedType); result[assetType] = properlyTypedAssets; @@ -176,7 +184,7 @@ async function installKibanaIndexPatterns({ } export function toAssetReference({ id, type }: SavedObject) { - const reference: AssetReference = { id, type }; + const reference: AssetReference = { id, type: type as KibanaSavedObjectType }; return reference; } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts index 2f60c74d3514f9..60309a756d4d11 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ElasticsearchAssetType, Installation, KibanaAssetType } from '../../../types'; +import { ElasticsearchAssetType, Installation, KibanaSavedObjectType } from '../../../types'; import { SavedObject } from 'src/core/server'; import { getInstallType } from './install'; @@ -14,7 +14,7 @@ const mockInstallation: SavedObject = { type: 'epm-packages', attributes: { id: 'test-pkg', - installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }], + installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], es_index_patterns: { pattern: 'pattern-name' }, name: 'test packagek', @@ -30,7 +30,7 @@ const mockInstallationUpdateFail: SavedObject = { type: 'epm-packages', attributes: { id: 'test-pkg', - installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }], + installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], es_index_patterns: { pattern: 'pattern-name' }, name: 'test packagek', diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index d00491afef72b1..a71d3cf1ca879d 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -54,6 +54,7 @@ export { AssetType, Installable, KibanaAssetType, + KibanaSavedObjectType, AssetParts, AssetsGroupedByServiceByType, CategoryId, From ae1e0fdb46941668f01fccad6d58ac3d046f40f0 Mon Sep 17 00:00:00 2001 From: Corey Robertson Date: Tue, 22 Sep 2020 16:38:47 -0400 Subject: [PATCH 06/11] Fix Test --- .../apis/epm/update_assets.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/update_assets.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/update_assets.ts index 9af27f5f985582..787d33a04cb499 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/update_assets.ts @@ -283,14 +283,14 @@ export default function (providerContext: FtrProviderContext) { id: 'sample_dashboard', type: 'dashboard', }, - { - id: 'sample_search2', - type: 'search', - }, { id: 'sample_visualization', type: 'visualization', }, + { + id: 'sample_search2', + type: 'search', + }, ], installed_es: [ { From f0ecbc4704cd82dc9fdf5bd93b5b309933d8cd4e Mon Sep 17 00:00:00 2001 From: John Schulz Date: Tue, 3 Nov 2020 15:42:13 -0500 Subject: [PATCH 07/11] Update install.ts Use new `dataTypes` const which replaced `DataType` enum --- .../server/services/epm/kibana/index_pattern/install.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts index 5f25305bfb8ab3..beca73838c3978 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts @@ -77,7 +77,7 @@ export interface IndexPatternField { readFromDocValues: boolean; } -export const indexPatternTypes = [DataType.logs, DataType.metrics]; +const indexPatternTypes = Object.values(dataTypes); // TODO: use a function overload and make pkgName and pkgVersion required for install/update // and not for an update removal. or separate out the functions From 40cf94425ce6bb395e5c65ee91093493f2822621 Mon Sep 17 00:00:00 2001 From: John Schulz Date: Tue, 3 Nov 2020 15:44:42 -0500 Subject: [PATCH 08/11] Update install.ts Remove unused `indexPatternTypes` from outer scope --- .../server/services/epm/kibana/index_pattern/install.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts index beca73838c3978..366c5306a29699 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts @@ -77,8 +77,6 @@ export interface IndexPatternField { readFromDocValues: boolean; } -const indexPatternTypes = Object.values(dataTypes); - // TODO: use a function overload and make pkgName and pkgVersion required for install/update // and not for an update removal. or separate out the functions export async function installIndexPatterns( From f84206afefdf1a1ce3562c16319955d2bf254885 Mon Sep 17 00:00:00 2001 From: John Schulz Date: Tue, 3 Nov 2020 16:28:48 -0500 Subject: [PATCH 09/11] Update install.ts fix (?) bad updates from before where new/correct value was used but result wasn't exported --- .../server/services/epm/kibana/index_pattern/install.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts index 366c5306a29699..ca4be05f39eddc 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts @@ -77,6 +77,7 @@ export interface IndexPatternField { readFromDocValues: boolean; } +const indexPatternTypes = Object.values(dataTypes); // TODO: use a function overload and make pkgName and pkgVersion required for install/update // and not for an update removal. or separate out the functions export async function installIndexPatterns( From 52daf8bad185bd35a20f8ba33c573c0686741caf Mon Sep 17 00:00:00 2001 From: John Schulz Date: Tue, 3 Nov 2020 16:29:56 -0500 Subject: [PATCH 10/11] Update install.ts --- .../server/services/epm/kibana/index_pattern/install.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts index ca4be05f39eddc..4d94e4db03b491 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts @@ -77,7 +77,7 @@ export interface IndexPatternField { readFromDocValues: boolean; } -const indexPatternTypes = Object.values(dataTypes); +export const indexPatternTypes = Object.values(dataTypes); // TODO: use a function overload and make pkgName and pkgVersion required for install/update // and not for an update removal. or separate out the functions export async function installIndexPatterns( From 5b83a9b65f4cb0f92097ec15e8c234012b23e9ec Mon Sep 17 00:00:00 2001 From: John Schulz Date: Tue, 3 Nov 2020 17:37:57 -0500 Subject: [PATCH 11/11] Update install.ts --- .../server/services/epm/kibana/index_pattern/install.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts index 4d94e4db03b491..49cd787880340d 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts @@ -122,7 +122,6 @@ export async function installIndexPatterns( const packageVersionsInfo = await Promise.all(packageVersionsFetchInfoPromise); // for each index pattern type, create an index pattern - const indexPatternTypes = Object.values(dataTypes); indexPatternTypes.forEach(async (indexPatternType) => { // if this is an update because a package is being uninstalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern if (!pkgName && installedPackages.length === 0) {