diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts index 512f4618792fbe..b1460a5fe5cd85 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts @@ -1012,6 +1012,410 @@ describe('successful migrations', () => { }); }); }); + + describe('7.15.0', () => { + test('security solution is migrated to saved object references if it has 1 exceptionsList', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', // extra data to ensure we do not overwrite other params + exceptionsList: [ + { + id: '123', + list_id: '456', + type: 'detection', + namespace_type: 'single', + }, + ], + }, + }); + + expect(migration7150(alert, migrationContext)).toEqual({ + ...alert, + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + ], + }); + }); + + test('security solution is migrated to saved object references if it has 2 exceptionsLists', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', // extra data to ensure we do not overwrite other params + exceptionsList: [ + { + id: '123', + list_id: '456', + type: 'detection', + namespace_type: 'agnostic', + }, + { + id: '789', + list_id: '0123', + type: 'detection', + namespace_type: 'single', + }, + ], + }, + }); + + expect(migration7150(alert, migrationContext)).toEqual({ + ...alert, + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list-agnostic', + }, + { + name: 'param:exceptionsList_1', + id: '789', + type: 'exception-list', + }, + ], + }); + }); + + test('security solution is migrated to saved object references if it has 3 exceptionsLists', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', // extra data to ensure we do not overwrite other params + exceptionsList: [ + { + id: '123', + list_id: '456', + type: 'detection', + namespace_type: 'single', + }, + { + id: '789', + list_id: '0123', + type: 'detection', + namespace_type: 'agnostic', + }, + { + id: '101112', + list_id: '777', + type: 'detection', + namespace_type: 'single', + }, + ], + }, + }); + + expect(migration7150(alert, migrationContext)).toEqual({ + ...alert, + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + { + name: 'param:exceptionsList_1', + id: '789', + type: 'exception-list-agnostic', + }, + { + name: 'param:exceptionsList_2', + id: '101112', + type: 'exception-list', + }, + ], + }); + }); + + test('security solution does not change anything if exceptionsList is missing', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', + }, + }); + + expect(migration7150(alert, migrationContext)).toEqual(alert); + }); + + test('security solution will keep existing references if we do not have an exceptionsList but we do already have references', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = { + ...getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', + }, + }), + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + ], + }; + + expect(migration7150(alert, migrationContext)).toEqual({ + ...alert, + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + ], + }); + }); + + test('security solution keep any foreign references if they exist but still migrate other references', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = { + ...getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', + exceptionsList: [ + { + id: '123', + list_id: '456', + type: 'detection', + namespace_type: 'single', + }, + { + id: '789', + list_id: '0123', + type: 'detection', + namespace_type: 'single', + }, + { + id: '101112', + list_id: '777', + type: 'detection', + namespace_type: 'single', + }, + ], + }, + }), + references: [ + { + name: 'foreign-name', + id: '999', + type: 'foreign-name', + }, + ], + }; + + expect(migration7150(alert, migrationContext)).toEqual({ + ...alert, + references: [ + { + name: 'foreign-name', + id: '999', + type: 'foreign-name', + }, + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + { + name: 'param:exceptionsList_1', + id: '789', + type: 'exception-list', + }, + { + name: 'param:exceptionsList_2', + id: '101112', + type: 'exception-list', + }, + ], + }); + }); + + test('security solution is idempotent and if re-run on the same migrated data will keep the same items', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = { + ...getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', + exceptionsList: [ + { + id: '123', + list_id: '456', + type: 'detection', + namespace_type: 'single', + }, + { + id: '789', + list_id: '0123', + type: 'detection', + namespace_type: 'single', + }, + ], + }, + }), + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + { + name: 'param:exceptionsList_1', + id: '789', + type: 'exception-list', + }, + ], + }; + + expect(migration7150(alert, migrationContext)).toEqual(alert); + }); + + test('security solution will migrate with only missing data if we have partially migrated data', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = { + ...getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', + exceptionsList: [ + { + id: '123', + list_id: '456', + type: 'detection', + namespace_type: 'single', + }, + { + id: '789', + list_id: '0123', + type: 'detection', + namespace_type: 'single', + }, + ], + }, + }), + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + ], + }; + + expect(migration7150(alert, migrationContext)).toEqual({ + ...alert, + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + { + name: 'param:exceptionsList_1', + id: '789', + type: 'exception-list', + }, + ], + }); + }); + + test('security solution will not migrate if exception list if it is invalid data', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = { + ...getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', + exceptionsList: [{ invalid: 'invalid' }], + }, + }), + }; + expect(migration7150(alert, migrationContext)).toEqual(alert); + }); + + test('security solution will migrate valid data if it is mixed with invalid data', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = { + ...getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', + exceptionsList: [ + { + id: '123', + list_id: '456', + type: 'detection', + namespace_type: 'single', + }, + { id: 555 }, // <-- Id is a number and not a string, and is invalid + { + id: '456', + list_id: '456', + type: 'detection', + namespace_type: 'single', + }, + ], + }, + }), + }; + expect(migration7150(alert, migrationContext)).toEqual({ + ...alert, + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + { + name: 'param:exceptionsList_1', + id: '456', + type: 'exception-list', + }, + ], + }); + }); + + test('security solution will not migrate if exception list is invalid data but will keep existing references', () => { + const migration7150 = getMigrations(encryptedSavedObjectsSetup)['7.15.0']; + const alert = { + ...getMockData({ + alertTypeId: 'siem.signals', + params: { + note: 'some note', + exceptionsList: [{ invalid: 'invalid' }], + }, + }), + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + ], + }; + expect(migration7150(alert, migrationContext)).toEqual({ + ...alert, + references: [ + { + name: 'param:exceptionsList_0', + id: '123', + type: 'exception-list', + }, + ], + }); + }); + }); }); describe('handles errors during migrations', () => { diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts index 944acbdca01822..6823a9b9b20da3 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { isString } from 'lodash/fp'; import { LogMeta, SavedObjectMigrationMap, @@ -13,6 +14,7 @@ import { SavedObjectMigrationContext, SavedObjectAttributes, SavedObjectAttribute, + SavedObjectReference, } from '../../../../../src/core/server'; import { RawAlert, RawAlertAction } from '../types'; import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server'; @@ -91,12 +93,19 @@ export function getMigrations( pipeMigrations(removeNullAuthorFromSecurityRules) ); + const migrationSecurityRules715 = createEsoMigration( + encryptedSavedObjects, + (doc): doc is SavedObjectUnsanitizedDoc => isSecuritySolutionRule(doc), + pipeMigrations(addExceptionListsToReferences) + ); + return { '7.10.0': executeMigrationWithErrorHandling(migrationWhenRBACWasIntroduced, '7.10.0'), '7.11.0': executeMigrationWithErrorHandling(migrationAlertUpdatedAtAndNotifyWhen, '7.11.0'), '7.11.2': executeMigrationWithErrorHandling(migrationActions7112, '7.11.2'), '7.13.0': executeMigrationWithErrorHandling(migrationSecurityRules713, '7.13.0'), '7.14.1': executeMigrationWithErrorHandling(migrationSecurityRules714, '7.14.1'), + '7.15.0': executeMigrationWithErrorHandling(migrationSecurityRules715, '7.15.0'), }; } @@ -467,6 +476,97 @@ function removeNullAuthorFromSecurityRules( }; } +/** + * This migrates exception list containers to saved object references on an upgrade. + * We only migrate if we find these conditions: + * - exceptionLists are an array and not null, undefined, or malformed data. + * - The exceptionList item is an object and id is a string and not null, undefined, or malformed data + * - The existing references do not already have an exceptionItem reference already found within it. + * Some of these issues could crop up during either user manual errors of modifying things, earlier migration + * issues, etc... + * @param doc The document that might have exceptionListItems to migrate + * @returns The document migrated with saved object references + */ +function addExceptionListsToReferences( + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { + const { + attributes: { + params: { exceptionsList }, + }, + references, + } = doc; + if (!Array.isArray(exceptionsList)) { + // early return if we are not an array such as being undefined or null or malformed. + return doc; + } else { + const exceptionsToTransform = removeMalformedExceptionsList(exceptionsList); + const newReferences = exceptionsToTransform.flatMap( + (exceptionItem, index) => { + const existingReferenceFound = references?.find((reference) => { + return ( + reference.id === exceptionItem.id && + ((reference.type === 'exception-list' && exceptionItem.namespace_type === 'single') || + (reference.type === 'exception-list-agnostic' && + exceptionItem.namespace_type === 'agnostic')) + ); + }); + if (existingReferenceFound) { + // skip if the reference already exists for some uncommon reason so we do not add an additional one. + // This enables us to be idempotent and you can run this migration multiple times and get the same output. + return []; + } else { + return [ + { + name: `param:exceptionsList_${index}`, + id: String(exceptionItem.id), + type: + exceptionItem.namespace_type === 'agnostic' + ? 'exception-list-agnostic' + : 'exception-list', + }, + ]; + } + } + ); + if (references == null && newReferences.length === 0) { + // Avoid adding an empty references array if the existing saved object never had one to begin with + return doc; + } else { + return { ...doc, references: [...(references ?? []), ...newReferences] }; + } + } +} + +/** + * This will do a flatMap reduce where we only return exceptionsLists and their items if: + * - exceptionLists are an array and not null, undefined, or malformed data. + * - The exceptionList item is an object and id is a string and not null, undefined, or malformed data + * + * Some of these issues could crop up during either user manual errors of modifying things, earlier migration + * issues, etc... + * @param exceptionsList The list of exceptions + * @returns The exception lists if they are a valid enough shape + */ +function removeMalformedExceptionsList( + exceptionsList: SavedObjectAttribute +): SavedObjectAttributes[] { + if (!Array.isArray(exceptionsList)) { + // early return if we are not an array such as being undefined or null or malformed. + return []; + } else { + return exceptionsList.flatMap((exceptionItem) => { + if (!(exceptionItem instanceof Object) || !isString(exceptionItem.id)) { + // return early if we are not an object such as being undefined or null or malformed + // or the exceptionItem.id is not a string from being malformed + return []; + } else { + return [exceptionItem]; + } + }); + } +} + function pipeMigrations(...migrations: AlertMigration[]): AlertMigration { return (doc: SavedObjectUnsanitizedDoc) => migrations.reduce((migratedDoc, nextMigration) => nextMigration(migratedDoc), doc); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts index 0eaeaf9a4b7e75..81b544ac97152b 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts @@ -11,6 +11,7 @@ import { FtrProviderContext } from '../../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export export default function createGetTests({ getService }: FtrProviderContext) { + const es = getService('es'); const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); @@ -175,5 +176,26 @@ export default function createGetTests({ getService }: FtrProviderContext) { }, ]); }); + + it('7.15.0 migrates security_solution alerts with exceptionLists to be saved object references', async () => { + // NOTE: We hae to use elastic search directly against the ".kibana" index because alerts do not expose the references which we want to test exists + const response = await es.get<{ references: [{}] }>({ + index: '.kibana', + id: 'alert:38482620-ef1b-11eb-ad71-7de7959be71c', + }); + expect(response.statusCode).to.eql(200); + expect(response.body._source?.references).to.eql([ + { + name: 'param:exceptionsList_0', + id: 'endpoint_list', + type: 'exception-list-agnostic', + }, + { + name: 'param:exceptionsList_1', + id: '50e3bd70-ef1b-11eb-ad71-7de7959be71c', + type: 'exception-list', + }, + ]); + }); }); } diff --git a/x-pack/test/functional/es_archives/alerts/data.json b/x-pack/test/functional/es_archives/alerts/data.json index 26f201c095dcaf..2ce6be7b4816c0 100644 --- a/x-pack/test/functional/es_archives/alerts/data.json +++ b/x-pack/test/functional/es_archives/alerts/data.json @@ -333,3 +333,57 @@ } } } + +{ + "type": "doc", + "value": { + "id": "alert:38482620-ef1b-11eb-ad71-7de7959be71c", + "index": ".kibana_1", + "source": { + "alert" : { + "name" : "test upgrade of exceptionsList", + "alertTypeId" : "siem.signals", + "consumer" : "alertsFixture", + "params" : { + "ruleId" : "4ec223b9-77fa-4895-8539-6b3e586a2858", + "exceptionsList" : [ + { + "id" : "endpoint_list", + "list_id" : "endpoint_list", + "namespace_type" : "agnostic", + "type" : "endpoint" + }, + { + "list_id" : "cd152d0d-3590-4a45-a478-eed04da7936b", + "namespace_type" : "single", + "id" : "50e3bd70-ef1b-11eb-ad71-7de7959be71c", + "type" : "detection" + } + ] + }, + "schedule" : { + "interval" : "1m" + }, + "enabled" : true, + "actions" : [ ], + "throttle" : null, + "apiKeyOwner" : null, + "apiKey" : null, + "createdBy" : "elastic", + "updatedBy" : "elastic", + "createdAt" : "2021-07-27T20:42:55.896Z", + "muteAll" : false, + "mutedInstanceIds" : [ ], + "scheduledTaskId" : null, + "tags": [] + }, + "type" : "alert", + "migrationVersion" : { + "alert" : "7.8.0" + }, + "updated_at" : "2021-08-13T23:00:11.985Z", + "references": [ + ] + } + } +}