From ec53f7fb41af1332e0eb529a4d31c7118c346437 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 15 Jul 2021 15:50:00 -0600 Subject: [PATCH 1/2] Made the Cypress tests work more resliant to fleet having rules that it pulls from online --- .../detection_rules/prebuilt_rules.spec.ts | 180 ++++++++++-------- .../security_solution/cypress/objects/rule.ts | 8 - .../cypress/tasks/alerts_detection_rules.ts | 7 +- 3 files changed, 109 insertions(+), 86 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/prebuilt_rules.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/prebuilt_rules.spec.ts index b259c0f1d9e336..8373c5838f63ee 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/prebuilt_rules.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/prebuilt_rules.spec.ts @@ -36,50 +36,64 @@ import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; import { ALERTS_URL } from '../../urls/navigation'; -import { totalNumberOfPrebuiltRules } from '../../objects/rule'; import { cleanKibana } from '../../tasks/common'; describe('Alerts rules, prebuilt rules', () => { beforeEach(() => { cleanKibana(); + cy.intercept('PUT', '/api/detection_engine/rules/prepackaged').as('prepackaged'); }); it('Loads prebuilt rules', () => { const rowsPerPage = 100; - const expectedNumberOfRules = totalNumberOfPrebuiltRules; - const expectedNumberOfPages = Math.ceil(totalNumberOfPrebuiltRules / rowsPerPage); - const expectedElasticRulesBtnText = `Elastic rules (${expectedNumberOfRules})`; - loginAndWaitForPageWithoutDateRange(ALERTS_URL); waitForAlertsIndexToBeCreated(); goToManageAlertsDetectionRules(); waitForRulesTableToBeLoaded(); loadPrebuiltDetectionRules(); waitForPrebuiltDetectionRulesToBeLoaded(); - - cy.get(ELASTIC_RULES_BTN).should('have.text', expectedElasticRulesBtnText); - - changeRowsPerPageTo(rowsPerPage); - - cy.get(SHOWING_RULES_TEXT).should('have.text', `Showing ${expectedNumberOfRules} rules`); - cy.get(pageSelector(expectedNumberOfPages)).should('exist'); + cy.wait('@prepackaged').then((interception) => { + if (interception.response == null) { + throw new Error( + 'response body should not be null for "/api/detection_engine/rules/prepackaged"' + ); + } + const total = interception.response.body.rules_installed; + const expectedNumberOfPages = Math.ceil(total / rowsPerPage); + const expectedElasticRulesBtnText = `Elastic rules (${total})`; + cy.get(ELASTIC_RULES_BTN).should('have.text', expectedElasticRulesBtnText); + + changeRowsPerPageTo(rowsPerPage); + + cy.get(SHOWING_RULES_TEXT).should('have.text', `Showing ${total} rules`); + cy.get(pageSelector(expectedNumberOfPages)).should('exist'); + }); }); }); describe('Actions with prebuilt rules', () => { beforeEach(() => { - const expectedNumberOfRules = totalNumberOfPrebuiltRules; - const expectedElasticRulesBtnText = `Elastic rules (${expectedNumberOfRules})`; - cleanKibana(); + cy.intercept('PUT', '/api/detection_engine/rules/prepackaged').as('prepackaged'); loginAndWaitForPageWithoutDateRange(ALERTS_URL); waitForAlertsIndexToBeCreated(); goToManageAlertsDetectionRules(); waitForRulesTableToBeLoaded(); loadPrebuiltDetectionRules(); waitForPrebuiltDetectionRulesToBeLoaded(); + }); - cy.get(ELASTIC_RULES_BTN).should('have.text', expectedElasticRulesBtnText); + it('Has the correct number of rules', () => { + cy.wait('@prepackaged').then((interception) => { + if (interception.response == null) { + throw new Error( + 'response body should not be null for "/api/detection_engine/rules/prepackaged"' + ); + } + const total = interception.response.body.rules_installed; + const expectedElasticRulesBtnText = `Elastic rules (${total})`; + cy.get(ELASTIC_RULES_BTN).should('have.text', expectedElasticRulesBtnText); + }); }); it('Allows to activate/deactivate all rules at once', () => { @@ -113,67 +127,83 @@ describe('Actions with prebuilt rules', () => { }); it('Deletes and recovers one rule', () => { - changeRowsPerPageTo100(); - - const expectedNumberOfRulesAfterDeletion = totalNumberOfPrebuiltRules - 1; - const expectedNumberOfRulesAfterRecovering = totalNumberOfPrebuiltRules; - - deleteFirstRule(); - cy.reload(); - changeRowsPerPageTo100(); - - cy.get(ELASTIC_RULES_BTN).should( - 'have.text', - `Elastic rules (${expectedNumberOfRulesAfterDeletion})` - ); - cy.get(RELOAD_PREBUILT_RULES_BTN).should('exist'); - cy.get(RELOAD_PREBUILT_RULES_BTN).should('have.text', 'Install 1 Elastic prebuilt rule '); - - reloadDeletedRules(); - - cy.get(RELOAD_PREBUILT_RULES_BTN).should('not.exist'); - - cy.reload(); - changeRowsPerPageTo100(); - - cy.get(ELASTIC_RULES_BTN).should( - 'have.text', - `Elastic rules (${expectedNumberOfRulesAfterRecovering})` - ); + cy.wait('@prepackaged').then((interception) => { + if (interception.response == null) { + throw new Error( + 'response body should not be null for "/api/detection_engine/rules/prepackaged"' + ); + } + const total = interception.response.body.rules_installed; + changeRowsPerPageTo100(); + + const expectedNumberOfRulesAfterDeletion = total - 1; + const expectedNumberOfRulesAfterRecovering = total; + + deleteFirstRule(); + cy.reload(); + changeRowsPerPageTo100(); + + cy.get(ELASTIC_RULES_BTN).should( + 'have.text', + `Elastic rules (${expectedNumberOfRulesAfterDeletion})` + ); + cy.get(RELOAD_PREBUILT_RULES_BTN).should('exist'); + cy.get(RELOAD_PREBUILT_RULES_BTN).should('have.text', 'Install 1 Elastic prebuilt rule '); + + reloadDeletedRules(); + + cy.get(RELOAD_PREBUILT_RULES_BTN).should('not.exist'); + + cy.reload(); + changeRowsPerPageTo100(); + + cy.get(ELASTIC_RULES_BTN).should( + 'have.text', + `Elastic rules (${expectedNumberOfRulesAfterRecovering})` + ); + }); }); it('Deletes and recovers more than one rule', () => { - changeRowsPerPageTo100(); - - const numberOfRulesToBeSelected = 2; - const expectedNumberOfRulesAfterDeletion = totalNumberOfPrebuiltRules - 2; - const expectedNumberOfRulesAfterRecovering = totalNumberOfPrebuiltRules; - - selectNumberOfRules(numberOfRulesToBeSelected); - deleteSelectedRules(); - cy.reload(); - changeRowsPerPageTo100(); - - cy.get(RELOAD_PREBUILT_RULES_BTN).should('exist'); - cy.get(RELOAD_PREBUILT_RULES_BTN).should( - 'have.text', - `Install ${numberOfRulesToBeSelected} Elastic prebuilt rules ` - ); - cy.get(ELASTIC_RULES_BTN).should( - 'have.text', - `Elastic rules (${expectedNumberOfRulesAfterDeletion})` - ); - - reloadDeletedRules(); - - cy.get(RELOAD_PREBUILT_RULES_BTN).should('not.exist'); - - cy.reload(); - changeRowsPerPageTo100(); - - cy.get(ELASTIC_RULES_BTN).should( - 'have.text', - `Elastic rules (${expectedNumberOfRulesAfterRecovering})` - ); + cy.wait('@prepackaged').then((interception) => { + if (interception.response == null) { + throw new Error( + 'response body should not be null for "/api/detection_engine/rules/prepackaged"' + ); + } + const total = interception.response.body.rules_installed; + changeRowsPerPageTo100(); + + const numberOfRulesToBeSelected = 2; + const expectedNumberOfRulesAfterDeletion = total - 2; + const expectedNumberOfRulesAfterRecovering = total; + + selectNumberOfRules(numberOfRulesToBeSelected); + deleteSelectedRules(); + cy.reload(); + changeRowsPerPageTo100(); + + cy.get(RELOAD_PREBUILT_RULES_BTN).should('exist'); + cy.get(RELOAD_PREBUILT_RULES_BTN).should( + 'have.text', + `Install ${numberOfRulesToBeSelected} Elastic prebuilt rules ` + ); + cy.get(ELASTIC_RULES_BTN).should( + 'have.text', + `Elastic rules (${expectedNumberOfRulesAfterDeletion})` + ); + + reloadDeletedRules(); + + cy.get(RELOAD_PREBUILT_RULES_BTN).should('not.exist'); + + cy.reload(); + changeRowsPerPageTo100(); + + cy.get(ELASTIC_RULES_BTN).should( + 'have.text', + `Elastic rules (${expectedNumberOfRulesAfterRecovering})` + ); + }); }); }); diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts index 7589c8fab3dae5..24a801ad11f68d 100644 --- a/x-pack/plugins/security_solution/cypress/objects/rule.ts +++ b/x-pack/plugins/security_solution/cypress/objects/rule.ts @@ -5,17 +5,9 @@ * 2.0. */ -/* eslint-disable @kbn/eslint/no-restricted-paths */ -import { rawRules } from '../../server/lib/detection_engine/rules/prepackaged_rules/index'; import { getMockThreatData } from '../../public/detections/mitre/mitre_tactics_techniques'; import { getTimeline, CompleteTimeline, getIndicatorMatchTimelineTemplate } from './timeline'; -export const totalNumberOfPrebuiltRules = rawRules.length; - -export const totalNumberOfPrebuiltRulesInEsArchive = 127; - -export const totalNumberOfPrebuiltRulesInEsArchiveCustomRule = 145; - const ccsRemoteName: string = Cypress.env('CCS_REMOTE_NAME'); interface MitreAttackTechnique { diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts index 6dbd1ae16c4add..5c27372ce72387 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts @@ -217,9 +217,10 @@ export const waitForRulesTableToBeAutoRefreshed = () => { }; export const waitForPrebuiltDetectionRulesToBeLoaded = () => { - cy.get(LOAD_PREBUILT_RULES_BTN).should('not.exist'); - cy.get(RULES_TABLE).should('exist'); - cy.get(RULES_TABLE_REFRESH_INDICATOR).should('not.exist'); + // Wait up to 5 minutes for the rules to load as in CI containers this can be very slow + cy.get(LOAD_PREBUILT_RULES_BTN, { timeout: 300000 }).should('not.exist'); + cy.get(RULES_TABLE, { timeout: 300000 }).should('exist'); + cy.get(RULES_TABLE_REFRESH_INDICATOR, { timeout: 300000 }).should('not.exist'); }; export const waitForRuleToChangeStatus = () => { From 5e78d9db72882a76ca4b6732f1bf4ac7301b63ee Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 15 Jul 2021 17:50:19 -0600 Subject: [PATCH 2/2] Implements softer >= than the intercept --- .../detection_rules/prebuilt_rules.spec.ts | 206 ++++++++++-------- .../security_solution/cypress/objects/rule.ts | 4 + 2 files changed, 116 insertions(+), 94 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/prebuilt_rules.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/prebuilt_rules.spec.ts index 8373c5838f63ee..ddbe01e595cae1 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/prebuilt_rules.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/prebuilt_rules.spec.ts @@ -37,44 +37,54 @@ import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; import { ALERTS_URL } from '../../urls/navigation'; import { cleanKibana } from '../../tasks/common'; +import { totalNumberOfPrebuiltRules } from '../../objects/rule'; describe('Alerts rules, prebuilt rules', () => { beforeEach(() => { cleanKibana(); - cy.intercept('PUT', '/api/detection_engine/rules/prepackaged').as('prepackaged'); }); it('Loads prebuilt rules', () => { const rowsPerPage = 100; + const expectedNumberOfRules = totalNumberOfPrebuiltRules; + const expectedNumberOfPages = Math.ceil(totalNumberOfPrebuiltRules / rowsPerPage); + loginAndWaitForPageWithoutDateRange(ALERTS_URL); waitForAlertsIndexToBeCreated(); goToManageAlertsDetectionRules(); waitForRulesTableToBeLoaded(); loadPrebuiltDetectionRules(); waitForPrebuiltDetectionRulesToBeLoaded(); - cy.wait('@prepackaged').then((interception) => { - if (interception.response == null) { - throw new Error( - 'response body should not be null for "/api/detection_engine/rules/prepackaged"' - ); - } - const total = interception.response.body.rules_installed; - const expectedNumberOfPages = Math.ceil(total / rowsPerPage); - const expectedElasticRulesBtnText = `Elastic rules (${total})`; - cy.get(ELASTIC_RULES_BTN).should('have.text', expectedElasticRulesBtnText); - changeRowsPerPageTo(rowsPerPage); - - cy.get(SHOWING_RULES_TEXT).should('have.text', `Showing ${total} rules`); - cy.get(pageSelector(expectedNumberOfPages)).should('exist'); - }); + // The result should be at least equal to the number of rules we have on disk + // but could be greater than this number if there are additional cloud rules + cy.get(ELASTIC_RULES_BTN) + .first() + .then((result) => { + expect(+result.text().replace(/[^0-9]/g, '')).to.be.least(expectedNumberOfRules); + expect(result.text()).to.match(/Elastic rules\s\([0-9]+\)/); + }); + + changeRowsPerPageTo(rowsPerPage); + + // The result should be at least equal to the number of rules we have on disk + // but could be greater than this number if there are additional cloud rules + cy.get(SHOWING_RULES_TEXT) + .first() + .then((result) => { + expect(+result.text().replace(/[^0-9]/g, '')).to.be.least(expectedNumberOfRules); + expect(result.text()).to.match(/Showing\s[0-9]+\srules/); + }); + + cy.get(pageSelector(expectedNumberOfPages)).should('exist'); }); }); describe('Actions with prebuilt rules', () => { + const expectedNumberOfRules = totalNumberOfPrebuiltRules; + beforeEach(() => { cleanKibana(); - cy.intercept('PUT', '/api/detection_engine/rules/prepackaged').as('prepackaged'); loginAndWaitForPageWithoutDateRange(ALERTS_URL); waitForAlertsIndexToBeCreated(); goToManageAlertsDetectionRules(); @@ -84,16 +94,14 @@ describe('Actions with prebuilt rules', () => { }); it('Has the correct number of rules', () => { - cy.wait('@prepackaged').then((interception) => { - if (interception.response == null) { - throw new Error( - 'response body should not be null for "/api/detection_engine/rules/prepackaged"' - ); - } - const total = interception.response.body.rules_installed; - const expectedElasticRulesBtnText = `Elastic rules (${total})`; - cy.get(ELASTIC_RULES_BTN).should('have.text', expectedElasticRulesBtnText); - }); + // The result should be at least equal to the number of rules we have on disk + // but could be greater than this number if there are additional cloud rules + cy.get(ELASTIC_RULES_BTN) + .first() + .then((result) => { + expect(+result.text().replace(/[^0-9]/g, '')).to.be.least(expectedNumberOfRules); + expect(result.text()).to.match(/Elastic rules\s\([0-9]+\)/); + }); }); it('Allows to activate/deactivate all rules at once', () => { @@ -127,83 +135,93 @@ describe('Actions with prebuilt rules', () => { }); it('Deletes and recovers one rule', () => { - cy.wait('@prepackaged').then((interception) => { - if (interception.response == null) { - throw new Error( - 'response body should not be null for "/api/detection_engine/rules/prepackaged"' - ); - } - const total = interception.response.body.rules_installed; - changeRowsPerPageTo100(); + changeRowsPerPageTo100(); - const expectedNumberOfRulesAfterDeletion = total - 1; - const expectedNumberOfRulesAfterRecovering = total; + const expectedNumberOfRulesAfterDeletion = totalNumberOfPrebuiltRules - 1; + const expectedNumberOfRulesAfterRecovering = totalNumberOfPrebuiltRules; - deleteFirstRule(); - cy.reload(); - changeRowsPerPageTo100(); + deleteFirstRule(); + cy.reload(); + changeRowsPerPageTo100(); - cy.get(ELASTIC_RULES_BTN).should( - 'have.text', - `Elastic rules (${expectedNumberOfRulesAfterDeletion})` - ); - cy.get(RELOAD_PREBUILT_RULES_BTN).should('exist'); - cy.get(RELOAD_PREBUILT_RULES_BTN).should('have.text', 'Install 1 Elastic prebuilt rule '); + // The result should be at least equal to the number of rules we have on disk + // but could be greater than this number if there are additional cloud rules. + cy.get(ELASTIC_RULES_BTN) + .first() + .then((result) => { + expect(+result.text().replace(/[^0-9]/g, '')).to.be.least( + expectedNumberOfRulesAfterDeletion + ); + expect(result.text()).to.match(/Elastic rules\s\([0-9]+\)/); + }); - reloadDeletedRules(); + cy.get(RELOAD_PREBUILT_RULES_BTN).should('exist'); + cy.get(RELOAD_PREBUILT_RULES_BTN).should('have.text', 'Install 1 Elastic prebuilt rule '); - cy.get(RELOAD_PREBUILT_RULES_BTN).should('not.exist'); + reloadDeletedRules(); - cy.reload(); - changeRowsPerPageTo100(); + cy.get(RELOAD_PREBUILT_RULES_BTN).should('not.exist'); - cy.get(ELASTIC_RULES_BTN).should( - 'have.text', - `Elastic rules (${expectedNumberOfRulesAfterRecovering})` - ); - }); + cy.reload(); + changeRowsPerPageTo100(); + + // The result should be at least equal to the number of rules we have on disk + // but could be greater than this number if there are additional cloud rules + cy.get(ELASTIC_RULES_BTN) + .first() + .then((result) => { + expect(+result.text().replace(/[^0-9]/g, '')).to.be.least( + expectedNumberOfRulesAfterRecovering + ); + expect(result.text()).to.match(/Elastic rules\s\([0-9]+\)/); + }); }); it('Deletes and recovers more than one rule', () => { - cy.wait('@prepackaged').then((interception) => { - if (interception.response == null) { - throw new Error( - 'response body should not be null for "/api/detection_engine/rules/prepackaged"' + changeRowsPerPageTo100(); + + const numberOfRulesToBeSelected = 2; + const expectedNumberOfRulesAfterDeletion = totalNumberOfPrebuiltRules - 2; + const expectedNumberOfRulesAfterRecovering = totalNumberOfPrebuiltRules; + + selectNumberOfRules(numberOfRulesToBeSelected); + deleteSelectedRules(); + cy.reload(); + changeRowsPerPageTo100(); + + cy.get(RELOAD_PREBUILT_RULES_BTN).should('exist'); + cy.get(RELOAD_PREBUILT_RULES_BTN).should( + 'have.text', + `Install ${numberOfRulesToBeSelected} Elastic prebuilt rules ` + ); + + // The result should be at least equal to the number of rules we have on disk + // but could be greater than this number if there are additional cloud rules + cy.get(ELASTIC_RULES_BTN) + .first() + .then((result) => { + expect(+result.text().replace(/[^0-9]/g, '')).to.be.least( + expectedNumberOfRulesAfterDeletion ); - } - const total = interception.response.body.rules_installed; - changeRowsPerPageTo100(); - - const numberOfRulesToBeSelected = 2; - const expectedNumberOfRulesAfterDeletion = total - 2; - const expectedNumberOfRulesAfterRecovering = total; - - selectNumberOfRules(numberOfRulesToBeSelected); - deleteSelectedRules(); - cy.reload(); - changeRowsPerPageTo100(); - - cy.get(RELOAD_PREBUILT_RULES_BTN).should('exist'); - cy.get(RELOAD_PREBUILT_RULES_BTN).should( - 'have.text', - `Install ${numberOfRulesToBeSelected} Elastic prebuilt rules ` - ); - cy.get(ELASTIC_RULES_BTN).should( - 'have.text', - `Elastic rules (${expectedNumberOfRulesAfterDeletion})` - ); - - reloadDeletedRules(); - - cy.get(RELOAD_PREBUILT_RULES_BTN).should('not.exist'); - - cy.reload(); - changeRowsPerPageTo100(); - - cy.get(ELASTIC_RULES_BTN).should( - 'have.text', - `Elastic rules (${expectedNumberOfRulesAfterRecovering})` - ); - }); + expect(result.text()).to.match(/Elastic rules\s\([0-9]+\)/); + }); + + reloadDeletedRules(); + + cy.get(RELOAD_PREBUILT_RULES_BTN).should('not.exist'); + + cy.reload(); + changeRowsPerPageTo100(); + + // The result should be at least equal to the number of rules we have on disk + // but could be greater than this number if there are additional cloud rules + cy.get(ELASTIC_RULES_BTN) + .first() + .then((result) => { + expect(+result.text().replace(/[^0-9]/g, '')).to.be.least( + expectedNumberOfRulesAfterRecovering + ); + expect(result.text()).to.match(/Elastic rules\s\([0-9]+\)/); + }); }); }); diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts index 24a801ad11f68d..af77a464eac5f6 100644 --- a/x-pack/plugins/security_solution/cypress/objects/rule.ts +++ b/x-pack/plugins/security_solution/cypress/objects/rule.ts @@ -5,9 +5,13 @@ * 2.0. */ +/* eslint-disable @kbn/eslint/no-restricted-paths */ +import { rawRules } from '../../server/lib/detection_engine/rules/prepackaged_rules/index'; import { getMockThreatData } from '../../public/detections/mitre/mitre_tactics_techniques'; import { getTimeline, CompleteTimeline, getIndicatorMatchTimelineTemplate } from './timeline'; +export const totalNumberOfPrebuiltRules = rawRules.length; + const ccsRemoteName: string = Cypress.env('CCS_REMOTE_NAME'); interface MitreAttackTechnique {