From 8d539574a572ee02df8389fb8f086b86dfbddb64 Mon Sep 17 00:00:00 2001 From: Vacha Date: Thu, 12 Aug 2021 00:29:50 -0700 Subject: [PATCH] Adding rolling upgrade and full restart upgrade bwc tests Signed-off-by: Vacha --- build.gradle | 122 +++++++++++++++--- .../ad/bwc/ADBackwardsCompatibilityIT.java | 64 ++++++--- 2 files changed, 148 insertions(+), 38 deletions(-) diff --git a/build.gradle b/build.gradle index 4c8c2aa94..dbef9cd69 100644 --- a/build.gradle +++ b/build.gradle @@ -267,7 +267,7 @@ testClusters.integTest { } String bwcVersion = "1.13.0.0"; -String baseName = "adMixedCluster" +String baseName = "adBwcCluster" String bwcFilePath = "src/test/resources/org/opensearch/ad/bwc/" testClusters { @@ -302,23 +302,7 @@ testClusters { } } -task "${baseName}#oldVersionClusterTask"(type: StandaloneRestIntegTestTask) { - useCluster testClusters."${baseName}" - if (System.getProperty("mixedCluster") != null) { - filter { - includeTestsMatching "org.opensearch.ad.bwc.*IT" - } - } - systemProperty 'tests.rest.bwcsuite', 'old_cluster' - systemProperty 'tests.plugin_bwc_version', bwcVersion - nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") - nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") -} - -task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { - dependsOn "${baseName}#oldVersionClusterTask" - useCluster testClusters."${baseName}" - List> plugins = [ +List> plugins = [ provider(new Callable(){ @Override RegularFile call() throws Exception { @@ -342,20 +326,120 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { } }) ] + +// Creates a test cluster with 3 nodes of the old version. +task "${baseName}#oldVersionClusterTask"(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}" + if (System.getProperty("mixedCluster") != null) { + filter { + includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster") + } + } + if (System.getProperty("rollingUpgradeCluster") != null) { + filter { + includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster") + } + } + if (System.getProperty("fullRestartCluster") != null) { + filter { + includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster") + } + } + systemProperty 'tests.rest.bwcsuite', 'old_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'old' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") +} + +// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version +// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. +// This is also used as a one third upgraded cluster for a rolling upgrade. +task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#oldVersionClusterTask" + useCluster testClusters."${baseName}" doFirst { testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) } if (System.getProperty("mixedCluster") != null) { filter { - includeTestsMatching "org.opensearch.ad.bwc.*IT" + includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster") + } + } + if (System.getProperty("rollingUpgradeCluster") != null) { + filter { + includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster") } } systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'first' systemProperty 'tests.plugin_bwc_version', bwcVersion nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } +// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded. +// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. +// This is used for rolling upgrade. +task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#mixedClusterTask" + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) + } + if (System.getProperty("rollingUpgradeCluster") != null) { + filter { + includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster") + } + } + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'second' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") +} + +// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded. +// This results in a fully upgraded cluster. +// This is used for rolling upgrade. +task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#twoThirdsUpgradedClusterTask" + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins) + } + if (System.getProperty("rollingUpgradeCluster") != null) { + filter { + includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster") + } + } + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'third' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") +} + +// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version +// at the same time resulting in a fully upgraded cluster. +tasks.register("${baseName}#fullRestartClusterTask", StandaloneRestIntegTestTask) { + dependsOn "${baseName}#oldVersionClusterTask" + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins) + } + if (System.getProperty("fullRestartCluster") != null) { + filter { + includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster") + } + } + systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") +} + + run { doFirst { // There seems to be an issue when running multi node run or integ tasks with unicast_hosts diff --git a/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java b/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java index ff72a5299..a78d33a55 100644 --- a/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java +++ b/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java @@ -55,10 +55,47 @@ protected final Settings restClientSettings() { .build(); } - @SuppressWarnings("unchecked") public void testPluginUpgradeInAMixedCluster() throws Exception { - Map> responseMap = (Map>) getAsMap("_nodes/" + CLUSTER_NAME + "-0/plugins") - .get("nodes"); + assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-0/plugins"); + } + + public void testPluginUpgradeInAnUpgradedCluster() throws Exception { + assertPluginUpgrade("_nodes/plugins"); + } + + public void testPluginUpgradeInARollingUpgradedCluster() throws Exception { + String round = System.getProperty("tests.rest.bwcsuite_round"); + if (round.equals("first") || round.equals("old")) { + assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-0/plugins"); + } else if (round.equals("second")) { + assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-1/plugins"); + } else if (round.equals("third")) { + assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-2/plugins"); + } + } + + private enum ClusterType { + OLD, + MIXED, + UPGRADED; + + public static ClusterType parse(String value) { + switch (value) { + case "old_cluster": + return OLD; + case "mixed_cluster": + return MIXED; + case "upgraded_cluster": + return UPGRADED; + default: + throw new AssertionError("unknown cluster type: " + value); + } + } + } + + @SuppressWarnings("unchecked") + private void assertPluginUpgrade(String uri) throws Exception { + Map> responseMap = (Map>) getAsMap(uri).get("nodes"); for (Map response : responseMap.values()) { List> plugins = (List>) response.get("plugins"); Set pluginNames = plugins.stream().map(map -> map.get("name")).collect(Collectors.toSet()); @@ -73,27 +110,16 @@ public void testPluginUpgradeInAMixedCluster() throws Exception { Assert.assertTrue(pluginNames.contains("opensearch-job-scheduler")); verifyAnomalyDetector(TestHelpers.LEGACY_OPENDISTRO_AD_BASE_DETECTORS_URI); break; + case UPGRADED: + Assert.assertTrue(pluginNames.contains("opensearch-anomaly-detection")); + Assert.assertTrue(pluginNames.contains("opensearch-job-scheduler")); + verifyAnomalyDetector(TestHelpers.AD_BASE_DETECTORS_URI); + break; } break; } } - private enum ClusterType { - OLD, - MIXED; - - public static ClusterType parse(String value) { - switch (value) { - case "old_cluster": - return OLD; - case "mixed_cluster": - return MIXED; - default: - throw new AssertionError("unknown cluster type: " + value); - } - } - } - private void createBasicAnomalyDetector() throws Exception { AnomalyDetector detector = TestHelpers.randomAnomalyDetector(TestHelpers.randomUiMetadata(), null); String indexName = detector.getIndices().get(0);