Skip to content

Commit

Permalink
Adding rolling upgrade and full restart upgrade bwc tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Vacha <vachshah@amazon.com>
  • Loading branch information
@VachaShah
VachaShah committed Aug 12, 2021
1 parent 88b07f4 commit 8d53957
Show file tree
Hide file tree
Showing 2 changed files with 148 additions and 38 deletions.
122 changes: 103 additions & 19 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ testClusters.integTest {
}

String bwcVersion = "1.13.0.0";
String baseName = "adMixedCluster"
String baseName = "adBwcCluster"
String bwcFilePath = "src/test/resources/org/opensearch/ad/bwc/"

testClusters {
Expand Down Expand Up @@ -302,23 +302,7 @@ testClusters {
}
}

task "${baseName}#oldVersionClusterTask"(type: StandaloneRestIntegTestTask) {
useCluster testClusters."${baseName}"
if (System.getProperty("mixedCluster") != null) {
filter {
includeTestsMatching "org.opensearch.ad.bwc.*IT"
}
}
systemProperty 'tests.rest.bwcsuite', 'old_cluster'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}

task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#oldVersionClusterTask"
useCluster testClusters."${baseName}"
List<Provider<RegularFile>> plugins = [
List<Provider<RegularFile>> plugins = [
provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
Expand All @@ -342,20 +326,120 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
}
})
]

// Creates a test cluster with 3 nodes of the old version.
task "${baseName}#oldVersionClusterTask"(type: StandaloneRestIntegTestTask) {
useCluster testClusters."${baseName}"
if (System.getProperty("mixedCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
}
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
if (System.getProperty("fullRestartCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'old_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'old'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}

// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
// This is also used as a one third upgraded cluster for a rolling upgrade.
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#oldVersionClusterTask"
useCluster testClusters."${baseName}"
doFirst {
testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("mixedCluster") != null) {
filter {
includeTestsMatching "org.opensearch.ad.bwc.*IT"
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
}
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'first'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}

// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#mixedClusterTask"
useCluster testClusters."${baseName}"
doFirst {
testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'second'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}

// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
// This results in a fully upgraded cluster.
// This is used for rolling upgrade.
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
useCluster testClusters."${baseName}"
doFirst {
testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'third'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}

// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
// at the same time resulting in a fully upgraded cluster.
tasks.register("${baseName}#fullRestartClusterTask", StandaloneRestIntegTestTask) {
dependsOn "${baseName}#oldVersionClusterTask"
useCluster testClusters."${baseName}"
doFirst {
testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins)
}
if (System.getProperty("fullRestartCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}


run {
doFirst {
// There seems to be an issue when running multi node run or integ tasks with unicast_hosts
Expand Down
64 changes: 45 additions & 19 deletions src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,47 @@ protected final Settings restClientSettings() {
.build();
}

@SuppressWarnings("unchecked")
public void testPluginUpgradeInAMixedCluster() throws Exception {
Map<String, Map<String, Object>> responseMap = (Map<String, Map<String, Object>>) getAsMap("_nodes/" + CLUSTER_NAME + "-0/plugins")
.get("nodes");
assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-0/plugins");
}

public void testPluginUpgradeInAnUpgradedCluster() throws Exception {
assertPluginUpgrade("_nodes/plugins");
}

public void testPluginUpgradeInARollingUpgradedCluster() throws Exception {
String round = System.getProperty("tests.rest.bwcsuite_round");
if (round.equals("first") || round.equals("old")) {
assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-0/plugins");
} else if (round.equals("second")) {
assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-1/plugins");
} else if (round.equals("third")) {
assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-2/plugins");
}
}

private enum ClusterType {
OLD,
MIXED,
UPGRADED;

public static ClusterType parse(String value) {
switch (value) {
case "old_cluster":
return OLD;
case "mixed_cluster":
return MIXED;
case "upgraded_cluster":
return UPGRADED;
default:
throw new AssertionError("unknown cluster type: " + value);
}
}
}

@SuppressWarnings("unchecked")
private void assertPluginUpgrade(String uri) throws Exception {
Map<String, Map<String, Object>> responseMap = (Map<String, Map<String, Object>>) getAsMap(uri).get("nodes");
for (Map<String, Object> response : responseMap.values()) {
List<Map<String, Object>> plugins = (List<Map<String, Object>>) response.get("plugins");
Set<Object> pluginNames = plugins.stream().map(map -> map.get("name")).collect(Collectors.toSet());
Expand All @@ -73,27 +110,16 @@ public void testPluginUpgradeInAMixedCluster() throws Exception {
Assert.assertTrue(pluginNames.contains("opensearch-job-scheduler"));
verifyAnomalyDetector(TestHelpers.LEGACY_OPENDISTRO_AD_BASE_DETECTORS_URI);
break;
case UPGRADED:
Assert.assertTrue(pluginNames.contains("opensearch-anomaly-detection"));
Assert.assertTrue(pluginNames.contains("opensearch-job-scheduler"));
verifyAnomalyDetector(TestHelpers.AD_BASE_DETECTORS_URI);
break;
}
break;
}
}

private enum ClusterType {
OLD,
MIXED;

public static ClusterType parse(String value) {
switch (value) {
case "old_cluster":
return OLD;
case "mixed_cluster":
return MIXED;
default:
throw new AssertionError("unknown cluster type: " + value);
}
}
}

private void createBasicAnomalyDetector() throws Exception {
AnomalyDetector detector = TestHelpers.randomAnomalyDetector(TestHelpers.randomUiMetadata(), null);
String indexName = detector.getIndices().get(0);
Expand Down

0 comments on commit 8d53957

Please sign in to comment.