Skip to content

Commit

Permalink
Adding mixed cluster, rolling upgrade, restart upgrade bwc tests (#158)
Browse files Browse the repository at this point in the history 
* Adding mixed cluster bwc tests

Signed-off-by: Vacha <vachshah@amazon.com>

* Adding anomaly detector assertions for bwc

Signed-off-by: Vacha <vachshah@amazon.com>

* Adding rolling upgrade and full restart upgrade bwc tests

Signed-off-by: Vacha <vachshah@amazon.com>
  • Loading branch information
@VachaShah @ohltyler
VachaShah authored and ohltyler committed Sep 1, 2021
1 parent e169347 commit e78f242
Show file tree
Hide file tree
Showing 10 changed files with 487 additions and 139 deletions.
181 changes: 181 additions & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

import java.util.concurrent.Callable
import org.opensearch.gradle.test.RestIntegTestTask
import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask

buildscript {
ext {
Expand Down Expand Up @@ -185,6 +186,12 @@ integTest {
}
}

if (System.getProperty("tests.rest.bwcsuite") == null) {
filter {
excludeTestsMatching "org.opensearch.ad.bwc.*IT"
}
}

// The 'doFirst' delays till execution time.
doFirst {
// Tell the test JVM if the cluster JVM is running under a debugger so that tests can
Expand Down Expand Up @@ -260,6 +267,180 @@ testClusters.integTest {
}
}

String bwcVersion = "1.13.0.0";
String baseName = "adBwcCluster"
String bwcFilePath = "src/test/resources/org/opensearch/ad/bwc/"

testClusters {
"${baseName}" {
testDistribution = "ARCHIVE"
versions = ["7.10.2","1.0.0"]
numberOfNodes = 3
plugin(provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return fileTree(bwcFilePath + "job-scheduler/" + bwcVersion).getSingleFile()
}
}
}
}))
plugin(provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return fileTree(bwcFilePath + "anomaly-detection/" + bwcVersion).getSingleFile()
}
}
}
}))
setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
setting 'http.content_type.required', 'true'
}
}

List<Provider<RegularFile>> plugins = [
provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return fileTree(bwcFilePath + "job-scheduler/" + project.version).getSingleFile()
}
}
}
}),
provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return fileTree(bwcFilePath + "anomaly-detection/" + project.version).getSingleFile()
}
}
}
})
]

// Creates a test cluster with 3 nodes of the old version.
task "${baseName}#oldVersionClusterTask"(type: StandaloneRestIntegTestTask) {
useCluster testClusters."${baseName}"
if (System.getProperty("mixedCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
}
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
if (System.getProperty("fullRestartCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'old_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'old'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}

// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
// This is also used as a one third upgraded cluster for a rolling upgrade.
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#oldVersionClusterTask"
useCluster testClusters."${baseName}"
doFirst {
testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("mixedCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
}
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'first'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}

// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#mixedClusterTask"
useCluster testClusters."${baseName}"
doFirst {
testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'second'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}

// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
// This results in a fully upgraded cluster.
// This is used for rolling upgrade.
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
useCluster testClusters."${baseName}"
doFirst {
testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'third'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}

// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
// at the same time resulting in a fully upgraded cluster.
tasks.register("${baseName}#fullRestartClusterTask", StandaloneRestIntegTestTask) {
dependsOn "${baseName}#oldVersionClusterTask"
useCluster testClusters."${baseName}"
doFirst {
testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(plugins)
}
if (System.getProperty("fullRestartCluster") != null) {
filter {
includeTest("org.opensearch.ad.bwc.ADBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
}


run {
doFirst {
// There seems to be an issue when running multi node run or integ tasks with unicast_hosts
Expand Down
Loading

0 comments on commit e78f242

Please sign in to comment.