CDK fixes for thousandeyes (#868)

* Only deploy MSK if capture proxy or replayer enabled Signed-off-by: Mikayla Thompson <thomika@amazon.com> * Guarantee that OS_USER_AND_SECRET is created before mig console stack Signed-off-by: Mikayla Thompson <thomika@amazon.com> * Skip msk utility stack Signed-off-by: Mikayla Thompson <thomika@amazon.com> * Turn off traffic replayer by default Signed-off-by: Mikayla Thompson <thomika@amazon.com> * pull broker endpoints conditionally Signed-off-by: Mikayla Thompson <thomika@amazon.com> * Address review comments Signed-off-by: Mikayla Thompson <thomika@amazon.com> * Explicitly add enabled replayer back to contexts Signed-off-by: Mikayla Thompson <thomika@amazon.com> --------- Signed-off-by: Mikayla Thompson <thomika@amazon.com>
opensearch-project · Aug 6, 2024 · d8b4b8f · d8b4b8f
1 parent b0d24fa
commit d8b4b8f
Show file tree

Hide file tree

Showing 10 changed files with 38 additions and 7 deletions.
diff --git a/deployment/cdk/opensearch-service-migration/README.md b/deployment/cdk/opensearch-service-migration/README.md
@@ -315,6 +315,7 @@ To give an example of this process, a user could decide to configure an addition
     "openAccessPolicyEnabled": true,
     "domainRemovalPolicy": "DESTROY",
     "enableDemoAdmin": true,
+    "trafficReplayerServiceEnabled": true,
     "trafficReplayerEnableClusterFGACAuth": true
   }
 ```

diff --git a/deployment/cdk/opensearch-service-migration/cdk.context.json b/deployment/cdk/opensearch-service-migration/cdk.context.json
@@ -8,6 +8,7 @@
     "domainRemovalPolicy": "DESTROY",
     "artifactBucketRemovalPolicy": "DESTROY",
     "enableDemoAdmin": true,
+    "trafficReplayerServiceEnabled": true,
     "trafficReplayerEnableClusterFGACAuth": true,
     "captureProxyESServiceEnabled": true,
     "reindexFromSnapshotEnabled": true,

diff --git a/deployment/cdk/opensearch-service-migration/default-values.json b/deployment/cdk/opensearch-service-migration/default-values.json
@@ -12,7 +12,7 @@
   "migrationAssistanceEnabled": true,
   "replayerOutputEFSRemovalPolicy": "DESTROY",
   "migrationConsoleServiceEnabled": true,
-  "trafficReplayerServiceEnabled": true,
+  "trafficReplayerServiceEnabled": false,
   "otelCollectorEnabled": true,
   "dpPipelineTemplatePath": "./dp_pipeline_template.yaml"
 }
diff --git a/deployment/cdk/opensearch-service-migration/lib/network-stack.ts b/deployment/cdk/opensearch-service-migration/lib/network-stack.ts
@@ -24,6 +24,8 @@ export interface NetworkStackProps extends StackPropsExt {
     readonly migrationAPIEnabled?: boolean;
     readonly sourceClusterEndpoint?: string;
     readonly targetClusterEndpoint?: string;
+    readonly targetClusterUsername?: string;
+    readonly targetClusterPasswordSecretArn?: string;
     readonly albAcmCertArn?: string;
     readonly env?: { [key: string]: any };
 }
@@ -257,6 +259,17 @@ export class NetworkStack extends Stack {
                     defaultDeployId: deployId,
                     parameter: MigrationSSMParameter.OS_CLUSTER_ENDPOINT
                 });
+                // This is a somewhat surprsing place for this non-network related set of parameters, but it pairs well with
+                // the OS_CLUSTER_ENDPOINT parameter and is helpful to ensure it happens. This probably isn't a long-term place
+                // for it, but is helpful for the time being.
+                if (props.targetClusterUsername && props.targetClusterPasswordSecretArn) {
+                    createMigrationStringParameter(this,
+                        `${props.targetClusterUsername} ${props.targetClusterPasswordSecretArn}`, {
+                        parameter: MigrationSSMParameter.OS_USER_AND_SECRET_ARN,
+                        defaultDeployId: deployId,
+                        stage: props.stage,
+                    });
+                }
             }
         }
     }

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts
@@ -153,10 +153,11 @@ export class MigrationConsoleStack extends MigrationServiceCore {
             ...props,
             parameter: MigrationSSMParameter.SOURCE_CLUSTER_ENDPOINT,
         });
-        const brokerEndpoints = getMigrationStringParameterValue(this, {
-            ...props,
-            parameter: MigrationSSMParameter.KAFKA_BROKERS,
-        });
+        const brokerEndpoints = props.streamingSourceType != StreamingSourceType.DISABLED ?
+            getMigrationStringParameterValue(this, {
+                ...props,
+                parameter: MigrationSSMParameter.KAFKA_BROKERS,
+            }) : "";
 
         const volumeName = "sharedReplayerOutputVolume"
         const volumeId = getMigrationStringParameterValue(this, {

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts
@@ -101,6 +101,8 @@ export class OpenSearchContainerStack extends MigrationServiceCore {
             ...props
         });
 
-        this.createSSMParameters(props.stage, deployId, adminUserName, adminUserSecret)
+        if (props.enableDemoAdmin) {
+            this.createSSMParameters(props.stage, deployId, adminUserName, adminUserSecret)
+        }
     }
 }
diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
@@ -234,7 +234,14 @@ export class StackComposer {
         }
 
         const fargateCpuArch = validateFargateCpuArch(defaultFargateCpuArch)
-        const streamingSourceType = determineStreamingSourceType(kafkaBrokerServiceEnabled)
+
+        let streamingSourceType
+        if (captureProxyServiceEnabled || captureProxyESServiceEnabled || trafficReplayerServiceEnabled || kafkaBrokerServiceEnabled) {
+            streamingSourceType = determineStreamingSourceType(kafkaBrokerServiceEnabled)
+        } else {
+            console.log("MSK is not enabled and will not be deployed.")
+            streamingSourceType = StreamingSourceType.DISABLED
+        }
 
         const engineVersion = this.getContextForType('engineVersion', 'string', defaultValues, contextJSON)
         version = this.getEngineVersion(engineVersion)
@@ -277,6 +284,8 @@ export class StackComposer {
                 targetClusterProxyServiceEnabled,
                 migrationAPIEnabled,
                 sourceClusterEndpoint: sourceClusterEndpoint,
+                targetClusterUsername: fineGrainedManagerUserName,
+                targetClusterPasswordSecretArn: fineGrainedManagerUserSecretManagerKeyARN,
                 env: props.env
             })
             this.stacks.push(networkStack)

diff --git a/deployment/cdk/opensearch-service-migration/test/msk-utility-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/msk-utility-stack.test.ts
@@ -21,6 +21,7 @@ describe('MSKUtilityStack Tests', () => {
     const contextOptions = {
       vpcEnabled: true,
       migrationAssistanceEnabled: true,
+      trafficReplayerServiceEnabled: true,
       mskEnablePublicEndpoints: true,
       mskRestrictPublicAccessTo: "10.0.0.0/32",
       mskRestrictPublicAccessType: "ipv4",
@@ -49,6 +50,7 @@ describe('MSKUtilityStack Tests', () => {
     const contextOptions = {
       vpcEnabled: true,
       migrationAssistanceEnabled: true,
+      trafficReplayerServiceEnabled: true,
       sourceClusterEndpoint: "https://test-cluster",
     }
 

diff --git a/jenkins/migrationIntegPipelines/ec2SourceE2EPipeline.groovy b/jenkins/migrationIntegPipelines/ec2SourceE2EPipeline.groovy
@@ -45,6 +45,7 @@ def migration_cdk_context = """
         "openAccessPolicyEnabled": true,
         "domainRemovalPolicy": "DESTROY",
         "artifactBucketRemovalPolicy": "DESTROY",
+        "trafficReplayerServiceEnabled": true,
         "trafficReplayerExtraArgs": "--speedup-factor 10.0",
         "fetchMigrationEnabled": true,
         "reindexFromSnapshotServiceEnabled": true,

diff --git a/test/defaultMigrationContext.json b/test/defaultMigrationContext.json
@@ -8,6 +8,7 @@
     "openAccessPolicyEnabled": true,
     "domainRemovalPolicy": "DESTROY",
     "artifactBucketRemovalPolicy": "DESTROY",
+    "trafficReplayerServiceEnabled": true,
     "trafficReplayerExtraArgs": "--speedup-factor 10.0",
     "fetchMigrationEnabled": true,
     "reindexFromSnapshotServiceEnabled": true,