Merge pull request #666 from AndreKurait/RemoveServiceConnect

Remove ECS ServiceConnect
opensearch-project · May 23, 2024 · 433025c · 433025c
2 parents cda3d2f + 73371a0
commit 433025c
Show file tree

Hide file tree

Showing 13 changed files with 40 additions and 75 deletions.
diff --git a/TrafficCapture/dockerSolution/src/main/docker/elasticsearchTestConsole/catIndices.sh b/TrafficCapture/dockerSolution/src/main/docker/elasticsearchTestConsole/catIndices.sh
@@ -1,10 +1,14 @@
 #!/bin/bash
 
-# Default values
-source_endpoint="https://capture-proxy-es:9200"
-source_auth_user_and_pass="admin:admin"
-source_no_auth=false
-target_no_auth=false
+
+# Check for the presence of SOURCE_DOMAIN_ENDPOINT environment variable
+if [ -n "$SOURCE_DOMAIN_ENDPOINT" ]; then
+    source_endpoint="${SOURCE_DOMAIN_ENDPOINT}"
+    source_auth_user_and_pass="admin:admin"
+else
+    source_endpoint="https://capture-proxy-es:9200"
+    source_auth_user_and_pass="admin:admin"
+fi
 
 # Check for the presence of MIGRATION_DOMAIN_ENDPOINT environment variable
 if [ -n "$MIGRATION_DOMAIN_ENDPOINT" ]; then
@@ -15,6 +19,10 @@ else
     target_auth_user_and_pass="admin:myStrongPassword123!"
 fi
 
+# Default values
+source_no_auth=false
+target_no_auth=false
+
 usage() {
   echo ""
   echo "Script to display all indices and doc counts for both the source cluster and target cluster."

diff --git a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts
@@ -35,7 +35,7 @@ export class FetchMigrationStack extends Stack {
         const targetClusterEndpoint = StringParameter.fromStringParameterName(this, "targetClusterEndpoint", `/migration/${props.stage}/${props.defaultDeployId}/osClusterEndpoint`)
         const domainAccessGroupId = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/osAccessSecurityGroupId`)
         // This SG allows outbound access for ECR access as well as communication with other services in the cluster
-        const serviceConnectGroupId = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)
+        const serviceGroupId = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceSecurityGroupId`)
 
         const ecsCluster = Cluster.fromClusterAttributes(this, 'ecsCluster', {
             clusterName: `migration-${props.stage}-ecs-cluster`,
@@ -106,7 +106,7 @@ export class FetchMigrationStack extends Stack {
         let networkConfigJson = {
             "awsvpcConfiguration": {
                 "subnets": props.vpc.privateSubnets.map(_ => _.subnetId),
-                "securityGroups": [domainAccessGroupId, serviceConnectGroupId]
+                "securityGroups": [domainAccessGroupId, serviceGroupId]
             }
         }
         let networkConfigString = JSON.stringify(networkConfigJson)

diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts
@@ -211,17 +211,17 @@ export class MigrationAssistanceStack extends Stack {
             stringValue: replayerOutputEFS.fileSystemId
         });
 
-        const serviceConnectSecurityGroup = new SecurityGroup(this, 'serviceConnectSecurityGroup', {
+        const serviceSecurityGroup = new SecurityGroup(this, 'serviceSecurityGroup', {
             vpc: props.vpc,
             // Required for retrieving ECR image at service startup
             allowAllOutbound: true,
         })
-        serviceConnectSecurityGroup.addIngressRule(serviceConnectSecurityGroup, Port.allTraffic());
+        serviceSecurityGroup.addIngressRule(serviceSecurityGroup, Port.allTraffic());
 
-        new StringParameter(this, 'SSMParameterServiceConnectGroupId', {
-            description: 'OpenSearch migration parameter for Service Connect security group id',
-            parameterName: `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`,
-            stringValue: serviceConnectSecurityGroup.securityGroupId
+        new StringParameter(this, 'SSMParameterServiceGroupId', {
+            description: 'OpenSearch migration parameter for service security group id',
+            parameterName: `/migration/${props.stage}/${props.defaultDeployId}/serviceSecurityGroupId`,
+            stringValue: serviceSecurityGroup.securityGroupId
         });
 
         const artifactBucket = new Bucket(this, 'migrationArtifactsS3', {
@@ -244,7 +244,6 @@ export class MigrationAssistanceStack extends Stack {
         ecsCluster.addDefaultCloudMapNamespace( {
             name: `migration.${props.stage}.local`,
             type: NamespaceType.DNS_PRIVATE,
-            useForServiceConnect: true,
             vpc: props.vpc
         })
         const cloudMapNamespaceId = ecsCluster.defaultCloudMapNamespace!.namespaceId

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts
@@ -1,6 +1,6 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {CpuArchitecture, PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
@@ -28,7 +28,7 @@ export class CaptureProxyESStack extends MigrationServiceCore {
     constructor(scope: Construct, id: string, props: CaptureProxyESProps) {
         super(scope, id, props)
         let securityGroups = [
-            SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)),
+            SecurityGroup.fromSecurityGroupId(this, "serviceSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceSecurityGroupId`)),
             SecurityGroup.fromSecurityGroupId(this, "trafficStreamSourceAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/trafficStreamSourceAccessSecurityGroupId`))
         ]
 
@@ -38,22 +38,12 @@ export class CaptureProxyESStack extends MigrationServiceCore {
             containerPort: 9200,
             protocol: Protocol.TCP
         }
-        const serviceConnectService: ServiceConnectService = {
-            portMappingName: "capture-proxy-es-connect",
-            dnsName: "capture-proxy-es",
-            port: 9200
-        }
         const esServicePort: PortMapping = {
             name: "es-connect",
             hostPort: 19200,
             containerPort: 19200,
             protocol: Protocol.TCP
         }
-        const esServiceConnectService: ServiceConnectService = {
-            portMappingName: "es-connect",
-            dnsName: "capture-proxy-es",
-            port: 19200
-        }
 
         const servicePolicies = props.streamingSourceType === StreamingSourceType.AWS_MSK ? createMSKProducerIAMPolicies(this, this.partition, this.region, this.account, props.stage, props.defaultDeployId) : []
 
@@ -73,7 +63,6 @@ export class CaptureProxyESStack extends MigrationServiceCore {
                 // Set Elasticsearch port to 19200 to allow capture proxy at port 9200
                 "http.port": "19200"
             },
-            serviceConnectServices: [serviceConnectService, esServiceConnectService],
             serviceDiscoveryEnabled: true,
             serviceDiscoveryPort: 19200,
             cpuArchitecture: props.fargateCpuArch,

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts
@@ -1,6 +1,6 @@
 import {StackPropsExt} from "../stack-composer";
-import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {CpuArchitecture, PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs";
+import {IVpc, Port, SecurityGroup} from "aws-cdk-lib/aws-ec2";
+import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
@@ -29,7 +29,7 @@ export class CaptureProxyStack extends MigrationServiceCore {
     constructor(scope: Construct, id: string, props: CaptureProxyProps) {
         super(scope, id, props)
         let securityGroups = [
-            SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)),
+            SecurityGroup.fromSecurityGroupId(this, "serviceSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceSecurityGroupId`)),
             SecurityGroup.fromSecurityGroupId(this, "trafficStreamSourceAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/trafficStreamSourceAccessSecurityGroupId`))
         ]
 
@@ -39,11 +39,6 @@ export class CaptureProxyStack extends MigrationServiceCore {
             containerPort: 9200,
             protocol: Protocol.TCP
         }
-        const serviceConnectService: ServiceConnectService = {
-            portMappingName: "capture-proxy-connect",
-            dnsName: "capture-proxy",
-            port: 9200
-        }
 
         const servicePolicies = props.streamingSourceType === StreamingSourceType.AWS_MSK ? createMSKProducerIAMPolicies(this, this.partition, this.region, this.account, props.stage, props.defaultDeployId) : []
 
@@ -60,7 +55,6 @@ export class CaptureProxyStack extends MigrationServiceCore {
             securityGroups: securityGroups,
             taskRolePolicies: servicePolicies,
             portMappings: [servicePort],
-            serviceConnectServices: [serviceConnectService],
             cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 512,
             taskMemoryLimitMiB: 2048,

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts
@@ -1,6 +1,6 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {CpuArchitecture, PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
@@ -22,7 +22,7 @@ export class ElasticsearchStack extends MigrationServiceCore {
     constructor(scope: Construct, id: string, props: ElasticsearchProps) {
         super(scope, id, props)
         let securityGroups = [
-            SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)),
+            SecurityGroup.fromSecurityGroupId(this, "serviceSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceSecurityGroupId`)),
         ]
 
         const servicePort: PortMapping = {
@@ -31,18 +31,12 @@ export class ElasticsearchStack extends MigrationServiceCore {
             containerPort: 9200,
             protocol: Protocol.TCP
         }
-        const serviceConnectService: ServiceConnectService = {
-            portMappingName: "elasticsearch-connect",
-            dnsName: "elasticsearch",
-            port: 9200
-        }
 
         this.createService({
             serviceName: "elasticsearch",
             dockerDirectoryPath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/src/main/docker/elasticsearchWithSearchGuard"),
             securityGroups: securityGroups,
             portMappings: [servicePort],
-            serviceConnectServices: [serviceConnectService],
             serviceDiscoveryEnabled: true,
             serviceDiscoveryPort: 9200,
             cpuArchitecture: props.fargateCpuArch,

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-stack.ts
@@ -1,6 +1,6 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {CpuArchitecture, PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {MigrationServiceCore} from "./migration-service-core";
 import {StringParameter} from "aws-cdk-lib/aws-ssm";
@@ -19,7 +19,7 @@ export class KafkaStack extends MigrationServiceCore {
     constructor(scope: Construct, id: string, props: KafkaBrokerProps) {
         super(scope, id, props)
         let securityGroups = [
-            SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)),
+            SecurityGroup.fromSecurityGroupId(this, "serviceSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceSecurityGroupId`)),
             SecurityGroup.fromSecurityGroupId(this, "trafficStreamSourceAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/trafficStreamSourceAccessSecurityGroupId`))
         ]
 
@@ -29,11 +29,6 @@ export class KafkaStack extends MigrationServiceCore {
             containerPort: 9092,
             protocol: Protocol.TCP
         }
-        const serviceConnectService: ServiceConnectService = {
-            portMappingName: "kafka-connect",
-            dnsName: "kafka",
-            port: 9092
-        }
 
         new StringParameter(this, 'SSMParameterKafkaBrokers', {
             description: 'OpenSearch Migration Parameter for Kafka brokers',
@@ -63,7 +58,6 @@ export class KafkaStack extends MigrationServiceCore {
                 "KAFKA_LOG_DIRS": '/tmp/kraft-combined-logs'
             },
             portMappings: [servicePort],
-            serviceConnectServices: [serviceConnectService],
             cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 256,
             taskMemoryLimitMiB: 2048,

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts
@@ -1,5 +1,5 @@
 import {StackPropsExt} from "../stack-composer";
-import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
+import {IVpc, SecurityGroup, Port, ISecurityGroup} from "aws-cdk-lib/aws-ec2";
 import {CpuArchitecture, MountPoint, Volume} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
@@ -107,7 +107,7 @@ export class MigrationConsoleStack extends MigrationServiceCore {
     constructor(scope: Construct, id: string, props: MigrationConsoleProps) {
         super(scope, id, props)
         let securityGroups = [
-            SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)),
+            SecurityGroup.fromSecurityGroupId(this, "serviceSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceSecurityGroupId`)),
             SecurityGroup.fromSecurityGroupId(this, "trafficStreamSourceAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/trafficStreamSourceAccessSecurityGroupId`)),
             SecurityGroup.fromSecurityGroupId(this, "defaultDomainAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/osAccessSecurityGroupId`)),
             SecurityGroup.fromSecurityGroupId(this, "replayerOutputAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/replayerOutputAccessSecurityGroupId`))
@@ -182,6 +182,8 @@ export class MigrationConsoleStack extends MigrationServiceCore {
 
         const environment: { [key: string]: string; } = {
             "MIGRATION_DOMAIN_ENDPOINT": osClusterEndpoint,
+            // Temporary fix for source domain endpoint until we move to either alb or migration console yaml configuration
+            "SOURCE_DOMAIN_ENDPOINT": `https://capture-proxy-es.migration.${props.stage}.local:9200`,
             "MIGRATION_KAFKA_BROKER_ENDPOINTS": brokerEndpoints,
             "MIGRATION_STAGE": props.stage,
             "MIGRATION_SOLUTION_VERSION": props.migrationsSolutionVersion

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts
@@ -9,7 +9,6 @@ import {
     LogDrivers,
     MountPoint,
     PortMapping,
-    ServiceConnectService,
     Ulimit,
     OperatingSystemFamily,
     Volume,
@@ -42,7 +41,6 @@ export interface MigrationServiceCoreProps extends StackPropsExt {
     readonly environment?: {
         [key: string]: string;
     },
-    readonly serviceConnectServices?: ServiceConnectService[],
     readonly serviceDiscoveryEnabled?: boolean,
     readonly serviceDiscoveryPort?: number,
     readonly taskCpuUnits?: number,
@@ -221,14 +219,6 @@ export class MigrationServiceCore extends Stack {
             enableExecuteCommand: true,
             securityGroups: props.securityGroups,
             vpcSubnets: props.vpc.selectSubnets({subnetType: SubnetType.PRIVATE_WITH_EGRESS}),
-            serviceConnectConfiguration: {
-                namespace: `migration.${props.stage}.local`,
-                services: props.serviceConnectServices ? props.serviceConnectServices : undefined,
-                logDriver: LogDrivers.awsLogs({
-                    streamPrefix: "service-connect-logs",
-                    logGroup: serviceLogGroup
-                })
-            },
             cloudMapOptions: cloudMapOptions
         });
 

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts
@@ -44,7 +44,7 @@ export class OpenSearchContainerStack extends MigrationServiceCore {
         const deployId = props.addOnMigrationDeployId ? props.addOnMigrationDeployId : props.defaultDeployId
 
         let securityGroups = [
-            SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)),
+            SecurityGroup.fromSecurityGroupId(this, "serviceSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceSecurityGroupId`)),
         ]
 
         let adminUserSecret: ISecret|undefined = props.fineGrainedManagerUserSecretManagerKeyARN ?
@@ -65,11 +65,7 @@ export class OpenSearchContainerStack extends MigrationServiceCore {
             containerPort: 9200,
             protocol: Protocol.TCP
         }
-        const serviceConnectService: ServiceConnectService = {
-            portMappingName: "opensearch-connect",
-            dnsName: dnsNameForContainer,
-            port: 9200
-        }
+
         const ulimits: Ulimit[] = [
             {
                 name: UlimitName.MEMLOCK,
@@ -96,7 +92,6 @@ export class OpenSearchContainerStack extends MigrationServiceCore {
                 "OPENSEARCH_INITIAL_ADMIN_PASSWORD": opensearch_target_initial_admin_password
             },
             portMappings: [servicePort],
-            serviceConnectServices: [serviceConnectService],
             taskCpuUnits: 1024,
             taskMemoryLimitMiB: 4096,
             cpuArchitecture: props.fargateCpuArch,

diff --git a/...oyment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts b/...oyment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts
@@ -25,7 +25,7 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore {
     constructor(scope: Construct, id: string, props: ReindexFromSnapshotProps) {
         super(scope, id, props)
         let securityGroups = [
-            SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)),
+            SecurityGroup.fromSecurityGroupId(this, "serviceSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceSecurityGroupId`)),
             SecurityGroup.fromSecurityGroupId(this, "defaultDomainAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/osAccessSecurityGroupId`)),
         ]