Add custom cloudformation resource for instance ports.

cmd/empire/factories.go

@@ -33,12 +33,7 @@ func newDB(c *cli.Context) (*empire.DB, error) {
 
 // Empire ===============================
 
-func newEmpire(c *cli.Context) (*empire.Empire, error) {
-	db, err := newDB(c)
-	if err != nil {
-		return nil, err
-	}
-
+func newEmpire(db *empire.DB, c *cli.Context) (*empire.Empire, error) {
 	docker, err := newDockerClient(c)
 	if err != nil {
 		return nil, err
@@ -134,6 +129,7 @@ func newCloudFormationScheduler(db *empire.DB, c *cli.Context) (scheduler.Schedu
 		ExternalSubnetIDs:       c.StringSlice(FlagEC2SubnetsPublic),
 		HostedZone:              zone,
 		ServiceRole:             c.String(FlagECSServiceRole),
+		CustomResourcesTopic:    c.String(FlagCustomResourcesTopic),
 		LogConfiguration:        logConfiguration,
 	}
 

cmd/empire/main.go

@@ -35,13 +35,15 @@ const (
 	FlagDockerCert   = "docker.cert"
 	FlagDockerAuth   = "docker.auth"
 
-	FlagScheduler        = "scheduler"
-	FlagAWSDebug         = "aws.debug"
-	FlagS3TemplateBucket = "s3.templatebucket"
-	FlagECSCluster       = "ecs.cluster"
-	FlagECSServiceRole   = "ecs.service.role"
-	FlagECSLogDriver     = "ecs.logdriver"
-	FlagECSLogOpts       = "ecs.logopt"
+	FlagScheduler            = "scheduler"
+	FlagAWSDebug             = "aws.debug"
+	FlagS3TemplateBucket     = "s3.templatebucket"
+	FlagCustomResourcesTopic = "customresources.topic"
+	FlagCustomResourcesQueue = "customresources.queue"
+	FlagECSCluster           = "ecs.cluster"
+	FlagECSServiceRole       = "ecs.service.role"
+	FlagECSLogDriver         = "ecs.logdriver"
+	FlagECSLogOpts           = "ecs.logopt"
 
 	FlagELBSGPrivate = "elb.sg.private"
 	FlagELBSGPublic  = "elb.sg.public"
@@ -200,6 +202,16 @@ var EmpireFlags = []cli.Flag{
 		Usage:  "When using the cloudformation backend, this is the bucket where templates will be stored",
 		EnvVar: "EMPIRE_S3_TEMPLATE_BUCKET",
 	},
+	cli.StringFlag{
+		Name:   FlagCustomResourcesTopic,
+		Usage:  "The ARN of the SNS topic used to create custom resources when using the CloudFormation backend.",
+		EnvVar: "EMPIRE_CUSTOM_RESOURCES_TOPIC",
+	},
+	cli.StringFlag{
+		Name:   FlagCustomResourcesQueue,
+		Usage:  "The queue url of the SQS queue to pull CloudFormation Custom Resource requests from.",
+		EnvVar: "EMPIRE_CUSTOM_RESOURCES_QUEUE",
+	},
 	cli.StringFlag{
 		Name:   FlagECSCluster,
 		Value:  "default",

cmd/empire/server.go

@@ -14,6 +14,7 @@ import (
 	"github.com/remind101/empire/server"
 	"github.com/remind101/empire/server/auth"
 	githubauth "github.com/remind101/empire/server/auth/github"
+	"github.com/remind101/empire/server/cloudformation"
 	"github.com/remind101/empire/server/github"
 	"golang.org/x/oauth2"
 )
@@ -25,11 +26,20 @@ func runServer(c *cli.Context) {
 		runMigrate(c)
 	}
 
-	e, err := newEmpire(c)
+	db, err := newDB(c)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	e, err := newEmpire(db, c)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	p := cloudformation.NewCustomResourceProvisioner(db.DB.DB(), newConfigProvider(c))
+	p.QueueURL = c.String(FlagCustomResourcesQueue)
+	go p.Start()
+
 	s := newServer(c, e)
 	log.Printf("Starting on port %s", port)
 	log.Fatal(http.ListenAndServe(":"+port, s))

docs/cloudformation.json

@@ -305,6 +305,20 @@
         "PolicyDocument": {
           "Version": "2012-10-17",
           "Statement": [
+            {
+              "Effect": "Allow",
+              "Action": [
+                "sqs:*"
+              ],
+              "Resource": { "Fn::GetAtt": ["CustomResourcesQueue", "Arn"] }
+            },
+            {
+              "Effect": "Allow",
+              "Action": [
+                "sns:Publish"
+              ],
+              "Resource": { "Ref": "CustomResourcesTopic" }
+            },
             {
               "Effect": "Allow",
               "Action": [
@@ -629,6 +643,17 @@
       "Properties": {
         "RetentionInDays": 7
       }
+    },
+
+    "CustomResourcesTopic": {
+      "Type": "AWS::SNS::Topic",
+      "Properties": {
+        "DisplayName": "Empire Custom Resources"
+      }
+    },
+
+    "CustomResourcesQueue": {
+      "Type": "AWS::SQS::Queue"
     }
   },
 
@@ -682,6 +707,14 @@
     "TemplateBucket": {
       "Description": "The s3 bucket where stack templates will be stored",
       "Value": { "Ref": "TemplateBucket" }
+    },
+    "CustomResourcesTopic": {
+      "Description": "The ARN of the SNS topic to use as the ServiceToken for custom CloudFormation resources.",
+      "Value": { "Ref": "CustomResourcesTopic" }
+    },
+    "CustomResourcesQueue": {
+      "Description": "The queue that Empire will listen on to provision custom CloudFormation resources.",
+      "Value": { "Ref": "CustomResourcesQueue" }
     }
   }
 }

scheduler/cloudformation/template.go

@@ -54,6 +54,9 @@ type EmpireTemplate struct {
 	// The name of the ECS Service IAM role.
 	ServiceRole string
 
+	// The ARN of the SNS topic to provision instance ports.
+	CustomResourcesTopic string
+
 	LogConfiguration *ecs.LogConfiguration
 }
 
@@ -87,7 +90,7 @@ func (t *EmpireTemplate) Build(app *scheduler.App) (interface{}, error) {
 		r := regexp.MustCompile("[^a-zA-Z0-9]")
 		key := r.ReplaceAllString(p.Type, "")
 
-		portMappings := []map[string]int64{}
+		portMappings := []map[string]interface{}{}
 
 		loadBalancers := []map[string]interface{}{}
 		if p.Exposure != nil {
@@ -101,12 +104,25 @@ func (t *EmpireTemplate) Build(app *scheduler.App) (interface{}, error) {
 				subnets = t.ExternalSubnetIDs
 			}
 
-			instancePort := int64(9000) // TODO: Allocate a port
+			instancePort := fmt.Sprintf("%s%dInstancePort", key, ContainerPort)
+			resources[instancePort] = map[string]interface{}{
+				"Type":    "Custom::InstancePort",
+				"Version": "1.0",
+				"Properties": map[string]interface{}{
+					"ServiceToken": t.CustomResourcesTopic,
+				},
+			}
+
 			listeners := []map[string]interface{}{
 				map[string]interface{}{
 					"LoadBalancerPort": 80,
 					"Protocol":         "http",
-					"InstancePort":     instancePort,
+					"InstancePort": map[string][]string{
+						"Fn::GetAtt": []string{
+							instancePort,
+							"InstancePort",
+						},
+					},
 					"InstanceProtocol": "http",
 				},
 			}
@@ -115,15 +131,25 @@ func (t *EmpireTemplate) Build(app *scheduler.App) (interface{}, error) {
 				listeners = append(listeners, map[string]interface{}{
 					"LoadBalancerPort": 80,
 					"Protocol":         "http",
-					"InstancePort":     instancePort,
+					"InstancePort": map[string][]string{
+						"Fn::GetAtt": []string{
+							instancePort,
+							"InstancePort",
+						},
+					},
 					"SSLCertificateId": e.Cert,
 					"InstanceProtocol": "http",
 				})
 			}
 
-			portMappings = append(portMappings, map[string]int64{
+			portMappings = append(portMappings, map[string]interface{}{
 				"ContainerPort": ContainerPort,
-				"HostPort":      instancePort,
+				"HostPort": map[string][]string{
+					"Fn::GetAtt": []string{
+						instancePort,
+						"InstancePort",
+					},
+				},
 			})
 			cd.Environment = append(cd.Environment, &ecs.KeyValuePair{
 				Name:  aws.String("PORT"),

scheduler/cloudformation/template_test.go

@@ -83,6 +83,7 @@ func TestEmpireTemplate(t *testing.T) {
 			ExternalSecurityGroupID: "sg-1938737f",
 			InternalSubnetIDs:       []string{"subnet-bb01c4cd", "subnet-c85f4091"},
 			ExternalSubnetIDs:       []string{"subnet-ca96f4cd", "subnet-a13b909c"},
+			CustomResourcesTopic:    "sns topic arn",
 			HostedZone: &route53.HostedZone{
 				Id:   aws.String("Z3DG6IL3SJCGPX"),
 				Name: aws.String("empire"),

scheduler/cloudformation/templates/basic.json

@@ -39,6 +39,13 @@
       },
       "Type": "AWS::ECS::Service"
     },
+    "web8080InstancePort": {
+      "Properties": {
+        "ServiceToken": "sns topic arn"
+      },
+      "Type": "Custom::InstancePort",
+      "Version": "1.0"
+    },
     "webLoadBalancer": {
       "Properties": {
         "ConnectionDrainingPolicy": {
@@ -48,7 +55,12 @@
         "CrossZone": true,
         "Listeners": [
           {
-            "InstancePort": 9000,
+            "InstancePort": {
+              "Fn::GetAtt": [
+                "web8080InstancePort",
+                "InstancePort"
+              ]
+            },
             "InstanceProtocol": "http",
             "LoadBalancerPort": 80,
             "Protocol": "http"
@@ -95,7 +107,12 @@
             "PortMappings": [
               {
                 "ContainerPort": 8080,
-                "HostPort": 9000
+                "HostPort": {
+                  "Fn::GetAtt": [
+                    "web8080InstancePort",
+                    "InstancePort"
+                  ]
+                }
               }
             ],
             "Ulimits": [

scheduler/cloudformation/templates/https.json

@@ -39,6 +39,13 @@
       },
       "Type": "AWS::ECS::Service"
     },
+    "web8080InstancePort": {
+      "Properties": {
+        "ServiceToken": "sns topic arn"
+      },
+      "Type": "Custom::InstancePort",
+      "Version": "1.0"
+    },
     "webLoadBalancer": {
       "Properties": {
         "ConnectionDrainingPolicy": {
@@ -48,13 +55,23 @@
         "CrossZone": true,
         "Listeners": [
           {
-            "InstancePort": 9000,
+            "InstancePort": {
+              "Fn::GetAtt": [
+                "web8080InstancePort",
+                "InstancePort"
+              ]
+            },
             "InstanceProtocol": "http",
             "LoadBalancerPort": 80,
             "Protocol": "http"
           },
           {
-            "InstancePort": 9000,
+            "InstancePort": {
+              "Fn::GetAtt": [
+                "web8080InstancePort",
+                "InstancePort"
+              ]
+            },
             "InstanceProtocol": "http",
             "LoadBalancerPort": 80,
             "Protocol": "http",
@@ -100,7 +117,12 @@
             "PortMappings": [
               {
                 "ContainerPort": 8080,
-                "HostPort": 9000
+                "HostPort": {
+                  "Fn::GetAtt": [
+                    "web8080InstancePort",
+                    "InstancePort"
+                  ]
+                }
               }
             ],
             "Ulimits": []

server/cloudformation/README.md

@@ -0,0 +1,3 @@
+This provides an interface for CloudFormation Custom Resources so that Empire can provision things for CloudFormation. It supports the following resources:
+
+* `Custom::InstancePort`: Allocates an instance port from the pool of instance ports.