Implement CloudFormation backend.

apps.go

@@ -149,10 +149,6 @@ func (s *appsService) Scale(ctx context.Context, db *gorm.DB, opts ScaleOpts) (*
 		return nil, &ValidationError{Err: fmt.Errorf("no %s process type in release", t)}
 	}
 
-	if err := s.Scheduler.Scale(ctx, release.AppID, opts.Process, uint(quantity)); err != nil {
-		return nil, err
-	}
-
 	event := opts.Event()
 	event.PreviousQuantity = p.Quantity
 	event.PreviousConstraints = p.Constraints()

bin/bootstrap

@@ -127,6 +127,7 @@ else
   echo "AWS_REGION=$AWS_DEFAULT_REGION" > .env
   echo "AWS_ACCESS_KEY_ID=$(output AccessKeyId)" >> .env
   echo "AWS_SECRET_ACCESS_KEY=$(output SecretAccessKey)" >> .env
+  echo "EMPIRE_S3_TEMPLATE_BUCKET=$(output TemplateBucket)" >> .env
   echo "EMPIRE_ELB_VPC_ID=$(output VPC)" >> .env
   echo "EMPIRE_ELB_SG_PRIVATE=$(output InternalELBSG)" >> .env
   echo "EMPIRE_ELB_SG_PUBLIC=$(output ExternalELBSG)" >> .env

cmd/empire/factories.go

@@ -2,13 +2,15 @@ package main
 
 import (
 	"fmt"
+	"html/template"
 	"log"
 	"net/url"
 	"os"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/session"
+	cf "github.com/aws/aws-sdk-go/service/cloudformation"
 	"github.com/aws/aws-sdk-go/service/ecr"
 	"github.com/codegangsta/cli"
 	"github.com/inconshreveable/log15"
@@ -19,6 +21,7 @@ import (
 	"github.com/remind101/empire/pkg/ecsutil"
 	"github.com/remind101/empire/pkg/runner"
 	"github.com/remind101/empire/scheduler"
+	"github.com/remind101/empire/scheduler/cloudformation"
 	"github.com/remind101/empire/scheduler/ecs"
 	"github.com/remind101/pkg/reporter"
 	"github.com/remind101/pkg/reporter/hb"
@@ -32,12 +35,7 @@ func newDB(c *cli.Context) (*empire.DB, error) {
 
 // Empire ===============================
 
-func newEmpire(c *cli.Context) (*empire.Empire, error) {
-	db, err := newDB(c)
-	if err != nil {
-		return nil, err
-	}
-
+func newEmpire(db *empire.DB, c *cli.Context) (*empire.Empire, error) {
 	docker, err := newDockerClient(c)
 	if err != nil {
 		return nil, err
@@ -86,7 +84,90 @@ func newEmpire(c *cli.Context) (*empire.Empire, error) {
 // Scheduler ============================
 
 func newScheduler(db *empire.DB, c *cli.Context) (scheduler.Scheduler, error) {
-	return newECSScheduler(db, c)
+	r, err := newDockerRunner(c)
+	if err != nil {
+		return nil, err
+	}
+
+	var s scheduler.Scheduler
+	name := c.String(FlagScheduler)
+	switch name {
+	case "cloudformation":
+		s, err = newCloudFormationScheduler(db, c)
+	case "ecs":
+		s, err = newECSScheduler(db, c)
+	default:
+		panic(fmt.Sprintf("unknown scheduler: %v", name))
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &scheduler.AttachedRunner{
+		Scheduler: s,
+		Runner:    r,
+	}, nil
+}
+
+func newCloudFormationScheduler(db *empire.DB, c *cli.Context) (scheduler.Scheduler, error) {
+	logDriver := c.String(FlagECSLogDriver)
+	logOpts := c.StringSlice(FlagECSLogOpts)
+	logConfiguration := ecsutil.NewLogConfiguration(logDriver, logOpts)
+
+	config := newConfigProvider(c)
+
+	zoneID := c.String(FlagRoute53InternalZoneID)
+	zone, err := cloudformation.HostedZone(config, zoneID)
+	if err != nil {
+		return nil, err
+	}
+
+	t := &cloudformation.EmpireTemplate{
+		Cluster:                 c.String(FlagECSCluster),
+		InternalSecurityGroupID: c.String(FlagELBSGPrivate),
+		ExternalSecurityGroupID: c.String(FlagELBSGPublic),
+		InternalSubnetIDs:       c.StringSlice(FlagEC2SubnetsPrivate),
+		ExternalSubnetIDs:       c.StringSlice(FlagEC2SubnetsPublic),
+		HostedZone:              zone,
+		ServiceRole:             c.String(FlagECSServiceRole),
+		CustomResourcesTopic:    c.String(FlagCustomResourcesTopic),
+		LogConfiguration:        logConfiguration,
+	}
+
+	if err := t.Validate(); err != nil {
+		return nil, fmt.Errorf("error validating CloudFormation template: %v", err)
+	}
+
+	var tags []*cf.Tag
+	if env := c.String(FlagEnvironment); env != "" {
+		tags = append(tags, &cf.Tag{Key: aws.String("environment"), Value: aws.String(env)})
+	}
+
+	s := cloudformation.NewScheduler(db.DB.DB(), config)
+	s.Cluster = c.String(FlagECSCluster)
+	s.Template = t
+	s.StackNameTemplate = prefixedStackName(c.String(FlagEnvironment))
+	s.Bucket = c.String(FlagS3TemplateBucket)
+	s.Tags = tags
+
+	log.Println("Using CloudFormation backend with the following configuration:")
+	log.Println(fmt.Sprintf("  Cluster: %v", s.Cluster))
+	log.Println(fmt.Sprintf("  InternalSecurityGroupID: %v", t.InternalSecurityGroupID))
+	log.Println(fmt.Sprintf("  ExternalSecurityGroupID: %v", t.ExternalSecurityGroupID))
+	log.Println(fmt.Sprintf("  InternalSubnetIDs: %v", t.InternalSubnetIDs))
+	log.Println(fmt.Sprintf("  ExternalSubnetIDs: %v", t.ExternalSubnetIDs))
+	log.Println(fmt.Sprintf("  ZoneID: %v", zoneID))
+	log.Println(fmt.Sprintf("  LogConfiguration: %v", t.LogConfiguration))
+
+	return s, nil
+}
+
+// prefixedStackName returns a text/template that prefixes the stack name with
+// the given prefix, if it's set.
+func prefixedStackName(prefix string) *template.Template {
+	t := `{{ if "` + prefix + `" }}{{"` + prefix + `"}}-{{ end }}{{.Name}}`
+	return template.Must(template.New("stack_name").Parse(t))
 }
 
 func newECSScheduler(db *empire.DB, c *cli.Context) (scheduler.Scheduler, error) {
@@ -111,11 +192,6 @@ func newECSScheduler(db *empire.DB, c *cli.Context) (scheduler.Scheduler, error)
 		return nil, err
 	}
 
-	r, err := newDockerRunner(c)
-	if err != nil {
-		return nil, err
-	}
-
 	log.Println("Using ECS backend with the following configuration:")
 	log.Println(fmt.Sprintf("  Cluster: %v", config.Cluster))
 	log.Println(fmt.Sprintf("  ServiceRole: %v", config.ServiceRole))
@@ -126,10 +202,7 @@ func newECSScheduler(db *empire.DB, c *cli.Context) (scheduler.Scheduler, error)
 	log.Println(fmt.Sprintf("  ZoneID: %v", config.ZoneID))
 	log.Println(fmt.Sprintf("  LogConfiguration: %v", logConfiguration))
 
-	return &scheduler.AttachedRunner{
-		Scheduler: s,
-		Runner:    r,
-	}, nil
+	return s, nil
 }
 
 func newConfigProvider(c *cli.Context) client.ConfigProvider {

cmd/empire/main.go

@@ -35,11 +35,15 @@ const (
 	FlagDockerCert   = "docker.cert"
 	FlagDockerAuth   = "docker.auth"
 
-	FlagAWSDebug       = "aws.debug"
-	FlagECSCluster     = "ecs.cluster"
-	FlagECSServiceRole = "ecs.service.role"
-	FlagECSLogDriver   = "ecs.logdriver"
-	FlagECSLogOpts     = "ecs.logopt"
+	FlagScheduler            = "scheduler"
+	FlagAWSDebug             = "aws.debug"
+	FlagS3TemplateBucket     = "s3.templatebucket"
+	FlagCustomResourcesTopic = "customresources.topic"
+	FlagCustomResourcesQueue = "customresources.queue"
+	FlagECSCluster           = "ecs.cluster"
+	FlagECSServiceRole       = "ecs.service.role"
+	FlagECSLogDriver         = "ecs.logdriver"
+	FlagECSLogOpts           = "ecs.logopt"
 
 	FlagELBSGPrivate = "elb.sg.private"
 	FlagELBSGPublic  = "elb.sg.public"
@@ -182,11 +186,32 @@ var EmpireFlags = []cli.Flag{
 		Usage:  "Path to a docker registry auth file (~/.dockercfg)",
 		EnvVar: "DOCKER_AUTH_PATH",
 	},
+	cli.StringFlag{
+		Name:   FlagScheduler,
+		Value:  "ecs",
+		Usage:  "The scheduler backend to use. Can be `ecs` or `cloudformation`.",
+		EnvVar: "EMPIRE_SCHEDULER",
+	},
 	cli.BoolFlag{
 		Name:   FlagAWSDebug,
 		Usage:  "Enable verbose debug output for AWS integration.",
 		EnvVar: "EMPIRE_AWS_DEBUG",
 	},
+	cli.StringFlag{
+		Name:   FlagS3TemplateBucket,
+		Usage:  "When using the cloudformation backend, this is the bucket where templates will be stored",
+		EnvVar: "EMPIRE_S3_TEMPLATE_BUCKET",
+	},
+	cli.StringFlag{
+		Name:   FlagCustomResourcesTopic,
+		Usage:  "The ARN of the SNS topic used to create custom resources when using the CloudFormation backend.",
+		EnvVar: "EMPIRE_CUSTOM_RESOURCES_TOPIC",
+	},
+	cli.StringFlag{
+		Name:   FlagCustomResourcesQueue,
+		Usage:  "The queue url of the SQS queue to pull CloudFormation Custom Resource requests from.",
+		EnvVar: "EMPIRE_CUSTOM_RESOURCES_QUEUE",
+	},
 	cli.StringFlag{
 		Name:   FlagECSCluster,
 		Value:  "default",

cmd/empire/server.go

@@ -14,6 +14,7 @@ import (
 	"github.com/remind101/empire/server"
 	"github.com/remind101/empire/server/auth"
 	githubauth "github.com/remind101/empire/server/auth/github"
+	"github.com/remind101/empire/server/cloudformation"
 	"github.com/remind101/empire/server/github"
 	"golang.org/x/oauth2"
 )
@@ -25,11 +26,25 @@ func runServer(c *cli.Context) {
 		runMigrate(c)
 	}
 
-	e, err := newEmpire(c)
+	db, err := newDB(c)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	e, err := newEmpire(db, c)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if c.String(FlagCustomResourcesQueue) != "" {
+		p, err := newCloudFormationCustomResourceProvisioner(db, c)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Printf("Starting CloudFormation custom resource provisioner")
+		go p.Start()
+	}
+
 	s := newServer(c, e)
 	log.Printf("Starting on port %s", port)
 	log.Fatal(http.ListenAndServe(":"+port, s))
@@ -46,6 +61,19 @@ func newServer(c *cli.Context, e *empire.Empire) http.Handler {
 	return server.New(e, opts)
 }
 
+func newCloudFormationCustomResourceProvisioner(db *empire.DB, c *cli.Context) (*cloudformation.CustomResourceProvisioner, error) {
+	r, err := newReporter(c)
+	if err != nil {
+		return nil, err
+	}
+
+	p := cloudformation.NewCustomResourceProvisioner(db.DB.DB(), newConfigProvider(c))
+	p.Logger = newLogger()
+	p.Reporter = r
+	p.QueueURL = c.String(FlagCustomResourcesQueue)
+	return p, nil
+}
+
 func newImageBuilder(c *cli.Context) github.ImageBuilder {
 	builder := c.String(FlagGithubDeploymentsImageBuilder)
 

docker-compose.yml

@@ -13,6 +13,7 @@ server:
   environment:
     EMPIRE_DATABASE_URL: postgres://postgres:postgres@postgres/postgres?sslmode=disable
     DOCKER_HOST: unix:///var/run/docker.sock
+    EMPIRE_SCHEDULER: cloudformation
 postgres:
   image: postgres
   ports:

docs/cloudformation.json

@@ -289,6 +289,13 @@
       }
     },
 
+    "TemplateBucket": {
+      "Type": "AWS::S3::Bucket",
+      "Properties": {
+        "AccessControl": "Private"
+      }
+    },
+
     "InstancePolicies": {
       "Type": "AWS::IAM::Policy",
       "Properties": {
@@ -301,6 +308,22 @@
             {
               "Effect": "Allow",
               "Action": [
+                "sqs:*"
+              ],
+              "Resource": { "Fn::GetAtt": ["CustomResourcesQueue", "Arn"] }
+            },
+            {
+              "Effect": "Allow",
+              "Action": [
+                "sns:Publish"
+              ],
+              "Resource": { "Ref": "CustomResourcesTopic" }
+            },
+            {
+              "Effect": "Allow",
+              "Action": [
+                "cloudformation:*",
+                "s3:*",
                 "ec2:Describe*",
                 "elasticloadbalancing:*",
                 "ecs:*",
@@ -490,6 +513,10 @@
                 "Name": "EMPIRE_DATABASE_URL",
                 "Value": "postgres://postgres:postgres@postgres/postgres?sslmode=disable"
               },
+              {
+                "Name": "EMPIRE_S3_TEMPLATE_BUCKET",
+                "Value": { "Ref": "TemplateBucket" }
+              },
               {
                 "Name": "EMPIRE_ECS_CLUSTER",
                 "Value": { "Ref": "Cluster"  }
@@ -616,6 +643,17 @@
       "Properties": {
         "RetentionInDays": 7
       }
+    },
+
+    "CustomResourcesTopic": {
+      "Type": "AWS::SNS::Topic",
+      "Properties": {
+        "DisplayName": "Empire Custom Resources"
+      }
+    },
+
+    "CustomResourcesQueue": {
+      "Type": "AWS::SQS::Queue"
     }
   },
 
@@ -665,6 +703,18 @@
     "InternalZoneID": {
       "Description": "The zone ID for the internal hosted zone.",
       "Value": { "Ref": "InternalDomain" }
+    },
+    "TemplateBucket": {
+      "Description": "The s3 bucket where stack templates will be stored",
+      "Value": { "Ref": "TemplateBucket" }
+    },
+    "CustomResourcesTopic": {
+      "Description": "The ARN of the SNS topic to use as the ServiceToken for custom CloudFormation resources.",
+      "Value": { "Ref": "CustomResourcesTopic" }
+    },
+    "CustomResourcesQueue": {
+      "Description": "The queue that Empire will listen on to provision custom CloudFormation resources.",
+      "Value": { "Ref": "CustomResourcesQueue" }
     }
   }
 }

migrations.go

@@ -515,4 +515,21 @@ ALTER TABLE apps ADD COLUMN exposure TEXT NOT NULL default 'private'`,
 			`CREATE UNIQUE INDEX index_processes_on_release_id_and_type ON processes USING btree (release_id, "type")`,
 		}),
 	},
+
+	// This migration changes that way we store process configuration for
+	// releases and slugs, to instead store a Formation in JSON format.
+	{
+		ID: 16,
+		Up: migrate.Queries([]string{
+			`CREATE TABLE stacks (
+  app_id text NOT NULL,
+  stack_name text NOT NULL
+)`,
+			`CREATE UNIQUE INDEX index_stacks_on_app_id ON stacks USING btree (app_id)`,
+			`CREATE UNIQUE INDEX index_stacks_on_stack_name ON stacks USING btree (stack_name)`,
+		}),
+		Down: migrate.Queries([]string{
+			`DROP TABLE stacks`,
+		}),
+	},
 }