From 4d13975063cedf96e7ef1b42e07f3555faa4b14b Mon Sep 17 00:00:00 2001 From: Billy Batista Date: Wed, 12 Jun 2024 19:04:21 -0400 Subject: [PATCH] Reapply "Even more remote heartbeat improvements (#3594)" (#3619) * Restart remote builder machine if it's stopped * Fix tests * Reapply "Even more remote heartbeat improvements (#3594)" This reverts commit 4fb6bd46640e97b32ca21c22551e627cefc3a232. Ben T made some changes that should let this work again. We'll need to add a preflight test to ensure that this doesn't break in the future, however. * Add the recreate builder flag This allows users forcing recreating the remote builder, even if it's valid * Add a test for deploying and recreating the remote builder * Catch and trace errors --- .github/workflows/checks.yml | 1 - internal/build/imgsrc/docker.go | 39 +- internal/build/imgsrc/ensure_builder.go | 389 +++++++++++++++++++ internal/build/imgsrc/ensure_builder_test.go | 255 ++++++++++++ internal/build/imgsrc/nixpacks_builder.go | 11 +- internal/build/imgsrc/resolver.go | 4 +- internal/command/command_run.go | 2 +- internal/command/deploy/deploy.go | 6 +- internal/command/deploy/deploy_build.go | 4 +- internal/command/launch/plan_builder.go | 4 +- internal/flag/flag.go | 12 + internal/flyutil/client.go | 3 + internal/mock/client.go | 184 +++++---- test/preflight/fly_deploy_test.go | 9 + 14 files changed, 799 insertions(+), 124 deletions(-) create mode 100644 internal/build/imgsrc/ensure_builder.go create mode 100644 internal/build/imgsrc/ensure_builder_test.go diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b3c026e689..004e4ff83a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -42,4 +42,3 @@ jobs: with: version: v1.54 working-directory: . - skip-pkg-cache: true diff --git a/internal/build/imgsrc/docker.go b/internal/build/imgsrc/docker.go index 129fc9636e..e70eeebda6 100644 --- a/internal/build/imgsrc/docker.go +++ b/internal/build/imgsrc/docker.go @@ -53,14 +53,14 @@ type dockerClientFactory struct { appName string } -func newDockerClientFactory(daemonType DockerDaemonType, apiClient flyutil.Client, appName string, streams *iostreams.IOStreams, connectOverWireguard bool) *dockerClientFactory { +func newDockerClientFactory(daemonType DockerDaemonType, apiClient flyutil.Client, appName string, streams *iostreams.IOStreams, connectOverWireguard, recreateBuilder bool) *dockerClientFactory { remoteFactory := func() *dockerClientFactory { terminal.Debug("trying remote docker daemon") return &dockerClientFactory{ mode: daemonType, remote: true, buildFn: func(ctx context.Context, build *build) (*dockerclient.Client, error) { - return newRemoteDockerClient(ctx, apiClient, appName, streams, build, cachedDocker, connectOverWireguard) + return newRemoteDockerClient(ctx, apiClient, appName, streams, build, cachedDocker, connectOverWireguard, recreateBuilder) }, apiClient: apiClient, appName: appName, @@ -203,7 +203,7 @@ func logClearLinesAbove(streams *iostreams.IOStreams, count int) { } } -func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appName string, streams *iostreams.IOStreams, build *build, cachedClient *dockerclient.Client, connectOverWireguard bool) (c *dockerclient.Client, err error) { +func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appName string, streams *iostreams.IOStreams, build *build, cachedClient *dockerclient.Client, connectOverWireguard, recreateBuilder bool) (c *dockerclient.Client, err error) { ctx, span := tracing.GetTracer().Start(ctx, "build_remote_docker_client", trace.WithAttributes( attribute.Bool("connect_over_wireguard", connectOverWireguard), )) @@ -223,8 +223,8 @@ func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appNam var host string var app *fly.App - var machine *fly.GqlMachine - machine, app, err = remoteBuilderMachine(ctx, apiClient, appName) + var machine *fly.Machine + machine, app, err = remoteBuilderMachine(ctx, apiClient, appName, recreateBuilder) if err != nil { tracing.RecordError(span, err, "failed to init remote builder machine") return nil, err @@ -271,7 +271,7 @@ func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appNam } fmt.Fprintln(streams.Out, streams.ColorScheme().Yellow("🔧 creating fresh remote builder, (this might take a while ...)")) - machine, app, err = remoteBuilderMachine(ctx, apiClient, appName) + machine, app, err = remoteBuilderMachine(ctx, apiClient, appName, false) if err != nil { tracing.RecordError(span, err, "failed to init remote builder machine") return nil, err @@ -321,13 +321,7 @@ func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appNam ) } - for _, ip := range machine.IPs.Nodes { - terminal.Debugf("checking ip %+v\n", ip) - if ip.Kind == "privatenet" { - host = "tcp://[" + ip.IP + "]:2375" - break - } - } + host = "tcp://[" + machine.PrivateIP + "]:2375" if !connectOverWireguard { oldHost := host @@ -700,20 +694,14 @@ func ResolveDockerfile(cwd string) string { return "" } -func EagerlyEnsureRemoteBuilder(ctx context.Context, apiClient flyutil.Client, orgSlug string) { +func EagerlyEnsureRemoteBuilder(ctx context.Context, apiClient flyutil.Client, org *fly.Organization, recreateBuilder bool) { // skip if local docker is available if _, err := NewLocalDockerClient(); err == nil { return } - org, err := apiClient.GetOrganizationBySlug(ctx, orgSlug) - if err != nil { - terminal.Debugf("error resolving organization for slug %s: %s", orgSlug, err) - return - } - region := os.Getenv("FLY_REMOTE_BUILDER_REGION") - _, app, err := apiClient.EnsureRemoteBuilder(ctx, org.ID, "", region) + _, app, err := EnsureBuilder(ctx, org, region, recreateBuilder) if err != nil { terminal.Debugf("error ensuring remote builder for organization: %s", err) return @@ -722,13 +710,18 @@ func EagerlyEnsureRemoteBuilder(ctx context.Context, apiClient flyutil.Client, o terminal.Debugf("remote builder %s is being prepared", app.Name) } -func remoteBuilderMachine(ctx context.Context, apiClient flyutil.Client, appName string) (*fly.GqlMachine, *fly.App, error) { +func remoteBuilderMachine(ctx context.Context, apiClient flyutil.Client, appName string, recreateBuilder bool) (*fly.Machine, *fly.App, error) { if v := os.Getenv("FLY_REMOTE_BUILDER_HOST"); v != "" { return nil, nil, nil } region := os.Getenv("FLY_REMOTE_BUILDER_REGION") - return apiClient.EnsureRemoteBuilder(ctx, "", appName, region) + org, err := apiClient.GetOrganizationByApp(ctx, appName) + if err != nil { + return nil, nil, err + } + builderMachine, builderApp, err := EnsureBuilder(ctx, org, region, recreateBuilder) + return builderMachine, builderApp, err } func (d *dockerClientFactory) IsRemote() bool { diff --git a/internal/build/imgsrc/ensure_builder.go b/internal/build/imgsrc/ensure_builder.go new file mode 100644 index 0000000000..dce60d3722 --- /dev/null +++ b/internal/build/imgsrc/ensure_builder.go @@ -0,0 +1,389 @@ +package imgsrc + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/samber/lo" + "github.com/superfly/fly-go" + "github.com/superfly/fly-go/flaps" + "github.com/superfly/flyctl/internal/flapsutil" + "github.com/superfly/flyctl/internal/flyutil" + "github.com/superfly/flyctl/internal/haikunator" + "github.com/superfly/flyctl/internal/tracing" +) + +func EnsureBuilder(ctx context.Context, org *fly.Organization, region string, recreateBuilder bool) (*fly.Machine, *fly.App, error) { + ctx, span := tracing.GetTracer().Start(ctx, "ensure_builder") + defer span.End() + + if !recreateBuilder { + builderApp := org.RemoteBuilderApp + if builderApp != nil { + flaps, err := flapsutil.NewClientWithOptions(ctx, flaps.NewClientOpts{ + AppName: builderApp.Name, + // TOOD(billy) make a utility function for App -> AppCompact + AppCompact: &fly.AppCompact{ + ID: builderApp.ID, + Name: builderApp.Name, + Status: builderApp.Status, + Deployed: builderApp.Deployed, + Hostname: builderApp.Hostname, + AppURL: builderApp.AppURL, + Organization: &fly.OrganizationBasic{ + ID: builderApp.Organization.ID, + Name: builderApp.Organization.Name, + Slug: builderApp.Organization.Slug, + RawSlug: builderApp.Organization.RawSlug, + PaidPlan: builderApp.Organization.PaidPlan, + }, + PlatformVersion: builderApp.PlatformVersion, + PostgresAppRole: builderApp.PostgresAppRole, + }, + OrgSlug: builderApp.Organization.Slug, + }) + if err != nil { + tracing.RecordError(span, err, "error creating flaps client") + return nil, nil, err + } + ctx = flapsutil.NewContextWithClient(ctx, flaps) + } + + builderMachine, err := validateBuilder(ctx, builderApp) + if err == nil { + span.AddEvent("builder app already exists and is valid") + return builderMachine, builderApp, nil + } + + var validateBuilderErr ValidateBuilderError + if !errors.As(err, &validateBuilderErr) { + return nil, nil, err + } + + if validateBuilderErr == BuilderMachineNotStarted { + span.AddEvent("builder machine not started, restarting") + flapsClient := flapsutil.ClientFromContext(ctx) + + if err := flapsClient.Restart(ctx, fly.RestartMachineInput{ + ID: builderMachine.ID, + }, ""); err != nil { + tracing.RecordError(span, err, "error restarting builder machine") + return nil, nil, err + } + + return builderMachine, builderApp, nil + } + + if validateBuilderErr != NoBuilderApp { + span.AddEvent(fmt.Sprintf("deleting existing invalid builder due to %s", validateBuilderErr)) + client := flyutil.ClientFromContext(ctx) + err := client.DeleteApp(ctx, builderApp.Name) + if err != nil { + tracing.RecordError(span, err, "error deleting invalid builder app") + return nil, nil, err + } + } + } else { + span.AddEvent("recreating builder") + if org.RemoteBuilderApp != nil { + client := flyutil.ClientFromContext(ctx) + err := client.DeleteApp(ctx, org.RemoteBuilderApp.Name) + if err != nil { + tracing.RecordError(span, err, "error deleting existing builder app") + return nil, nil, err + } + } + } + + builderName := "fly-builder-" + haikunator.Haikunator().Build() + // we want to lauch the machine to the builder + flapsClient, err := flapsutil.NewClientWithOptions(ctx, flaps.NewClientOpts{ + AppName: builderName, + OrgSlug: org.Slug, + }) + if err != nil { + tracing.RecordError(span, err, "error creating flaps client") + return nil, nil, err + } + ctx = flapsutil.NewContextWithClient(ctx, flapsClient) + app, machine, err := createBuilder(ctx, org, region, builderName) + if err != nil { + tracing.RecordError(span, err, "error creating builder") + return nil, nil, err + } + return machine, app, nil +} + +type ValidateBuilderError int + +func (e ValidateBuilderError) Error() string { + switch e { + case NoBuilderApp: + return "no builder app" + case NoBuilderVolume: + return "no builder volume" + case InvalidMachineCount: + return "invalid machine count" + case BuilderMachineNotStarted: + return "builder machine not started" + default: + return "unknown error validating builder" + } +} + +const ( + NoBuilderApp ValidateBuilderError = iota + NoBuilderVolume + InvalidMachineCount + BuilderMachineNotStarted +) + +func validateBuilder(ctx context.Context, app *fly.App) (*fly.Machine, error) { + ctx, span := tracing.GetTracer().Start(ctx, "validate_builder") + defer span.End() + + if app == nil { + tracing.RecordError(span, NoBuilderApp, "no builder app") + return nil, NoBuilderApp + } + + flapsClient := flapsutil.ClientFromContext(ctx) + + if _, err := validateBuilderVolumes(ctx, flapsClient); err != nil { + tracing.RecordError(span, err, "error validating builder volumes") + return nil, err + } + machine, err := validateBuilderMachines(ctx, flapsClient) + if err != nil { + tracing.RecordError(span, err, "error validating builder machines") + return nil, err + } + + if machine.State != "started" { + tracing.RecordError(span, BuilderMachineNotStarted, "builder machine not started") + return machine, BuilderMachineNotStarted + } + + return machine, nil + +} + +func validateBuilderVolumes(ctx context.Context, flapsClient flapsutil.FlapsClient) (*fly.Volume, error) { + ctx, span := tracing.GetTracer().Start(ctx, "validate_builder_volumes") + defer span.End() + + var volumes []fly.Volume + numRetries := 0 + + for { + var err error + + volumes, err = flapsClient.GetVolumes(ctx) + if err == nil { + break + } + + var flapsErr *flaps.FlapsError + // if it isn't a server error, no point in retrying + if errors.As(err, &flapsErr) && flapsErr.ResponseStatusCode >= 500 && flapsErr.ResponseStatusCode < 600 { + span.AddEvent(fmt.Sprintf("non-server error %d", flapsErr.ResponseStatusCode)) + numRetries += 1 + + if numRetries >= 3 { + tracing.RecordError(span, err, "error getting volumes") + return nil, err + } + time.Sleep(1 * time.Second) + } else { + tracing.RecordError(span, err, "error getting volumes") + return nil, err + } + } + + if len(volumes) == 0 { + tracing.RecordError(span, NoBuilderVolume, "the existing builder app has no volume") + return nil, NoBuilderVolume + } + + return &volumes[0], nil +} + +func validateBuilderMachines(ctx context.Context, flapsClient flapsutil.FlapsClient) (*fly.Machine, error) { + ctx, span := tracing.GetTracer().Start(ctx, "validate_builder_machines") + defer span.End() + + var machines []*fly.Machine + numRetries := 0 + for { + var err error + + machines, err = flapsClient.List(ctx, "") + if err == nil { + break + } + + var flapsErr *flaps.FlapsError + // if it isn't a server error, no point in retrying + if errors.As(err, &flapsErr) && flapsErr.ResponseStatusCode >= 500 && flapsErr.ResponseStatusCode < 600 { + span.AddEvent(fmt.Sprintf("non-server error %d", flapsErr.ResponseStatusCode)) + numRetries += 1 + + if numRetries >= 3 { + tracing.RecordError(span, err, "error listing machines") + return nil, err + } + time.Sleep(1 * time.Second) + } else { + tracing.RecordError(span, err, "error listing machines") + return nil, err + } + } + + if len(machines) != 1 { + span.AddEvent(fmt.Sprintf("invalid machine count %d", len(machines))) + tracing.RecordError(span, InvalidMachineCount, "the existing builder app has an invalid number of machines") + return nil, InvalidMachineCount + } + + return machines[0], nil +} + +func createBuilder(ctx context.Context, org *fly.Organization, region, builderName string) (app *fly.App, mach *fly.Machine, retErr error) { + ctx, span := tracing.GetTracer().Start(ctx, "create_builder") + defer span.End() + + client := flyutil.ClientFromContext(ctx) + flapsClient := flapsutil.ClientFromContext(ctx) + + app, retErr = client.CreateApp(ctx, fly.CreateAppInput{ + OrganizationID: org.ID, + Name: builderName, + AppRoleID: "remote-docker-builder", + Machines: true, + PreferredRegion: fly.StringPointer(region), + }) + if retErr != nil { + tracing.RecordError(span, retErr, "error creating app") + return nil, nil, retErr + } + + defer func() { + if retErr != nil { + span.AddEvent("cleaning up new builder app due to error") + client.DeleteApp(ctx, builderName) + } + }() + + _, retErr = client.AllocateIPAddress(ctx, app.Name, "shared_v4", "", org, "") + if retErr != nil { + tracing.RecordError(span, retErr, "error allocating ip address") + return nil, nil, retErr + } + + guest := fly.MachineGuest{ + CPUKind: "shared", + CPUs: 4, + MemoryMB: 4096, + } + + retErr = flapsClient.WaitForApp(ctx, app.Name) + if retErr != nil { + tracing.RecordError(span, retErr, "error waiting for builder") + return nil, nil, fmt.Errorf("waiting for app %s: %w", app.Name, retErr) + } + + var volume *fly.Volume + numRetries := 0 + for { + volume, retErr = flapsClient.CreateVolume(ctx, fly.CreateVolumeRequest{ + Name: "machine_data", + SizeGb: fly.IntPointer(50), + AutoBackupEnabled: fly.BoolPointer(false), + ComputeRequirements: &guest, + Region: region, + }) + if retErr == nil { + break + } + + var flapsErr *flaps.FlapsError + if errors.As(retErr, &flapsErr) && flapsErr.ResponseStatusCode >= 500 && flapsErr.ResponseStatusCode < 600 { + span.AddEvent(fmt.Sprintf("non-server error %d", flapsErr.ResponseStatusCode)) + numRetries += 1 + + if numRetries >= 5 { + tracing.RecordError(span, retErr, "error creating volume") + return nil, nil, retErr + } + time.Sleep(1 * time.Second) + } else { + tracing.RecordError(span, retErr, "error creating volume") + return nil, nil, retErr + } + } + + defer func() { + if retErr != nil { + span.AddEvent("cleaning up new volume due to error") + flapsClient.DeleteVolume(ctx, volume.ID) + } + }() + + mach, retErr = flapsClient.Launch(ctx, fly.LaunchMachineInput{ + Region: region, + Config: &fly.MachineConfig{ + Env: map[string]string{ + "ALLOW_ORG_SLUG": org.Slug, + "DATA_DIR": "/data", + "LOG_LEVEL": "debug", + }, + Guest: &guest, + Mounts: []fly.MachineMount{ + { + Path: "/data", + Volume: volume.ID, + Name: app.Name, + }, + }, + Services: []fly.MachineService{ + { + Protocol: "tcp", + InternalPort: 8080, + Autostop: fly.BoolPointer(false), + Autostart: fly.BoolPointer(true), + MinMachinesRunning: fly.IntPointer(0), + Ports: []fly.MachinePort{ + { + Port: fly.IntPointer(80), + Handlers: []string{"http"}, + ForceHTTPS: true, + HTTPOptions: &fly.HTTPOptions{ + H2Backend: fly.BoolPointer(true), + }, + }, + { + Port: fly.IntPointer(443), + Handlers: []string{"http", "tls"}, + TLSOptions: &fly.TLSOptions{ + ALPN: []string{"h2"}, + }, + HTTPOptions: &fly.HTTPOptions{ + H2Backend: fly.BoolPointer(true), + }, + }, + }, + ForceInstanceKey: nil, + }, + }, + Image: lo.Ternary(org.RemoteBuilderImage != "", org.RemoteBuilderImage, "docker-hub-mirror.fly.io/flyio/rchab:sha-9346699"), + }, + }) + if retErr != nil { + tracing.RecordError(span, retErr, "error launching builder machine") + return nil, nil, retErr + } + + return +} diff --git a/internal/build/imgsrc/ensure_builder_test.go b/internal/build/imgsrc/ensure_builder_test.go new file mode 100644 index 0000000000..458c551055 --- /dev/null +++ b/internal/build/imgsrc/ensure_builder_test.go @@ -0,0 +1,255 @@ +package imgsrc + +import ( + "context" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + fly "github.com/superfly/fly-go" + "github.com/superfly/fly-go/flaps" + "github.com/superfly/flyctl/internal/flapsutil" + "github.com/superfly/flyctl/internal/flyutil" + "github.com/superfly/flyctl/internal/mock" +) + +func TestValidateBuilder(t *testing.T) { + t.Parallel() + ctx := context.Background() + + hasVolumes := false + hasMachines := false + flapsClient := mock.FlapsClient{ + GetVolumesFunc: func(ctx context.Context) ([]fly.Volume, error) { + if hasVolumes { + return []fly.Volume{{ + ID: "bigvolume", + }}, nil + } else { + return []fly.Volume{}, nil + } + }, + ListFunc: func(ctx context.Context, state string) ([]*fly.Machine, error) { + if hasMachines { + return []*fly.Machine{{ + ID: "bigmachine", + State: "started", + }}, nil + } else { + return []*fly.Machine{}, nil + } + }, + } + ctx = flapsutil.NewContextWithClient(ctx, &flapsClient) + + _, err := validateBuilder(ctx, nil) + assert.EqualError(t, err, NoBuilderApp.Error()) + + _, err = validateBuilder(ctx, &fly.App{}) + assert.EqualError(t, err, NoBuilderVolume.Error()) + + hasVolumes = true + _, err = validateBuilder(ctx, &fly.App{}) + assert.EqualError(t, err, InvalidMachineCount.Error()) + + hasMachines = true + _, err = validateBuilder(ctx, &fly.App{}) + assert.NoError(t, err) +} + +func TestValidateBuilderAPIErrors(t *testing.T) { + t.Parallel() + ctx := context.Background() + + maxVolumeRetries := 3 + volumeRetries := 0 + volumesShouldFail := false + + maxMachineRetries := 3 + machineRetries := 0 + machinesShouldFail := false + + responseStatusCode := 500 + + flapsClient := mock.FlapsClient{ + GetVolumesFunc: func(ctx context.Context) ([]fly.Volume, error) { + if volumesShouldFail { + volumeRetries += 1 + + if volumeRetries < maxVolumeRetries { + return nil, &flaps.FlapsError{ + ResponseStatusCode: responseStatusCode, + ResponseBody: []byte("internal server error"), + } + } + } + return []fly.Volume{{ + ID: "bigvolume", + }}, nil + + }, + ListFunc: func(ctx context.Context, state string) ([]*fly.Machine, error) { + if machinesShouldFail { + machineRetries += 1 + + if machineRetries < maxMachineRetries { + return nil, &flaps.FlapsError{ + ResponseStatusCode: responseStatusCode, + ResponseBody: []byte("internal server error"), + } + } + } + return []*fly.Machine{{ + ID: "bigmachine", + State: "started", + }}, nil + }, + } + ctx = flapsutil.NewContextWithClient(ctx, &flapsClient) + + volumesShouldFail = true + _, err := validateBuilder(ctx, &fly.App{}) + assert.NoError(t, err) + + volumeRetries = 0 + maxVolumeRetries = 7 + _, err = validateBuilder(ctx, &fly.App{}) + assert.Error(t, err) + + volumeRetries = 0 + responseStatusCode = 404 + // we should only try once if the error is not a server error + _, err = validateBuilder(ctx, &fly.App{}) + var flapsErr *flaps.FlapsError + assert.True(t, errors.As(err, &flapsErr)) + assert.Equal(t, 404, flapsErr.ResponseStatusCode) + assert.Equal(t, 1, volumeRetries) + + volumesShouldFail = false + machinesShouldFail = true + responseStatusCode = 500 + _, err = validateBuilder(ctx, &fly.App{}) + assert.NoError(t, err) + + machineRetries = 0 + maxMachineRetries = 7 + _, err = validateBuilder(ctx, &fly.App{}) + assert.Error(t, err) + + machineRetries = 0 + responseStatusCode = 404 + // we should only try once if the error is not a server error + _, err = validateBuilder(ctx, &fly.App{}) + assert.True(t, errors.As(err, &flapsErr)) + assert.Equal(t, 404, flapsErr.ResponseStatusCode) + assert.Equal(t, 1, machineRetries) +} + +func TestCreateBuilder(t *testing.T) { + t.Parallel() + ctx := context.Background() + org := &fly.Organization{ + Slug: "bigorg", + } + + createAppShouldFail := false + allocateIPAddressShouldFail := false + apiClient := mock.Client{ + CreateAppFunc: func(ctx context.Context, input fly.CreateAppInput) (*fly.App, error) { + if createAppShouldFail { + return nil, errors.New("create app failed") + } + return &fly.App{ + Name: input.Name, + }, nil + }, + DeleteAppFunc: func(ctx context.Context, appName string) error { + return nil + }, + AllocateIPAddressFunc: func(ctx context.Context, appName string, addrType string, region string, org *fly.Organization, network string) (*fly.IPAddress, error) { + if allocateIPAddressShouldFail { + return nil, errors.New("allocate ip address failed") + } + return &fly.IPAddress{}, nil + }, + } + + waitForAppShouldFail := false + launchShouldFail := false + + createVolumeShouldFail := false + maxCreateVolumeAttempts := 3 + createVolumeAttempts := 0 + + flapsClient := mock.FlapsClient{ + WaitForAppFunc: func(ctx context.Context, name string) error { + if waitForAppShouldFail { + return errors.New("wait for app failed") + } + return nil + }, + CreateVolumeFunc: func(ctx context.Context, req fly.CreateVolumeRequest) (*fly.Volume, error) { + if createVolumeShouldFail { + createVolumeAttempts += 1 + + if createVolumeAttempts < maxCreateVolumeAttempts { + return nil, &flaps.FlapsError{ + ResponseStatusCode: 500, + ResponseBody: []byte("internal server error"), + } + } + } + return &fly.Volume{ + ID: "bigvolume", + }, nil + }, + DeleteVolumeFunc: func(ctx context.Context, volumeId string) (*fly.Volume, error) { + return nil, nil + }, + LaunchFunc: func(ctx context.Context, input fly.LaunchMachineInput) (*fly.Machine, error) { + if launchShouldFail { + return nil, errors.New("launch machine failed") + } + return &fly.Machine{ + ID: "bigmachine", + State: "started", + }, nil + }, + } + ctx = flyutil.NewContextWithClient(ctx, &apiClient) + ctx = flapsutil.NewContextWithClient(ctx, &flapsClient) + + app, machine, err := createBuilder(ctx, org, "ord", "builder") + assert.NoError(t, err) + assert.Equal(t, "bigmachine", machine.ID) + assert.Equal(t, app.Name, "builder") + + createAppShouldFail = true + _, _, err = createBuilder(ctx, org, "ord", "builder") + assert.Error(t, err) + + createAppShouldFail = false + allocateIPAddressShouldFail = true + _, _, err = createBuilder(ctx, org, "ord", "builder") + assert.Error(t, err) + + allocateIPAddressShouldFail = false + waitForAppShouldFail = true + _, _, err = createBuilder(ctx, org, "ord", "builder") + assert.Error(t, err) + + waitForAppShouldFail = false + createVolumeShouldFail = true + _, _, err = createBuilder(ctx, org, "ord", "builder") + assert.NoError(t, err) + + createVolumeAttempts = 0 + maxCreateVolumeAttempts = 7 + _, _, err = createBuilder(ctx, org, "ord", "builder") + assert.Error(t, err) + + createVolumeShouldFail = false + launchShouldFail = true + _, _, err = createBuilder(ctx, org, "ord", "builder") + assert.Error(t, err) +} diff --git a/internal/build/imgsrc/nixpacks_builder.go b/internal/build/imgsrc/nixpacks_builder.go index 17436aec67..23930d35ae 100644 --- a/internal/build/imgsrc/nixpacks_builder.go +++ b/internal/build/imgsrc/nixpacks_builder.go @@ -130,21 +130,14 @@ func (*nixpacksBuilder) Run(ctx context.Context, dockerFactory *dockerClientFact return nil, "", err } - machine, app, err := remoteBuilderMachine(ctx, dockerFactory.apiClient, dockerFactory.appName) + machine, app, err := remoteBuilderMachine(ctx, dockerFactory.apiClient, dockerFactory.appName, false) if err != nil { build.BuilderInitFinish() build.BuildFinish() return nil, "", err } - var remoteHost string - for _, ip := range machine.IPs.Nodes { - terminal.Debugf("checking ip %+v\n", ip) - if ip.Kind == "privatenet" { - remoteHost = ip.IP - break - } - } + remoteHost := machine.PrivateIP if remoteHost == "" { build.BuilderInitFinish() diff --git a/internal/build/imgsrc/resolver.go b/internal/build/imgsrc/resolver.go index 04d0e818d2..238820859e 100644 --- a/internal/build/imgsrc/resolver.go +++ b/internal/build/imgsrc/resolver.go @@ -726,9 +726,9 @@ func (s *StopSignal) Stop() { }) } -func NewResolver(daemonType DockerDaemonType, apiClient flyutil.Client, appName string, iostreams *iostreams.IOStreams, connectOverWireguard bool) *Resolver { +func NewResolver(daemonType DockerDaemonType, apiClient flyutil.Client, appName string, iostreams *iostreams.IOStreams, connectOverWireguard, recreateBuilder bool) *Resolver { return &Resolver{ - dockerFactory: newDockerClientFactory(daemonType, apiClient, appName, iostreams, connectOverWireguard), + dockerFactory: newDockerClientFactory(daemonType, apiClient, appName, iostreams, connectOverWireguard, recreateBuilder), apiClient: apiClient, heartbeatFn: heartbeat, } diff --git a/internal/command/command_run.go b/internal/command/command_run.go index b54ce29bd5..2a902601b0 100644 --- a/internal/command/command_run.go +++ b/internal/command/command_run.go @@ -37,7 +37,7 @@ func DetermineImage(ctx context.Context, appName string, imageOrPath string) (im ) daemonType := imgsrc.NewDockerDaemonType(!flag.GetBool(ctx, "build-remote-only"), !flag.GetBool(ctx, "build-local-only"), env.IsCI(), flag.GetBool(ctx, "build-nixpacks")) - resolver := imgsrc.NewResolver(daemonType, client, appName, io, flag.GetWireguard(ctx)) + resolver := imgsrc.NewResolver(daemonType, client, appName, io, flag.GetWireguard(ctx), false) // build if relative or absolute path if strings.HasPrefix(imageOrPath, ".") || strings.HasPrefix(imageOrPath, "/") { diff --git a/internal/command/deploy/deploy.go b/internal/command/deploy/deploy.go index 6d80a3004c..3e09cc5d60 100644 --- a/internal/command/deploy/deploy.go +++ b/internal/command/deploy/deploy.go @@ -53,6 +53,7 @@ var CommonFlags = flag.Set{ flag.BuildOnly(), flag.BpDockerHost(), flag.BpVolume(), + flag.RecreateBuilder(), flag.Yes(), flag.VMSizeFlags, flag.StringArray{ @@ -298,13 +299,14 @@ func DeployWithConfig(ctx context.Context, appConfig *appconfig.Config, forceYes httpFailover := flag.GetHTTPSFailover(ctx) usingWireguard := flag.GetWireguard(ctx) + recreateBuilder := flag.GetRecreateBuilder(ctx) // Fetch an image ref or build from source to get the final image reference to deploy - img, err := determineImage(ctx, appConfig, usingWireguard) + img, err := determineImage(ctx, appConfig, usingWireguard, recreateBuilder) if err != nil && usingWireguard && httpFailover { span.SetAttributes(attribute.String("builder.failover_error", err.Error())) span.AddEvent("using http failover") - img, err = determineImage(ctx, appConfig, false) + img, err = determineImage(ctx, appConfig, false, recreateBuilder) } if err != nil { diff --git a/internal/command/deploy/deploy_build.go b/internal/command/deploy/deploy_build.go index 26b4f7dd53..94e1e4c8e0 100644 --- a/internal/command/deploy/deploy_build.go +++ b/internal/command/deploy/deploy_build.go @@ -48,7 +48,7 @@ func multipleDockerfile(ctx context.Context, appConfig *appconfig.Config) error // determineImage picks the deployment strategy, builds the image and returns a // DeploymentImage struct -func determineImage(ctx context.Context, appConfig *appconfig.Config, useWG bool) (img *imgsrc.DeploymentImage, err error) { +func determineImage(ctx context.Context, appConfig *appconfig.Config, useWG, recreateBuilder bool) (img *imgsrc.DeploymentImage, err error) { ctx, span := tracing.GetTracer().Start(ctx, "determine_image") defer span.End() @@ -67,7 +67,7 @@ func determineImage(ctx context.Context, appConfig *appconfig.Config, useWG bool terminal.Warnf("%s\n", err.Error()) } - resolver := imgsrc.NewResolver(daemonType, client, appConfig.AppName, io, useWG) + resolver := imgsrc.NewResolver(daemonType, client, appConfig.AppName, io, useWG, recreateBuilder) var imageRef string if imageRef, err = fetchImageRef(ctx, appConfig); err != nil { diff --git a/internal/command/launch/plan_builder.go b/internal/command/launch/plan_builder.go index e9317ee5a2..c36f55ac8f 100644 --- a/internal/command/launch/plan_builder.go +++ b/internal/command/launch/plan_builder.go @@ -255,7 +255,7 @@ func stateFromManifest(ctx context.Context, m LaunchManifest, optionalCache *pla client = flyutil.ClientFromContext(ctx) ) - org, err := client.GetOrganizationBySlug(ctx, m.Plan.OrgSlug) + org, err := client.GetOrganizationRemoteBuilderBySlug(ctx, m.Plan.OrgSlug) if err != nil { return nil, err } @@ -263,7 +263,7 @@ func stateFromManifest(ctx context.Context, m LaunchManifest, optionalCache *pla // If we potentially are deploying, launch a remote builder to prepare for deployment. if !flag.GetBool(ctx, "no-deploy") { // TODO: determine if eager remote builder is still required here - go imgsrc.EagerlyEnsureRemoteBuilder(ctx, client, org.Slug) + go imgsrc.EagerlyEnsureRemoteBuilder(ctx, client, org, flag.GetRecreateBuilder(ctx)) } var ( diff --git a/internal/flag/flag.go b/internal/flag/flag.go index 07b63029a7..5e2cbd8d4f 100644 --- a/internal/flag/flag.go +++ b/internal/flag/flag.go @@ -588,6 +588,18 @@ This option may set DOCKER_HOST environment variable for the build container if } } +func RecreateBuilder() Bool { + return Bool{ + Name: "recreate-builder", + Description: "Recreate the builder app, if it exists", + Default: false, + } +} + +func GetRecreateBuilder(ctx context.Context) bool { + return GetBool(ctx, "recreate-builder") +} + // BuildpacksVolume the host volume that will be mounted to the buildpacks build container const BuildpacksVolume = "buildpacks-volume" diff --git a/internal/flyutil/client.go b/internal/flyutil/client.go index 073a3ec956..2b5381783c 100644 --- a/internal/flyutil/client.go +++ b/internal/flyutil/client.go @@ -44,6 +44,7 @@ type Client interface { ExportDNSRecords(ctx context.Context, domainId string) (string, error) FinishBuild(ctx context.Context, input fly.FinishBuildInput) (*fly.FinishBuildResponse, error) GetApp(ctx context.Context, appName string) (*fly.App, error) + GetAppRemoteBuilder(ctx context.Context, appName string) (*fly.App, error) GetAppBasic(ctx context.Context, appName string) (*fly.AppBasic, error) GetAppCertificates(ctx context.Context, appName string) ([]fly.AppCertificateCompact, error) GetAppCompact(ctx context.Context, appName string) (*fly.AppCompact, error) @@ -71,6 +72,8 @@ type Client interface { GetMachine(ctx context.Context, machineId string) (*fly.GqlMachine, error) GetNearestRegion(ctx context.Context) (*fly.Region, error) GetOrganizationBySlug(ctx context.Context, slug string) (*fly.Organization, error) + GetOrganizationByApp(ctx context.Context, appName string) (*fly.Organization, error) + GetOrganizationRemoteBuilderBySlug(ctx context.Context, slug string) (*fly.Organization, error) GetOrganizations(ctx context.Context, filters ...fly.OrganizationFilter) ([]fly.Organization, error) GetSnapshotsFromVolume(ctx context.Context, volID string) ([]fly.VolumeSnapshot, error) GetWireGuardPeer(ctx context.Context, slug, name string) (*fly.WireGuardPeer, error) diff --git a/internal/mock/client.go b/internal/mock/client.go index 5796040387..3eea775864 100644 --- a/internal/mock/client.go +++ b/internal/mock/client.go @@ -14,88 +14,92 @@ import ( var _ flyutil.Client = (*Client)(nil) type Client struct { - AddCertificateFunc func(ctx context.Context, appName, hostname string) (*fly.AppCertificate, *fly.HostnameCheck, error) - AllocateIPAddressFunc func(ctx context.Context, appName string, addrType string, region string, org *fly.Organization, network string) (*fly.IPAddress, error) - AllocateSharedIPAddressFunc func(ctx context.Context, appName string) (net.IP, error) - AppNameAvailableFunc func(ctx context.Context, appName string) (bool, error) - AttachPostgresClusterFunc func(ctx context.Context, input fly.AttachPostgresClusterInput) (*fly.AttachPostgresClusterPayload, error) - AuthenticatedFunc func() bool - CanPerformBluegreenDeploymentFunc func(ctx context.Context, appName string) (bool, error) - CheckAppCertificateFunc func(ctx context.Context, appName, hostname string) (*fly.AppCertificate, *fly.HostnameCheck, error) - CheckDomainFunc func(ctx context.Context, name string) (*fly.CheckDomainResult, error) - ClosestWireguardGatewayRegionFunc func(ctx context.Context) (*fly.Region, error) - CreateAndRegisterDomainFunc func(organizationID string, name string) (*fly.Domain, error) - CreateAppFunc func(ctx context.Context, input fly.CreateAppInput) (*fly.App, error) - CreateBuildFunc func(ctx context.Context, input fly.CreateBuildInput) (*fly.CreateBuildResponse, error) - CreateDelegatedWireGuardTokenFunc func(ctx context.Context, org *fly.Organization, name string) (*fly.DelegatedWireGuardToken, error) - CreateDoctorUrlFunc func(ctx context.Context) (putUrl string, err error) - CreateDomainFunc func(organizationID string, name string) (*fly.Domain, error) - CreateOrganizationFunc func(ctx context.Context, organizationname string) (*fly.Organization, error) - CreateOrganizationInviteFunc func(ctx context.Context, id, email string) (*fly.Invitation, error) - CreateReleaseFunc func(ctx context.Context, input fly.CreateReleaseInput) (*fly.CreateReleaseResponse, error) - CreateWireGuardPeerFunc func(ctx context.Context, org *fly.Organization, region, name, pubkey, network string) (*fly.CreatedWireGuardPeer, error) - DeleteAppFunc func(ctx context.Context, appName string) error - DeleteCertificateFunc func(ctx context.Context, appName, hostname string) (*fly.DeleteCertificatePayload, error) - DeleteDelegatedWireGuardTokenFunc func(ctx context.Context, org *fly.Organization, name, token *string) error - DeleteOrganizationFunc func(ctx context.Context, id string) (deletedid string, err error) - DeleteOrganizationMembershipFunc func(ctx context.Context, orgId, userId string) (string, string, error) - DetachPostgresClusterFunc func(ctx context.Context, input fly.DetachPostgresClusterInput) error - EnablePostgresConsulFunc func(ctx context.Context, appName string) (*fly.PostgresEnableConsulPayload, error) - EnsureRemoteBuilderFunc func(ctx context.Context, orgID, appName, region string) (*fly.GqlMachine, *fly.App, error) - ExportDNSRecordsFunc func(ctx context.Context, domainId string) (string, error) - FinishBuildFunc func(ctx context.Context, input fly.FinishBuildInput) (*fly.FinishBuildResponse, error) - GetAppFunc func(ctx context.Context, appName string) (*fly.App, error) - GetAppBasicFunc func(ctx context.Context, appName string) (*fly.AppBasic, error) - GetAppCertificatesFunc func(ctx context.Context, appName string) ([]fly.AppCertificateCompact, error) - GetAppCompactFunc func(ctx context.Context, appName string) (*fly.AppCompact, error) - GetAppCurrentReleaseMachinesFunc func(ctx context.Context, appName string) (*fly.Release, error) - GetAppHostIssuesFunc func(ctx context.Context, appName string) ([]fly.HostIssue, error) - GetAppLimitedAccessTokensFunc func(ctx context.Context, appName string) ([]fly.LimitedAccessToken, error) - GetAppLogsFunc func(ctx context.Context, appName, token, region, instanceID string) (entries []fly.LogEntry, nextToken string, err error) - GetAppNameFromVolumeFunc func(ctx context.Context, volID string) (*string, error) - GetAppNameStateFromVolumeFunc func(ctx context.Context, volID string) (*string, *string, error) - GetAppNetworkFunc func(ctx context.Context, appName string) (*string, error) - GetAppReleasesMachinesFunc func(ctx context.Context, appName, status string, limit int) ([]fly.Release, error) - GetAppSecretsFunc func(ctx context.Context, appName string) ([]fly.Secret, error) - GetAppsFunc func(ctx context.Context, role *string) ([]fly.App, error) - GetAppsForOrganizationFunc func(ctx context.Context, orgID string) ([]fly.App, error) - GetCurrentUserFunc func(ctx context.Context) (*fly.User, error) - GetDNSRecordsFunc func(ctx context.Context, domainName string) ([]*fly.DNSRecord, error) - GetDelegatedWireGuardTokensFunc func(ctx context.Context, slug string) ([]*fly.DelegatedWireGuardTokenHandle, error) - GetDetailedOrganizationBySlugFunc func(ctx context.Context, slug string) (*fly.OrganizationDetails, error) - GetDomainFunc func(ctx context.Context, name string) (*fly.Domain, error) - GetDomainsFunc func(ctx context.Context, organizationSlug string) ([]*fly.Domain, error) - GetIPAddressesFunc func(ctx context.Context, appName string) ([]fly.IPAddress, error) - GetLatestImageDetailsFunc func(ctx context.Context, image string) (*fly.ImageVersion, error) - GetLatestImageTagFunc func(ctx context.Context, repository string, snapshotId *string) (string, error) - GetLoggedCertificatesFunc func(ctx context.Context, slug string) ([]fly.LoggedCertificate, error) - GetMachineFunc func(ctx context.Context, machineId string) (*fly.GqlMachine, error) - GetNearestRegionFunc func(ctx context.Context) (*fly.Region, error) - GetOrganizationBySlugFunc func(ctx context.Context, slug string) (*fly.Organization, error) - GetOrganizationsFunc func(ctx context.Context, filters ...fly.OrganizationFilter) ([]fly.Organization, error) - GetSnapshotsFromVolumeFunc func(ctx context.Context, volID string) ([]fly.VolumeSnapshot, error) - GetWireGuardPeerFunc func(ctx context.Context, slug, name string) (*fly.WireGuardPeer, error) - GetWireGuardPeersFunc func(ctx context.Context, slug string) ([]*fly.WireGuardPeer, error) - GenqClientFunc func() genq.Client - LatestImageFunc func(ctx context.Context, appName string) (string, error) - ImportDNSRecordsFunc func(ctx context.Context, domainId string, zonefile string) ([]fly.ImportDnsWarning, []fly.ImportDnsChange, error) - IssueSSHCertificateFunc func(ctx context.Context, org fly.OrganizationImpl, principals []string, appNames []string, valid_hours *int, publicKey ed25519.PublicKey) (*fly.IssuedCertificate, error) - ListPostgresClusterAttachmentsFunc func(ctx context.Context, appName, postgresAppName string) ([]*fly.PostgresClusterAttachment, error) - LoggerFunc func() fly.Logger - MoveAppFunc func(ctx context.Context, appName string, orgID string) (*fly.App, error) - NewRequestFunc func(q string) *graphql.Request - PlatformRegionsFunc func(ctx context.Context) ([]fly.Region, *fly.Region, error) - ReleaseIPAddressFunc func(ctx context.Context, appName string, ip string) error - RemoveWireGuardPeerFunc func(ctx context.Context, org *fly.Organization, name string) error - ResolveImageForAppFunc func(ctx context.Context, appName, imageRef string) (*fly.Image, error) - RevokeLimitedAccessTokenFunc func(ctx context.Context, id string) error - RunFunc func(req *graphql.Request) (fly.Query, error) - RunWithContextFunc func(ctx context.Context, req *graphql.Request) (fly.Query, error) - SetGenqClientFunc func(client genq.Client) - SetSecretsFunc func(ctx context.Context, appName string, secrets map[string]string) (*fly.Release, error) - UpdateReleaseFunc func(ctx context.Context, input fly.UpdateReleaseInput) (*fly.UpdateReleaseResponse, error) - UnsetSecretsFunc func(ctx context.Context, appName string, keys []string) (*fly.Release, error) - ValidateWireGuardPeersFunc func(ctx context.Context, peerIPs []string) (invalid []string, err error) + AddCertificateFunc func(ctx context.Context, appName, hostname string) (*fly.AppCertificate, *fly.HostnameCheck, error) + AllocateIPAddressFunc func(ctx context.Context, appName string, addrType string, region string, org *fly.Organization, network string) (*fly.IPAddress, error) + AllocateSharedIPAddressFunc func(ctx context.Context, appName string) (net.IP, error) + AppNameAvailableFunc func(ctx context.Context, appName string) (bool, error) + AttachPostgresClusterFunc func(ctx context.Context, input fly.AttachPostgresClusterInput) (*fly.AttachPostgresClusterPayload, error) + AuthenticatedFunc func() bool + CanPerformBluegreenDeploymentFunc func(ctx context.Context, appName string) (bool, error) + CheckAppCertificateFunc func(ctx context.Context, appName, hostname string) (*fly.AppCertificate, *fly.HostnameCheck, error) + CheckDomainFunc func(ctx context.Context, name string) (*fly.CheckDomainResult, error) + ClosestWireguardGatewayRegionFunc func(ctx context.Context) (*fly.Region, error) + CreateAndRegisterDomainFunc func(organizationID string, name string) (*fly.Domain, error) + CreateAppFunc func(ctx context.Context, input fly.CreateAppInput) (*fly.App, error) + CreateBuildFunc func(ctx context.Context, input fly.CreateBuildInput) (*fly.CreateBuildResponse, error) + CreateDelegatedWireGuardTokenFunc func(ctx context.Context, org *fly.Organization, name string) (*fly.DelegatedWireGuardToken, error) + CreateDoctorUrlFunc func(ctx context.Context) (putUrl string, err error) + CreateDomainFunc func(organizationID string, name string) (*fly.Domain, error) + CreateOrganizationFunc func(ctx context.Context, organizationname string) (*fly.Organization, error) + CreateOrganizationInviteFunc func(ctx context.Context, id, email string) (*fly.Invitation, error) + CreateReleaseFunc func(ctx context.Context, input fly.CreateReleaseInput) (*fly.CreateReleaseResponse, error) + CreateWireGuardPeerFunc func(ctx context.Context, org *fly.Organization, region, name, pubkey, network string) (*fly.CreatedWireGuardPeer, error) + DeleteAppFunc func(ctx context.Context, appName string) error + DeleteCertificateFunc func(ctx context.Context, appName, hostname string) (*fly.DeleteCertificatePayload, error) + DeleteDelegatedWireGuardTokenFunc func(ctx context.Context, org *fly.Organization, name, token *string) error + DeleteOrganizationFunc func(ctx context.Context, id string) (deletedid string, err error) + DeleteOrganizationMembershipFunc func(ctx context.Context, orgId, userId string) (string, string, error) + DetachPostgresClusterFunc func(ctx context.Context, input fly.DetachPostgresClusterInput) error + EnablePostgresConsulFunc func(ctx context.Context, appName string) (*fly.PostgresEnableConsulPayload, error) + EnsureRemoteBuilderFunc func(ctx context.Context, orgID, appName, region string) (*fly.GqlMachine, *fly.App, error) + ExportDNSRecordsFunc func(ctx context.Context, domainId string) (string, error) + FinishBuildFunc func(ctx context.Context, input fly.FinishBuildInput) (*fly.FinishBuildResponse, error) + GetAppFunc func(ctx context.Context, appName string) (*fly.App, error) + GetAppRemoteBuilderFunc func(ctx context.Context, appName string) (*fly.App, error) + GetAppBasicFunc func(ctx context.Context, appName string) (*fly.AppBasic, error) + GetAppCertificatesFunc func(ctx context.Context, appName string) ([]fly.AppCertificateCompact, error) + GetAppCompactFunc func(ctx context.Context, appName string) (*fly.AppCompact, error) + GetAppCurrentReleaseMachinesFunc func(ctx context.Context, appName string) (*fly.Release, error) + GetAppHostIssuesFunc func(ctx context.Context, appName string) ([]fly.HostIssue, error) + GetAppLimitedAccessTokensFunc func(ctx context.Context, appName string) ([]fly.LimitedAccessToken, error) + GetAppLogsFunc func(ctx context.Context, appName, token, region, instanceID string) (entries []fly.LogEntry, nextToken string, err error) + GetAppNameFromVolumeFunc func(ctx context.Context, volID string) (*string, error) + GetAppNameStateFromVolumeFunc func(ctx context.Context, volID string) (*string, *string, error) + GetAppNetworkFunc func(ctx context.Context, appName string) (*string, error) + GetAppReleasesMachinesFunc func(ctx context.Context, appName, status string, limit int) ([]fly.Release, error) + GetAppSecretsFunc func(ctx context.Context, appName string) ([]fly.Secret, error) + GetAppsFunc func(ctx context.Context, role *string) ([]fly.App, error) + GetAppsForOrganizationFunc func(ctx context.Context, orgID string) ([]fly.App, error) + GetCurrentUserFunc func(ctx context.Context) (*fly.User, error) + GetDNSRecordsFunc func(ctx context.Context, domainName string) ([]*fly.DNSRecord, error) + GetDelegatedWireGuardTokensFunc func(ctx context.Context, slug string) ([]*fly.DelegatedWireGuardTokenHandle, error) + GetDetailedOrganizationBySlugFunc func(ctx context.Context, slug string) (*fly.OrganizationDetails, error) + GetDomainFunc func(ctx context.Context, name string) (*fly.Domain, error) + GetDomainsFunc func(ctx context.Context, organizationSlug string) ([]*fly.Domain, error) + GetIPAddressesFunc func(ctx context.Context, appName string) ([]fly.IPAddress, error) + GetLatestImageDetailsFunc func(ctx context.Context, image string) (*fly.ImageVersion, error) + GetLatestImageTagFunc func(ctx context.Context, repository string, snapshotId *string) (string, error) + GetLoggedCertificatesFunc func(ctx context.Context, slug string) ([]fly.LoggedCertificate, error) + GetMachineFunc func(ctx context.Context, machineId string) (*fly.GqlMachine, error) + GetNearestRegionFunc func(ctx context.Context) (*fly.Region, error) + GetOrganizationBySlugFunc func(ctx context.Context, slug string) (*fly.Organization, error) + GetOrganizationRemoteBuilderBySlugFunc func(ctx context.Context, slug string) (*fly.Organization, error) + GetOrganizationByAppFunc func(ctx context.Context, appName string) (*fly.Organization, error) + GetOrganizationsFunc func(ctx context.Context, filters ...fly.OrganizationFilter) ([]fly.Organization, error) + GetSnapshotsFromVolumeFunc func(ctx context.Context, volID string) ([]fly.VolumeSnapshot, error) + GetWireGuardPeerFunc func(ctx context.Context, slug, name string) (*fly.WireGuardPeer, error) + GetWireGuardPeersFunc func(ctx context.Context, slug string) ([]*fly.WireGuardPeer, error) + GenqClientFunc func() genq.Client + ImportDNSRecordsFunc func(ctx context.Context, domainId string, zonefile string) ([]fly.ImportDnsWarning, []fly.ImportDnsChange, error) + IssueSSHCertificateFunc func(ctx context.Context, org fly.OrganizationImpl, principals []string, appNames []string, valid_hours *int, publicKey ed25519.PublicKey) (*fly.IssuedCertificate, error) + LatestImageFunc func(ctx context.Context, appName string) (string, error) + ListPostgresClusterAttachmentsFunc func(ctx context.Context, appName, postgresAppName string) ([]*fly.PostgresClusterAttachment, error) + LoggerFunc func() fly.Logger + MoveAppFunc func(ctx context.Context, appName string, orgID string) (*fly.App, error) + NewRequestFunc func(q string) *graphql.Request + PlatformRegionsFunc func(ctx context.Context) ([]fly.Region, *fly.Region, error) + ReleaseIPAddressFunc func(ctx context.Context, appName string, ip string) error + RemoveWireGuardPeerFunc func(ctx context.Context, org *fly.Organization, name string) error + ResolveImageForAppFunc func(ctx context.Context, appName, imageRef string) (*fly.Image, error) + RevokeLimitedAccessTokenFunc func(ctx context.Context, id string) error + RunFunc func(req *graphql.Request) (fly.Query, error) + RunWithContextFunc func(ctx context.Context, req *graphql.Request) (fly.Query, error) + SetGenqClientFunc func(client genq.Client) + SetRemoteBuilderFunc func(ctx context.Context, appName string) error + SetSecretsFunc func(ctx context.Context, appName string, secrets map[string]string) (*fly.Release, error) + UpdateReleaseFunc func(ctx context.Context, input fly.UpdateReleaseInput) (*fly.UpdateReleaseResponse, error) + UnsetSecretsFunc func(ctx context.Context, appName string, keys []string) (*fly.Release, error) + ValidateWireGuardPeersFunc func(ctx context.Context, peerIPs []string) (invalid []string, err error) } func (m *Client) AddCertificate(ctx context.Context, appName, hostname string) (*fly.AppCertificate, *fly.HostnameCheck, error) { @@ -222,6 +226,10 @@ func (m *Client) GetApp(ctx context.Context, appName string) (*fly.App, error) { return m.GetAppFunc(ctx, appName) } +func (m *Client) GetAppRemoteBuilder(ctx context.Context, appName string) (*fly.App, error) { + return m.GetAppRemoteBuilderFunc(ctx, appName) +} + func (m *Client) GetAppBasic(ctx context.Context, appName string) (*fly.AppBasic, error) { return m.GetAppBasicFunc(ctx, appName) } @@ -330,6 +338,14 @@ func (m *Client) GetOrganizationBySlug(ctx context.Context, slug string) (*fly.O return m.GetOrganizationBySlugFunc(ctx, slug) } +func (m *Client) GetOrganizationRemoteBuilderBySlug(ctx context.Context, slug string) (*fly.Organization, error) { + return m.GetOrganizationRemoteBuilderBySlugFunc(ctx, slug) +} + +func (m *Client) GetOrganizationByApp(ctx context.Context, appName string) (*fly.Organization, error) { + return m.GetOrganizationByAppFunc(ctx, appName) +} + func (m *Client) GetOrganizations(ctx context.Context, filters ...fly.OrganizationFilter) ([]fly.Organization, error) { return m.GetOrganizationsFunc(ctx, filters...) } @@ -410,6 +426,10 @@ func (m *Client) SetGenqClient(client genq.Client) { m.SetGenqClientFunc(client) } +func (m *Client) SetRemoteBuilder(ctx context.Context, appName string) error { + return m.SetRemoteBuilderFunc(ctx, appName) +} + func (m *Client) SetSecrets(ctx context.Context, appName string, secrets map[string]string) (*fly.Release, error) { return m.SetSecretsFunc(ctx, appName, secrets) } diff --git a/test/preflight/fly_deploy_test.go b/test/preflight/fly_deploy_test.go index 8f9fbc550c..da81693c31 100644 --- a/test/preflight/fly_deploy_test.go +++ b/test/preflight/fly_deploy_test.go @@ -267,3 +267,12 @@ func TestFlyDeploy_DeployMachinesCheckCanary(t *testing.T) { output := deployRes.StdOutString() require.Contains(f, output, "Test Machine") } + +func TestFlyDeploy_CreateBuilderWDeployToken(t *testing.T) { + f := testlib.NewTestEnvFromEnv(t) + appName := f.CreateRandomAppName() + + f.Fly("launch --org %s --name %s --region %s --image nginx --internal-port 80 --ha=false --strategy canary", f.OrgSlug(), appName, f.PrimaryRegion()) + f.OverrideAuthAccessToken(f.Fly("tokens deploy").StdOutString()) + f.Fly("deploy") +}