fix fail in unfreeze

whywaita · May 9, 2024 · d8f2deb · d8f2deb
1 parent 7aee2fc
commit d8f2deb
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 5 deletions.
diff --git a/pool-agent/go.mod b/pool-agent/go.mod
@@ -11,6 +11,7 @@ require (
 	github.com/prometheus/client_golang v1.12.1
 	github.com/spf13/cobra v1.3.0
 	github.com/whywaita/shoes-lxd-multi/server v0.0.0-20240416053912-6df6153e11ef
+	golang.org/x/sync v0.6.0
 )
 
 require (
@@ -60,7 +61,6 @@ require (
 	golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
 	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.6.0 // indirect
 	golang.org/x/text v0.8.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

diff --git a/server/pkg/api/pool.go b/server/pkg/api/pool.go
@@ -156,7 +156,7 @@ func findInstances(ctx context.Context, targets []lxdclient.LXDHost, match func(
 
 func findInstanceByJob(ctx context.Context, targets []lxdclient.LXDHost, runnerName string, l *slog.Logger) (*lxdclient.LXDHost, string, bool) {
 	s := findInstances(ctx, targets, func(i api.Instance) bool {
-		return i.Config[lxdclient.ConfigKeyRunnerName] == runnerName
+		return i.Config[lxdclient.ConfigKeyRunnerName] == runnerName && i.StatusCode == api.Frozen
 	}, 0, l)
 	if len(s) < 1 {
 		return nil, "", false
@@ -230,16 +230,32 @@ func allocateInstance(host *lxdclient.LXDHost, instanceName, runnerName string,
 	return nil
 }
 
-func unfreezeInstance(c lxd.InstanceServer, name string) error {
-	state, etag, err := c.GetInstanceState(name)
+func recoverInvalidInstance(c lxd.InstanceServer, instanceName string) error {
+	i, etag, err := c.GetInstance(instanceName)
+	if err != nil {
+		return fmt.Errorf("get instance: %w", err)
+	}
+	i.InstancePut.Config[lxdclient.ConfigKeyRunnerName] = ""
+	op, err := c.UpdateInstance(instanceName, i.InstancePut, etag)
+	if err != nil {
+		return fmt.Errorf("update instance: %w", err)
+	}
+	if err := op.Wait(); err != nil {
+		return fmt.Errorf("waiting operation: %w", err)
+	}
+	return nil
+}
+
+func unfreezeInstance(c lxd.InstanceServer, instanceName string) error {
+	state, etag, err := c.GetInstanceState(instanceName)
 	if err != nil {
 		return fmt.Errorf("get instance state: %w", err)
 	}
 	switch state.StatusCode {
 	case api.Running:
 		// do nothing
 	case api.Frozen:
-		op, err := c.UpdateInstanceState(name, api.InstanceStatePut{
+		op, err := c.UpdateInstanceState(instanceName, api.InstanceStatePut{
 			Action:  "unfreeze",
 			Timeout: -1,
 		}, etag)
@@ -250,6 +266,9 @@ func unfreezeInstance(c lxd.InstanceServer, name string) error {
 			return fmt.Errorf("waiting operation: %w", err)
 		}
 	default:
+		if err := recoverInvalidInstance(c, instanceName); err != nil {
+			return fmt.Errorf("failed to recover invalid instance: %w", err)
+		}
 		return fmt.Errorf("unexpected instance state: %s", state.StatusCode.String())
 	}
 	return nil