From aae239b083b047922a570d46025c34b51e18aa7a Mon Sep 17 00:00:00 2001 From: takonomura Date: Thu, 14 Sep 2023 15:36:02 +0900 Subject: [PATCH] Implement pool mode --- .goreleaser.yml | 5 + README.md | 1 + pool-agent/README.md | 46 ++++ pool-agent/agent.go | 201 +++++++++++++++++ pool-agent/config.go | 83 +++++++ pool-agent/config_keys.go | 7 + pool-agent/create.go | 110 +++++++++ pool-agent/go.mod | 60 +++++ pool-agent/go.sum | 271 +++++++++++++++++++++++ pool-agent/main.go | 52 +++++ server/main.go | 4 +- server/pkg/api/pool.go | 229 +++++++++++++++++++ server/pkg/api/server.go | 6 +- server/pkg/api/server_add_instance.go | 131 +++++++++-- server/pkg/api/server_delete_instance.go | 4 - server/pkg/config/config.go | 26 ++- 16 files changed, 1197 insertions(+), 39 deletions(-) create mode 100644 pool-agent/README.md create mode 100644 pool-agent/agent.go create mode 100644 pool-agent/config.go create mode 100644 pool-agent/config_keys.go create mode 100644 pool-agent/create.go create mode 100644 pool-agent/go.mod create mode 100644 pool-agent/go.sum create mode 100644 pool-agent/main.go create mode 100644 server/pkg/api/pool.go diff --git a/.goreleaser.yml b/.goreleaser.yml index f4a6e51..92510a3 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -9,6 +9,11 @@ builds: binary: shoes-lxd-multi-server env: - CGO_ENABLED=0 + - id: "shoes-lxd-multi-pool-agent" + dir: pool-agent + binary: shoes-lxd-multi-pool-agent + env: + - CGO_ENABLED=0 archives: - format: binary name_template: "{{ .Binary }}-{{ .Os }}-{{ .Arch }}" diff --git a/README.md b/README.md index 33361ca..2531396 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,4 @@ - shoes-lxd-multi: shoes-provider - server: Server-side +- pool-agent: stadium agent for pool mode diff --git a/pool-agent/README.md b/pool-agent/README.md new file mode 100644 index 0000000..0c4d4f9 --- /dev/null +++ b/pool-agent/README.md @@ -0,0 +1,46 @@ +# Pool Agent + +Stadium Agent for pool mode + +## Setup + +- `LXD_MULTI_RESOURCE_TYPES` + +```json +[ + { + "name": "nano", + "cpu": 1, + "memory": "1GB", + "count": 3 + }, + { + "name": "micro", + "cpu": 2, + "memory": "2GB", + "count": 1 + }, + ... +] +``` + +- `LXD_MULTI_IMAGE_ALIAS` + - Image to pool + +### Optional values + +- `LXD_SOCKET` + - Path to LXD socket + - default: `/var/lib/lxd/unix.socket` +- `LXD_MULTI_CHECK_INTERVAL` + - Interval to check instances + - default: `2s` +- `LXD_MULTI_CONCURRENT_CREATE_LIMIT` + - Limit concurrency for creating instance + - default: `3` +- `LXD_MULTI_WAIT_IDLE_TIME` + - Duration to wait instance idle after `systemctl is-system-running --wait` + - default: `5s` +- `LXD_MULTI_ZOMBIE_ALLOW_TIME` + - Duration to delete zombie instances after instance created + - default: `5m` diff --git a/pool-agent/agent.go b/pool-agent/agent.go new file mode 100644 index 0000000..190bad7 --- /dev/null +++ b/pool-agent/agent.go @@ -0,0 +1,201 @@ +package main + +import ( + "context" + "crypto/rand" + "fmt" + "log" + "sync" + "time" + + lxd "github.com/lxc/lxd/client" + "github.com/lxc/lxd/shared/api" + "golang.org/x/sync/semaphore" +) + +type Agent struct { + ImageAlias string + InstanceSource api.InstanceSource + + ResourceTypes []ResourceType + Client lxd.InstanceServer + + CheckInterval time.Duration + ConcurrentCreateLimit int64 + WaitIdleTime time.Duration + ZombieAllowTime time.Duration + + wg *sync.WaitGroup + createLimit *semaphore.Weighted + creatingInstances map[string]map[string]struct{} + deletingInstances map[string]struct{} +} + +func (a *Agent) Run(ctx context.Context) error { + ticker := time.NewTicker(a.CheckInterval) + + a.wg = new(sync.WaitGroup) + a.createLimit = semaphore.NewWeighted(a.ConcurrentCreateLimit) + a.deletingInstances = make(map[string]struct{}) + + a.creatingInstances = make(map[string]map[string]struct{}, len(a.ResourceTypes)) + for _, rt := range a.ResourceTypes { + a.creatingInstances[rt.Name] = make(map[string]struct{}) + } + + log.Printf("Started agent") + + for { + select { + case <-ticker.C: + if err := a.checkInstances(ctx); err != nil { + log.Printf("failed to check instances: %+v", err) + } + case <-ctx.Done(): + log.Printf("Stopping agent...") + a.wg.Wait() + return ctx.Err() + } + } +} + +func (a *Agent) countPooledInstances(instances []api.Instance, resourceTypeName string) int { + var count int + for _, i := range instances { + if i.StatusCode != api.Frozen { + continue + } + if i.Config[configKeyImageAlias] != a.ImageAlias { + continue + } + if i.Config[configKeyResourceType] != resourceTypeName { + continue + } + if _, ok := i.Config[configKeyRunnerName]; ok { + continue + } + count++ + } + return count +} + +func (a *Agent) generateInstanceName() (string, error) { + var b [4]byte + _, err := rand.Read(b[:]) + if err != nil { + return "", fmt.Errorf("generate random id: %w", err) + } + return fmt.Sprintf("myshoes-runner-%x", b), nil +} + +func (a *Agent) checkInstances(ctx context.Context) error { + s, err := a.Client.GetInstances(api.InstanceTypeAny) + if err != nil { + return fmt.Errorf("get instances: %w", err) + } + + for _, rt := range a.ResourceTypes { + current := a.countPooledInstances(s, rt.Name) + creating := len(a.creatingInstances[rt.Name]) + createCount := rt.PoolCount - current - creating + if createCount < 1 { + continue + } + log.Printf("Create %d instances for %q", createCount, rt.Name) + for i := 0; i < createCount; i++ { + name, err := a.generateInstanceName() + if err != nil { + return fmt.Errorf("generate instance name: %w", err) + } + a.creatingInstances[rt.Name][name] = struct{}{} + go func(name string, rt ResourceType) { + a.createLimit.Acquire(context.Background(), 1) + defer a.createLimit.Release(1) + + defer delete(a.creatingInstances[rt.Name], name) + + a.wg.Add(1) + defer a.wg.Done() + select { + case <-ctx.Done(): + // context cancelled, stop creating immediately + return + default: + // context is not cancelled, continue + } + + if err := a.createInstance(name, rt); err != nil { + log.Printf("failed to create instance %q: %+v", name, err) + } + }(name, rt) + } + } + + for _, i := range s { + if a.isZombieInstance(i) { + if _, ok := a.deletingInstances[i.Name]; ok { + continue + } + log.Printf("Deleting zombie instance %q...", i.Name) + a.deletingInstances[i.Name] = struct{}{} + a.wg.Add(1) + go func(i api.Instance) { + defer a.wg.Done() + defer delete(a.deletingInstances, i.Name) + + if err := a.deleteZombieInstance(i); err != nil { + log.Printf("failed to delete zombie instance %q: %+v", i.Name, err) + } + log.Printf("Deleted zombie instance %q", i.Name) + }(i) + } + } + + return nil +} + +func (a *Agent) isZombieInstance(i api.Instance) bool { + if i.StatusCode == api.Frozen { + return false + } + if _, ok := i.Config[configKeyRunnerName]; ok { + return false + } + if i.Config[configKeyImageAlias] != a.ImageAlias { + return false + } + if i.CreatedAt.Add(a.ZombieAllowTime).After(time.Now()) { + return false + } + if rt, ok := i.Config[configKeyResourceType]; !ok { + return false + } else if _, ok := a.creatingInstances[rt][i.Name]; ok { + return false + } + return true +} + +func (a *Agent) deleteZombieInstance(i api.Instance) error { + if i.StatusCode == api.Running { + op, err := a.Client.UpdateInstanceState(i.Name, api.InstanceStatePut{ + Action: "stop", + Timeout: -1, + }, "") + if err != nil { + return fmt.Errorf("stop: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("stop operation: %w", err) + } + } + + op, err := a.Client.DeleteInstance(i.Name) + if err != nil { + return fmt.Errorf("delete: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("delete operation: %w", err) + } + + return nil +} diff --git a/pool-agent/config.go b/pool-agent/config.go new file mode 100644 index 0000000..e19b9f2 --- /dev/null +++ b/pool-agent/config.go @@ -0,0 +1,83 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "time" + + "github.com/lxc/lxd/shared/api" + slm "github.com/whywaita/shoes-lxd-multi/server/pkg/api" +) + +type ResourceType struct { + Name string `json:"name"` + + CPUCore int `json:"cpu"` + Memory string `json:"memory"` + + PoolCount int `json:"count"` +} + +func LoadResourceTypes() ([]ResourceType, error) { + env := os.Getenv("LXD_MULTI_RESOURCE_TYPES") + if env == "" { + return nil, fmt.Errorf("LXD_MULTI_RESOURCE_TYPES is not set") + } + var s []ResourceType + if err := json.Unmarshal([]byte(env), &s); err != nil { + return nil, fmt.Errorf("parse LXD_MULTI_RESOURCE_TYPES: %w", err) + } + return s, nil +} + +func LoadImageAlias() (string, api.InstanceSource, error) { + env := os.Getenv("LXD_MULTI_IMAGE_ALIAS") + if env == "" { + return "", api.InstanceSource{}, fmt.Errorf("LXD_MULTI_IMAGE_ALIAS is not set") + } + source, err := slm.ParseAlias(env) + if err != nil { + return "", api.InstanceSource{}, fmt.Errorf("parse LXD_MULTI_IMAGE_ALIAS: %w", err) + } + return env, *source, nil +} + +func LoadParams() (checkInterval time.Duration, concurrentCreateLimit int64, waitIdleTime time.Duration, zombieAllowTime time.Duration, err error) { + checkInterval, err = loadDurationEnv("LXD_MULTI_CHECK_INTERVAL", 2*time.Second) + if err != nil { + return + } + waitIdleTime, err = loadDurationEnv("LXD_MULTI_WAIT_IDLE_TIME", 5*time.Second) + if err != nil { + return + } + zombieAllowTime, err = loadDurationEnv("LXD_MULTI_ZOMBIE_ALLOW_TIME", 5*time.Minute) + if err != nil { + return + } + + if env := os.Getenv("LXD_MULTI_CONCURRENT_CREATE_LIMIT"); env != "" { + concurrentCreateLimit, err = strconv.ParseInt(env, 10, 64) + if err != nil { + return + } + } else { + concurrentCreateLimit = 3 + } + + return +} + +func loadDurationEnv(name string, def time.Duration) (time.Duration, error) { + env := os.Getenv(name) + if env == "" { + return def, nil + } + d, err := time.ParseDuration(env) + if err != nil { + return 0, fmt.Errorf("parse %s: %w", name, err) + } + return d, nil +} diff --git a/pool-agent/config_keys.go b/pool-agent/config_keys.go new file mode 100644 index 0000000..16526a2 --- /dev/null +++ b/pool-agent/config_keys.go @@ -0,0 +1,7 @@ +package main + +const ( + configKeyResourceType = "user.myshoes_resource_type" + configKeyImageAlias = "user.myshoes_image_alias" + configKeyRunnerName = "user.myshoes_runner_name" +) diff --git a/pool-agent/create.go b/pool-agent/create.go new file mode 100644 index 0000000..f159925 --- /dev/null +++ b/pool-agent/create.go @@ -0,0 +1,110 @@ +package main + +import ( + "fmt" + "log" + "strconv" + "strings" + "time" + + "github.com/lxc/lxd/shared/api" +) + +func (a *Agent) createInstance(name string, rt ResourceType) error { + log.Printf("Creating instance %q...", name) + op, err := a.Client.CreateInstance(api.InstancesPost{ + Name: name, + InstancePut: api.InstancePut{ + Config: map[string]string{ + "limits.cpu": strconv.Itoa(rt.CPUCore), + "limits.memory": rt.Memory, + "security.nesting": "true", + "security.privileged": "true", + "raw.lxc": strings.Join([]string{ + "lxc.apparmor.profile = unconfined", + "lxc.cgroup.devices.allow = a", + "lxc.cap.drop=", + }, "\n"), + configKeyImageAlias: a.ImageAlias, + configKeyResourceType: rt.Name, + }, + Devices: map[string]map[string]string{ + "kmsg": { + "path": "/dev/kmsg", + "source": "/dev/kmsg", + "type": "unix-char", + }, + }, + }, + Source: a.InstanceSource, + }) + if err != nil { + return fmt.Errorf("create: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("create operation: %w", err) + } + + log.Printf("Starting instance %q...", name) + op, err = a.Client.UpdateInstanceState(name, api.InstanceStatePut{ + Action: "start", + Timeout: -1, + }, "") + if err != nil { + return fmt.Errorf("start: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("start operation: %w", err) + } + + log.Printf("Waiting system bus in instance %q...", name) + op, err = a.Client.ExecInstance(name, api.InstanceExecPost{ + Command: []string{"bash", "-c", "until test -e /var/run/dbus/system_bus_socket; do sleep 0.5; done"}, + }, nil) + if err != nil { + return fmt.Errorf("wait system bus: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("wait system bus operation: %w", err) + } + + log.Printf("Waiting system running for instance %q...", name) + op, err = a.Client.ExecInstance(name, api.InstanceExecPost{ + Command: []string{"systemctl", "is-system-running", "--wait"}, + }, nil) + if err != nil { + return fmt.Errorf("wait system running: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("wait system running operation: %w", err) + } + + log.Printf("Disabling systemd service watchdogs in instance %q...", name) + op, err = a.Client.ExecInstance(name, api.InstanceExecPost{ + Command: []string{"systemctl", "service-watchdogs", "no"}, + }, nil) + if err != nil { + return fmt.Errorf("disable systemd service watchdogs: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("disable systemd service watchdogs operation: %w", err) + } + + log.Printf("Waiting for instance %q idle...", name) + time.Sleep(a.WaitIdleTime) + + log.Printf("Freezing instance %q...", name) + op, err = a.Client.UpdateInstanceState(name, api.InstanceStatePut{ + Action: "freeze", + Timeout: -1, + }, "") + if err != nil { + return fmt.Errorf("freeze: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("freeze operation: %w", err) + } + + log.Printf("Created instance %q successfully", name) + return nil +} diff --git a/pool-agent/go.mod b/pool-agent/go.mod new file mode 100644 index 0000000..fd5a639 --- /dev/null +++ b/pool-agent/go.mod @@ -0,0 +1,60 @@ +module github.com/whywaita/shoes-lxd-multi/pool-agent + +go 1.22.0 + +toolchain go1.22.2 + +require ( + github.com/lxc/lxd v0.0.0-20211202222358-a293da71aeb0 + github.com/whywaita/shoes-lxd-multi/server v0.0.0-20240416053912-6df6153e11ef + golang.org/x/sync v0.6.0 +) + +require ( + github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/fatih/color v1.7.0 // indirect + github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 // indirect + github.com/go-macaroon-bakery/macaroonpb v1.0.0 // indirect + github.com/golang-jwt/jwt/v4 v4.0.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-github/v41 v41.0.0 // indirect + github.com/google/go-github/v47 v47.1.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/go-hclog v0.14.1 // indirect + github.com/hashicorp/go-plugin v1.4.3 // indirect + github.com/hashicorp/go-version v1.4.0 // indirect + github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect + github.com/juju/webbrowser v1.0.0 // indirect + github.com/julienschmidt/httprouter v1.3.0 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/m4ns0ur/httpcache v0.0.0-20200426190423-1040e2e8823f // indirect + github.com/mattn/go-colorable v0.1.11 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect + github.com/oklog/run v1.0.0 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pborman/uuid v1.2.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rogpeppe/fastuuid v1.2.0 // indirect + github.com/satori/go.uuid v1.2.0 // indirect + github.com/whywaita/myshoes v1.14.0 // indirect + github.com/whywaita/shoes-lxd-multi/proto.go v0.0.0-20230331051154-d763b94b0dd7 // indirect + github.com/whywaita/xsemaphore v0.0.0-20240305080042-cf6ba671d2e7 // indirect + golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/oauth2 v0.4.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230330200707-38013875ee22 // indirect + google.golang.org/grpc v1.54.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/errgo.v1 v1.0.1 // indirect + gopkg.in/httprequest.v1 v1.2.1 // indirect + gopkg.in/macaroon-bakery.v2 v2.3.0 // indirect + gopkg.in/macaroon.v2 v2.1.0 // indirect +) diff --git a/pool-agent/go.sum b/pool-agent/go.sum new file mode 100644 index 0000000..2acfe4b --- /dev/null +++ b/pool-agent/go.sum @@ -0,0 +1,271 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 h1:tXKVfhE7FcSkhkv0UwkLvPDeZ4kz6OXd0PKPlFqf81M= +github.com/bradleyfalzon/ghinstallation/v2 v2.0.4/go.mod h1:B40qPqJxWE0jDZgOR1JmaMy+4AY1eBP+IByOvqyAKp0= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 h1:fmFk0Wt3bBxxwZnu48jqMdaOR/IZ4vdtJFuaFV8MpIE= +github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3/go.mod h1:bJWSKrZyQvfTnb2OudyUjurSG4/edverV7n82+K3JiM= +github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= +github.com/frankban/quicktest v1.1.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= +github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= +github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/go-macaroon-bakery/macaroonpb v1.0.0 h1:It9exBaRMZ9iix1iJ6gwzfwsDE6ExNuwtAJ9e09v6XE= +github.com/go-macaroon-bakery/macaroonpb v1.0.0/go.mod h1:UzrGOcbiwTXISFP2XDLDPjfhMINZa+fX/7A2lMd31zc= +github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg= +github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg= +github.com/google/go-github/v47 v47.1.0 h1:Cacm/WxQBOa9lF0FT0EMjZ2BWMetQ1TQfyurn4yF1z8= +github.com/google/go-github/v47 v47.1.0/go.mod h1:VPZBXNbFSJGjyjFRUKo9vZGawTajnWzC/YjGw/oFKi0= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/go-hclog v0.14.1 h1:nQcJDQwIAGnmoUWp8ubocEX40cCml/17YkF6csQLReU= +github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM= +github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= +github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4= +github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= +github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= +github.com/juju/mgotest v1.0.1/go.mod h1:vTaDufYul+Ps8D7bgseHjq87X8eu0ivlKLp9mVc/Bfc= +github.com/juju/postgrestest v1.1.0/go.mod h1:/n17Y2T6iFozzXwSCO0JYJ5gSiz2caEtSwAjh/uLXDM= +github.com/juju/qthttptest v0.0.1/go.mod h1://LCf/Ls22/rPw2u1yWukUJvYtfPY4nYpWUl2uZhryo= +github.com/juju/qthttptest v0.1.1 h1:JPju5P5CDMCy8jmBJV2wGLjDItUsx2KKL514EfOYueM= +github.com/juju/qthttptest v0.1.1/go.mod h1:aTlAv8TYaflIiTDIQYzxnl1QdPjAg8Q8qJMErpKy6A4= +github.com/juju/schema v1.0.0/go.mod h1:Y+ThzXpUJ0E7NYYocAbuvJ7vTivXfrof/IfRPq/0abI= +github.com/juju/webbrowser v0.0.0-20160309143629-54b8c57083b4/go.mod h1:G6PCelgkM6cuvyD10iYJsjLBsSadVXtJ+nBxFAxE2BU= +github.com/juju/webbrowser v1.0.0 h1:JLdmbFtCGY6Qf2jmS6bVaenJFGIFkdF1/BjUm76af78= +github.com/juju/webbrowser v1.0.0/go.mod h1:RwVlbBcF91Q4vS+iwlkJ6bZTE3EwlrjbYlM3WMVD6Bc= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lxc/lxd v0.0.0-20211202222358-a293da71aeb0 h1:sgUGXZalzqdRNdl8Vvz3n0M8sPepVkMGHblztsYG6AE= +github.com/lxc/lxd v0.0.0-20211202222358-a293da71aeb0/go.mod h1:fSXF962PCUBIAFgaxF7FI7djUU7rjaQl5k4tCVT1ebE= +github.com/m4ns0ur/httpcache v0.0.0-20200426190423-1040e2e8823f h1:MBcrTbmCf7CZa9yAwcB7ArveQb9TPVy4zFnQGz/LiUU= +github.com/m4ns0ur/httpcache v0.0.0-20200426190423-1040e2e8823f/go.mod h1:UawoqorwkpZ58qWiL+nVJM0Po7FrzAdCxYVh9GgTTaA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 h1:7GoSOOW2jpsfkntVKaS2rAr1TJqfcxotyaUcuxoZSzg= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/whywaita/myshoes v1.14.0 h1:wIJoInogzSErsdVEwiKjWJuqXzg0mKpxxf5YKeaT334= +github.com/whywaita/myshoes v1.14.0/go.mod h1:Fea/XeUlwjWSZdqfG/Ub3+U278kGilIG9o9f6DxRpq8= +github.com/whywaita/shoes-lxd-multi/proto.go v0.0.0-20230331051154-d763b94b0dd7 h1:MdNKtHc/T+46wn7JfEi5P72Q/GHVPHLInqLldZB4eWE= +github.com/whywaita/shoes-lxd-multi/proto.go v0.0.0-20230331051154-d763b94b0dd7/go.mod h1:2Z9+TYX1eQMUrCOnFBG71JaVcq5D8enmYJH6rnCWW88= +github.com/whywaita/shoes-lxd-multi/server v0.0.0-20240416053912-6df6153e11ef h1:QxSdADc/MvMpCIKUrPnnFRCt/uFY9ah5X3/3/FNB5qo= +github.com/whywaita/shoes-lxd-multi/server v0.0.0-20240416053912-6df6153e11ef/go.mod h1:HKTConnuygXlLPUtIjZZ5z24nrEpuju5SwwwB60nt7k= +github.com/whywaita/xsemaphore v0.0.0-20240305080042-cf6ba671d2e7 h1:1dG8mw/qyLMKtzfkdQnsZABm07YXxOfPI+2GRIXileo= +github.com/whywaita/xsemaphore v0.0.0-20240305080042-cf6ba671d2e7/go.mod h1:DxG5uRIYgWQPhHggrUwSxLdCzhmKD91mUZYxq/det6o= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20150829230318-ea47fc708ee3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181008205924-a2b3f7f249e9/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230330200707-38013875ee22 h1:n3ThVoQnHbCbnkhZZ1fx3+3fBAisViSwrpbtLV7vydY= +google.golang.org/genproto v0.0.0-20230330200707-38013875ee22/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk= +gopkg.in/errgo.v1 v1.0.1 h1:oQFRXzZ7CkBGdm1XZm/EbQYaYNNEElNBOd09M6cqNso= +gopkg.in/errgo.v1 v1.0.1/go.mod h1:3NjfXwocQRYAPTq4/fzX+CwUhPRcR/azYRhj8G+LqMo= +gopkg.in/httprequest.v1 v1.2.0/go.mod h1:T61ZUaJLpMnzvoJDO03ZD8yRXD4nZzBeDoW5e9sffjg= +gopkg.in/httprequest.v1 v1.2.1 h1:pEPLMdF/gjWHnKxLpuCYaHFjc8vAB2wrYjXrqDVC16E= +gopkg.in/httprequest.v1 v1.2.1/go.mod h1:x2Otw96yda5+8+6ZeWwHIJTFkEHWP/qP8pJOzqEtWPM= +gopkg.in/juju/environschema.v1 v1.0.0/go.mod h1:WTgU3KXKCVoO9bMmG/4KHzoaRvLeoxfjArpgd1MGWFA= +gopkg.in/macaroon-bakery.v2 v2.3.0 h1:b40knPgPTke1QLTE8BSYeH7+R/hiIozB1A8CTLYN0Ic= +gopkg.in/macaroon-bakery.v2 v2.3.0/go.mod h1:/8YhtPARXeRzbpEPLmRB66+gQE8/pzBBkWwg7Vz/guc= +gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= +gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pool-agent/main.go b/pool-agent/main.go new file mode 100644 index 0000000..7ccd1ea --- /dev/null +++ b/pool-agent/main.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "log" + "os/signal" + "syscall" + + lxd "github.com/lxc/lxd/client" +) + +func main() { + resourceTypes, err := LoadResourceTypes() + if err != nil { + log.Fatal(err) + } + + imageAlias, source, err := LoadImageAlias() + if err != nil { + log.Fatal(err) + } + + checkInterval, concurrentCreateLimit, waitIdleTime, zombieAllowTime, err := LoadParams() + if err != nil { + log.Fatal(err) + } + + c, err := lxd.ConnectLXDUnix("", nil) + if err != nil { + log.Fatalf("failed to connect lxd: %+v", err) + } + + agent := &Agent{ + ImageAlias: imageAlias, + InstanceSource: source, + + ResourceTypes: resourceTypes, + Client: c, + + CheckInterval: checkInterval, + ConcurrentCreateLimit: concurrentCreateLimit, + WaitIdleTime: waitIdleTime, + ZombieAllowTime: zombieAllowTime, + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + if err := agent.Run(ctx); err != nil && err != context.Canceled { + log.Fatal(err) + } +} diff --git a/server/main.go b/server/main.go index 2b6d629..5fc6151 100644 --- a/server/main.go +++ b/server/main.go @@ -29,7 +29,7 @@ func main() { func run() error { ctx := context.Background() - hostConfigs, mapping, periodSec, listenPort, overCommitPercent, err := config.Load() + hostConfigs, mapping, periodSec, listenPort, overCommitPercent, poolMode, err := config.Load() if err != nil { return fmt.Errorf("failed to create server: %w", err) } @@ -44,7 +44,7 @@ func run() error { }) go resourcecache.RunLXDResourceCacheTicker(ctx, hcs, periodSec) - server, err := api.New(hostConfigs, mapping, overCommitPercent) + server, err := api.New(hostConfigs, mapping, overCommitPercent, poolMode) if err != nil { return fmt.Errorf("failed to create server: %w", err) } diff --git a/server/pkg/api/pool.go b/server/pkg/api/pool.go new file mode 100644 index 0000000..94522d7 --- /dev/null +++ b/server/pkg/api/pool.go @@ -0,0 +1,229 @@ +package api + +import ( + "errors" + "fmt" + "log" + "math/rand" + "sort" + "strconv" + "sync" + "time" + + lxd "github.com/lxc/lxd/client" + "github.com/lxc/lxd/shared/api" + + "github.com/whywaita/shoes-lxd-multi/server/pkg/lxdclient" +) + +const ( + configKeyResourceType = "user.myshoes_resource_type" + configKeyImageAlias = "user.myshoes_image_alias" + configKeyRunnerName = "user.myshoes_runner_name" + configKeyAllocatedAt = "user.myshoes_allocated_at" +) + +func getInstancesWithTimeout(c lxd.InstanceServer, d time.Duration) (s []api.Instance, overCommitPercent uint64, err error) { + done := make(chan struct{}) + go func() { + defer close(done) + s, err = c.GetInstances(api.InstanceTypeAny) + if err != nil { + return + } + var r *api.Resources + r, err = c.GetServerResources() + if err != nil { + return + } + + var used uint64 + for _, i := range s { + if i.StatusCode != api.Running { + continue + } + instanceCPU := i.Config["limits.cpu"] + if instanceCPU == "" { + continue + } + cpu, err := strconv.Atoi(i.Config["limits.cpu"]) + if err != nil { + err = fmt.Errorf("failed to parse limits.cpu: %w", err) + return + } + used += uint64(cpu) + } + overCommitPercent = uint64(float64(used) / float64(r.CPU.Total) * 100) + }() + select { + case <-done: + return + case <-time.After(d): + return nil, 0, errors.New("timed out") + } +} + +type instance struct { + Host *lxdclient.LXDHost + InstanceName string +} + +func findInstances(targets []lxdclient.LXDHost, match func(api.Instance) bool, limitOverCommit uint64) []instance { + type result struct { + host *lxdclient.LXDHost + overCommitPercent uint64 + instances []string + } + rs := make([]result, len(targets)) + + wg := new(sync.WaitGroup) + wg.Add(len(targets)) + for i, target := range targets { + go func(i int, target lxdclient.LXDHost) { + defer wg.Done() + + s, overCommitPercent, err := getInstancesWithTimeout(target.Client, 2*time.Second) + if err != nil { + log.Printf("failed to find instance in host %q: %+v", target.HostConfig.LxdHost, err) + return + } + if limitOverCommit > 0 && overCommitPercent >= limitOverCommit { + log.Printf("host %q reached over commit limit: current=%d limit=%d", target.HostConfig.LxdHost, overCommitPercent, limitOverCommit) + return + } + + var instances []string + for _, i := range s { + if match(i) { + instances = append(instances, i.Name) + } + } + + // Shuffle instances to reduce conflicting + rand.Shuffle(len(instances), func(i, j int) { + instances[i], instances[j] = instances[j], instances[i] + }) + + rs[i] = result{ + host: &target, + overCommitPercent: overCommitPercent, + instances: instances, + } + }(i, target) + } + wg.Wait() + + sort.Slice(rs, func(i, j int) bool { + return rs[i].overCommitPercent < rs[j].overCommitPercent + }) + + var instances []instance + for _, r := range rs { + for _, i := range r.instances { + instances = append(instances, instance{ + Host: r.host, + InstanceName: i, + }) + } + } + + return instances +} + +func findInstanceByJob(targets []lxdclient.LXDHost, runnerName string) (*lxdclient.LXDHost, string, bool) { + s := findInstances(targets, func(i api.Instance) bool { + return i.Config[configKeyRunnerName] == runnerName + }, 0) + if len(s) < 1 { + return nil, "", false + } + return s[0].Host, s[0].InstanceName, true +} + +func allocatePooledInstance(targets []lxdclient.LXDHost, resourceType, imageAlias string, limitOverCommit uint64, runnerName string) (*lxdclient.LXDHost, string, error) { + s := findInstances(targets, func(i api.Instance) bool { + if i.StatusCode != api.Frozen { + return false + } + if i.Config[configKeyResourceType] != resourceType { + return false + } + if i.Config[configKeyImageAlias] != imageAlias { + return false + } + if _, ok := i.Config[configKeyRunnerName]; ok { + return false + } + return true + }, limitOverCommit) + + for _, i := range s { + if err := allocateInstance(*i.Host, i.InstanceName, runnerName); err != nil { + log.Printf("failed to allocate instance %q in host %q (trying another instance): %+v", i.InstanceName, i.Host.HostConfig.LxdHost, err) + continue + } + return i.Host, i.InstanceName, nil + } + + return nil, "", fmt.Errorf("no available instance for resource_type=%q image_alias=%q", resourceType, imageAlias) +} + +func allocateInstance(host lxdclient.LXDHost, instanceName, runnerName string) error { + i, etag, err := host.Client.GetInstance(instanceName) + if err != nil { + return fmt.Errorf("get instance: %w", err) + } + + if _, ok := i.Config[configKeyRunnerName]; ok { + return fmt.Errorf("already allocated instance %q in host %q", instanceName, host.HostConfig.LxdHost) + } + + log.Printf("Allocating %q to %q", instanceName, runnerName) + + i.InstancePut.Config[configKeyRunnerName] = runnerName + i.InstancePut.Config[configKeyAllocatedAt] = time.Now().UTC().Format(time.RFC3339Nano) + + op, err := host.Client.UpdateInstance(instanceName, i.InstancePut, etag) + if err != nil { + return fmt.Errorf("update instance: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("waiting operation: %w", err) + } + + // Workaround for https://github.com/canonical/lxd/issues/12189 + i, _, err = host.Client.GetInstance(instanceName) + if err != nil { + return fmt.Errorf("get instance: %w", err) + } + if i.Config[configKeyRunnerName] != runnerName { + return fmt.Errorf("updated instance config mismatch: got=%q expected=%q", i.Config[configKeyRunnerName], runnerName) + } + + return nil +} + +func unfreezeInstance(c lxd.InstanceServer, name string) error { + state, etag, err := c.GetInstanceState(name) + if err != nil { + return fmt.Errorf("get instance state: %w", err) + } + switch state.StatusCode { + case api.Running: + // do nothing + case api.Frozen: + op, err := c.UpdateInstanceState(name, api.InstanceStatePut{ + Action: "unfreeze", + Timeout: -1, + }, etag) + if err != nil { + return fmt.Errorf("update instance state: %w", err) + } + if err := op.Wait(); err != nil { + return fmt.Errorf("waiting operation: %w", err) + } + default: + return fmt.Errorf("unexpected instance state: %s", state.StatusCode.String()) + } + return nil +} diff --git a/server/pkg/api/server.go b/server/pkg/api/server.go index 3d27ada..a5dc41c 100644 --- a/server/pkg/api/server.go +++ b/server/pkg/api/server.go @@ -22,16 +22,18 @@ type ShoesLXDMultiServer struct { overCommitPercent uint64 - mu sync.Mutex + mu sync.Mutex + poolMode bool } // New create gRPC server -func New(hostConfigs *config.HostConfigMap, mapping map[myshoespb.ResourceType]config.Mapping, overCommitPercent uint64) (*ShoesLXDMultiServer, error) { +func New(hostConfigs *config.HostConfigMap, mapping map[myshoespb.ResourceType]config.Mapping, overCommitPercent uint64, poolMode bool) (*ShoesLXDMultiServer, error) { return &ShoesLXDMultiServer{ hostConfigs: hostConfigs, resourceMapping: mapping, overCommitPercent: overCommitPercent, mu: sync.Mutex{}, + poolMode: poolMode, }, nil } diff --git a/server/pkg/api/server_add_instance.go b/server/pkg/api/server_add_instance.go index cebddf3..8aa773c 100644 --- a/server/pkg/api/server_add_instance.go +++ b/server/pkg/api/server_add_instance.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "sync" + "time" myshoespb "github.com/whywaita/myshoes/api/proto.go" @@ -18,6 +19,7 @@ import ( lxd "github.com/lxc/lxd/client" "github.com/lxc/lxd/shared/api" + "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/runner" pb "github.com/whywaita/shoes-lxd-multi/proto.go" "github.com/whywaita/shoes-lxd-multi/server/pkg/lxdclient" @@ -35,42 +37,74 @@ func (s *ShoesLXDMultiServer) AddInstance(ctx context.Context, req *pb.AddInstan if _, err := runner.ToUUID(req.RunnerName); err != nil { return nil, status.Errorf(codes.InvalidArgument, "failed to parse request name: %+v", err) } - instanceName := req.RunnerName - l = l.With("runnerName", instanceName) + l = l.With("runnerName", req.RunnerName) - instanceSource, err := ParseAlias(req.ImageAlias) + targetLXDHosts, err := s.validateTargetHosts(req.TargetHosts, l) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "failed to parse image alias: %+v", err) + return nil, status.Errorf(codes.InvalidArgument, "failed to validate target hosts: %+v", err) } - targetLXDHosts, err := s.validateTargetHosts(req.TargetHosts, l) + var host *lxdclient.LXDHost + var instanceName string + + if s.poolMode { + host, instanceName, err = s.addInstancePoolMode(targetLXDHosts, req, l) + if err != nil { + return nil, err + } + } else { + host, instanceName, err = s.addInstanceCreateMode(ctx, targetLXDHosts, req, l) + if err != nil { + return nil, err + } + } + i, _, err := host.Client.GetInstance(instanceName) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "failed to validate target hosts: %+v", err) + return nil, status.Errorf(codes.Internal, "failed to retrieve instance information: %+v", err) + } + + l.Info("Success AddInstance", "host", host.HostConfig.LxdHost, "runnerName", i.Name) + + return &pb.AddInstanceResponse{ + CloudId: i.Name, + ShoesType: "lxd", + IpAddress: "", + ResourceType: req.ResourceType, + }, nil +} + +func (s *ShoesLXDMultiServer) addInstanceCreateMode(ctx context.Context, targetLXDHosts []lxdclient.LXDHost, req *pb.AddInstanceRequest, l *slog.Logger) (*lxdclient.LXDHost, string, error) { + instanceName := req.RunnerName + + instanceSource, err := ParseAlias(req.ImageAlias) + if err != nil { + return nil, "", status.Errorf(codes.InvalidArgument, "failed to parse image alias: %+v", err) } host, err := s.isExistInstance(targetLXDHosts, instanceName, l) if err != nil && !errors.Is(err, ErrInstanceIsNotFound) { - return nil, status.Errorf(codes.Internal, "failed to get instance: %+v", err) + return nil, "", status.Errorf(codes.Internal, "failed to get instance: %+v", err) } var client lxd.InstanceServer - var reqInstance *api.InstancesPost if errors.Is(err, ErrInstanceIsNotFound) { + var reqInstance *api.InstancesPost host, reqInstance, err = s.setLXDStatusCache(ctx, targetLXDHosts, instanceName, instanceSource, req) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to set LXD status cache: %+v", err) + return nil, "", status.Errorf(codes.Internal, "failed to set LXD status cache: %+v", err) } client = host.Client op, err := client.CreateInstance(*reqInstance) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to create instance: %+v", err) + return nil, "", status.Errorf(codes.Internal, "failed to create instance: %+v", err) } if err := op.Wait(); err != nil { - return nil, status.Errorf(codes.Internal, "failed to wait creating instance: %+v", err) + return nil, "", status.Errorf(codes.Internal, "failed to wait creating instance: %+v", err) } } else { client = host.Client } + l = l.With("host", host.HostConfig.LxdHost) reqState := api.InstanceStatePut{ @@ -84,13 +118,13 @@ func (s *ShoesLXDMultiServer) AddInstance(ctx context.Context, req *pb.AddInstan op, err := client.DeleteInstance(instanceName) if err != nil { l.Warn("failed to delete instance", "err", err.Error()) - return nil, status.Errorf(codes.Internal, "failed to wait starting instance: %+v", err) + return nil, "", status.Errorf(codes.Internal, "failed to wait starting instance: %+v", err) } if err := op.Wait(); err != nil { l.Warn("failed to wait deleting instance", "err", err.Error()) } - return nil, status.Errorf(codes.Internal, "failed to start instance: %+v", err) + return nil, "", status.Errorf(codes.Internal, "failed to start instance: %+v", err) } if err := op.Wait(); err != nil && !strings.EqualFold(err.Error(), "The instance is already running") { // Do rollback @@ -98,27 +132,76 @@ func (s *ShoesLXDMultiServer) AddInstance(ctx context.Context, req *pb.AddInstan op, err := client.DeleteInstance(instanceName) if err != nil { l.Warn("failed to delete instance", "err", err.Error()) - return nil, status.Errorf(codes.Internal, "failed to wait starting instance: %+v", err) + return nil, "", status.Errorf(codes.Internal, "failed to wait starting instance: %+v", err) } if err := op.Wait(); err != nil { l.Warn("failed to wait deleting instance", "err", err.Error()) } - return nil, status.Errorf(codes.Internal, "failed to wait starting instance: %+v", err) + return nil, "", status.Errorf(codes.Internal, "failed to wait starting instance: %+v", err) } - i, _, err := client.GetInstance(instanceName) + return host, instanceName, nil +} + +func (s *ShoesLXDMultiServer) addInstancePoolMode(targets []lxdclient.LXDHost, req *pb.AddInstanceRequest, l *slog.Logger) (*lxdclient.LXDHost, string, error) { + host, instanceName, found := findInstanceByJob(targets, req.RunnerName) + if !found { + resourceTypeName := datastore.UnmarshalResourceTypePb(req.ResourceType).String() + retried := 0 + for { + var err error + host, instanceName, err = allocatePooledInstance(targets, resourceTypeName, req.ImageAlias, s.overCommitPercent, req.RunnerName) + if err != nil { + if retried < 10 { + retried++ + l.Info("AddInstance failed allocating instance", "retrying", retried, "err", err.Error()) + time.Sleep(1 * time.Second) + continue + } else { + return nil, "", status.Errorf(codes.Internal, "can not allocate instance") + } + } + break + } + } + l.Info("AddInstance for pool mode", "runnerName", instanceName, "host", host.HostConfig.LxdHost) + client := host.Client + + err := unfreezeInstance(client, instanceName) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to retrieve instance information: %+v", err) + return nil, "", status.Errorf(codes.Internal, "failed to unfreeze instance: %+v", err) } - l.Info("Success AddInstance", "host", host.HostConfig.LxdHost, "runnerName", i.Name) - return &pb.AddInstanceResponse{ - CloudId: i.Name, - ShoesType: "lxd", - IpAddress: "", - ResourceType: req.ResourceType, - }, nil + scriptFilename := fmt.Sprintf("/tmp/myshoes_setup_script.%d", rand.Int()) + err = client.CreateInstanceFile(instanceName, scriptFilename, lxd.InstanceFileArgs{ + Content: strings.NewReader(req.SetupScript), + Mode: 0744, + Type: "file", + WriteMode: "overwrite", + }) + if err != nil { + return nil, "", status.Errorf(codes.Internal, "failed to copy setup script: %+v", err) + } + op, err := client.ExecInstance(instanceName, api.InstanceExecPost{ + Command: []string{ + "systemd-run", + "--unit", "myshoes-setup", + "--property", "After=multi-user.target", + "--property", "StandardOutput=journal+console", + "--property", fmt.Sprintf("ExecStartPre=/usr/bin/hostnamectl set-hostname %s", req.RunnerName), + "--property", fmt.Sprintf("ExecStartPre=/bin/sh -c 'echo 127.0.1.1 %s >> /etc/hosts'", req.RunnerName), + scriptFilename, + }, + }, nil) + if err != nil { + return nil, "", status.Errorf(codes.Internal, "failed to execute setup script: %+v", err) + } + if err := op.Wait(); err != nil { + return nil, "", status.Errorf(codes.Internal, "failed to wait executing setup script: %+v", err) + } + + return host, instanceName, nil } func (s *ShoesLXDMultiServer) setLXDStatusCache( diff --git a/server/pkg/api/server_delete_instance.go b/server/pkg/api/server_delete_instance.go index 9bf048f..430eb25 100644 --- a/server/pkg/api/server_delete_instance.go +++ b/server/pkg/api/server_delete_instance.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/lxc/lxd/shared/api" - "github.com/whywaita/myshoes/pkg/runner" pb "github.com/whywaita/shoes-lxd-multi/proto.go" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -17,9 +16,6 @@ import ( func (s *ShoesLXDMultiServer) DeleteInstance(ctx context.Context, req *pb.DeleteInstanceRequest) (*pb.DeleteInstanceResponse, error) { slog.Info("DeleteInstance", "req", req) l := slog.With("method", "DeleteInstance") - if _, err := runner.ToUUID(req.CloudId); err != nil { - return nil, status.Errorf(codes.InvalidArgument, "failed to parse request id: %+v", err) - } instanceName := req.CloudId l = l.With("instanceName", instanceName) targetLXDHosts, err := s.validateTargetHosts(req.TargetHosts, l) diff --git a/server/pkg/config/config.go b/server/pkg/config/config.go index 4ce1298..ef415e7 100644 --- a/server/pkg/config/config.go +++ b/server/pkg/config/config.go @@ -23,6 +23,8 @@ const ( EnvPort = "LXD_MULTI_PORT" // EnvOverCommit will set percent of over commit in CPU EnvOverCommit = "LXD_MULTI_OVER_COMMIT_PERCENT" + // EnvMode will set running mode + EnvMode = "LXD_MULTI_MODE" ) // Mapping is resource mapping @@ -33,10 +35,10 @@ type Mapping struct { } // Load load config from Environment values -func Load() (*HostConfigMap, map[myshoespb.ResourceType]Mapping, int64, int, uint64, error) { +func Load() (*HostConfigMap, map[myshoespb.ResourceType]Mapping, int64, int, uint64, bool, error) { hostConfigs, err := loadHostConfigs() if err != nil { - return nil, nil, 0, -1, 0, fmt.Errorf("failed to load host config: %w", err) + return nil, nil, 0, -1, 0, false, fmt.Errorf("failed to load host config: %w", err) } envMappingJSON := os.Getenv(EnvLXDResourceTypeMapping) @@ -44,7 +46,7 @@ func Load() (*HostConfigMap, map[myshoespb.ResourceType]Mapping, int64, int, uin if envMappingJSON != "" { m, err = readResourceTypeMapping(envMappingJSON) if err != nil { - return nil, nil, 0, -1, 0, fmt.Errorf("failed to read %s: %w", EnvLXDResourceTypeMapping, err) + return nil, nil, 0, -1, 0, false, fmt.Errorf("failed to read %s: %w", EnvLXDResourceTypeMapping, err) } } @@ -55,7 +57,7 @@ func Load() (*HostConfigMap, map[myshoespb.ResourceType]Mapping, int64, int, uin } else { periodSec, err = strconv.ParseInt(envPeriodSec, 10, 64) if err != nil { - return nil, nil, 0, -1, 0, fmt.Errorf("failed to parse %s, need to uint: %w", EnvOverCommit, err) + return nil, nil, 0, -1, 0, false, fmt.Errorf("failed to parse %s, need to uint: %w", EnvOverCommit, err) } } log.Printf("periodSec: %d\n", periodSec) @@ -67,7 +69,7 @@ func Load() (*HostConfigMap, map[myshoespb.ResourceType]Mapping, int64, int, uin } else { port, err = strconv.Atoi(envPort) if err != nil { - return nil, nil, 0, -1, 0, fmt.Errorf("failed to parse %s, need to int: %w", EnvPort, err) + return nil, nil, 0, -1, 0, false, fmt.Errorf("failed to parse %s, need to int: %w", EnvPort, err) } } @@ -78,12 +80,22 @@ func Load() (*HostConfigMap, map[myshoespb.ResourceType]Mapping, int64, int, uin } else { overCommitPercent, err = strconv.ParseUint(envOCP, 10, 64) if err != nil { - return nil, nil, 0, -1, 0, fmt.Errorf("failed to parse %s, need to uint: %w", EnvOverCommit, err) + return nil, nil, 0, -1, 0, false, fmt.Errorf("failed to parse %s, need to uint: %w", EnvOverCommit, err) } } log.Printf("overCommitPercent: %d\n", overCommitPercent) - return hostConfigs, m, periodSec, port, overCommitPercent, nil + var poolMode bool + switch os.Getenv(EnvMode) { + case "", "create": + poolMode = false + case "pool": + poolMode = true + default: + return nil, nil, 0, -1, 0, false, fmt.Errorf(`unknown mode %q (expected "create" or "pool")`, os.Getenv(EnvMode)) + } + + return hostConfigs, m, periodSec, port, overCommitPercent, poolMode, nil } func readResourceTypeMapping(env string) (map[myshoespb.ResourceType]Mapping, error) {