Implement

.gitignore

@@ -0,0 +1,2 @@
+/server/serer
+/shoes-lxd-multi/shoes-lxd-multi

server/Dockerfile

@@ -0,0 +1,21 @@
+FROM golang:1.16 AS builder
+
+WORKDIR /go/src/github.com/whywaita/shoes-lxd-multi/server
+
+ENV CGO_ENABLED=0
+ENV GOOS=linux
+ENV GOARCH=amd64
+
+COPY . .
+RUN go build .
+
+FROM alpine
+
+RUN apk update \
+  && apk update
+RUN apk add --no-cache ca-certificates \
+  && update-ca-certificates 2>/dev/null || true
+
+COPY --from=builder /go/src/github.com/whywaita/shoes-lxd-multi/server/server /app
+
+CMD ["/app"]

server/README.md

@@ -0,0 +1,40 @@
+# Server
+
+Server-side implementation for shoes-lxd-multi
+
+## Setup
+
+- `LXD_MULTI_HOSTS`
+
+```json
+[
+  {
+    "host": "https://192.0.2.100:8443",
+    "client_cert": "./node1/client.crt",
+    "client_key": "./node1/client.key"
+  },
+  ...
+]
+```
+
+### Optional values
+- `LXD_MULTI_IMAGE_ALIAS`
+    - set runner image alias
+    - default: `ubuntu:bionic`
+    - e.g.) for remote image server: `https://192.0.2.110:8443/ubuntu-custom`
+- `LXD_MULTI_RESOURCE_TYPE_MAPPING`
+    - mapping `resource_type` and CPU / Memory.
+    - need JSON format. keys is `resource_type_name`, `cpu`, `memory`.
+    - e.g.) `[{"resource_type_name": "nano", "cpu": 1, "memory": "1GB"}, {"resource_type_name": "micro", "cpu": 2, "memory": "2GB"}]`
+    - become no limit if not set resource_type.
+- `LXD_MULTI_PORT`
+    - Port of listen gRPC Server
+    - default: `8080`
+- `LXD_MULTI_OVER_COMMIT_PERCENT`
+    - Percent of able over commit in CPU
+    - default: `100`
+
+## Note
+LXD Server can't use `zfs` in storageclass if use `--privileged`. ref: https://discuss.linuxcontainers.org/t/docker-with-overlay-driver-in-lxd-cluster-not-working/9243
+
+We recommend using `btrfs`.

server/go.mod

@@ -0,0 +1,14 @@
+module github.com/whywaita/shoes-lxd-multi/server
+
+go 1.16
+
+require (
+	github.com/docker/go-units v0.4.0
+	github.com/lxc/lxd v0.0.0-20211202222358-a293da71aeb0
+	github.com/prometheus/client_golang v1.11.0
+	github.com/whywaita/myshoes v1.10.4
+	github.com/whywaita/shoes-lxd-multi/proto.go v0.0.0-20211203151606-53728ef694c2
+	google.golang.org/grpc v1.42.0
+)
+
+//replace github.com/flosch/pongo2 => github.com/flosch/pongo2/v4 v4.0.2

server/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"math/rand"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/whywaita/shoes-lxd-multi/server/pkg/api"
+	"github.com/whywaita/shoes-lxd-multi/server/pkg/config"
+	"github.com/whywaita/shoes-lxd-multi/server/pkg/metric"
+)
+
+func main() {
+	rand.Seed(time.Now().UnixNano())
+
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run() error {
+	hostConfigs, mapping, instanceSource, listenPort, overCommitPercent, err := config.Load()
+	if err != nil {
+		return fmt.Errorf("failed to create server: %w", err)
+	}
+
+	go serveMetrics(context.Background(), hostConfigs)
+
+	server, err := api.New(hostConfigs, mapping, instanceSource, overCommitPercent)
+	if err != nil {
+		return fmt.Errorf("failed to create server: %w", err)
+	}
+
+	if err := server.Run(listenPort); err != nil {
+		return fmt.Errorf("faied to run server: %w", err)
+	}
+
+	return nil
+}
+
+func serveMetrics(ctx context.Context, hostConfigs *config.HostConfigMap) {
+	var hcs []config.HostConfig
+	hostConfigs.Range(func(key string, value config.HostConfig) bool {
+		hcs = append(hcs, value)
+		return true
+	})
+
+	registry := prometheus.NewRegistry()
+	registry.MustRegister(metric.NewCollector(ctx, hcs))
+	gatherers := prometheus.Gatherers{
+		prometheus.DefaultGatherer,
+		registry,
+	}
+
+	http.Handle("/metrics", promhttp.HandlerFor(
+		gatherers,
+		promhttp.HandlerOpts{
+			EnableOpenMetrics: true,
+		},
+	))
+
+	if err := http.ListenAndServe(":9090", nil); err != nil {
+		log.Fatal(err)
+	}
+}

server/pkg/api/server.go

@@ -0,0 +1,93 @@
+package api
+
+import (
+	"fmt"
+	"log"
+	"net"
+
+	"github.com/lxc/lxd/shared/api"
+
+	pb "github.com/whywaita/shoes-lxd-multi/proto.go"
+	"github.com/whywaita/shoes-lxd-multi/server/pkg/config"
+	"github.com/whywaita/shoes-lxd-multi/server/pkg/lxdclient"
+	"google.golang.org/grpc"
+)
+
+type ShoesLXDMultiServer struct {
+	pb.UnimplementedShoesLXDMultiServer
+
+	hostConfigs     *config.HostConfigMap
+	resourceMapping map[pb.ResourceType]config.Mapping
+	instanceSource  *api.InstanceSource
+
+	overCommitPercent uint64
+}
+
+// New create gRPC server
+func New(hostConfigs *config.HostConfigMap, mapping map[pb.ResourceType]config.Mapping, instanceSource *api.InstanceSource, overCommitPercent uint64) (*ShoesLXDMultiServer, error) {
+	return &ShoesLXDMultiServer{
+		hostConfigs:       hostConfigs,
+		resourceMapping:   mapping,
+		instanceSource:    instanceSource,
+		overCommitPercent: overCommitPercent,
+	}, nil
+}
+
+// Run run gRPC server
+func (s *ShoesLXDMultiServer) Run(listenPort int) error {
+	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", listenPort))
+	if err != nil {
+		return fmt.Errorf("failed to listen: %w", err)
+	}
+	log.Printf("start listen :%d\n", listenPort)
+
+	grpcServer := grpc.NewServer()
+	pb.RegisterShoesLXDMultiServer(grpcServer, s)
+
+	if err := grpcServer.Serve(lis); err != nil {
+		return fmt.Errorf("failed to serve gRPC: %w", err)
+	}
+	return nil
+}
+
+// isExistInstance search created instance in same name
+func (s *ShoesLXDMultiServer) isExistInstance(targetLXDHosts []lxdclient.LXDHost, instanceName string) *lxdclient.LXDHost {
+	for _, lxdHost := range targetLXDHosts {
+		_, _, err := lxdHost.Client.GetInstance(instanceName)
+		if err == nil {
+			// found LXD worker
+			return &lxdHost
+		}
+	}
+
+	return nil
+}
+
+func (s *ShoesLXDMultiServer) validateTargetHosts(targetHosts []string) ([]lxdclient.LXDHost, error) {
+	var hostConfigs []config.HostConfig
+
+	for _, target := range targetHosts {
+		host, err := s.hostConfigs.Load(target)
+		if err != nil {
+			log.Printf("ignore host in target (target: %s): %+v\n", target, err)
+			continue
+		}
+
+		hostConfigs = append(hostConfigs, *host)
+	}
+
+	if len(hostConfigs) == 0 {
+		return nil, fmt.Errorf("valid target host is not found")
+	}
+
+	targetLXDHosts, err := lxdclient.ConnectLXDs(hostConfigs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect LXD: %w", err)
+	}
+
+	if len(targetLXDHosts) == 0 {
+		return nil, fmt.Errorf("all target host can't connect")
+	}
+
+	return targetLXDHosts, nil
+}