diff --git a/pkg/apis/devices.harvesterhci.io/v1beta1/common.go b/pkg/apis/devices.harvesterhci.io/v1beta1/common.go new file mode 100644 index 00000000..dd4a741d --- /dev/null +++ b/pkg/apis/devices.harvesterhci.io/v1beta1/common.go @@ -0,0 +1,10 @@ +package v1beta1 + +const ( + DeviceAllocationKey = "harvesterhci.io/deviceAllocationDetails" +) + +type AllocationDetails struct { + GPUs map[string][]string `json:"gpus,omitempty"` + HostDevices map[string][]string `json:"hostdevices,omitempty"` +} diff --git a/pkg/codegen/main.go b/pkg/codegen/main.go index 29190e46..6077b91d 100644 --- a/pkg/codegen/main.go +++ b/pkg/codegen/main.go @@ -37,6 +37,8 @@ func main() { kubevirtv1.SchemeGroupVersion.Group: { Types: []interface{}{ kubevirtv1.KubeVirt{}, + kubevirtv1.VirtualMachineInstance{}, + kubevirtv1.VirtualMachine{}, }, GenerateTypes: false, GenerateClients: true, diff --git a/pkg/config/factory_magement.go b/pkg/config/factory_magement.go index 16c218bb..e6c521fb 100644 --- a/pkg/config/factory_magement.go +++ b/pkg/config/factory_magement.go @@ -1,11 +1,12 @@ package config import ( - ctlnetwork "github.com/harvester/harvester-network-controller/pkg/generated/controllers/network.harvesterhci.io" ctlcore "github.com/rancher/wrangler/pkg/generated/controllers/core" "k8s.io/client-go/rest" "kubevirt.io/client-go/kubecli" + ctlnetwork "github.com/harvester/harvester-network-controller/pkg/generated/controllers/network.harvesterhci.io" + ctldevices "github.com/harvester/pcidevices/pkg/generated/controllers/devices.harvesterhci.io" ctlkubevirt "github.com/harvester/pcidevices/pkg/generated/controllers/kubevirt.io" ) diff --git a/pkg/controller/setup.go b/pkg/controller/setup.go index bdbfbce5..7d0db5fb 100644 --- a/pkg/controller/setup.go +++ b/pkg/controller/setup.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - ctlnetwork "github.com/harvester/harvester-network-controller/pkg/generated/controllers/network.harvesterhci.io" "github.com/rancher/lasso/pkg/cache" "github.com/rancher/lasso/pkg/client" "github.com/rancher/lasso/pkg/controller" @@ -18,6 +17,8 @@ import ( "k8s.io/client-go/util/workqueue" "kubevirt.io/client-go/kubecli" + ctlnetwork "github.com/harvester/harvester-network-controller/pkg/generated/controllers/network.harvesterhci.io" + "github.com/harvester/pcidevices/pkg/config" "github.com/harvester/pcidevices/pkg/controller/gpudevice" "github.com/harvester/pcidevices/pkg/controller/nodecleanup" @@ -25,6 +26,7 @@ import ( "github.com/harvester/pcidevices/pkg/controller/pcideviceclaim" "github.com/harvester/pcidevices/pkg/controller/sriovdevice" "github.com/harvester/pcidevices/pkg/controller/usbdevice" + "github.com/harvester/pcidevices/pkg/controller/virtualmachine" "github.com/harvester/pcidevices/pkg/crd" ctldevices "github.com/harvester/pcidevices/pkg/generated/controllers/devices.harvesterhci.io" ctlkubevirt "github.com/harvester/pcidevices/pkg/generated/controllers/kubevirt.io" @@ -103,6 +105,7 @@ func Setup(ctx context.Context, cfg *rest.Config, _ *runtime.Scheme) error { sriovdevice.Register, nodecleanup.Register, gpudevice.Register, + virtualmachine.Register, } for _, register := range registers { @@ -111,7 +114,7 @@ func Setup(ctx context.Context, cfg *rest.Config, _ *runtime.Scheme) error { } } - if err := start.All(ctx, 2, coreFactory, networkFactory, deviceFactory); err != nil { + if err := start.All(ctx, 2, coreFactory, networkFactory, deviceFactory, kubevirtFactory); err != nil { return fmt.Errorf("error starting controllers :%v", err) } diff --git a/pkg/controller/virtualmachine/virtualmachine.go b/pkg/controller/virtualmachine/virtualmachine.go new file mode 100644 index 00000000..ee7e40cd --- /dev/null +++ b/pkg/controller/virtualmachine/virtualmachine.go @@ -0,0 +1,462 @@ +package virtualmachine + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "os" + "reflect" + "slices" + "strings" + + ctlcorev1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/rest" + kubevirtv1 "kubevirt.io/api/core/v1" + "kubevirt.io/kubevirt/pkg/util" + + "github.com/harvester/pcidevices/pkg/apis/devices.harvesterhci.io/v1beta1" + "github.com/harvester/pcidevices/pkg/config" + "github.com/harvester/pcidevices/pkg/deviceplugins" + ctldevicesv1beta1 "github.com/harvester/pcidevices/pkg/generated/controllers/devices.harvesterhci.io/v1beta1" + ctlkubevirtv1 "github.com/harvester/pcidevices/pkg/generated/controllers/kubevirt.io/v1" + "github.com/harvester/pcidevices/pkg/util/executor" +) + +const ( + kubevirtVMLabelKey = "vm.kubevirt.io/name" +) + +type Handler struct { + ctx context.Context + vmCache ctlkubevirtv1.VirtualMachineCache + vmClient ctlkubevirtv1.VirtualMachineClient + vmi ctlkubevirtv1.VirtualMachineInstanceController + pod ctlcorev1.PodController + vgpuCache ctldevicesv1beta1.VGPUDeviceCache + pciDeviceCache ctldevicesv1beta1.PCIDeviceCache + config *rest.Config + nodeName string +} + +func Register(ctx context.Context, management *config.FactoryManager) error { + vmCache := management.KubevirtFactory.Kubevirt().V1().VirtualMachine().Cache() + vmClient := management.KubevirtFactory.Kubevirt().V1().VirtualMachine() + vmi := management.KubevirtFactory.Kubevirt().V1().VirtualMachineInstance() + pod := management.CoreFactory.Core().V1().Pod() + vgpuCache := management.DeviceFactory.Devices().V1beta1().VGPUDevice().Cache() + pciDeviceCache := management.DeviceFactory.Devices().V1beta1().PCIDevice().Cache() + nodeName := os.Getenv(v1beta1.NodeEnvVarName) + h := Handler{ + ctx: ctx, + vmCache: vmCache, + vmClient: vmClient, + vmi: vmi, + pod: pod, + vgpuCache: vgpuCache, + pciDeviceCache: pciDeviceCache, + config: management.Cfg, + nodeName: nodeName, + } + vmi.OnChange(ctx, "virtual-machine-instance-handler", h.OnVMIChange) + vmi.OnRemove(ctx, "virtual-machine-deletion", h.OnVMIDeletion) + return nil +} + +// OnVMIChange attempts to reconcile devices allocated to launcher pod associated with a running VMI +// and annotate the VM object with device annotations +func (h *Handler) OnVMIChange(name string, vmi *kubevirtv1.VirtualMachineInstance) (*kubevirtv1.VirtualMachineInstance, error) { + if vmi == nil || vmi.DeletionTimestamp != nil { + logrus.WithFields(logrus.Fields{ + "name": name, + }).Debug("skipping object, either does not exist or marked for deletion") + return vmi, nil + } + + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + }).Debug("reconcilling vmi device allocation") + + if vmi.Status.Phase != kubevirtv1.Running { + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + }).Debug("skipping vmi as it is not running") + return vmi, nil + } + + if len(vmi.Spec.Domain.Devices.HostDevices) == 0 && len(vmi.Spec.Domain.Devices.GPUs) == 0 { + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + }).Debug("skipping vmi as it does not request any host devices or gpus") + return vmi, h.checkAndClearDeviceAllocation(vmi) + } + + if vmi.Status.NodeName != h.nodeName { + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + "hostname": vmi.Status.NodeName, + }).Debug("skipping vmi as it is not scheduled on current node") + return vmi, nil + } + return vmi, h.trackDevices(vmi) +} + +// trackDevices reconciles GPU and HostDevices info +func (h *Handler) trackDevices(vmi *kubevirtv1.VirtualMachineInstance) error { + envMap, err := h.generatePodEnvMap(vmi) + if err != nil { + return err + } + return h.reconcileDeviceAllocationDetails(vmi, envMap) +} + +// reconcileDeviceAllocationDetails will reconcile envMap into device allocation annotation on vmi +// has been split into its own method to simplify testing +func (h *Handler) reconcileDeviceAllocationDetails(vmi *kubevirtv1.VirtualMachineInstance, envMap map[string]string) error { + var pciDeviceMap, vGPUMap map[string]string + selector := map[string]string{ + "nodename": vmi.Status.NodeName, + } + if len(vmi.Spec.Domain.Devices.HostDevices) > 0 { + deviceList, err := h.pciDeviceCache.List(labels.SelectorFromSet(selector)) + if err != nil { + return fmt.Errorf("error listing pcidevices from cache: %v", err) + } + pciDeviceMap = buildPCIDeviceMap(deviceList) + } + if len(vmi.Spec.Domain.Devices.GPUs) > 0 { + deviceList, err := h.vgpuCache.List(labels.SelectorFromSet(selector)) + if err != nil { + return fmt.Errorf("error listing vgpudevices from cache: %v", err) + } + vGPUMap = buildVGPUMap(deviceList) + } + // map to hold device details + hostDeviceMap := make(map[string][]string) + gpuMap := make(map[string][]string) + for _, device := range vmi.Spec.Domain.Devices.HostDevices { + val, ok := envMap[util.ResourceNameToEnvVar(deviceplugins.PCIResourcePrefix, device.DeviceName)] + if ok { + deviceInfo := hostDeviceMap[device.DeviceName] + devices := strings.Split(val, ",") + for _, v := range devices { + deviceInfo = append(deviceInfo, pciDeviceMap[v]) + } + hostDeviceMap[device.DeviceName] = deviceInfo + } + } + + for _, device := range vmi.Spec.Domain.Devices.GPUs { + val, ok := envMap[util.ResourceNameToEnvVar(deviceplugins.VGPUPrefix, device.DeviceName)] + if ok { + deviceInfo := gpuMap[device.DeviceName] + devices := strings.Split(val, ",") + for _, v := range devices { + deviceInfo = append(deviceInfo, vGPUMap[v]) + } + gpuMap[device.DeviceName] = deviceInfo + } + } + + // generate allocation details + deviceDetails := generateAllocationDetails(hostDeviceMap, gpuMap) + deviceDetailsBytes, err := json.Marshal(deviceDetails) + if err != nil { + return fmt.Errorf("error marshalling deviceDetails: %v", err) + } + + return h.reconcileVMResourceAllocationAnnotation(vmi, string(deviceDetailsBytes)) +} + +// findPodForVMI leverages the fact that each pod associated with a VMI a label vm.kubevirt.io/name: $vmName +// this makes it easier to find the correct pod +func (h *Handler) findPodForVMI(vmi *kubevirtv1.VirtualMachineInstance) (*corev1.Pod, error) { + podList, err := h.pod.List(vmi.Namespace, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", kubevirtVMLabelKey, vmi.Name), + }) + if err != nil { + return nil, fmt.Errorf("error listing pods: %v", err) + } + + // if more than 1 pod is returned make sure only 1 is running + // if more than 1 is running then error out since there may be a migration + // running and reconcile again + var runningPod corev1.Pod + + var count int + for _, pod := range podList.Items { + if pod.Status.Phase == corev1.PodRunning { + runningPod = pod // we copy pod in case there is only 1 running, avoids having to iterate again + count++ + } + } + + if count != 1 { + return nil, fmt.Errorf("expected to find 1 pod, but found %d associated with vmi %s", count, vmi.Name) + } + + return &runningPod, nil +} + +func (h *Handler) getPodEnv(pod *corev1.Pod) ([]byte, error) { + e, err := executor.NewRemoteCommandExecutor(h.ctx, h.config, pod) + if err != nil { + return nil, fmt.Errorf("error setting up remote executor for pod %s: %v", pod.Name, err) + } + return e.Run("env", []string{}) +} + +// convertEnvToMap converts env info to a map to make it easier to find device +// allocation details +func convertEnvToMap(podEnv []byte) (map[string]string, error) { + var contents []string + bufReader := bufio.NewReader(bytes.NewReader(podEnv)) + + for { + line, err := bufReader.ReadBytes('\n') + if len(line) != 0 { + contents = append(contents, strings.TrimSuffix(string(line), "\n")) + } + + if err != nil { + if err != io.EOF { + return nil, err + } + break + } + } + + resultMap := make(map[string]string) + for _, line := range contents { + lineArr := strings.Split(line, "=") + resultMap[lineArr[0]] = strings.Join(lineArr[1:], "=") + } + return resultMap, nil +} + +// check and clear deviceAllocation annotations if needed +func (h *Handler) checkAndClearDeviceAllocation(vmi *kubevirtv1.VirtualMachineInstance) error { + vmObj, err := h.vmCache.Get(vmi.Namespace, vmi.Name) + if err != nil { + return fmt.Errorf("error fetching vm %s from cache: %s", vmi.Name, err) + } + + _, ok := vmObj.Annotations[v1beta1.DeviceAllocationKey] + // no key, nothing is needed + if !ok { + return nil + } + delete(vmObj.Annotations, v1beta1.DeviceAllocationKey) + _, err = h.vmClient.Update(vmObj) + return err +} + +func buildVGPUMap(vgpuDevices []*v1beta1.VGPUDevice) map[string]string { + result := make(map[string]string) + for _, vgpu := range vgpuDevices { + if vgpu.Status.UUID != "" { + result[vgpu.Status.UUID] = vgpu.Name + } + } + return result +} + +func buildPCIDeviceMap(pciDevices []*v1beta1.PCIDevice) map[string]string { + result := make(map[string]string) + for _, device := range pciDevices { + result[device.Status.Address] = device.Name + } + return result +} + +func generateAllocationDetails(hostDeviceMap, gpuMap map[string][]string) *v1beta1.AllocationDetails { + resp := &v1beta1.AllocationDetails{} + if len(hostDeviceMap) > 0 { + hostDeviceMap = dedupDevices(hostDeviceMap) + resp.HostDevices = hostDeviceMap + } + + if len(gpuMap) > 0 { + gpuMap = dedupDevices(gpuMap) + resp.GPUs = gpuMap + } + return resp +} + +// if multiple devices of same type are added to a VM, there may be duplicates in the deviceDetails values +// since env variable would have been looked up twice from envMap and added to deviceDetails +// so we dedup the unique device names +func dedupDevices(deviceMap map[string][]string) map[string][]string { + for key, val := range deviceMap { + deviceMap[key] = slices.Compact(val) + } + return deviceMap +} + +// generatePodEnvMap attempts to find the pod associated with vmi, exec into pod to fetch `env` output +// and converts the same to the map, to allow the controller to identify device allocated to pod by kubelet +// which can differ from the name in the vmi devices spec, since allocation is only performed by resourceName +func (h *Handler) generatePodEnvMap(vmi *kubevirtv1.VirtualMachineInstance) (map[string]string, error) { + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + }).Debug("looking up pod associated with vmi") + + pod, err := h.findPodForVMI(vmi) + if err != nil { + return nil, err + } + + logrus.WithFields(logrus.Fields{ + "name": pod.Name, + "namespace": pod.Namespace, + }).Debug("looking up pod env") + + podEnv, err := h.getPodEnv(pod) + if err != nil { + return nil, err + } + + envMap, err := convertEnvToMap(podEnv) + if err != nil { + return nil, err + } + + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + }).Debugf("found envMap: %v", envMap) + return envMap, nil +} + +func (h *Handler) reconcileVMResourceAllocationAnnotation(vmi *kubevirtv1.VirtualMachineInstance, deviceDetails string) error { + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + }).Debugf("device allocation details: %s", deviceDetails) + + vmObj, err := h.vmCache.Get(vmi.Namespace, vmi.Name) + if err != nil { + return fmt.Errorf("error fetching vm %s from cache: %v", vmi.Name, err) + } + + var currentAnnotationValue string + // update device allocation details + if vmObj.Annotations == nil { + vmObj.Annotations = make(map[string]string) + } else { + currentAnnotationValue = vmObj.Annotations[v1beta1.DeviceAllocationKey] + } + + if currentAnnotationValue != deviceDetails { + vmObj.Annotations[v1beta1.DeviceAllocationKey] = deviceDetails + _, err = h.vmClient.Update(vmObj) + } + return err +} + +// OnVMIDeletion will update the VM spec with actual device allocation details +// this simplifies deletion of devices from the VM object without needing any changes +// it needs to be done during the VM shutdown avoid the object generation to differ between VM and VMI object +// and this avoids the generation warning being reported in the UI +func (h *Handler) OnVMIDeletion(_ string, vmi *kubevirtv1.VirtualMachineInstance) (*kubevirtv1.VirtualMachineInstance, error) { + if vmi == nil || vmi.DeletionTimestamp == nil { + return vmi, nil + } + + // no host or GPU devices present so nothing needed to be done + if len(vmi.Spec.Domain.Devices.GPUs) == 0 && len(vmi.Spec.Domain.Devices.HostDevices) == 0 { + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + "hostname": vmi.Status.NodeName, + }).Debug("skipping vmi as it has no hostdevices or GPUs") + return vmi, nil + } + + vmObj, err := h.vmCache.Get(vmi.Namespace, vmi.Name) + if err != nil { + return vmi, fmt.Errorf("error fetching vm object for vmi %s/%s: %v", vmi.Namespace, vmi.Name, err) + } + g + if vmi.Status.NodeName != h.nodeName { + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + "hostname": vmi.Status.NodeName, + }).Debug("skipping vmi as it is not scheduled on current node") + return vmi, nil + } + + val, ok := vmObj.Annotations[v1beta1.DeviceAllocationKey] + if !ok { + // no device allocation annotations, nothing to do + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + "hostname": vmi.Status.NodeName, + }).Debug("skipping vmi as it has no device allocation annotation") + return vmi, nil + } + + allocationDetails := &v1beta1.AllocationDetails{} + err = json.Unmarshal([]byte(val), allocationDetails) + if err != nil { + return vmi, fmt.Errorf("error unmarshalling allocation details annotation: %v", err) + } + + vmObjCopy := vmObj.DeepCopy() + if len(vmi.Spec.Domain.Devices.HostDevices) > 0 { + patchHostDevices(vmObj, allocationDetails.HostDevices) + } + + if len(vmi.Spec.Domain.Devices.GPUs) > 0 { + patchGPUDevices(vmObj, allocationDetails.GPUs) + } + + if !reflect.DeepEqual(vmObj.Spec.Template.Spec.Domain.Devices, vmObjCopy.Spec.Template.Spec.Domain.Devices) { + logrus.WithFields(logrus.Fields{ + "name": vmi.Name, + "namespace": vmi.Namespace, + }).Debugf("updating vm device allocation details: %v", vmObj.Spec.Template.Spec.Domain.Devices) + _, err := h.vmClient.Update(vmObj) + return vmi, err + } + return vmi, nil +} + +func patchHostDevices(vm *kubevirtv1.VirtualMachine, deviceInfo map[string][]string) { + for i, device := range vm.Spec.Template.Spec.Domain.Devices.HostDevices { + actualDevices, ok := deviceInfo[device.DeviceName] + if !ok { + continue // should not ideally be possible but in case it does happen we ignore and continue + } + // pop first element of the actualDevices and set to deviceNames + vm.Spec.Template.Spec.Domain.Devices.HostDevices[i].Name = actualDevices[0] + actualDevices = actualDevices[1:] + deviceInfo[device.DeviceName] = actualDevices // update map to ensure same name is not reused + } +} + +func patchGPUDevices(vm *kubevirtv1.VirtualMachine, deviceInfo map[string][]string) { + for i, device := range vm.Spec.Template.Spec.Domain.Devices.GPUs { + actualDevices, ok := deviceInfo[device.DeviceName] + if !ok { + continue // should not ideally be possible but in case it does happen we ignore and continue + } + // pop first element of the actualDevices and set to deviceNames + vm.Spec.Template.Spec.Domain.Devices.GPUs[i].Name = actualDevices[0] + actualDevices = actualDevices[1:] + deviceInfo[device.DeviceName] = actualDevices // update map to ensure same name is not reused + } +} diff --git a/pkg/controller/virtualmachine/virtualmachine_test.go b/pkg/controller/virtualmachine/virtualmachine_test.go new file mode 100644 index 00000000..aa29f7d3 --- /dev/null +++ b/pkg/controller/virtualmachine/virtualmachine_test.go @@ -0,0 +1,81 @@ +package virtualmachine + +import ( + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubevirtv1 "kubevirt.io/api/core/v1" +) + +func Test_patchHostDevices(t *testing.T) { + assert := require.New(t) + + HostDevices := map[string][]string{ + "device.com/sample": {"node1dev1", "node1dev2"}, + } + vm := &kubevirtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Spec: kubevirtv1.VirtualMachineSpec{ + Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Domain: kubevirtv1.DomainSpec{ + Devices: kubevirtv1.Devices{ + HostDevices: []kubevirtv1.HostDevice{ + { + Name: "node1dev1", + DeviceName: "device.com/sample", + }, + { + Name: "randomDevice", + DeviceName: "device.com/sample", + }, + }, + }, + }, + }, + }, + }, + } + patchHostDevices(vm, HostDevices) + // expect randomDevice to be replaced with actual device name + assert.Equal(vm.Spec.Template.Spec.Domain.Devices.HostDevices[1].Name, "node1dev2") + +} + +func Test_patchGPUDevices(t *testing.T) { + assert := require.New(t) + + GPUDevices := map[string][]string{ + "nvidia.com/A2-Q2": {"node1dev1"}, + } + vm := &kubevirtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo", + Namespace: "default", + }, + Spec: kubevirtv1.VirtualMachineSpec{ + Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Domain: kubevirtv1.DomainSpec{ + Devices: kubevirtv1.Devices{ + GPUs: []kubevirtv1.GPU{ + { + Name: "sample", + DeviceName: "nvidia.com/A2-Q2", + }, + }, + }, + }, + }, + }, + }, + } + patchGPUDevices(vm, GPUDevices) + // expect randomDevice to be replaced with actual device name + assert.Equal(vm.Spec.Template.Spec.Domain.Devices.GPUs[0].Name, "node1dev1") + +} diff --git a/pkg/deviceplugins/vgpu_device_manager.go b/pkg/deviceplugins/vgpu_device_manager.go index 45930b07..aec99ed3 100644 --- a/pkg/deviceplugins/vgpu_device_manager.go +++ b/pkg/deviceplugins/vgpu_device_manager.go @@ -41,7 +41,7 @@ import ( ) const ( - vgpuPrefix = "MDEV_PCI_RESOURCE" + VGPUPrefix = "MDEV_PCI_RESOURCE" ) type VGPUDevicePlugin struct { @@ -212,7 +212,7 @@ func (dp *VGPUDevicePlugin) ListAndWatch(_ *pluginapi.Empty, s pluginapi.DeviceP func (dp *VGPUDevicePlugin) Allocate(_ context.Context, r *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) { logrus.Debugf("Allocate request %s", r.String()) - resourceNameEnvVar := util.ResourceNameToEnvVar(vgpuPrefix, dp.resourceName) + resourceNameEnvVar := util.ResourceNameToEnvVar(VGPUPrefix, dp.resourceName) allocatedDevices := []string{} resp := new(pluginapi.AllocateResponse) containerResponse := new(pluginapi.ContainerAllocateResponse) diff --git a/pkg/generated/controllers/kubevirt.io/v1/interface.go b/pkg/generated/controllers/kubevirt.io/v1/interface.go index 6563cf40..c30ecef6 100644 --- a/pkg/generated/controllers/kubevirt.io/v1/interface.go +++ b/pkg/generated/controllers/kubevirt.io/v1/interface.go @@ -31,6 +31,8 @@ func init() { type Interface interface { KubeVirt() KubeVirtController + VirtualMachine() VirtualMachineController + VirtualMachineInstance() VirtualMachineInstanceController } func New(controllerFactory controller.SharedControllerFactory) Interface { @@ -46,3 +48,9 @@ type version struct { func (c *version) KubeVirt() KubeVirtController { return NewKubeVirtController(schema.GroupVersionKind{Group: "kubevirt.io", Version: "v1", Kind: "KubeVirt"}, "kubevirts", true, c.controllerFactory) } +func (c *version) VirtualMachine() VirtualMachineController { + return NewVirtualMachineController(schema.GroupVersionKind{Group: "kubevirt.io", Version: "v1", Kind: "VirtualMachine"}, "virtualmachines", true, c.controllerFactory) +} +func (c *version) VirtualMachineInstance() VirtualMachineInstanceController { + return NewVirtualMachineInstanceController(schema.GroupVersionKind{Group: "kubevirt.io", Version: "v1", Kind: "VirtualMachineInstance"}, "virtualmachineinstances", true, c.controllerFactory) +} diff --git a/pkg/generated/controllers/kubevirt.io/v1/virtualmachine.go b/pkg/generated/controllers/kubevirt.io/v1/virtualmachine.go new file mode 100644 index 00000000..191e56cb --- /dev/null +++ b/pkg/generated/controllers/kubevirt.io/v1/virtualmachine.go @@ -0,0 +1,376 @@ +/* +Copyright 2022 Rancher Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by main. DO NOT EDIT. + +package v1 + +import ( + "context" + "time" + + "github.com/rancher/lasso/pkg/client" + "github.com/rancher/lasso/pkg/controller" + "github.com/rancher/wrangler/pkg/apply" + "github.com/rancher/wrangler/pkg/condition" + "github.com/rancher/wrangler/pkg/generic" + "github.com/rancher/wrangler/pkg/kv" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/cache" + v1 "kubevirt.io/api/core/v1" +) + +type VirtualMachineHandler func(string, *v1.VirtualMachine) (*v1.VirtualMachine, error) + +type VirtualMachineController interface { + generic.ControllerMeta + VirtualMachineClient + + OnChange(ctx context.Context, name string, sync VirtualMachineHandler) + OnRemove(ctx context.Context, name string, sync VirtualMachineHandler) + Enqueue(namespace, name string) + EnqueueAfter(namespace, name string, duration time.Duration) + + Cache() VirtualMachineCache +} + +type VirtualMachineClient interface { + Create(*v1.VirtualMachine) (*v1.VirtualMachine, error) + Update(*v1.VirtualMachine) (*v1.VirtualMachine, error) + UpdateStatus(*v1.VirtualMachine) (*v1.VirtualMachine, error) + Delete(namespace, name string, options *metav1.DeleteOptions) error + Get(namespace, name string, options metav1.GetOptions) (*v1.VirtualMachine, error) + List(namespace string, opts metav1.ListOptions) (*v1.VirtualMachineList, error) + Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error) + Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VirtualMachine, err error) +} + +type VirtualMachineCache interface { + Get(namespace, name string) (*v1.VirtualMachine, error) + List(namespace string, selector labels.Selector) ([]*v1.VirtualMachine, error) + + AddIndexer(indexName string, indexer VirtualMachineIndexer) + GetByIndex(indexName, key string) ([]*v1.VirtualMachine, error) +} + +type VirtualMachineIndexer func(obj *v1.VirtualMachine) ([]string, error) + +type virtualMachineController struct { + controller controller.SharedController + client *client.Client + gvk schema.GroupVersionKind + groupResource schema.GroupResource +} + +func NewVirtualMachineController(gvk schema.GroupVersionKind, resource string, namespaced bool, controller controller.SharedControllerFactory) VirtualMachineController { + c := controller.ForResourceKind(gvk.GroupVersion().WithResource(resource), gvk.Kind, namespaced) + return &virtualMachineController{ + controller: c, + client: c.Client(), + gvk: gvk, + groupResource: schema.GroupResource{ + Group: gvk.Group, + Resource: resource, + }, + } +} + +func FromVirtualMachineHandlerToHandler(sync VirtualMachineHandler) generic.Handler { + return func(key string, obj runtime.Object) (ret runtime.Object, err error) { + var v *v1.VirtualMachine + if obj == nil { + v, err = sync(key, nil) + } else { + v, err = sync(key, obj.(*v1.VirtualMachine)) + } + if v == nil { + return nil, err + } + return v, err + } +} + +func (c *virtualMachineController) Updater() generic.Updater { + return func(obj runtime.Object) (runtime.Object, error) { + newObj, err := c.Update(obj.(*v1.VirtualMachine)) + if newObj == nil { + return nil, err + } + return newObj, err + } +} + +func UpdateVirtualMachineDeepCopyOnChange(client VirtualMachineClient, obj *v1.VirtualMachine, handler func(obj *v1.VirtualMachine) (*v1.VirtualMachine, error)) (*v1.VirtualMachine, error) { + if obj == nil { + return obj, nil + } + + copyObj := obj.DeepCopy() + newObj, err := handler(copyObj) + if newObj != nil { + copyObj = newObj + } + if obj.ResourceVersion == copyObj.ResourceVersion && !equality.Semantic.DeepEqual(obj, copyObj) { + return client.Update(copyObj) + } + + return copyObj, err +} + +func (c *virtualMachineController) AddGenericHandler(ctx context.Context, name string, handler generic.Handler) { + c.controller.RegisterHandler(ctx, name, controller.SharedControllerHandlerFunc(handler)) +} + +func (c *virtualMachineController) AddGenericRemoveHandler(ctx context.Context, name string, handler generic.Handler) { + c.AddGenericHandler(ctx, name, generic.NewRemoveHandler(name, c.Updater(), handler)) +} + +func (c *virtualMachineController) OnChange(ctx context.Context, name string, sync VirtualMachineHandler) { + c.AddGenericHandler(ctx, name, FromVirtualMachineHandlerToHandler(sync)) +} + +func (c *virtualMachineController) OnRemove(ctx context.Context, name string, sync VirtualMachineHandler) { + c.AddGenericHandler(ctx, name, generic.NewRemoveHandler(name, c.Updater(), FromVirtualMachineHandlerToHandler(sync))) +} + +func (c *virtualMachineController) Enqueue(namespace, name string) { + c.controller.Enqueue(namespace, name) +} + +func (c *virtualMachineController) EnqueueAfter(namespace, name string, duration time.Duration) { + c.controller.EnqueueAfter(namespace, name, duration) +} + +func (c *virtualMachineController) Informer() cache.SharedIndexInformer { + return c.controller.Informer() +} + +func (c *virtualMachineController) GroupVersionKind() schema.GroupVersionKind { + return c.gvk +} + +func (c *virtualMachineController) Cache() VirtualMachineCache { + return &virtualMachineCache{ + indexer: c.Informer().GetIndexer(), + resource: c.groupResource, + } +} + +func (c *virtualMachineController) Create(obj *v1.VirtualMachine) (*v1.VirtualMachine, error) { + result := &v1.VirtualMachine{} + return result, c.client.Create(context.TODO(), obj.Namespace, obj, result, metav1.CreateOptions{}) +} + +func (c *virtualMachineController) Update(obj *v1.VirtualMachine) (*v1.VirtualMachine, error) { + result := &v1.VirtualMachine{} + return result, c.client.Update(context.TODO(), obj.Namespace, obj, result, metav1.UpdateOptions{}) +} + +func (c *virtualMachineController) UpdateStatus(obj *v1.VirtualMachine) (*v1.VirtualMachine, error) { + result := &v1.VirtualMachine{} + return result, c.client.UpdateStatus(context.TODO(), obj.Namespace, obj, result, metav1.UpdateOptions{}) +} + +func (c *virtualMachineController) Delete(namespace, name string, options *metav1.DeleteOptions) error { + if options == nil { + options = &metav1.DeleteOptions{} + } + return c.client.Delete(context.TODO(), namespace, name, *options) +} + +func (c *virtualMachineController) Get(namespace, name string, options metav1.GetOptions) (*v1.VirtualMachine, error) { + result := &v1.VirtualMachine{} + return result, c.client.Get(context.TODO(), namespace, name, result, options) +} + +func (c *virtualMachineController) List(namespace string, opts metav1.ListOptions) (*v1.VirtualMachineList, error) { + result := &v1.VirtualMachineList{} + return result, c.client.List(context.TODO(), namespace, result, opts) +} + +func (c *virtualMachineController) Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error) { + return c.client.Watch(context.TODO(), namespace, opts) +} + +func (c *virtualMachineController) Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (*v1.VirtualMachine, error) { + result := &v1.VirtualMachine{} + return result, c.client.Patch(context.TODO(), namespace, name, pt, data, result, metav1.PatchOptions{}, subresources...) +} + +type virtualMachineCache struct { + indexer cache.Indexer + resource schema.GroupResource +} + +func (c *virtualMachineCache) Get(namespace, name string) (*v1.VirtualMachine, error) { + obj, exists, err := c.indexer.GetByKey(namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(c.resource, name) + } + return obj.(*v1.VirtualMachine), nil +} + +func (c *virtualMachineCache) List(namespace string, selector labels.Selector) (ret []*v1.VirtualMachine, err error) { + + err = cache.ListAllByNamespace(c.indexer, namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1.VirtualMachine)) + }) + + return ret, err +} + +func (c *virtualMachineCache) AddIndexer(indexName string, indexer VirtualMachineIndexer) { + utilruntime.Must(c.indexer.AddIndexers(map[string]cache.IndexFunc{ + indexName: func(obj interface{}) (strings []string, e error) { + return indexer(obj.(*v1.VirtualMachine)) + }, + })) +} + +func (c *virtualMachineCache) GetByIndex(indexName, key string) (result []*v1.VirtualMachine, err error) { + objs, err := c.indexer.ByIndex(indexName, key) + if err != nil { + return nil, err + } + result = make([]*v1.VirtualMachine, 0, len(objs)) + for _, obj := range objs { + result = append(result, obj.(*v1.VirtualMachine)) + } + return result, nil +} + +type VirtualMachineStatusHandler func(obj *v1.VirtualMachine, status v1.VirtualMachineStatus) (v1.VirtualMachineStatus, error) + +type VirtualMachineGeneratingHandler func(obj *v1.VirtualMachine, status v1.VirtualMachineStatus) ([]runtime.Object, v1.VirtualMachineStatus, error) + +func RegisterVirtualMachineStatusHandler(ctx context.Context, controller VirtualMachineController, condition condition.Cond, name string, handler VirtualMachineStatusHandler) { + statusHandler := &virtualMachineStatusHandler{ + client: controller, + condition: condition, + handler: handler, + } + controller.AddGenericHandler(ctx, name, FromVirtualMachineHandlerToHandler(statusHandler.sync)) +} + +func RegisterVirtualMachineGeneratingHandler(ctx context.Context, controller VirtualMachineController, apply apply.Apply, + condition condition.Cond, name string, handler VirtualMachineGeneratingHandler, opts *generic.GeneratingHandlerOptions) { + statusHandler := &virtualMachineGeneratingHandler{ + VirtualMachineGeneratingHandler: handler, + apply: apply, + name: name, + gvk: controller.GroupVersionKind(), + } + if opts != nil { + statusHandler.opts = *opts + } + controller.OnChange(ctx, name, statusHandler.Remove) + RegisterVirtualMachineStatusHandler(ctx, controller, condition, name, statusHandler.Handle) +} + +type virtualMachineStatusHandler struct { + client VirtualMachineClient + condition condition.Cond + handler VirtualMachineStatusHandler +} + +func (a *virtualMachineStatusHandler) sync(key string, obj *v1.VirtualMachine) (*v1.VirtualMachine, error) { + if obj == nil { + return obj, nil + } + + origStatus := obj.Status.DeepCopy() + obj = obj.DeepCopy() + newStatus, err := a.handler(obj, obj.Status) + if err != nil { + // Revert to old status on error + newStatus = *origStatus.DeepCopy() + } + + if a.condition != "" { + if errors.IsConflict(err) { + a.condition.SetError(&newStatus, "", nil) + } else { + a.condition.SetError(&newStatus, "", err) + } + } + if !equality.Semantic.DeepEqual(origStatus, &newStatus) { + if a.condition != "" { + // Since status has changed, update the lastUpdatedTime + a.condition.LastUpdated(&newStatus, time.Now().UTC().Format(time.RFC3339)) + } + + var newErr error + obj.Status = newStatus + newObj, newErr := a.client.UpdateStatus(obj) + if err == nil { + err = newErr + } + if newErr == nil { + obj = newObj + } + } + return obj, err +} + +type virtualMachineGeneratingHandler struct { + VirtualMachineGeneratingHandler + apply apply.Apply + opts generic.GeneratingHandlerOptions + gvk schema.GroupVersionKind + name string +} + +func (a *virtualMachineGeneratingHandler) Remove(key string, obj *v1.VirtualMachine) (*v1.VirtualMachine, error) { + if obj != nil { + return obj, nil + } + + obj = &v1.VirtualMachine{} + obj.Namespace, obj.Name = kv.RSplit(key, "/") + obj.SetGroupVersionKind(a.gvk) + + return nil, generic.ConfigureApplyForObject(a.apply, obj, &a.opts). + WithOwner(obj). + WithSetID(a.name). + ApplyObjects() +} + +func (a *virtualMachineGeneratingHandler) Handle(obj *v1.VirtualMachine, status v1.VirtualMachineStatus) (v1.VirtualMachineStatus, error) { + if !obj.DeletionTimestamp.IsZero() { + return status, nil + } + + objs, newStatus, err := a.VirtualMachineGeneratingHandler(obj, status) + if err != nil { + return newStatus, err + } + + return newStatus, generic.ConfigureApplyForObject(a.apply, obj, &a.opts). + WithOwner(obj). + WithSetID(a.name). + ApplyObjects(objs...) +} diff --git a/pkg/generated/controllers/kubevirt.io/v1/virtualmachineinstance.go b/pkg/generated/controllers/kubevirt.io/v1/virtualmachineinstance.go new file mode 100644 index 00000000..c22a5d71 --- /dev/null +++ b/pkg/generated/controllers/kubevirt.io/v1/virtualmachineinstance.go @@ -0,0 +1,376 @@ +/* +Copyright 2022 Rancher Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by main. DO NOT EDIT. + +package v1 + +import ( + "context" + "time" + + "github.com/rancher/lasso/pkg/client" + "github.com/rancher/lasso/pkg/controller" + "github.com/rancher/wrangler/pkg/apply" + "github.com/rancher/wrangler/pkg/condition" + "github.com/rancher/wrangler/pkg/generic" + "github.com/rancher/wrangler/pkg/kv" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/cache" + v1 "kubevirt.io/api/core/v1" +) + +type VirtualMachineInstanceHandler func(string, *v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error) + +type VirtualMachineInstanceController interface { + generic.ControllerMeta + VirtualMachineInstanceClient + + OnChange(ctx context.Context, name string, sync VirtualMachineInstanceHandler) + OnRemove(ctx context.Context, name string, sync VirtualMachineInstanceHandler) + Enqueue(namespace, name string) + EnqueueAfter(namespace, name string, duration time.Duration) + + Cache() VirtualMachineInstanceCache +} + +type VirtualMachineInstanceClient interface { + Create(*v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error) + Update(*v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error) + UpdateStatus(*v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error) + Delete(namespace, name string, options *metav1.DeleteOptions) error + Get(namespace, name string, options metav1.GetOptions) (*v1.VirtualMachineInstance, error) + List(namespace string, opts metav1.ListOptions) (*v1.VirtualMachineInstanceList, error) + Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error) + Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VirtualMachineInstance, err error) +} + +type VirtualMachineInstanceCache interface { + Get(namespace, name string) (*v1.VirtualMachineInstance, error) + List(namespace string, selector labels.Selector) ([]*v1.VirtualMachineInstance, error) + + AddIndexer(indexName string, indexer VirtualMachineInstanceIndexer) + GetByIndex(indexName, key string) ([]*v1.VirtualMachineInstance, error) +} + +type VirtualMachineInstanceIndexer func(obj *v1.VirtualMachineInstance) ([]string, error) + +type virtualMachineInstanceController struct { + controller controller.SharedController + client *client.Client + gvk schema.GroupVersionKind + groupResource schema.GroupResource +} + +func NewVirtualMachineInstanceController(gvk schema.GroupVersionKind, resource string, namespaced bool, controller controller.SharedControllerFactory) VirtualMachineInstanceController { + c := controller.ForResourceKind(gvk.GroupVersion().WithResource(resource), gvk.Kind, namespaced) + return &virtualMachineInstanceController{ + controller: c, + client: c.Client(), + gvk: gvk, + groupResource: schema.GroupResource{ + Group: gvk.Group, + Resource: resource, + }, + } +} + +func FromVirtualMachineInstanceHandlerToHandler(sync VirtualMachineInstanceHandler) generic.Handler { + return func(key string, obj runtime.Object) (ret runtime.Object, err error) { + var v *v1.VirtualMachineInstance + if obj == nil { + v, err = sync(key, nil) + } else { + v, err = sync(key, obj.(*v1.VirtualMachineInstance)) + } + if v == nil { + return nil, err + } + return v, err + } +} + +func (c *virtualMachineInstanceController) Updater() generic.Updater { + return func(obj runtime.Object) (runtime.Object, error) { + newObj, err := c.Update(obj.(*v1.VirtualMachineInstance)) + if newObj == nil { + return nil, err + } + return newObj, err + } +} + +func UpdateVirtualMachineInstanceDeepCopyOnChange(client VirtualMachineInstanceClient, obj *v1.VirtualMachineInstance, handler func(obj *v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error)) (*v1.VirtualMachineInstance, error) { + if obj == nil { + return obj, nil + } + + copyObj := obj.DeepCopy() + newObj, err := handler(copyObj) + if newObj != nil { + copyObj = newObj + } + if obj.ResourceVersion == copyObj.ResourceVersion && !equality.Semantic.DeepEqual(obj, copyObj) { + return client.Update(copyObj) + } + + return copyObj, err +} + +func (c *virtualMachineInstanceController) AddGenericHandler(ctx context.Context, name string, handler generic.Handler) { + c.controller.RegisterHandler(ctx, name, controller.SharedControllerHandlerFunc(handler)) +} + +func (c *virtualMachineInstanceController) AddGenericRemoveHandler(ctx context.Context, name string, handler generic.Handler) { + c.AddGenericHandler(ctx, name, generic.NewRemoveHandler(name, c.Updater(), handler)) +} + +func (c *virtualMachineInstanceController) OnChange(ctx context.Context, name string, sync VirtualMachineInstanceHandler) { + c.AddGenericHandler(ctx, name, FromVirtualMachineInstanceHandlerToHandler(sync)) +} + +func (c *virtualMachineInstanceController) OnRemove(ctx context.Context, name string, sync VirtualMachineInstanceHandler) { + c.AddGenericHandler(ctx, name, generic.NewRemoveHandler(name, c.Updater(), FromVirtualMachineInstanceHandlerToHandler(sync))) +} + +func (c *virtualMachineInstanceController) Enqueue(namespace, name string) { + c.controller.Enqueue(namespace, name) +} + +func (c *virtualMachineInstanceController) EnqueueAfter(namespace, name string, duration time.Duration) { + c.controller.EnqueueAfter(namespace, name, duration) +} + +func (c *virtualMachineInstanceController) Informer() cache.SharedIndexInformer { + return c.controller.Informer() +} + +func (c *virtualMachineInstanceController) GroupVersionKind() schema.GroupVersionKind { + return c.gvk +} + +func (c *virtualMachineInstanceController) Cache() VirtualMachineInstanceCache { + return &virtualMachineInstanceCache{ + indexer: c.Informer().GetIndexer(), + resource: c.groupResource, + } +} + +func (c *virtualMachineInstanceController) Create(obj *v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error) { + result := &v1.VirtualMachineInstance{} + return result, c.client.Create(context.TODO(), obj.Namespace, obj, result, metav1.CreateOptions{}) +} + +func (c *virtualMachineInstanceController) Update(obj *v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error) { + result := &v1.VirtualMachineInstance{} + return result, c.client.Update(context.TODO(), obj.Namespace, obj, result, metav1.UpdateOptions{}) +} + +func (c *virtualMachineInstanceController) UpdateStatus(obj *v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error) { + result := &v1.VirtualMachineInstance{} + return result, c.client.UpdateStatus(context.TODO(), obj.Namespace, obj, result, metav1.UpdateOptions{}) +} + +func (c *virtualMachineInstanceController) Delete(namespace, name string, options *metav1.DeleteOptions) error { + if options == nil { + options = &metav1.DeleteOptions{} + } + return c.client.Delete(context.TODO(), namespace, name, *options) +} + +func (c *virtualMachineInstanceController) Get(namespace, name string, options metav1.GetOptions) (*v1.VirtualMachineInstance, error) { + result := &v1.VirtualMachineInstance{} + return result, c.client.Get(context.TODO(), namespace, name, result, options) +} + +func (c *virtualMachineInstanceController) List(namespace string, opts metav1.ListOptions) (*v1.VirtualMachineInstanceList, error) { + result := &v1.VirtualMachineInstanceList{} + return result, c.client.List(context.TODO(), namespace, result, opts) +} + +func (c *virtualMachineInstanceController) Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error) { + return c.client.Watch(context.TODO(), namespace, opts) +} + +func (c *virtualMachineInstanceController) Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (*v1.VirtualMachineInstance, error) { + result := &v1.VirtualMachineInstance{} + return result, c.client.Patch(context.TODO(), namespace, name, pt, data, result, metav1.PatchOptions{}, subresources...) +} + +type virtualMachineInstanceCache struct { + indexer cache.Indexer + resource schema.GroupResource +} + +func (c *virtualMachineInstanceCache) Get(namespace, name string) (*v1.VirtualMachineInstance, error) { + obj, exists, err := c.indexer.GetByKey(namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(c.resource, name) + } + return obj.(*v1.VirtualMachineInstance), nil +} + +func (c *virtualMachineInstanceCache) List(namespace string, selector labels.Selector) (ret []*v1.VirtualMachineInstance, err error) { + + err = cache.ListAllByNamespace(c.indexer, namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1.VirtualMachineInstance)) + }) + + return ret, err +} + +func (c *virtualMachineInstanceCache) AddIndexer(indexName string, indexer VirtualMachineInstanceIndexer) { + utilruntime.Must(c.indexer.AddIndexers(map[string]cache.IndexFunc{ + indexName: func(obj interface{}) (strings []string, e error) { + return indexer(obj.(*v1.VirtualMachineInstance)) + }, + })) +} + +func (c *virtualMachineInstanceCache) GetByIndex(indexName, key string) (result []*v1.VirtualMachineInstance, err error) { + objs, err := c.indexer.ByIndex(indexName, key) + if err != nil { + return nil, err + } + result = make([]*v1.VirtualMachineInstance, 0, len(objs)) + for _, obj := range objs { + result = append(result, obj.(*v1.VirtualMachineInstance)) + } + return result, nil +} + +type VirtualMachineInstanceStatusHandler func(obj *v1.VirtualMachineInstance, status v1.VirtualMachineInstanceStatus) (v1.VirtualMachineInstanceStatus, error) + +type VirtualMachineInstanceGeneratingHandler func(obj *v1.VirtualMachineInstance, status v1.VirtualMachineInstanceStatus) ([]runtime.Object, v1.VirtualMachineInstanceStatus, error) + +func RegisterVirtualMachineInstanceStatusHandler(ctx context.Context, controller VirtualMachineInstanceController, condition condition.Cond, name string, handler VirtualMachineInstanceStatusHandler) { + statusHandler := &virtualMachineInstanceStatusHandler{ + client: controller, + condition: condition, + handler: handler, + } + controller.AddGenericHandler(ctx, name, FromVirtualMachineInstanceHandlerToHandler(statusHandler.sync)) +} + +func RegisterVirtualMachineInstanceGeneratingHandler(ctx context.Context, controller VirtualMachineInstanceController, apply apply.Apply, + condition condition.Cond, name string, handler VirtualMachineInstanceGeneratingHandler, opts *generic.GeneratingHandlerOptions) { + statusHandler := &virtualMachineInstanceGeneratingHandler{ + VirtualMachineInstanceGeneratingHandler: handler, + apply: apply, + name: name, + gvk: controller.GroupVersionKind(), + } + if opts != nil { + statusHandler.opts = *opts + } + controller.OnChange(ctx, name, statusHandler.Remove) + RegisterVirtualMachineInstanceStatusHandler(ctx, controller, condition, name, statusHandler.Handle) +} + +type virtualMachineInstanceStatusHandler struct { + client VirtualMachineInstanceClient + condition condition.Cond + handler VirtualMachineInstanceStatusHandler +} + +func (a *virtualMachineInstanceStatusHandler) sync(key string, obj *v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error) { + if obj == nil { + return obj, nil + } + + origStatus := obj.Status.DeepCopy() + obj = obj.DeepCopy() + newStatus, err := a.handler(obj, obj.Status) + if err != nil { + // Revert to old status on error + newStatus = *origStatus.DeepCopy() + } + + if a.condition != "" { + if errors.IsConflict(err) { + a.condition.SetError(&newStatus, "", nil) + } else { + a.condition.SetError(&newStatus, "", err) + } + } + if !equality.Semantic.DeepEqual(origStatus, &newStatus) { + if a.condition != "" { + // Since status has changed, update the lastUpdatedTime + a.condition.LastUpdated(&newStatus, time.Now().UTC().Format(time.RFC3339)) + } + + var newErr error + obj.Status = newStatus + newObj, newErr := a.client.UpdateStatus(obj) + if err == nil { + err = newErr + } + if newErr == nil { + obj = newObj + } + } + return obj, err +} + +type virtualMachineInstanceGeneratingHandler struct { + VirtualMachineInstanceGeneratingHandler + apply apply.Apply + opts generic.GeneratingHandlerOptions + gvk schema.GroupVersionKind + name string +} + +func (a *virtualMachineInstanceGeneratingHandler) Remove(key string, obj *v1.VirtualMachineInstance) (*v1.VirtualMachineInstance, error) { + if obj != nil { + return obj, nil + } + + obj = &v1.VirtualMachineInstance{} + obj.Namespace, obj.Name = kv.RSplit(key, "/") + obj.SetGroupVersionKind(a.gvk) + + return nil, generic.ConfigureApplyForObject(a.apply, obj, &a.opts). + WithOwner(obj). + WithSetID(a.name). + ApplyObjects() +} + +func (a *virtualMachineInstanceGeneratingHandler) Handle(obj *v1.VirtualMachineInstance, status v1.VirtualMachineInstanceStatus) (v1.VirtualMachineInstanceStatus, error) { + if !obj.DeletionTimestamp.IsZero() { + return status, nil + } + + objs, newStatus, err := a.VirtualMachineInstanceGeneratingHandler(obj, status) + if err != nil { + return newStatus, err + } + + return newStatus, generic.ConfigureApplyForObject(a.apply, obj, &a.opts). + WithOwner(obj). + WithSetID(a.name). + ApplyObjects(objs...) +} diff --git a/pkg/util/common/common.go b/pkg/util/common/common.go index 37041880..1ec919f3 100644 --- a/pkg/util/common/common.go +++ b/pkg/util/common/common.go @@ -1,11 +1,16 @@ package common import ( + "encoding/json" "fmt" "os" "path/filepath" "strconv" "strings" + + kubevirtv1 "kubevirt.io/api/core/v1" + + "github.com/harvester/pcidevices/pkg/apis/devices.harvesterhci.io/v1beta1" ) const ( @@ -77,3 +82,79 @@ func GetVFList(pfDir string) (vfList []string, err error) { } return } + +// VMByHostDeviceName indexes VM's by host device name. +// It could be usb device claim or pci device claim name. +func VMByHostDeviceName(obj *kubevirtv1.VirtualMachine) ([]string, error) { + if obj.Annotations == nil { + return nil, nil + } + + allocationDetails, ok := obj.Annotations[v1beta1.DeviceAllocationKey] + if !ok { + return nil, nil + } + + allocatedHostDevices, err := generateHostDeviceAllocation(obj, allocationDetails) + if err != nil { + return nil, err + } + + return allocatedHostDevices, nil +} + +// VMByVGPUDevice indexes VM's by vgpu names +func VMByVGPUDevice(obj *kubevirtv1.VirtualMachine) ([]string, error) { + // find and add vgpu info from the DeviceAllocationKey annotation if present on the vm + if obj.Annotations == nil { + return nil, nil + } + allocationDetails, ok := obj.Annotations[v1beta1.DeviceAllocationKey] + if !ok { + return nil, nil + } + + allocatedGPUs, err := generateGPUDeviceAllocation(obj, allocationDetails) + if err != nil { + return nil, err + } + return allocatedGPUs, nil +} + +func generateDeviceAllocationDetails(allocationDetails string) (*v1beta1.AllocationDetails, error) { + currentAllocation := &v1beta1.AllocationDetails{} + err := json.Unmarshal([]byte(allocationDetails), currentAllocation) + return currentAllocation, err +} + +func generateDeviceInfo(devices map[string][]string) []string { + var allDevices []string + for _, v := range devices { + allDevices = append(allDevices, v...) + } + return allDevices +} + +func generateGPUDeviceAllocation(obj *kubevirtv1.VirtualMachine, allocationDetails string) ([]string, error) { + allocation, err := generateDeviceAllocationDetails(allocationDetails) + if err != nil { + return nil, fmt.Errorf("error generating device allocation details %s/%s: %v", obj.Name, obj.Namespace, err) + } + + if allocation.GPUs != nil { + return generateDeviceInfo(allocation.GPUs), nil + } + return nil, nil +} + +func generateHostDeviceAllocation(obj *kubevirtv1.VirtualMachine, allocationDetails string) ([]string, error) { + allocation, err := generateDeviceAllocationDetails(allocationDetails) + if err != nil { + return nil, fmt.Errorf("error generating device allocation details %s/%s: %v", obj.Name, obj.Namespace, err) + } + + if allocation.HostDevices != nil { + return generateDeviceInfo(allocation.HostDevices), nil + } + return nil, nil +} diff --git a/pkg/util/executor/remote.go b/pkg/util/executor/remote.go index 45b91868..027f4264 100644 --- a/pkg/util/executor/remote.go +++ b/pkg/util/executor/remote.go @@ -64,7 +64,11 @@ func (r *RemoteCommandExecutor) Run(cmd string, args []string) ([]byte, error) { Executor: &exec.DefaultRemoteExecutor{}, } - cmdString := fmt.Sprintf("%s %s", cmd, strings.Join(args, " ")) + var argString string + if len(args) > 0 { + argString = strings.Join(args, " ") + } + cmdString := fmt.Sprintf("%s %s", cmd, argString) options.Command = []string{"/bin/sh", "-c", cmdString} err := options.Run() if err != nil { diff --git a/pkg/util/fakeclients/vm.go b/pkg/util/fakeclients/vm.go index 7cdde62f..cfe4fdc2 100644 --- a/pkg/util/fakeclients/vm.go +++ b/pkg/util/fakeclients/vm.go @@ -2,6 +2,7 @@ package fakeclients import ( "context" + "slices" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -11,6 +12,8 @@ import ( kubevirtv1 "github.com/harvester/harvester/pkg/generated/clientset/versioned/typed/kubevirt.io/v1" kubevirtctlv1 "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1" + + "github.com/harvester/pcidevices/pkg/util/common" ) const ( @@ -88,10 +91,12 @@ func (c VirtualMachineCache) GetByIndex(indexName, key string) ([]*kubevirtv1api } for _, vm := range vmList { - for _, hostDevice := range vm.Spec.Template.Spec.Domain.Devices.HostDevices { - if hostDevice.Name == key { - vms = append(vms, vm) - } + deviceInfo, err := common.VMByHostDeviceName(vm) + if err != nil { + return nil, err + } + if slices.Contains(deviceInfo, key) { + vms = append(vms, vm) } } return vms, nil @@ -103,10 +108,12 @@ func (c VirtualMachineCache) GetByIndex(indexName, key string) ([]*kubevirtv1api } for _, vm := range vmList { - for _, gpuDevice := range vm.Spec.Template.Spec.Domain.Devices.GPUs { - if gpuDevice.Name == key { - vms = append(vms, vm) - } + deviceInfo, err := common.VMByVGPUDevice(vm) + if err != nil { + return nil, err + } + if slices.Contains(deviceInfo, key) { + vms = append(vms, vm) } } return vms, nil diff --git a/pkg/util/gousb/usbid/load_data.go b/pkg/util/gousb/usbid/load_data.go index 2b9a42e8..cbc5ed3c 100644 --- a/pkg/util/gousb/usbid/load_data.go +++ b/pkg/util/gousb/usbid/load_data.go @@ -27,8 +27,8 @@ import "time" // // The baked-in data was last generated: // -// 2024-09-09 09:38:25.925192 +0800 CST m=+1.282289042 -var LastUpdate = time.Unix(0, 1725845905925192000) +// 2024-09-12 10:14:18.151722033 +1000 AEST m=+1.350937835 +var LastUpdate = time.Unix(0, 1726100058151722033) const usbIDListData = `# # List of USB ID's diff --git a/pkg/webhook/indexer.go b/pkg/webhook/indexer.go index 97ee5c91..d51885f8 100644 --- a/pkg/webhook/indexer.go +++ b/pkg/webhook/indexer.go @@ -6,6 +6,7 @@ import ( kubevirtv1 "kubevirt.io/api/core/v1" "github.com/harvester/pcidevices/pkg/apis/devices.harvesterhci.io/v1beta1" + "github.com/harvester/pcidevices/pkg/util/common" ) const ( @@ -21,9 +22,9 @@ const ( func RegisterIndexers(clients *Clients) { vmCache := clients.KubevirtFactory.Kubevirt().V1().VirtualMachine().Cache() vmCache.AddIndexer(VMByName, vmByName) - vmCache.AddIndexer(VMByPCIDeviceClaim, vmByHostDeviceName) - vmCache.AddIndexer(VMByUSBDeviceClaim, vmByHostDeviceName) - vmCache.AddIndexer(VMByVGPU, vmByVGPUDevice) + vmCache.AddIndexer(VMByPCIDeviceClaim, common.VMByHostDeviceName) + vmCache.AddIndexer(VMByUSBDeviceClaim, common.VMByHostDeviceName) + vmCache.AddIndexer(VMByVGPU, common.VMByVGPUDevice) deviceCache := clients.DeviceFactory.Devices().V1beta1().PCIDevice().Cache() deviceCache.AddIndexer(PCIDeviceByResourceName, pciDeviceByResourceName) deviceCache.AddIndexer(IommuGroupByNode, iommuGroupByNodeName) @@ -45,25 +46,6 @@ func iommuGroupByNodeName(obj *v1beta1.PCIDevice) ([]string, error) { return []string{fmt.Sprintf("%s-%s", obj.Status.NodeName, obj.Status.IOMMUGroup)}, nil } -// vmByHostDeviceName indexes VM's by host device name. -// It could be usb device claim or pci device claim name. -func vmByHostDeviceName(obj *kubevirtv1.VirtualMachine) ([]string, error) { - hostDeviceName := make([]string, 0, len(obj.Spec.Template.Spec.Domain.Devices.HostDevices)) - for _, hostDevice := range obj.Spec.Template.Spec.Domain.Devices.HostDevices { - hostDeviceName = append(hostDeviceName, hostDevice.Name) - } - return hostDeviceName, nil -} - -// vmByVGPUDevice indexes VM's by vgpu names -func vmByVGPUDevice(obj *kubevirtv1.VirtualMachine) ([]string, error) { - gpuNames := make([]string, 0, len(obj.Spec.Template.Spec.Domain.Devices.GPUs)) - for _, gpuDevice := range obj.Spec.Template.Spec.Domain.Devices.GPUs { - gpuNames = append(gpuNames, gpuDevice.Name) - } - return gpuNames, nil -} - func usbDeviceClaimByAddress(obj *v1beta1.USBDeviceClaim) ([]string, error) { return []string{fmt.Sprintf("%s-%s", obj.Status.NodeName, obj.Status.PCIAddress)}, nil } diff --git a/pkg/webhook/usbdeviceclaim_test.go b/pkg/webhook/usbdeviceclaim_test.go index e60922d4..2ec253de 100644 --- a/pkg/webhook/usbdeviceclaim_test.go +++ b/pkg/webhook/usbdeviceclaim_test.go @@ -36,6 +36,9 @@ var ( ObjectMeta: metav1.ObjectMeta{ Name: "vm-with-usb-devices", Namespace: "default", + Annotations: map[string]string{ + devicesv1beta1.DeviceAllocationKey: `{"hostdevices":{"fake.com/device1":["usbdevice1"]}}`, + }, }, Spec: kubevirtv1.VirtualMachineSpec{ Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ diff --git a/pkg/webhook/vgpu.go b/pkg/webhook/vgpu.go index 67bdd915..4b65291d 100644 --- a/pkg/webhook/vgpu.go +++ b/pkg/webhook/vgpu.go @@ -6,11 +6,12 @@ import ( "github.com/sirupsen/logrus" - kubevirtctl "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1" - "github.com/harvester/harvester/pkg/webhook/types" admissionregv1 "k8s.io/api/admissionregistration/v1" "k8s.io/apimachinery/pkg/runtime" + kubevirtctl "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1" + "github.com/harvester/harvester/pkg/webhook/types" + devicesv1beta1 "github.com/harvester/pcidevices/pkg/apis/devices.harvesterhci.io/v1beta1" ) diff --git a/pkg/webhook/vgpu_test.go b/pkg/webhook/vgpu_test.go index 00cf234b..cb8999ce 100644 --- a/pkg/webhook/vgpu_test.go +++ b/pkg/webhook/vgpu_test.go @@ -78,6 +78,9 @@ var ( ObjectMeta: metav1.ObjectMeta{ Name: "vgpu-vm", Namespace: "default", + Annotations: map[string]string{ + devicesv1beta1.DeviceAllocationKey: `{"gpus":{"nvidia.com/fakevgpu":["vgpu1"]}}`, + }, }, Spec: kubevirtv1.VirtualMachineSpec{ Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ @@ -173,6 +176,7 @@ func Test_VGPUDeletion(t *testing.T) { vGPUValidator := NewVGPUValidator(virtualMachineCache) for _, v := range testCases { err := vGPUValidator.Delete(nil, v.gpu) + t.Log(err) if v.expectError { assert.Error(err, fmt.Sprintf("expected to find error for test case %s", v.name)) } else { diff --git a/pkg/webhook/vm_test.go b/pkg/webhook/vm_test.go index aa74bf09..b301ac3c 100644 --- a/pkg/webhook/vm_test.go +++ b/pkg/webhook/vm_test.go @@ -107,6 +107,9 @@ var ( ObjectMeta: metav1.ObjectMeta{ Name: "vm-with-iommu-devices", Namespace: "default", + Annotations: map[string]string{ + devicesv1beta1.DeviceAllocationKey: `{"hostdevices":{"fake.com/device1":["node1dev1"]}}`, + }, }, Spec: kubevirtv1.VirtualMachineSpec{ Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ @@ -130,6 +133,9 @@ var ( ObjectMeta: metav1.ObjectMeta{ Name: "vm-with-iommu-devices", Namespace: "default", + Annotations: map[string]string{ + devicesv1beta1.DeviceAllocationKey: `{"hostdevices":{"fake.com/device1":["node1dev1"],"fake.com/device2":["node1dev2"]}}`, + }, }, Spec: kubevirtv1.VirtualMachineSpec{ Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{