diff --git a/pkg/karmadactl/addons/addons.go b/pkg/karmadactl/addons/addons.go index cf5c457eb5ec..1a62afa4549a 100644 --- a/pkg/karmadactl/addons/addons.go +++ b/pkg/karmadactl/addons/addons.go @@ -18,7 +18,8 @@ var ( 1. karmada-descheduler 2. karmada-scheduler-estimator - 3. karmada-search`) + 3. karmada-search + 4. karmada-metrics-adapter`) addonsExamples = templates.Examples(` # Enable or disable Karmada addons to the karmada-host cluster diff --git a/pkg/karmadactl/addons/enable.go b/pkg/karmadactl/addons/enable.go index 835a9a04a23b..ce3b8461a87f 100644 --- a/pkg/karmadactl/addons/enable.go +++ b/pkg/karmadactl/addons/enable.go @@ -69,6 +69,8 @@ func NewCmdAddonsEnable(parentCommand string) *cobra.Command { flags.IntVar(&opts.WaitComponentReadyTimeout, "pod-timeout", options.WaitComponentReadyTimeout, "Wait pod ready timeout.") flags.IntVar(&opts.WaitAPIServiceReadyTimeout, "apiservice-timeout", 30, "Wait apiservice ready timeout.") flags.StringVar(&opts.KarmadaSearchImage, "karmada-search-image", addoninit.DefaultKarmadaSearchImage, "karmada search image") + flags.Int32Var(&opts.KarmadaMetricsAdapterReplicas, "karmada-metrics-adapter-replicas", 1, "Karmada search replica set") + flags.StringVar(&opts.KarmadaMetricsAdapterImage, "karmada-metrics-adapter-image", addoninit.DefaultKarmadaMetricsAdapterImage, "karmada metrics adapter image") flags.Int32Var(&opts.KarmadaSearchReplicas, "karmada-search-replicas", 1, "Karmada search replica set") flags.StringVar(&opts.KarmadaDeschedulerImage, "karmada-descheduler-image", addoninit.DefaultKarmadaDeschedulerImage, "karmada descheduler image") flags.Int32Var(&opts.KarmadaDeschedulerReplicas, "karmada-descheduler-replicas", 1, "Karmada descheduler replica set") diff --git a/pkg/karmadactl/addons/init/addon.go b/pkg/karmadactl/addons/init/addon.go index 7bcd5c9d17d0..a29bf69c0d3c 100644 --- a/pkg/karmadactl/addons/init/addon.go +++ b/pkg/karmadactl/addons/init/addon.go @@ -23,6 +23,9 @@ const ( // SearchResourceName define Search Addon and component installed name SearchResourceName = "karmada-search" + + // MetricsAdapterResourceName define metrics-adapter Addon and component installed name + MetricsAdapterResourceName = "karmada-metrics-adapter" ) // Addons hosts the optional components that support by karmada diff --git a/pkg/karmadactl/addons/init/enable_option.go b/pkg/karmadactl/addons/init/enable_option.go index 92938cf6f087..82ae92f32f51 100644 --- a/pkg/karmadactl/addons/init/enable_option.go +++ b/pkg/karmadactl/addons/init/enable_option.go @@ -27,6 +27,10 @@ type CommandAddonsEnableOption struct { KarmadaDeschedulerReplicas int32 + KarmadaMetricsAdapterImage string + + KarmadaMetricsAdapterReplicas int32 + KarmadaSchedulerEstimatorImage string KarmadaEstimatorReplicas int32 @@ -51,8 +55,10 @@ var ( DefaultKarmadaDeschedulerImage string // DefaultKarmadaSchedulerEstimatorImage Karmada scheduler-estimator image DefaultKarmadaSchedulerEstimatorImage string - // DefaultKarmadaSearchImage Karmada search estimator image + // DefaultKarmadaSearchImage Karmada search image DefaultKarmadaSearchImage string + // DefaultKarmadaMetricsAdapterImage Karmada metrics adapter image + DefaultKarmadaMetricsAdapterImage string karmadaRelease string ) @@ -68,6 +74,7 @@ func init() { DefaultKarmadaDeschedulerImage = fmt.Sprintf("docker.io/karmada/karmada-descheduler:%s", releaseVer.ReleaseVersion()) DefaultKarmadaSchedulerEstimatorImage = fmt.Sprintf("docker.io/karmada/karmada-scheduler-estimator:%s", releaseVer.ReleaseVersion()) DefaultKarmadaSearchImage = fmt.Sprintf("docker.io/karmada/karmada-search:%s", releaseVer.ReleaseVersion()) + DefaultKarmadaMetricsAdapterImage = fmt.Sprintf("docker.io/karmada/karmada-metrics-adapter:%s", releaseVer.ReleaseVersion()) } // KarmadaDeschedulerImage get karmada descheduler image @@ -94,6 +101,14 @@ func KarmadaSearchImage(o *CommandAddonsEnableOption) string { return o.KarmadaSearchImage } +// KarmadaMetricsAdapterImage get karmada metrics adapter image +func KarmadaMetricsAdapterImage(o *CommandAddonsEnableOption) string { + if o.ImageRegistry != "" && o.KarmadaMetricsAdapterImage == DefaultKarmadaMetricsAdapterImage { + return o.ImageRegistry + "/karmada-metrics-adapter:" + karmadaRelease + } + return o.KarmadaMetricsAdapterImage +} + // Complete the conditions required to be able to run enable. func (o *CommandAddonsEnableOption) Complete() error { err := o.GlobalCommandOptions.Complete() diff --git a/pkg/karmadactl/addons/install/install.go b/pkg/karmadactl/addons/install/install.go index 5462e4a21985..b9e83e5827cf 100644 --- a/pkg/karmadactl/addons/install/install.go +++ b/pkg/karmadactl/addons/install/install.go @@ -4,6 +4,7 @@ import ( "github.com/karmada-io/karmada/pkg/karmadactl/addons/descheduler" "github.com/karmada-io/karmada/pkg/karmadactl/addons/estimator" addonsinit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + "github.com/karmada-io/karmada/pkg/karmadactl/addons/metricsadapter" "github.com/karmada-io/karmada/pkg/karmadactl/addons/search" ) @@ -12,4 +13,5 @@ func Install() { addonsinit.Addons["karmada-search"] = search.AddonSearch addonsinit.Addons["karmada-descheduler"] = descheduler.AddonDescheduler addonsinit.Addons["karmada-scheduler-estimator"] = estimator.AddonEstimator + addonsinit.Addons["karmada-metrics-adapter"] = metricsadapter.AddonMetricsAdapter } diff --git a/pkg/karmadactl/addons/metricsadapter/manifests.go b/pkg/karmadactl/addons/metricsadapter/manifests.go new file mode 100644 index 000000000000..7c14f15a79a8 --- /dev/null +++ b/pkg/karmadactl/addons/metricsadapter/manifests.go @@ -0,0 +1,150 @@ +package metricsadapter + +const ( + karmadaMetricsAdapterDeployment = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karmada-metrics-adapter + namespace: {{ .Namespace }} + labels: + app: karmada-metrics-adapter + apiserver: "true" +spec: + selector: + matchLabels: + app: karmada-metrics-adapter + apiserver: "true" + replicas: {{ .Replicas }} + template: + metadata: + labels: + app: karmada-metrics-adapter + apiserver: "true" + spec: + automountServiceAccountToken: false + containers: + - name: karmada-metrics-adapter + image: {{ .Image }} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true + - name: kubeconfig + subPath: kubeconfig + mountPath: /etc/kubeconfig + command: + - /bin/karmada-metrics-adapter + - --kubeconfig=/etc/kubeconfig + - --authentication-kubeconfig=/etc/kubeconfig + - --authorization-kubeconfig=/etc/kubeconfig + - --client-ca-file=/etc/karmada/pki/ca.crt + - --audit-log-path=- + - --audit-log-maxage=0 + - --audit-log-maxbackup=0 + readinessProbe: + httpGet: + path: /readyz + port: 443 + scheme: HTTPS + initialDelaySeconds: 1 + failureThreshold: 3 + periodSeconds: 3 + timeoutSeconds: 15 + livenessProbe: + httpGet: + path: /healthz + port: 443 + scheme: HTTPS + initialDelaySeconds: 10 + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 15 + resources: + requests: + cpu: 100m + volumes: + - name: k8s-certs + secret: + secretName: karmada-cert + - name: kubeconfig + secret: + secretName: kubeconfig +` + + karmadaMetricsAdapterService = ` +apiVersion: v1 +kind: Service +metadata: + name: karmada-metrics-adapter + namespace: {{ .Namespace }} + labels: + app: karmada-metrics-adapter + apiserver: "true" +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + app: karmada-metrics-adapter +` + + karmadaMetricsAdapterAAAPIService = ` +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: {{ .Name }} +spec: + service: + name: karmada-metrics-adapter + namespace: {{ .Namespace }} + group: {{ .Group }} + version: {{ .Version }} + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 200 +` + + karmadaMetricsAdapterAAService = ` +apiVersion: v1 +kind: Service +metadata: + name: karmada-metrics-adapter + namespace: {{ .Namespace }} +spec: + type: ExternalName + externalName: karmada-metrics-adapter.{{ .Namespace }}.svc.{{ .HostClusterDomain }} +` +) + +// DeploymentReplace is a struct to help to concrete +// the karmada-metrics-adapter deployment bytes with the deployment template +type DeploymentReplace struct { + Namespace string + Replicas *int32 + Image string +} + +// ServiceReplace is a struct to help to concrete +// the karmada-metrics-adapter Service bytes with the Service template +type ServiceReplace struct { + Namespace string +} + +// AAApiServiceReplace is a struct to help to concrete +// the karmada-metrics-adapter ApiService bytes with the AAApiService template +type AAApiServiceReplace struct { + Name string + Namespace string + Group string + Version string +} + +// AAServiceReplace is a struct to help to concrete +// the karmada-metrics-adapter AA Service bytes with the AAService template +type AAServiceReplace struct { + Namespace string + HostClusterDomain string +} diff --git a/pkg/karmadactl/addons/metricsadapter/metricsadapter.go b/pkg/karmadactl/addons/metricsadapter/metricsadapter.go new file mode 100644 index 000000000000..eeacad2ed5b0 --- /dev/null +++ b/pkg/karmadactl/addons/metricsadapter/metricsadapter.go @@ -0,0 +1,206 @@ +package metricsadapter + +import ( + "context" + "fmt" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + apiregistrationv1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" + + addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init" + addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" + initkarmada "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada" + cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" +) + +const ( + // aaAPIServiceName define apiservice name install on karmada control plane + aaAPIServiceV1beta1Name = "v1beta1.metrics.k8s.io" + aaAPIServiceCustomV1beta1Name = "v1beta1.custom.metrics.k8s.io" + aaAPIServiceCustomV1beta2Name = "v1beta2.custom.metrics.k8s.io" +) + +// AddonMetricsAdapter describe the metrics-adapter addon command process +var AddonMetricsAdapter = &addoninit.Addon{ + Name: addoninit.MetricsAdapterResourceName, + Status: status, + Enable: enableMetricsAdapter, + Disable: disableMetricsAdapter, +} + +var status = func(opts *addoninit.CommandAddonsListOption) (string, error) { + // check karmada-metrics-adapter deployment status on host cluster + deployClient := opts.KubeClientSet.AppsV1().Deployments(opts.Namespace) + deployment, err := deployClient.Get(context.TODO(), addoninit.MetricsAdapterResourceName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return addoninit.AddonDisabledStatus, nil + } + return addoninit.AddonUnknownStatus, err + } + if deployment.Status.Replicas != deployment.Status.ReadyReplicas || + deployment.Status.Replicas != deployment.Status.AvailableReplicas { + return addoninit.AddonUnhealthyStatus, nil + } + + // check metrics.k8s.io apiservice is available on karmada control plane + for _, aaAPIServiceName := range []string{aaAPIServiceV1beta1Name, aaAPIServiceCustomV1beta1Name, aaAPIServiceCustomV1beta2Name} { + apiService, err := opts.KarmadaAggregatorClientSet.ApiregistrationV1().APIServices().Get(context.TODO(), aaAPIServiceName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return addoninit.AddonDisabledStatus, nil + } + return addoninit.AddonUnknownStatus, err + } + if !apiregistrationv1helper.IsAPIServiceConditionTrue(apiService, apiregistrationv1.Available) { + return addoninit.AddonUnhealthyStatus, nil + } + } + + return addoninit.AddonEnabledStatus, nil +} + +var enableMetricsAdapter = func(opts *addoninit.CommandAddonsEnableOption) error { + if err := installComponentsOnHostCluster(opts); err != nil { + return err + } + + if err := installComponentsOnKarmadaControlPlane(opts); err != nil { + return err + } + + return nil +} + +var disableMetricsAdapter = func(opts *addoninit.CommandAddonsDisableOption) error { + // delete karmada metrics adapter service on host cluster + serviceClient := opts.KubeClientSet.CoreV1().Services(opts.Namespace) + if err := serviceClient.Delete(context.TODO(), addoninit.MetricsAdapterResourceName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + klog.Infof("Uninstall karmada metrics adapter service on host cluster successfully") + + // delete karmada metrics adapter deployment on host cluster + deployClient := opts.KubeClientSet.AppsV1().Deployments(opts.Namespace) + if err := deployClient.Delete(context.TODO(), addoninit.MetricsAdapterResourceName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + klog.Infof("Uninstall karmada metrics adapter deployment on host cluster successfully") + + // delete karmada metrics adapter aa service on karmada control plane + karmadaServiceClient := opts.KarmadaKubeClientSet.CoreV1().Services(opts.Namespace) + if err := karmadaServiceClient.Delete(context.TODO(), addoninit.MetricsAdapterResourceName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + klog.Infof("Uninstall karmada metrics adapter AA service on karmada control plane successfully") + for _, aaAPIServiceName := range []string{aaAPIServiceV1beta1Name, aaAPIServiceCustomV1beta1Name, aaAPIServiceCustomV1beta2Name} { + // delete karmada metrics adapter aa apiservice on karmada control plane + if err := opts.KarmadaAggregatorClientSet.ApiregistrationV1().APIServices().Delete(context.TODO(), aaAPIServiceName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + + klog.Infof("Uninstall karmada metrics adapter AA apiservice on karmada control plane successfully") + return nil +} + +func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) error { + // install karmada metrics adapter service on host cluster + karmadaMetricsAdapterServiceBytes, err := addonutils.ParseTemplate(karmadaMetricsAdapterService, ServiceReplace{ + Namespace: opts.Namespace, + }) + if err != nil { + return fmt.Errorf("error when parsing karmada metrics adapter service template :%v", err) + } + + karmadaMetricsAdapterService := &corev1.Service{} + if err = kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaMetricsAdapterServiceBytes, karmadaMetricsAdapterService); err != nil { + return fmt.Errorf("decode karmada metrics adapter service error: %v", err) + } + + if err = cmdutil.CreateService(opts.KubeClientSet, karmadaMetricsAdapterService); err != nil { + return fmt.Errorf("create karmada metrics adapter service error: %v", err) + } + + klog.Infof("Install karmada metrics adapter service on host cluster successfully") + + // install karmada metrics adapter deployment on host clusters + karmadaMetricsAdapterDeploymentBytes, err := addonutils.ParseTemplate(karmadaMetricsAdapterDeployment, DeploymentReplace{ + Namespace: opts.Namespace, + Replicas: &opts.KarmadaMetricsAdapterReplicas, + Image: addoninit.KarmadaMetricsAdapterImage(opts), + }) + if err != nil { + return fmt.Errorf("error when parsing karmada metrics adapter deployment template :%v", err) + } + + karmadaMetricsAdapterDeployment := &appsv1.Deployment{} + if err = kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaMetricsAdapterDeploymentBytes, karmadaMetricsAdapterDeployment); err != nil { + return fmt.Errorf("decode karmada metrics adapter deployment error: %v", err) + } + if err = cmdutil.CreateOrUpdateDeployment(opts.KubeClientSet, karmadaMetricsAdapterDeployment); err != nil { + return fmt.Errorf("create karmada metrics adapter deployment error: %v", err) + } + + if err = cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaMetricsAdapterDeployment, opts.WaitComponentReadyTimeout); err != nil { + return fmt.Errorf("wait karmada metrics adapter pod status ready timeout: %v", err) + } + + klog.Infof("Install karmada metrics adapter deployment on host cluster successfully") + return nil +} + +func installComponentsOnKarmadaControlPlane(opts *addoninit.CommandAddonsEnableOption) error { + // install karmada metrics adapter AA service on karmada control plane + aaServiceBytes, err := addonutils.ParseTemplate(karmadaMetricsAdapterAAService, AAServiceReplace{ + Namespace: opts.Namespace, + HostClusterDomain: opts.HostClusterDomain, + }) + if err != nil { + return fmt.Errorf("error when parsing karmada metrics adapter AA service template :%v", err) + } + + aaService := &corev1.Service{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), aaServiceBytes, aaService); err != nil { + return fmt.Errorf("decode karmada metrics adapter AA service error: %v", err) + } + if err := cmdutil.CreateService(opts.KarmadaKubeClientSet, aaService); err != nil { + return fmt.Errorf("create karmada metrics adapter AA service error: %v", err) + } + for _, aaAPIServiceName := range []string{aaAPIServiceV1beta1Name, aaAPIServiceCustomV1beta1Name, aaAPIServiceCustomV1beta2Name} { + // install karmada metrics adapter apiservice on karmada control plane + gv := strings.SplitN(aaAPIServiceName, ".", 2) + aaAPIServiceBytes, err := addonutils.ParseTemplate(karmadaMetricsAdapterAAAPIService, AAApiServiceReplace{ + Name: aaAPIServiceName, + Namespace: opts.Namespace, + Group: gv[1], + Version: gv[0], + }) + if err != nil { + return fmt.Errorf("error when parsing karmada metrics adapter AA apiservice template :%v", err) + } + aaAPIService := &apiregistrationv1.APIService{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), aaAPIServiceBytes, aaAPIService); err != nil { + return fmt.Errorf("decode karmada metrics adapter AA apiservice error: %v", err) + } + + if err = cmdutil.CreateOrUpdateAPIService(opts.KarmadaAggregatorClientSet, aaAPIService); err != nil { + return fmt.Errorf("create karmada metrics adapter AA apiservice error: %v", err) + } + + if err := initkarmada.WaitAPIServiceReady(opts.KarmadaAggregatorClientSet, aaAPIServiceName, time.Duration(opts.WaitAPIServiceReadyTimeout)*time.Second); err != nil { + return err + } + } + klog.Infof("Install karmada metrics adapter api server on karmada control plane successfully") + return nil +}