From af7637a9611fa7fd31b46a39c8c131d66f81e9ad Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Tue, 17 Sep 2024 19:28:52 +0100 Subject: [PATCH] feat: Support node taints per nodepool and control plane --- .../caren.nutanix.com_awsclusterconfigs.yaml | 28 +++++ ...aren.nutanix.com_awsworkernodeconfigs.yaml | 29 +++++ ...aren.nutanix.com_dockerclusterconfigs.yaml | 28 +++++ .../caren.nutanix.com_dockernodeconfigs.yaml | 29 +++++ ...ren.nutanix.com_nutanixclusterconfigs.yaml | 28 +++++ .../caren.nutanix.com_nutanixnodeconfigs.yaml | 28 +++++ api/v1alpha1/nodeconfig_types.go | 52 +++++++++ api/v1alpha1/zz_generated.deepcopy.go | 43 ++++++- docs/content/customization/generic/taints.md | 88 ++++++++++++++ .../aws/mutation/metapatch_handler.go | 26 ++--- .../docker/mutation/metapatch_handler.go | 12 +- pkg/handlers/generic/handlers.go | 1 + pkg/handlers/generic/mutation/handlers.go | 13 +++ .../generic/mutation/metapatch_handler.go | 15 ++- .../mutation/taints/inject_controlplane.go | 107 +++++++++++++++++ .../taints/inject_controlplane_test.go | 71 ++++++++++++ .../mutation/taints/inject_suite_test.go | 16 +++ .../generic/mutation/taints/inject_worker.go | 109 ++++++++++++++++++ .../mutation/taints/inject_worker_test.go | 69 +++++++++++ .../generic/mutation/taints/variables_test.go | 38 ++++++ .../nutanix/mutation/metapatch_handler.go | 18 +-- 21 files changed, 818 insertions(+), 30 deletions(-) create mode 100644 docs/content/customization/generic/taints.md create mode 100644 pkg/handlers/generic/mutation/taints/inject_controlplane.go create mode 100644 pkg/handlers/generic/mutation/taints/inject_controlplane_test.go create mode 100644 pkg/handlers/generic/mutation/taints/inject_suite_test.go create mode 100644 pkg/handlers/generic/mutation/taints/inject_worker.go create mode 100644 pkg/handlers/generic/mutation/taints/inject_worker_test.go create mode 100644 pkg/handlers/generic/mutation/taints/variables_test.go diff --git a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml index 9373306e8..b88d19abb 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml @@ -343,6 +343,34 @@ spec: default: m5.xlarge type: string type: object + taints: + description: Taints specifies the taints the Node API object should be registered with. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + default: NoSchedule + description: |- + The effect of the taint on pods that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: The taint key to be applied to a node. + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array type: object encryptionAtRest: description: |- diff --git a/api/v1alpha1/crds/caren.nutanix.com_awsworkernodeconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_awsworkernodeconfigs.yaml index a359f8e40..cb05f6593 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_awsworkernodeconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_awsworkernodeconfigs.yaml @@ -90,6 +90,35 @@ spec: description: The AWS instance type to use for the cluster Machines. type: string type: object + taints: + description: Taints specifies the taints the Node API object should + be registered with. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + default: NoSchedule + description: |- + The effect of the taint on pods that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: The taint key to be applied to a node. + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array type: object type: object served: true diff --git a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml index 572b1307a..e1d1950b5 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml @@ -260,6 +260,34 @@ spec: pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*(:[\w][\w.-]{0,127})?(@[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][0-9A-Fa-f]{32,})?$ type: string type: object + taints: + description: Taints specifies the taints the Node API object should be registered with. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + default: NoSchedule + description: |- + The effect of the taint on pods that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: The taint key to be applied to a node. + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array type: object docker: type: object diff --git a/api/v1alpha1/crds/caren.nutanix.com_dockernodeconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_dockernodeconfigs.yaml index 121b41bd9..04141c4ed 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_dockernodeconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_dockernodeconfigs.yaml @@ -48,6 +48,35 @@ spec: pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*(:[\w][\w.-]{0,127})?(@[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][0-9A-Fa-f]{32,})?$ type: string type: object + taints: + description: Taints specifies the taints the Node API object should + be registered with. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + default: NoSchedule + description: |- + The effect of the taint on pods that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: The taint key to be applied to a node. + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array type: object type: object served: true diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml index 4a02fa26f..bfad4a986 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml @@ -413,6 +413,34 @@ spec: required: - machineDetails type: object + taints: + description: Taints specifies the taints the Node API object should be registered with. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + default: NoSchedule + description: |- + The effect of the taint on pods that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: The taint key to be applied to a node. + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array type: object encryptionAtRest: description: |- diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixnodeconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixnodeconfigs.yaml index 26db4e45d..25ed585bf 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_nutanixnodeconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_nutanixnodeconfigs.yaml @@ -201,6 +201,34 @@ spec: required: - machineDetails type: object + taints: + description: Taints specifies the taints the Node API object should be registered with. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + default: NoSchedule + description: |- + The effect of the taint on pods that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: The taint key to be applied to a node. + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array type: object type: object served: true diff --git a/api/v1alpha1/nodeconfig_types.go b/api/v1alpha1/nodeconfig_types.go index 8c05e489d..734ec825f 100644 --- a/api/v1alpha1/nodeconfig_types.go +++ b/api/v1alpha1/nodeconfig_types.go @@ -50,6 +50,8 @@ func (s AWSWorkerNodeConfig) VariableSchema() clusterv1.VariableSchema { //nolin type AWSWorkerNodeConfigSpec struct { // +kubebuilder:validation:Optional AWS *AWSWorkerNodeSpec `json:"aws,omitempty"` + + GenericNodeSpec `json:",inline"` } // AWSControlPlaneConfigSpec defines the desired state of AWSNodeConfig. @@ -58,6 +60,8 @@ type AWSWorkerNodeConfigSpec struct { type AWSControlPlaneNodeConfigSpec struct { // +kubebuilder:validation:Optional AWS *AWSControlPlaneNodeSpec `json:"aws,omitempty"` + + GenericNodeSpec `json:",inline"` } // +kubebuilder:object:root=true @@ -79,6 +83,8 @@ func (s DockerNodeConfig) VariableSchema() clusterv1.VariableSchema { //nolint:g type DockerNodeConfigSpec struct { // +kubebuilder:validation:Optional Docker *DockerNodeSpec `json:"docker,omitempty"` + + GenericNodeSpec `json:",inline"` } // +kubebuilder:object:root=true @@ -100,8 +106,54 @@ func (s NutanixNodeConfig) VariableSchema() clusterv1.VariableSchema { //nolint: type NutanixNodeConfigSpec struct { // +kubebuilder:validation:Optional Nutanix *NutanixNodeSpec `json:"nutanix,omitempty"` + + GenericNodeSpec `json:",inline"` +} + +type GenericNodeSpec struct { + // Taints specifies the taints the Node API object should be registered with. + // +kubebuilder:validation:Optional + Taints []Taint `json:"taints,omitempty"` +} + +// The node this Taint is attached to has the "effect" on +// any pod that does not tolerate the Taint. +type Taint struct { + // The taint key to be applied to a node. + // +kubebuilder:validation:Required + Key string `json:"key"` + + // The taint value corresponding to the taint key. + // +kubebuilder:validation:Optional + Value string `json:"value,omitempty"` + + // The effect of the taint on pods that do not tolerate the taint. + // Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + // +kubebuilder:validation:Required + // +kubebuilder:default=NoSchedule + // +kubebuilder:validation:Enum:=NoSchedule;PreferNoSchedule;NoExecute + Effect TaintEffect `json:"effect"` } +type TaintEffect string + +const ( + // Do not allow new pods to schedule onto the node unless they tolerate the taint, + // but allow all pods submitted to Kubelet without going through the scheduler + // to start, and allow all already-running pods to continue running. + // Enforced by the scheduler. + TaintEffectNoSchedule TaintEffect = "NoSchedule" + + // Like TaintEffectNoSchedule, but the scheduler tries not to schedule + // new pods onto the node, rather than prohibiting new pods from scheduling + // onto the node entirely. Enforced by the scheduler. + TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule" + + // Evict any already-running pods that do not tolerate the taint. + // Currently enforced by NodeController. + TaintEffectNoExecute TaintEffect = "NoExecute" +) + //nolint:gochecknoinits // Idiomatic to use init functions to register APIs with scheme. func init() { SchemeBuilder.Register(&AWSWorkerNodeConfig{}, &DockerNodeConfig{}, &NutanixNodeConfig{}) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ec3acbe69..ed5835fb6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -189,6 +189,7 @@ func (in *AWSControlPlaneNodeConfigSpec) DeepCopyInto(out *AWSControlPlaneNodeCo *out = new(AWSControlPlaneNodeSpec) (*in).DeepCopyInto(*out) } + in.GenericNodeSpec.DeepCopyInto(&out.GenericNodeSpec) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSControlPlaneNodeConfigSpec. @@ -351,6 +352,7 @@ func (in *AWSWorkerNodeConfigSpec) DeepCopyInto(out *AWSWorkerNodeConfigSpec) { *out = new(AWSWorkerNodeSpec) (*in).DeepCopyInto(*out) } + in.GenericNodeSpec.DeepCopyInto(&out.GenericNodeSpec) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSWorkerNodeConfigSpec. @@ -742,6 +744,7 @@ func (in *DockerNodeConfigSpec) DeepCopyInto(out *DockerNodeConfigSpec) { *out = new(DockerNodeSpec) (*in).DeepCopyInto(*out) } + in.GenericNodeSpec.DeepCopyInto(&out.GenericNodeSpec) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DockerNodeConfigSpec. @@ -1002,6 +1005,26 @@ func (in *GenericClusterConfigSpec) DeepCopy() *GenericClusterConfigSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GenericNodeSpec) DeepCopyInto(out *GenericNodeSpec) { + *out = *in + if in.Taints != nil { + in, out := &in.Taints, &out.Taints + *out = make([]Taint, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericNodeSpec. +func (in *GenericNodeSpec) DeepCopy() *GenericNodeSpec { + if in == nil { + return nil + } + out := new(GenericNodeSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlobalImageRegistryMirror) DeepCopyInto(out *GlobalImageRegistryMirror) { *out = *in @@ -1305,6 +1328,7 @@ func (in *NutanixNodeConfigSpec) DeepCopyInto(out *NutanixNodeConfigSpec) { *out = new(NutanixNodeSpec) (*in).DeepCopyInto(*out) } + in.GenericNodeSpec.DeepCopyInto(&out.GenericNodeSpec) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NutanixNodeConfigSpec. @@ -1334,7 +1358,9 @@ func (in *NutanixNodeSpec) DeepCopy() *NutanixNodeSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto(out *NutanixPrismCentralEndpointCredentials) { +func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto( + out *NutanixPrismCentralEndpointCredentials, +) { *out = *in out.SecretRef = in.SecretRef } @@ -1587,6 +1613,21 @@ func (in Subnets) DeepCopy() Subnets { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Taint) DeepCopyInto(out *Taint) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Taint. +func (in *Taint) DeepCopy() *Taint { + if in == nil { + return nil + } + out := new(Taint) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *User) DeepCopyInto(out *User) { *out = *in diff --git a/docs/content/customization/generic/taints.md b/docs/content/customization/generic/taints.md new file mode 100644 index 000000000..275703b0c --- /dev/null +++ b/docs/content/customization/generic/taints.md @@ -0,0 +1,88 @@ ++++ +title = "Tainting nodes" ++++ + +Tainting nodes prevents pods from being scheduled on them unless they explicitly tolerate the taints applied to the +nodes. See the [Kubernetes Taints and Tolerations] documentation for more details. + +This customization will be available when the +[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. + +## Example + +To configure taints for the control plane nodes, specify the following configuration: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + controlPlane: + taints: + - key: some-key + effect: NoSchedule + value: some-value +``` + +Taints for individual nodepools can be configured similarly: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + workers: + machineDeployments: + - class: default-worker + name: md-0 + variables: + overrides: + - name: workerConfig + value: + taints: + - key: some-key + effect: NoSchedule + value: some-value +``` + +Applying this configuration will result in the following value being set: + +- `KubeadmControlPlaneTemplate`: + + - ```yaml + spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + taints: + - key: some-key + effect: NoSchedule + value: some-value + joinConfiguration: + nodeRegistration: + taints: + - key: some-key + effect: NoSchedule + value: some-value + ``` + +- `KubeadmConfigTemplate`: + + - ```yaml + spec: + joinConfiguration: + nodeRegistration: + taints: + - key: some-key + effect: NoSchedule + value: some-value + ``` + +[Kubernetes Taints and Tolerations]: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ diff --git a/pkg/handlers/aws/mutation/metapatch_handler.go b/pkg/handlers/aws/mutation/metapatch_handler.go index 5a57a03b5..1bb2fb89c 100644 --- a/pkg/handlers/aws/mutation/metapatch_handler.go +++ b/pkg/handlers/aws/mutation/metapatch_handler.go @@ -21,19 +21,18 @@ import ( // MetaPatchHandler returns a meta patch handler for mutating CAPA clusters. func MetaPatchHandler(mgr manager.Manager) handlers.Named { - patchHandlers := append( - []mutation.MetaMutator{ - calico.NewPatch(), - region.NewPatch(), - network.NewPatch(), - controlplaneloadbalancer.NewPatch(), - iaminstanceprofile.NewControlPlanePatch(), - instancetype.NewControlPlanePatch(), - ami.NewControlPlanePatch(), - securitygroups.NewControlPlanePatch(), - }, - genericmutation.MetaMutators(mgr)..., - ) + patchHandlers := []mutation.MetaMutator{ + calico.NewPatch(), + region.NewPatch(), + network.NewPatch(), + controlplaneloadbalancer.NewPatch(), + iaminstanceprofile.NewControlPlanePatch(), + instancetype.NewControlPlanePatch(), + ami.NewControlPlanePatch(), + securitygroups.NewControlPlanePatch(), + } + patchHandlers = append(patchHandlers, genericmutation.MetaMutators(mgr)...) + patchHandlers = append(patchHandlers, genericmutation.ControlPlaneMetaMutators()...) return mutation.NewMetaGeneratePatchesHandler( "awsClusterConfigPatch", @@ -50,6 +49,7 @@ func MetaWorkerPatchHandler(mgr manager.Manager) handlers.Named { ami.NewWorkerPatch(), securitygroups.NewWorkerPatch(), } + patchHandlers = append(patchHandlers, genericmutation.WorkerMetaMutators()...) return mutation.NewMetaGeneratePatchesHandler( "awsWorkerConfigPatch", diff --git a/pkg/handlers/docker/mutation/metapatch_handler.go b/pkg/handlers/docker/mutation/metapatch_handler.go index 74d61bcaf..aef6b54a0 100644 --- a/pkg/handlers/docker/mutation/metapatch_handler.go +++ b/pkg/handlers/docker/mutation/metapatch_handler.go @@ -14,12 +14,11 @@ import ( // MetaPatchHandler returns a meta patch handler for mutating CAPD clusters. func MetaPatchHandler(mgr manager.Manager) handlers.Named { - patchHandlers := append( - []mutation.MetaMutator{ - customimage.NewControlPlanePatch(), - }, - genericmutation.MetaMutators(mgr)..., - ) + patchHandlers := []mutation.MetaMutator{ + customimage.NewControlPlanePatch(), + } + patchHandlers = append(patchHandlers, genericmutation.MetaMutators(mgr)...) + patchHandlers = append(patchHandlers, genericmutation.ControlPlaneMetaMutators()...) return mutation.NewMetaGeneratePatchesHandler( "dockerClusterConfigPatch", @@ -33,6 +32,7 @@ func MetaWorkerPatchHandler(mgr manager.Manager) handlers.Named { patchHandlers := []mutation.MetaMutator{ customimage.NewWorkerPatch(), } + patchHandlers = append(patchHandlers, genericmutation.WorkerMetaMutators()...) return mutation.NewMetaGeneratePatchesHandler( "dockerWorkerConfigPatch", diff --git a/pkg/handlers/generic/handlers.go b/pkg/handlers/generic/handlers.go index f985a9c76..b8728065d 100644 --- a/pkg/handlers/generic/handlers.go +++ b/pkg/handlers/generic/handlers.go @@ -21,5 +21,6 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { return []handlers.Named{ genericclusterconfig.NewVariable(), genericmutation.MetaPatchHandler(mgr), + genericmutation.MetaWorkerPatchHandler(mgr), } } diff --git a/pkg/handlers/generic/mutation/handlers.go b/pkg/handlers/generic/mutation/handlers.go index 9da8d1d73..169a9a2a3 100644 --- a/pkg/handlers/generic/mutation/handlers.go +++ b/pkg/handlers/generic/mutation/handlers.go @@ -19,6 +19,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries/credentials" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubernetesimagerepository" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/taints" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/users" ) @@ -50,3 +51,15 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator { containerdapplypatchesandrestart.NewPatch(), } } + +func ControlPlaneMetaMutators() []mutation.MetaMutator { + return []mutation.MetaMutator{ + taints.NewControlPlanePatch(), + } +} + +func WorkerMetaMutators() []mutation.MetaMutator { + return []mutation.MetaMutator{ + taints.NewWorkerPatch(), + } +} diff --git a/pkg/handlers/generic/mutation/metapatch_handler.go b/pkg/handlers/generic/mutation/metapatch_handler.go index 83397c4c2..adff84088 100644 --- a/pkg/handlers/generic/mutation/metapatch_handler.go +++ b/pkg/handlers/generic/mutation/metapatch_handler.go @@ -12,9 +12,22 @@ import ( // MetaPatchHandler returns a meta patch handler for mutating generic Kubernetes clusters. func MetaPatchHandler(mgr manager.Manager) handlers.Named { + patchHandlers := MetaMutators(mgr) + patchHandlers = append(patchHandlers, ControlPlaneMetaMutators()...) return mutation.NewMetaGeneratePatchesHandler( "genericClusterConfigPatch", mgr.GetClient(), - MetaMutators(mgr)..., + patchHandlers..., + ) +} + +// MetaWorkerPatchHandler returns a meta patch handler for mutating generic workers. +func MetaWorkerPatchHandler(mgr manager.Manager) handlers.Named { + patchHandlers := WorkerMetaMutators() + + return mutation.NewMetaGeneratePatchesHandler( + "genericWorkerConfigPatch", + mgr.GetClient(), + patchHandlers..., ) } diff --git a/pkg/handlers/generic/mutation/taints/inject_controlplane.go b/pkg/handlers/generic/mutation/taints/inject_controlplane.go new file mode 100644 index 000000000..55f7dc772 --- /dev/null +++ b/pkg/handlers/generic/mutation/taints/inject_controlplane.go @@ -0,0 +1,107 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package taints + +import ( + "context" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" +) + +type taintsControlPlanePatchHandler struct { + variableName string + variableFieldPath []string +} + +func NewControlPlanePatch() *taintsControlPlanePatchHandler { + return newTaintsControlPlanePatchHandler( + v1alpha1.ClusterConfigVariableName, + v1alpha1.ControlPlaneConfigVariableName, + VariableName, + ) +} + +func newTaintsControlPlanePatchHandler( + variableName string, + variableFieldPath ...string, +) *taintsControlPlanePatchHandler { + return &taintsControlPlanePatchHandler{ + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *taintsControlPlanePatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + _ ctrlclient.ObjectKey, + _ mutation.ClusterGetter, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + + taintsVar, err := variables.Get[[]v1alpha1.Taint]( + vars, + h.variableName, + h.variableFieldPath..., + ) + if err != nil { + if variables.IsNotFoundError(err) { + log.V(5).Info("Taints variable for worker not defined") + return nil + } + return err + } + + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + taintsVar, + ) + + return patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding taints to worker node kubeadm config template") + if obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration != nil { + obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = &bootstrapv1.InitConfiguration{} + } + if obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration != nil { + obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration = &bootstrapv1.JoinConfiguration{} + } + coreTaints := toCoreTaints(taintsVar) + + obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.Taints = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.Taints, + coreTaints..., + ) + obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.Taints = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration.NodeRegistration.Taints, + coreTaints..., + ) + + return nil + }) +} diff --git a/pkg/handlers/generic/mutation/taints/inject_controlplane_test.go b/pkg/handlers/generic/mutation/taints/inject_controlplane_test.go new file mode 100644 index 000000000..f2bea81f9 --- /dev/null +++ b/pkg/handlers/generic/mutation/taints/inject_controlplane_test.go @@ -0,0 +1,71 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package taints + +import ( + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +var _ = Describe("Generate taints patches for Control Plane", func() { + patchGenerator := func() mutation.GeneratePatches { + return mutation.NewMetaGeneratePatchesHandler( + "", helpers.TestEnv.Client, NewControlPlanePatch(), + ).(mutation.GeneratePatches) + } + + testDefs := []capitest.PatchTestDef{ + { + Name: "unset variable", + }, + { + Name: "taints for control plane set", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + []v1alpha1.Taint{{ + Key: "key", + Effect: v1alpha1.TaintEffectNoExecute, + Value: "value", + }}, + v1alpha1.ControlPlaneConfigVariableName, + VariableName, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{ + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/taints", + ValueMatcher: gomega.ConsistOf( + map[string]interface{}{"key": "key", "effect": "NoExecute", "value": "value"}, + ), + }, { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/taints", + ValueMatcher: gomega.ConsistOf( + map[string]interface{}{"key": "key", "effect": "NoExecute", "value": "value"}, + ), + }}, + }, + } + + // create test node for each case + for testIdx := range testDefs { + tt := testDefs[testIdx] + It(tt.Name, func() { + capitest.AssertGeneratePatches( + GinkgoT(), + patchGenerator, + &tt, + ) + }) + } +}) diff --git a/pkg/handlers/generic/mutation/taints/inject_suite_test.go b/pkg/handlers/generic/mutation/taints/inject_suite_test.go new file mode 100644 index 000000000..869674c0e --- /dev/null +++ b/pkg/handlers/generic/mutation/taints/inject_suite_test.go @@ -0,0 +1,16 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package taints + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestInstanceTypePatch(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Taints patches for ControlPlane and Workers suite") +} diff --git a/pkg/handlers/generic/mutation/taints/inject_worker.go b/pkg/handlers/generic/mutation/taints/inject_worker.go new file mode 100644 index 000000000..3973cb5e8 --- /dev/null +++ b/pkg/handlers/generic/mutation/taints/inject_worker.go @@ -0,0 +1,109 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package taints + +import ( + "context" + + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" +) + +const VariableName = "taints" + +type taintsWorkerPatchHandler struct { + variableName string + variableFieldPath []string +} + +func NewWorkerPatch() *taintsWorkerPatchHandler { + return newTaintsWorkerPatchHandler( + v1alpha1.WorkerConfigVariableName, + VariableName, + ) +} + +func newTaintsWorkerPatchHandler( + variableName string, + variableFieldPath ...string, +) *taintsWorkerPatchHandler { + return &taintsWorkerPatchHandler{ + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *taintsWorkerPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + _ ctrlclient.ObjectKey, + _ mutation.ClusterGetter, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + + taintsVar, err := variables.Get[[]v1alpha1.Taint]( + vars, + h.variableName, + h.variableFieldPath..., + ) + if err != nil { + if variables.IsNotFoundError(err) { + log.V(5).Info("Taints variable for worker not defined") + return nil + } + return err + } + + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + taintsVar, + ) + + return patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, + func(obj *bootstrapv1.KubeadmConfigTemplate) error { + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding taints to worker node kubeadm config template") + if obj.Spec.Template.Spec.JoinConfiguration != nil { + obj.Spec.Template.Spec.JoinConfiguration = &bootstrapv1.JoinConfiguration{} + } + obj.Spec.Template.Spec.JoinConfiguration.NodeRegistration.Taints = append( + obj.Spec.Template.Spec.JoinConfiguration.NodeRegistration.Taints, + toCoreTaints(taintsVar)..., + ) + return nil + }) +} + +func toCoreTaints(taints []v1alpha1.Taint) []v1.Taint { + return lo.Map(taints, func(t v1alpha1.Taint, _ int) v1.Taint { + return v1.Taint{ + Key: t.Key, + Effect: v1.TaintEffect(t.Effect), + Value: t.Value, + } + }) +} diff --git a/pkg/handlers/generic/mutation/taints/inject_worker_test.go b/pkg/handlers/generic/mutation/taints/inject_worker_test.go new file mode 100644 index 000000000..1873a61c4 --- /dev/null +++ b/pkg/handlers/generic/mutation/taints/inject_worker_test.go @@ -0,0 +1,69 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package taints + +import ( + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +var _ = Describe("Generate taints patches for Worker", func() { + patchGenerator := func() mutation.GeneratePatches { + return mutation.NewMetaGeneratePatchesHandler("", helpers.TestEnv.Client, NewWorkerPatch()).(mutation.GeneratePatches) + } + + testDefs := []capitest.PatchTestDef{ + { + Name: "unset variable", + }, + { + Name: "taints for workers set", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.WorkerConfigVariableName, + []v1alpha1.Taint{{ + Key: "key", + Effect: v1alpha1.TaintEffectNoExecute, + Value: "value", + }}, + VariableName, + ), + capitest.VariableWithValue( + "builtin", + apiextensionsv1.JSON{ + Raw: []byte(`{"machineDeployment": {"class": "a-worker"}}`), + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{ + Operation: "add", + Path: "/spec/template/spec/joinConfiguration/nodeRegistration/taints", + ValueMatcher: gomega.ConsistOf( + map[string]interface{}{"key": "key", "effect": "NoExecute", "value": "value"}, + ), + }}, + }, + } + + // create test node for each case + for testIdx := range testDefs { + tt := testDefs[testIdx] + It(tt.Name, func() { + capitest.AssertGeneratePatches( + GinkgoT(), + patchGenerator, + &tt, + ) + }) + } +}) diff --git a/pkg/handlers/generic/mutation/taints/variables_test.go b/pkg/handlers/generic/mutation/taints/variables_test.go new file mode 100644 index 000000000..b99228a51 --- /dev/null +++ b/pkg/handlers/generic/mutation/taints/variables_test.go @@ -0,0 +1,38 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package taints + +import ( + "testing" + + "k8s.io/utils/ptr" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + nutanixclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/clusterconfig" +) + +func TestVariableValidation(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.ClusterConfigVariableName, + ptr.To(v1alpha1.NutanixClusterConfig{}.VariableSchema()), + true, + nutanixclusterconfig.NewVariable, + capitest.VariableTestDef{ + Name: "specified instance type", + Vals: v1alpha1.NutanixClusterConfigSpec{ + ControlPlane: &v1alpha1.NutanixNodeConfigSpec{ + GenericNodeSpec: v1alpha1.GenericNodeSpec{ + Taints: []v1alpha1.Taint{{ + Key: "key", + Effect: v1alpha1.TaintEffectNoExecute, + Value: "value", + }}, + }, + }, + }, + }, + ) +} diff --git a/pkg/handlers/nutanix/mutation/metapatch_handler.go b/pkg/handlers/nutanix/mutation/metapatch_handler.go index dff362b3b..7e975ad43 100644 --- a/pkg/handlers/nutanix/mutation/metapatch_handler.go +++ b/pkg/handlers/nutanix/mutation/metapatch_handler.go @@ -18,15 +18,14 @@ import ( // MetaPatchHandler returns a meta patch handler for mutating CAPX clusters. func MetaPatchHandler(mgr manager.Manager, cfg *controlplanevirtualip.Config) handlers.Named { - patchHandlers := append( - []mutation.MetaMutator{ - controlplaneendpoint.NewPatch(), - nutanixcontrolplanevirtualip.NewPatch(mgr.GetClient(), cfg), - prismcentralendpoint.NewPatch(), - machinedetails.NewControlPlanePatch(), - }, - genericmutation.MetaMutators(mgr)..., - ) + patchHandlers := []mutation.MetaMutator{ + controlplaneendpoint.NewPatch(), + nutanixcontrolplanevirtualip.NewPatch(mgr.GetClient(), cfg), + prismcentralendpoint.NewPatch(), + machinedetails.NewControlPlanePatch(), + } + patchHandlers = append(patchHandlers, genericmutation.MetaMutators(mgr)...) + patchHandlers = append(patchHandlers, genericmutation.ControlPlaneMetaMutators()...) return mutation.NewMetaGeneratePatchesHandler( "nutanixClusterConfigPatch", @@ -40,6 +39,7 @@ func MetaWorkerPatchHandler(mgr manager.Manager) handlers.Named { patchHandlers := []mutation.MetaMutator{ machinedetails.NewWorkerPatch(), } + patchHandlers = append(patchHandlers, genericmutation.WorkerMetaMutators()...) return mutation.NewMetaGeneratePatchesHandler( "nutanixWorkerConfigPatch",