diff --git a/src/go/k8s/apis/redpanda/v1alpha1/cluster_types.go b/src/go/k8s/apis/redpanda/v1alpha1/cluster_types.go index 331613bfe421..3c2f834bbac6 100644 --- a/src/go/k8s/apis/redpanda/v1alpha1/cluster_types.go +++ b/src/go/k8s/apis/redpanda/v1alpha1/cluster_types.go @@ -142,6 +142,15 @@ type ClusterSpec struct { // DNS name. // http://www.dns-sd.org/trailingdotsindomainnames.html DNSTrailingDotDisabled bool `json:"dnsTrailingDotDisabled,omitempty"` + // RestartConfig allows to control the behavior of the cluster when restarting + RestartConfig *RestartConfig `json:"restartConfig,omitempty"` +} + +// RestartConfig contains strategies to configure how the cluster behaves when restarting, because of upgrades +// or other lifecycle events. +type RestartConfig struct { + // DisableMaintenanceModeHooks deactivates the preStop and postStart hooks that force nodes to enter maintenance mode when stopping and exit maintenance mode when up again + DisableMaintenanceModeHooks *bool `json:"disableMaintenanceModeHooks,omitempty"` } // PDBConfig specifies how the PodDisruptionBudget should be created for the @@ -793,6 +802,16 @@ func (r *Cluster) IsSchemaRegistryMutualTLSEnabled() bool { r.Spec.Configuration.SchemaRegistry.TLS.RequireClientAuth } +// IsUsingMaintenanceModeHooks tells if the cluster is configured to use maintenance mode hooks on the pods. +// Maintenance mode feature needs to be enabled for this to be relevant. +func (r *Cluster) IsUsingMaintenanceModeHooks() bool { + // enabled unless explicitly stated + if r.Spec.RestartConfig != nil && r.Spec.RestartConfig.DisableMaintenanceModeHooks != nil { + return !*r.Spec.RestartConfig.DisableMaintenanceModeHooks + } + return true +} + // ClusterStatus // IsRestarting tells if the cluster is restarting due to a change in configuration or an upgrade in progress diff --git a/src/go/k8s/apis/redpanda/v1alpha1/zz_generated.deepcopy.go b/src/go/k8s/apis/redpanda/v1alpha1/zz_generated.deepcopy.go index 677ad886c299..57a60f5c37b9 100644 --- a/src/go/k8s/apis/redpanda/v1alpha1/zz_generated.deepcopy.go +++ b/src/go/k8s/apis/redpanda/v1alpha1/zz_generated.deepcopy.go @@ -200,6 +200,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { (*out)[key] = val } } + if in.RestartConfig != nil { + in, out := &in.RestartConfig, &out.RestartConfig + *out = new(RestartConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. @@ -507,6 +512,26 @@ func (in *RedpandaResourceRequirements) DeepCopy() *RedpandaResourceRequirements return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RestartConfig) DeepCopyInto(out *RestartConfig) { + *out = *in + if in.DisableMaintenanceModeHooks != nil { + in, out := &in.DisableMaintenanceModeHooks, &out.DisableMaintenanceModeHooks + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestartConfig. +func (in *RestartConfig) DeepCopy() *RestartConfig { + if in == nil { + return nil + } + out := new(RestartConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SchemaRegistryAPI) DeepCopyInto(out *SchemaRegistryAPI) { *out = *in diff --git a/src/go/k8s/config/crd/bases/redpanda.vectorized.io_clusters.yaml b/src/go/k8s/config/crd/bases/redpanda.vectorized.io_clusters.yaml index 008802ef0fbd..ef3383c82941 100644 --- a/src/go/k8s/config/crd/bases/redpanda.vectorized.io_clusters.yaml +++ b/src/go/k8s/config/crd/bases/redpanda.vectorized.io_clusters.yaml @@ -648,6 +648,16 @@ spec: to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + restartConfig: + description: RestartConfig allows to control the behavior of the cluster + when restarting + properties: + disableMaintenanceModeHooks: + description: DisableMaintenanceModeHooks deactivates the preStop + and postStart hooks that force nodes to enter maintenance mode + when stopping and exit maintenance mode when up again + type: boolean + type: object sidecars: description: Sidecars is list of sidecars run alongside redpanda container properties: diff --git a/src/go/k8s/pkg/resources/statefulset.go b/src/go/k8s/pkg/resources/statefulset.go index 87e633bea3ca..867642960072 100644 --- a/src/go/k8s/pkg/resources/statefulset.go +++ b/src/go/k8s/pkg/resources/statefulset.go @@ -512,6 +512,13 @@ func (r *StatefulSetResource) obj( }, } + if featuregates.MaintenanceMode(r.pandaCluster.Spec.Version) && r.pandaCluster.IsUsingMaintenanceModeHooks() { + ss.Spec.Template.Spec.Containers[0].Lifecycle = &corev1.Lifecycle{ + PreStop: r.getPreStopHook(), + PostStart: r.getPostStartHook(), + } + } + if featuregates.CentralizedConfiguration(r.pandaCluster.Spec.Version) { ss.Spec.Template.Spec.Containers[0].VolumeMounts = append(ss.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ Name: "configmap-dir", @@ -535,6 +542,76 @@ func (r *StatefulSetResource) obj( return ss, nil } +// getPrestopHook creates a hook that drains the node before shutting down. +func (r *StatefulSetResource) getPreStopHook() *corev1.Handler { + // TODO replace scripts with proper RPK calls + curlCommand := r.composeCURLMaintenanceCommand(`-X PUT --silent -o /dev/null -w "%{http_code}"`, nil) + genericMaintenancePath := "/v1/maintenance" + curlGetCommand := r.composeCURLMaintenanceCommand(`--silent`, &genericMaintenancePath) + cmd := strings.Join( + []string{ + fmt.Sprintf(`until [ "${status:-}" = "200" ]; do status=$(%s); sleep 0.5; done`, curlCommand), + fmt.Sprintf(`until [ "${finished:-}" = "true" ]; do finished=$(%s | grep -o '\"finished\":[^,}]*' | grep -o '[^: ]*$'); sleep 0.5; done`, curlGetCommand), + }, " && ") + + return &corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/bin/bash", + "-c", + cmd, + }, + }, + } +} + +// getPostStartHook creates a hook that removes maintenance mode after startup. +func (r *StatefulSetResource) getPostStartHook() *corev1.Handler { + // TODO replace scripts with proper RPK calls + curlCommand := r.composeCURLMaintenanceCommand(`-X DELETE --silent -o /dev/null -w "%{http_code}"`, nil) + // HTTP code 400 is returned by v22 nodes during an upgrade from v21 until the new version reaches quorum and the maintenance mode feature is enabled + cmd := fmt.Sprintf(`until [ "${status:-}" = "200" ] || [ "${status:-}" = "400" ]; do status=$(%s); sleep 0.5; done`, curlCommand) + + return &corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/bin/bash", + "-c", + cmd, + }, + }, + } +} + +// nolint:goconst // no need +func (r *StatefulSetResource) composeCURLMaintenanceCommand( + options string, urlOverwrite *string, +) string { + adminAPI := r.pandaCluster.AdminAPIInternal() + + cmd := fmt.Sprintf(`curl %s `, options) + + tlsConfig := adminAPI.GetTLS() + proto := "http" + if tlsConfig != nil && tlsConfig.Enabled { + proto = "https" + if tlsConfig.RequireClientAuth { + cmd += "--cacert /etc/tls/certs/admin/ca/ca.crt --cert /etc/tls/certs/admin/tls.crt --key /etc/tls/certs/admin/tls.key " + } else { + cmd += "--cacert /etc/tls/certs/admin/tls.crt " + } + } + cmd += fmt.Sprintf("%s://${POD_NAME}.%s.%s.svc.cluster.local:%d", proto, r.pandaCluster.Name, r.pandaCluster.Namespace, adminAPI.Port) + + if urlOverwrite == nil { + prefixLen := len(r.pandaCluster.Name) + 1 + cmd += fmt.Sprintf("/v1/brokers/${POD_NAME:%d}/maintenance", prefixLen) + } else { + cmd += *urlOverwrite + } + return cmd +} + // setCloudStorage manipulates v1.StatefulSet object in order to add cloud storage specific // properties to Redpanda pod. func setCloudStorage(