operator: mark downscaling as alpha feature and add a startup flag

We should enable downscaling as feature gate when issue with reusable node IDs is fixed.

src/go/k8s/apis/redpanda/v1alpha1/cluster_webhook.go

@@ -43,6 +43,11 @@ const (
 	defaultSchemaRegistryPort = 8081
 )
 
+// AllowDownscalingInWebhook controls the downscaling alpha feature in the Cluster custom resource.
+// Downscaling is not stable since nodeIDs are currently not reusable, so adding to a cluster a node
+// that has previously been decommissioned can cause issues.
+var AllowDownscalingInWebhook = false
+
 type resourceField struct {
 	resources *corev1.ResourceRequirements
 	path      *field.Path
@@ -177,6 +182,8 @@ func (r *Cluster) ValidateUpdate(old runtime.Object) error {
 
 	allErrs = append(allErrs, r.validateScaling()...)
 
+	allErrs = append(allErrs, r.validateDownscaling(oldCluster)...)
+
 	allErrs = append(allErrs, r.validateKafkaListeners()...)
 
 	allErrs = append(allErrs, r.validateAdminListeners()...)
@@ -227,6 +234,17 @@ func (r *Cluster) validateScaling() field.ErrorList {
 	return allErrs
 }
 
+func (r *Cluster) validateDownscaling(old *Cluster) field.ErrorList {
+	var allErrs field.ErrorList
+	if !AllowDownscalingInWebhook && old.Spec.Replicas != nil && r.Spec.Replicas != nil && *r.Spec.Replicas < *old.Spec.Replicas {
+		allErrs = append(allErrs,
+			field.Invalid(field.NewPath("spec").Child("replicas"),
+				r.Spec.Replicas,
+				"downscaling is an alpha feature: set --allow-downscaling in the controller parameters to enable it"))
+	}
+	return allErrs
+}
+
 func (r *Cluster) validateAdminListeners() field.ErrorList {
 	var allErrs field.ErrorList
 	externalAdmin := r.AdminAPIExternal()

src/go/k8s/main.go

@@ -70,6 +70,7 @@ func main() {
 	flag.StringVar(&configuratorTag, "configurator-tag", "latest", "Set the configurator tag")
 	flag.StringVar(&configuratorImagePullPolicy, "configurator-image-pull-policy", "Always", "Set the configurator image pull policy")
 	flag.DurationVar(&decommissionWaitInterval, "decommission-wait-interval", 8*time.Second, "Set the time to wait for a node decommission to happen in the cluster")
+	flag.BoolVar(&redpandav1alpha1.AllowDownscalingInWebhook, "allow-downscaling", false, "Allow to reduce the number of replicas in existing clusters (alpha feature)")
 
 	opts := zap.Options{
 		Development: true,

src/go/k8s/tests/e2e/decommission/00-assert.yaml

@@ -1,10 +1,8 @@
-apiVersion: redpanda.vectorized.io/v1alpha1
-kind: Cluster
-metadata:
-  name: decommissioning
-status:
-  replicas: 3
-  currentReplicas: 3
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - command: kubectl rollout status deployment redpanda-controller-manager -n redpanda-system
+  - command: hack/wait-for-webhook-ready.sh
 ---
 
 apiVersion: kuttl.dev/v1beta1

src/go/k8s/tests/e2e/decommission/00-enable-downscaling.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+# Set the --allow-downscaling=true parameter in the controller manager
+- command: kubectl patch deployment redpanda-controller-manager -n redpanda-system --type='json' -p='[{"op":"add", "path":"/spec/template/spec/containers/1/args/-", "value":"--allow-downscaling=true"}]'
+- command: kubectl get deployment redpanda-controller-manager -n redpanda-system -o json

src/go/k8s/tests/e2e/decommission/01-assert.yaml

@@ -1,16 +1,10 @@
-apiVersion: v1
-kind: Pod
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
 metadata:
-  labels:
-    job-name: wait-for-3-brokers
+  name: decommissioning
 status:
-  containerStatuses:
-    - name: curl
-      state:
-        terminated:
-          message: |
-            3
-  phase: Succeeded
+  replicas: 3
+  currentReplicas: 3
 ---
 
 apiVersion: kuttl.dev/v1beta1

src/go/k8s/tests/e2e/decommission/02-assert.yaml

@@ -1,9 +1,16 @@
-apiVersion: redpanda.vectorized.io/v1alpha1
-kind: Cluster
+apiVersion: v1
+kind: Pod
 metadata:
-  name: decommissioning
+  labels:
+    job-name: wait-for-3-brokers
 status:
-  replicas: 2
+  containerStatuses:
+    - name: curl
+      state:
+        terminated:
+          message: |
+            3
+  phase: Succeeded
 ---
 
 apiVersion: kuttl.dev/v1beta1

src/go/k8s/tests/e2e/decommission/03-assert.yaml

@@ -1,16 +1,9 @@
-apiVersion: v1
-kind: Pod
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
 metadata:
-  labels:
-    job-name: wait-for-2-brokers
+  name: decommissioning
 status:
-  containerStatuses:
-    - name: curl
-      state:
-        terminated:
-          message: |
-            2
-  phase: Succeeded
+  replicas: 2
 ---
 
 apiVersion: kuttl.dev/v1beta1

src/go/k8s/tests/e2e/decommission/04-assert.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    job-name: wait-for-2-brokers
+status:
+  containerStatuses:
+    - name: curl
+      state:
+        terminated:
+          message: |
+            2
+  phase: Succeeded
+---
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+collectors:
+  - type: pod
+    selector: app.kubernetes.io/name=redpanda
+    tail: -1
+  - type: pod
+    namespace: redpanda-system
+    selector: control-plane=controller-manager
+    tail: -1
+  - type: command
+    command: kubectl get clusters -o jsonpath={@} -n $NAMESPACE
+  - type: command
+    command: kubectl get pods -o jsonpath={@} -n $NAMESPACE

src/go/k8s/tests/e2e/decommission/05-restore-downscaling.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+# Downscaling will be set to the default value
+- command: kubectl patch deployment redpanda-controller-manager -n redpanda-system --type='json' -p='[{"op":"remove", "path":"/spec/template/spec/containers/1/args/4"}]'

src/go/k8s/tests/e2e/decommission/06-assert.yaml

@@ -0,0 +1,22 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - command: kubectl rollout status deployment redpanda-controller-manager -n redpanda-system
+  - command: hack/wait-for-webhook-ready.sh
+
+---
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+collectors:
+  - type: pod
+    selector: app.kubernetes.io/name=redpanda
+    tail: -1
+  - type: pod
+    namespace: redpanda-system
+    selector: control-plane=controller-manager
+    tail: -1
+  - type: command
+    command: kubectl get clusters -o jsonpath={@} -n $NAMESPACE
+  - type: command
+    command: kubectl get pods -o jsonpath={@} -n $NAMESPACE