operator: add kuttl test for decommission

src/go/k8s/tests/e2e/decommission/00-assert.yaml

@@ -0,0 +1,22 @@
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
+metadata:
+  name: decommissioning
+status:
+  replicas: 3
+---
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+collectors:
+  - type: pod
+    selector: app.kubernetes.io/name=redpanda
+    tail: -1
+  - type: pod
+    namespace: redpanda-system
+    selector: control-plane=controller-manager
+    tail: -1
+  - type: command
+    command: kubectl get clusters -o jsonpath={@} -n $NAMESPACE
+  - type: command
+    command: kubectl get pods -o jsonpath={@} -n $NAMESPACE

src/go/k8s/tests/e2e/decommission/00-redpanda-cluster.yaml

@@ -0,0 +1,25 @@
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
+metadata:
+  name: decommissioning
+spec:
+  image: "localhost/redpanda"
+  version: "dev"
+  replicas: 3
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+    limits:
+      cpu: 1
+      memory: 500Mi
+  configuration:
+    rpcServer:
+      port: 33145
+    kafkaApi:
+    - port: 9092
+    adminApi:
+    - port: 9644
+    pandaproxyApi:
+     - port: 8082
+    developerMode: true

src/go/k8s/tests/e2e/decommission/01-assert.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    job-name: wait-for-3-brokers
+status:
+  containerStatuses:
+    - name: curl
+      state:
+        terminated:
+          message: |
+            3
+  phase: Succeeded
+---
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+collectors:
+  - type: pod
+    selector: app.kubernetes.io/name=redpanda
+    tail: -1
+  - type: pod
+    namespace: redpanda-system
+    selector: control-plane=controller-manager
+    tail: -1
+  - type: command
+    command: kubectl get clusters -o jsonpath={@} -n $NAMESPACE
+  - type: command
+    command: kubectl get pods -o jsonpath={@} -n $NAMESPACE

src/go/k8s/tests/e2e/decommission/01-probe.yaml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: wait-for-3-brokers
+spec:
+  backoffLimit: 10
+  template:
+    spec:
+      activeDeadlineSeconds: 90
+      containers:
+        - name: curl
+          image: curlimages/curl:latest
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /bin/sh
+            - -c
+            - -ex
+          args:
+            - >
+              url=http://decommissioning-0.decommissioning.$NAMESPACE.svc.cluster.local:9644/v1/brokers
+              res=$(curl --silent -L $url | tr '{' '\n' | grep node_id | wc -l) &&
+              echo $res > /dev/termination-log &&
+              if [[ "$res" != "3" ]]; then
+                exit 1;
+              fi
+      restartPolicy: Never

src/go/k8s/tests/e2e/decommission/02-assert.yaml

@@ -0,0 +1,22 @@
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
+metadata:
+  name: decommissioning
+status:
+  replicas: 2
+---
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+collectors:
+  - type: pod
+    selector: app.kubernetes.io/name=redpanda
+    tail: -1
+  - type: pod
+    namespace: redpanda-system
+    selector: control-plane=controller-manager
+    tail: -1
+  - type: command
+    command: kubectl get clusters -o jsonpath={@} -n $NAMESPACE
+  - type: command
+    command: kubectl get pods -o jsonpath={@} -n $NAMESPACE

src/go/k8s/tests/e2e/decommission/02-redpanda-downscale.yaml

@@ -0,0 +1,6 @@
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
+metadata:
+  name: decommissioning
+spec:
+  replicas: 2

src/go/k8s/tests/e2e/decommission/03-assert.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    job-name: wait-for-2-brokers
+status:
+  containerStatuses:
+    - name: curl
+      state:
+        terminated:
+          message: |
+            2
+  phase: Succeeded
+---
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+collectors:
+  - type: pod
+    selector: app.kubernetes.io/name=redpanda
+    tail: -1
+  - type: pod
+    namespace: redpanda-system
+    selector: control-plane=controller-manager
+    tail: -1
+  - type: command
+    command: kubectl get clusters -o jsonpath={@} -n $NAMESPACE
+  - type: command
+    command: kubectl get pods -o jsonpath={@} -n $NAMESPACE

src/go/k8s/tests/e2e/decommission/03-probe.yaml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: wait-for-2-brokers
+spec:
+  backoffLimit: 10
+  template:
+    spec:
+      activeDeadlineSeconds: 90
+      containers:
+        - name: curl
+          image: curlimages/curl:latest
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /bin/sh
+            - -c
+            - -ex
+          args:
+            - >
+              url=http://decommissioning-0.decommissioning.$NAMESPACE.svc.cluster.local:9644/v1/brokers
+              res=$(curl --silent -L $url | tr '{' '\n' | grep node_id | wc -l) &&
+              echo $res > /dev/termination-log &&
+              if [[ "$res" != "2" ]]; then
+                exit 1;
+              fi
+      restartPolicy: Never