operator: add kuttl test for rolling upgrade

src/go/k8s/tests/e2e/rolling-restart/00-assert.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: rolling-0
+status:
+  phase: "Running"
+---
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: rolling-1
+status:
+  phase: "Running"
+---
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: rolling-2
+status:
+  phase: "Running"

src/go/k8s/tests/e2e/rolling-restart/00-cluster.yaml

@@ -0,0 +1,25 @@
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
+metadata:
+  name: rolling
+spec:
+  image: "localhost/redpanda"
+  version: "dev"
+  replicas: 3
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+    limits:
+      cpu: 1
+      memory: 500Mi
+  configuration:
+    rpcServer:
+      port: 33145
+    kafkaApi:
+      - port: 9092
+    adminApi:
+      - port: 9644
+    pandaproxyApi:
+      - port: 8082
+    developerMode: true

src/go/k8s/tests/e2e/rolling-restart/01-assert.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    job-name: message-sender
+status:
+  phase: Running
+---
+
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    job-name: message-receiver
+status:
+  phase: Running

src/go/k8s/tests/e2e/rolling-restart/01-clients.yaml

@@ -0,0 +1,158 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: message-scripts
+data:
+  common.sh: |
+    # cluster objects
+    brokers=rolling.$NAMESPACE.svc.cluster.local:9092
+    topic=test1
+    replicas=3
+    partitions=1
+    
+    # sender
+    sender_timeout=30s
+    delay_between_messages_seconds=1.1
+    
+    # receiver
+    receiver_timeout=30s
+    max_message_delay_millis=40000
+    
+    # utility functions
+    halt=false
+    handle_term() {
+      echo "Terminating ..."
+      halt=true
+    }
+
+  sender.sh: |
+    #!/bin/bash
+    
+    set -ex
+    root=$(dirname $0)
+    cd $root
+    source ./common.sh
+    trap handle_term SIGTERM
+    
+    rpk topic create $topic --brokers $brokers --partitions $partitions --replicas $replicas
+    
+    rpk_options="--key 0 --brokers $brokers --delivery-timeout $sender_timeout"
+    
+    counter=0
+    while true
+    do
+      if [ "$halt" = "true" ]; then break; fi
+    
+      echo $counter | rpk topic produce $topic $rpk_options
+      counter=$((counter+1))
+      sleep $delay_between_messages_seconds
+    done
+    
+    echo EOM | rpk topic produce $topic $rpk_options
+    
+    echo "Done sending!"
+
+  receiver.sh: |
+    #!/bin/bash
+    
+    set -ex
+    root=$(dirname $0)
+    cd $root
+    source ./common.sh
+    trap handle_term SIGTERM
+    
+    catched_up=false
+    counter=0
+    while true
+    do
+      if [ "$halt" = "true" ]; then break; fi
+    
+      msg=$(rpk topic consume $topic --fetch-max-wait $receiver_timeout --brokers $brokers -n 1 --offset $counter --format "%v %d")
+      echo "Received: $msg"
+    
+      IFS=" " read -r -a msg_array <<< "$msg"
+      val=${msg_array[0]}
+      message_time=${msg_array[1]}
+    
+      if [ "$val" = "EOM" ]; then break; fi
+    
+      if [ "$val" != "$counter" ]; then
+        echo "Invalid value received: expected $counter, got $val" >> /dev/termination-log
+        exit 1
+      fi
+    
+      current_time=$(date +%s%N | cut -b1-13)
+      
+      delay=$((current_time - message_time))
+      if ((delay < max_message_delay_millis)); then
+        echo "Message arrived in time: $delay milliseconds"
+        catched_up=true
+      elif [ "$catched_up" = "true" ]; then
+        
+        echo "Too much delay for receiving a message: $delay milliseconds"
+        exit 1
+      fi
+      
+      counter=$((counter+1))
+    done
+    
+    echo "Received all messages correctly!"
+
+---
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: message-sender
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      volumes:
+        - name: scripts
+          configMap:
+            defaultMode: 0555
+            name: message-scripts
+      containers:
+        - name: sender
+          image: localhost/redpanda:dev
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /opt/scripts/sender.sh
+          volumeMounts:
+            - name: scripts
+              mountPath: /opt/scripts
+      restartPolicy: Never
+---
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: message-receiver
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      volumes:
+        - name: scripts
+          configMap:
+            defaultMode: 0555
+            name: message-scripts
+      containers:
+        - name: receiver
+          image: localhost/redpanda:dev
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /opt/scripts/receiver.sh
+          volumeMounts:
+            - name: scripts
+              mountPath: /opt/scripts
+      restartPolicy: Never

src/go/k8s/tests/e2e/rolling-restart/02-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
+metadata:
+  name: rolling
+status:
+  restarting: true

src/go/k8s/tests/e2e/rolling-restart/02-cluster-restart.yaml

@@ -0,0 +1,25 @@
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
+metadata:
+  name: rolling
+spec:
+  image: "localhost/redpanda"
+  version: "dev"
+  replicas: 3
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+    limits:
+      cpu: 1
+      memory: 505Mi
+  configuration:
+    rpcServer:
+      port: 33145
+    kafkaApi:
+      - port: 9092
+    adminApi:
+      - port: 9644
+    pandaproxyApi:
+      - port: 8082
+    developerMode: true

src/go/k8s/tests/e2e/rolling-restart/03-assert.yaml

@@ -0,0 +1,24 @@
+apiVersion: redpanda.vectorized.io/v1alpha1
+kind: Cluster
+metadata:
+  name: rolling
+status:
+  restarting: false
+---
+
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    job-name: message-sender
+status:
+  phase: Running
+---
+
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    job-name: message-receiver
+status:
+  phase: Running

src/go/k8s/tests/e2e/rolling-restart/04-assert.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    job-name: message-receiver
+status:
+  phase: Succeeded

src/go/k8s/tests/e2e/rolling-restart/04-stop-sender.yaml

@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: message-sender
+spec:
+  activeDeadlineSeconds: 1
+  backoffLimit: 0
+  template:
+    spec:
+      volumes:
+        - name: scripts
+          configMap:
+            defaultMode: 0555
+            name: message-scripts
+      containers:
+        - name: sender
+          image: localhost/redpanda:dev
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /opt/scripts/sender.sh
+          volumeMounts:
+            - name: scripts
+              mountPath: /opt/scripts
+      restartPolicy: Never