Merge pull request #5111 from nicolaferraro/fix-unstable-tests

operator: use relaxed kuttl assertions in e2e tests to reduce flakiness

src/go/k8s/controllers/redpanda/cluster_controller_configuration.go

@@ -193,7 +193,7 @@ func (r *ClusterReconciler) applyPatchIfNeeded(
 	}
 
 	log.Info("Applying patch to the cluster configuration", "patch", patch.String())
-	_, err = adminAPI.PatchClusterConfig(ctx, patch.Upsert, patch.Remove)
+	wr, err := adminAPI.PatchClusterConfig(ctx, patch.Upsert, patch.Remove)
 	if err != nil {
 		var conditionData *redpandav1alpha1.ClusterCondition
 		conditionData, err = tryMapErrorToCondition(err)
@@ -220,6 +220,7 @@ func (r *ClusterReconciler) applyPatchIfNeeded(
 		// Patch issue is due to user error, so it's unrecoverable
 		return false, nil
 	}
+	log.Info("Patch written to the cluster", "config_version", wr.ConfigVersion)
 	return true, nil
 }
 
@@ -318,18 +319,24 @@ func (r *ClusterReconciler) synchronizeStatusWithCluster(
 	}
 	conditionData := mapStatusToCondition(status)
 	conditionChanged := redpandaCluster.Status.SetCondition(conditionData.Type, conditionData.Status, conditionData.Reason, conditionData.Message)
-	stsNeedsRestart := needsRestart(status)
-	if conditionChanged || (stsNeedsRestart && !redpandaCluster.Status.IsRestarting()) {
-		// Trigger restart here if needed
-		if stsNeedsRestart {
+	clusterNeedsRestart := needsRestart(status, log)
+	clusterSafeToRestart := isSafeToRestart(status, log)
+	restartingCluster := clusterNeedsRestart && clusterSafeToRestart
+
+	log.Info("Synchronizing configuration state for cluster",
+		"status", conditionData.Status,
+		"reason", conditionData.Reason,
+		"message", conditionData.Message,
+		"needs_restart", clusterNeedsRestart,
+		"restarting", restartingCluster,
+	)
+	if conditionChanged || (restartingCluster && !redpandaCluster.Status.IsRestarting()) {
+		log.Info("Updating configuration state for cluster")
+		// Trigger restart here if needed and safe to do it
+		if restartingCluster {
 			redpandaCluster.Status.SetRestarting(true)
 		}
-		log.Info("Updating configuration state for cluster",
-			"status", conditionData.Status,
-			"reason", conditionData.Reason,
-			"message", conditionData.Message,
-			"restarting", redpandaCluster.Status.IsRestarting(),
-		)
+
 		if err := r.Status().Update(ctx, redpandaCluster); err != nil {
 			return nil, errorWithContext(err, "could not update condition on cluster")
 		}
@@ -342,7 +349,7 @@ func mapStatusToCondition(
 	clusterStatus admin.ConfigStatusResponse,
 ) redpandav1alpha1.ClusterCondition {
 	var condition *redpandav1alpha1.ClusterCondition
-	var configVersion int64
+	var configVersion int64 = -1
 	for _, nodeStatus := range clusterStatus {
 		if len(nodeStatus.Invalid) > 0 {
 			condition = &redpandav1alpha1.ClusterCondition{
@@ -365,7 +372,7 @@ func mapStatusToCondition(
 				Reason:  redpandav1alpha1.ClusterConfiguredReasonUpdating,
 				Message: fmt.Sprintf("Node %d needs restart", nodeStatus.NodeID),
 			}
-		} else if configVersion != 0 && nodeStatus.ConfigVersion != configVersion {
+		} else if configVersion >= 0 && nodeStatus.ConfigVersion != configVersion {
 			condition = &redpandav1alpha1.ClusterCondition{
 				Type:    redpandav1alpha1.ClusterConfiguredConditionType,
 				Status:  corev1.ConditionFalse,
@@ -387,13 +394,24 @@ func mapStatusToCondition(
 	return *condition
 }
 
-func needsRestart(clusterStatus admin.ConfigStatusResponse) bool {
+func needsRestart(clusterStatus admin.ConfigStatusResponse, log logr.Logger) bool {
+	nodeNeedsRestart := false
 	for i := range clusterStatus {
+		log.Info(fmt.Sprintf("Node %d restart status is %v", clusterStatus[i].NodeID, clusterStatus[i].Restart))
 		if clusterStatus[i].Restart {
-			return true
+			nodeNeedsRestart = true
 		}
 	}
-	return false
+	return nodeNeedsRestart
+}
+
+func isSafeToRestart(clusterStatus admin.ConfigStatusResponse, log logr.Logger) bool {
+	configVersions := make(map[int64]bool)
+	for i := range clusterStatus {
+		log.Info(fmt.Sprintf("Node %d is using config version %d", clusterStatus[i].NodeID, clusterStatus[i].ConfigVersion))
+		configVersions[clusterStatus[i].ConfigVersion] = true
+	}
+	return len(configVersions) == 1
 }
 
 // tryMapErrorToCondition tries to map validation errors received from the cluster to a condition

src/go/k8s/tests/e2e-unstable/centralized-configuration/03-assert.yaml

@@ -4,10 +4,6 @@ metadata:
   name: centralized-configuration
 status:
   restarting: true
-  conditions:
-    - type: ClusterConfigured
-      status: "False"
-
 ---
 
 apiVersion: kuttl.dev/v1beta1

src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-assert.yaml

@@ -13,7 +13,7 @@ metadata:
 spec:
   containers:
   - name: redpanda
-    image: "vectorized/redpanda:v21.11.11"
+    image: "vectorized/redpanda:v21.11.16"
 status:
   phase: "Running"
 ---
@@ -25,7 +25,7 @@ metadata:
 spec:
   containers:
     - name: redpanda
-      image: "vectorized/redpanda:v21.11.11"
+      image: "vectorized/redpanda:v21.11.16"
 status:
   phase: "Running"
 ---

src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-first-upgrade.yaml

@@ -4,7 +4,7 @@ metadata:
   name: centralized-configuration-upgrade
 spec:
   image: "vectorized/redpanda"
-  version: "v21.11.12"
+  version: "v21.11.16"
   replicas: 2
   resources:
     requests:

src/go/k8s/tests/e2e-unstable/update-conf-image/00-configurator.yaml → src/go/k8s/tests/e2e/update-conf-image/00-configurator.yaml

@@ -2,5 +2,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
 # Set specific configurator version - this will be reflected in all configurator containers of managed redpanda clusters (new or old)
-- command: kubectl patch deployment redpanda-controller-manager -n redpanda-system --type='json' -p='[{"op":"add", "path":"/spec/template/spec/containers/1/args/-", "value":"--configurator-tag=v21.6.6"}]'
+- command: kubectl patch deployment redpanda-controller-manager -n redpanda-system --type='json' -p='[{"op":"add", "path":"/spec/template/spec/containers/1/args/-", "value":"--configurator-tag=v22.1.3"}]'
 - command: kubectl get deployment redpanda-controller-manager -n redpanda-system -o json

src/go/k8s/tests/e2e-unstable/update-conf-image/01-assert.yaml → src/go/k8s/tests/e2e/update-conf-image/01-assert.yaml

@@ -6,7 +6,7 @@ spec:
   template:
     spec:
       initContainers:
-        - image: "vectorized/configurator:v21.6.6"
+        - image: "vectorized/configurator:v22.1.3"
       containers:
         - image: "localhost/redpanda:dev"
 status:

src/go/k8s/tests/e2e-unstable/update-conf-image/03-assert.yaml → src/go/k8s/tests/e2e/update-conf-image/03-assert.yaml

@@ -19,7 +19,7 @@ metadata:
 spec:
   initContainers:
     - name: redpanda-configurator
-      image: "vectorized/configurator:latest" 
+      image: "vectorized/configurator:latest"
 status:
   phase: "Running"