From e0491d142bdbb39c6d4852a85b0990574f485e44 Mon Sep 17 00:00:00 2001 From: Rafal Korepta Date: Sat, 26 Nov 2022 23:58:33 +0100 Subject: [PATCH 1/5] Get cluster health before an update As per https://github.com/redpanda-data/redpanda/issues/3023 the cluster should be healthy before starting put node in maintanance mode and after POD is restarted. --- src/go/k8s/controllers/redpanda/suite_test.go | 6 +++ src/go/k8s/pkg/admin/admin.go | 2 + .../resources/featuregates/featuregates.go | 6 +++ .../k8s/pkg/resources/statefulset_update.go | 37 +++++++++++++++++++ 4 files changed, 51 insertions(+) diff --git a/src/go/k8s/controllers/redpanda/suite_test.go b/src/go/k8s/controllers/redpanda/suite_test.go index ddac7a8b1d00..c709a408b24e 100644 --- a/src/go/k8s/controllers/redpanda/suite_test.go +++ b/src/go/k8s/controllers/redpanda/suite_test.go @@ -558,6 +558,12 @@ func (m *mockAdminAPI) DisableMaintenanceMode(_ context.Context, _ int) error { return nil } +func (m *mockAdminAPI) GetHealthOverview(_ context.Context) (admin.ClusterHealthOverview, error) { + return admin.ClusterHealthOverview{ + IsHealthy: true, + }, nil +} + //nolint:goerr113 // test code func (m *mockAdminAPI) SetBrokerStatus( id int, status admin.MembershipStatus, diff --git a/src/go/k8s/pkg/admin/admin.go b/src/go/k8s/pkg/admin/admin.go index 728b8db67ca4..4962683bcddd 100644 --- a/src/go/k8s/pkg/admin/admin.go +++ b/src/go/k8s/pkg/admin/admin.go @@ -97,6 +97,8 @@ type AdminAPIClient interface { EnableMaintenanceMode(ctx context.Context, node int) error DisableMaintenanceMode(ctx context.Context, node int) error + + GetHealthOverview(ctx context.Context) (admin.ClusterHealthOverview, error) } var _ AdminAPIClient = &admin.AdminAPI{} diff --git a/src/go/k8s/pkg/resources/featuregates/featuregates.go b/src/go/k8s/pkg/resources/featuregates/featuregates.go index 794cf5b22fd6..69b146096508 100644 --- a/src/go/k8s/pkg/resources/featuregates/featuregates.go +++ b/src/go/k8s/pkg/resources/featuregates/featuregates.go @@ -38,6 +38,12 @@ func CentralizedConfiguration(version string) bool { return atLeastVersion(V22_1, version) } +// ClusterHealth feature gate should be removed when the operator +// will no longer support 21.x or older versions +func ClusterHealth(version string) bool { + return atLeastVersion(V22_1, version) +} + // MaintenanceMode feature gate should be removed when the operator // will no longer support 21.x or older versions func MaintenanceMode(version string) bool { diff --git a/src/go/k8s/pkg/resources/statefulset_update.go b/src/go/k8s/pkg/resources/statefulset_update.go index 8200463031d7..618cf33199a0 100644 --- a/src/go/k8s/pkg/resources/statefulset_update.go +++ b/src/go/k8s/pkg/resources/statefulset_update.go @@ -23,6 +23,7 @@ import ( "github.com/banzaicloud/k8s-objectmatcher/patch" "github.com/redpanda-data/redpanda/src/go/k8s/pkg/labels" + "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/featuregates" "github.com/redpanda-data/redpanda/src/go/k8s/pkg/utils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -78,10 +79,15 @@ func (r *StatefulSetResource) runUpdate( if err = r.updateRestartingStatus(ctx, true); err != nil { return fmt.Errorf("unable to turn on restarting status in cluster custom resource: %w", err) } + if err = r.updateStatefulSet(ctx, current, modified); err != nil { return err } + if err = r.isClusterHealthy(ctx); err != nil { + return err + } + if err = r.rollingUpdate(ctx, &modified.Spec.Template); err != nil { return err } @@ -94,6 +100,37 @@ func (r *StatefulSetResource) runUpdate( return nil } +func (r *StatefulSetResource) isClusterHealthy(ctx context.Context) error { + if !featuregates.ClusterHealth(r.pandaCluster.Status.Version) { + r.logger.V(debugLogLevel).Info("Cluster health endpoint is not available", "version", r.pandaCluster.Spec.Version) + return nil + } + + adminAPIClient, err := r.getAdminAPIClient(ctx) + if err != nil { + return fmt.Errorf("creating admin API client: %w", err) + } + + health, err := adminAPIClient.GetHealthOverview(ctx) + if err != nil { + return fmt.Errorf("getting cluster health overview: %w", err) + } + + restarting := "not restarting" + if r.pandaCluster.Status.IsRestarting() { + restarting = "restarting" + } + + if !health.IsHealthy { + return &RequeueAfterError{ + RequeueAfter: RequeueDuration, + Msg: fmt.Sprintf("wait for cluster to become healthy (cluster %s)", restarting), + } + } + + return nil +} + func (r *StatefulSetResource) rollingUpdate( ctx context.Context, template *corev1.PodTemplateSpec, ) error { From 00acf240fbbc6a7fcbf164cb033459ef8aa9b706 Mon Sep 17 00:00:00 2001 From: Rafal Korepta Date: Mon, 28 Nov 2022 10:43:14 +0100 Subject: [PATCH 2/5] k8s: Move admin API mock to shared package In the statefulset unit test the admin API needs to be mocked as cluster health should be available. --- src/go/k8s/controllers/redpanda/suite_test.go | 437 +---------------- src/go/k8s/pkg/admin/mock_admin.go | 445 ++++++++++++++++++ src/go/k8s/pkg/resources/statefulset_test.go | 5 +- 3 files changed, 454 insertions(+), 433 deletions(-) create mode 100644 src/go/k8s/pkg/admin/mock_admin.go diff --git a/src/go/k8s/controllers/redpanda/suite_test.go b/src/go/k8s/controllers/redpanda/suite_test.go index c709a408b24e..44f18c248eb4 100644 --- a/src/go/k8s/controllers/redpanda/suite_test.go +++ b/src/go/k8s/controllers/redpanda/suite_test.go @@ -10,18 +10,11 @@ package redpanda_test import ( - "bytes" "context" - "encoding/json" - "fmt" - "net/http" "path/filepath" - "sort" - "sync" "testing" "time" - "github.com/go-logr/logr" cmapiv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -31,7 +24,6 @@ import ( adminutils "github.com/redpanda-data/redpanda/src/go/k8s/pkg/admin" consolepkg "github.com/redpanda-data/redpanda/src/go/k8s/pkg/console" "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources" - "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/configuration" "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/types" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/api/admin" "k8s.io/client-go/kubernetes/scheme" @@ -51,7 +43,7 @@ var ( k8sClient client.Client testEnv *envtest.Environment cfg *rest.Config - testAdminAPI *mockAdminAPI + testAdminAPI *adminutils.MockAdminAPI testAdminAPIFactory adminutils.AdminAPIClientFactory testStore *consolepkg.Store testKafkaAdmin *mockKafkaAdmin @@ -93,7 +85,7 @@ var _ = BeforeSuite(func(done Done) { }) Expect(err).ToNot(HaveOccurred()) - testAdminAPI = &mockAdminAPI{log: ctrl.Log.WithName("testAdminAPI").WithName("mockAdminAPI")} + testAdminAPI = &adminutils.MockAdminAPI{Log: ctrl.Log.WithName("testAdminAPI").WithName("mockAdminAPI")} testAdminAPIFactory = func( _ context.Context, _ client.Reader, @@ -103,9 +95,9 @@ var _ = BeforeSuite(func(done Done) { ordinals ...int32, ) (adminutils.AdminAPIClient, error) { if len(ordinals) == 1 { - return &scopedMockAdminAPI{ - mockAdminAPI: testAdminAPI, - ordinal: ordinals[0], + return &adminutils.ScopedMockAdminAPI{ + MockAdminAPI: testAdminAPI, + Ordinal: ordinals[0], }, nil } return testAdminAPI, nil @@ -190,422 +182,3 @@ var _ = AfterSuite(func() { return testEnv.Stop() }, timeout, poll).ShouldNot(HaveOccurred()) }) - -type mockAdminAPI struct { - config admin.Config - schema admin.ConfigSchema - patches []configuration.CentralConfigurationPatch - unavailable bool - invalid []string - unknown []string - directValidation bool - brokers []admin.Broker - monitor sync.Mutex - log logr.Logger -} - -type scopedMockAdminAPI struct { - *mockAdminAPI - ordinal int32 -} - -var _ adminutils.AdminAPIClient = &mockAdminAPI{log: ctrl.Log.WithName("AdminAPIClient").WithName("mockAdminAPI")} - -type unavailableError struct{} - -func (*unavailableError) Error() string { - return "unavailable" -} - -func (m *mockAdminAPI) Config(context.Context, bool) (admin.Config, error) { - m.log.WithName("Config").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.unavailable { - return admin.Config{}, &unavailableError{} - } - var res admin.Config - makeCopy(m.config, &res) - return res, nil -} - -func (m *mockAdminAPI) ClusterConfigStatus( - _ context.Context, _ bool, -) (admin.ConfigStatusResponse, error) { - m.log.WithName("ClusterConfigStatus").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.unavailable { - return admin.ConfigStatusResponse{}, &unavailableError{} - } - node := admin.ConfigStatus{ - Invalid: append([]string{}, m.invalid...), - Unknown: append([]string{}, m.unknown...), - } - return []admin.ConfigStatus{node}, nil -} - -func (m *mockAdminAPI) ClusterConfigSchema( - _ context.Context, -) (admin.ConfigSchema, error) { - m.log.WithName("ClusterConfigSchema").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.unavailable { - return admin.ConfigSchema{}, &unavailableError{} - } - var res admin.ConfigSchema - makeCopy(m.schema, &res) - return res, nil -} - -func (m *mockAdminAPI) PatchClusterConfig( - _ context.Context, upsert map[string]interface{}, remove []string, -) (admin.ClusterConfigWriteResult, error) { - m.log.WithName("PatchClusterConfig").WithValues("upsert", upsert, "remove", remove).Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.unavailable { - return admin.ClusterConfigWriteResult{}, &unavailableError{} - } - m.patches = append(m.patches, configuration.CentralConfigurationPatch{ - Upsert: upsert, - Remove: remove, - }) - var newInvalid []string - var newUnknown []string - for k := range upsert { - if meta, ok := m.schema[k]; !ok { - newUnknown = append(newUnknown, k) - } else if meta.Description == "invalid" { - newInvalid = append(newInvalid, k) - } - } - invalidRequest := len(newInvalid)+len(newUnknown) > 0 - if m.directValidation && invalidRequest { - return admin.ClusterConfigWriteResult{}, &admin.HTTPResponseError{ - Method: http.MethodPut, - URL: "/v1/cluster_config", - Response: &http.Response{ - Status: "Bad Request", - StatusCode: 400, - }, - Body: []byte("Mock bad request message"), - } - } - if invalidRequest { - m.invalid = addAsSet(m.invalid, newInvalid...) - m.unknown = addAsSet(m.unknown, newUnknown...) - return admin.ClusterConfigWriteResult{}, nil - } - if m.config == nil { - m.config = make(map[string]interface{}) - } - for k, v := range upsert { - m.config[k] = v - } - for _, k := range remove { - delete(m.config, k) - for i := range m.invalid { - if m.invalid[i] == k { - m.invalid = append(m.invalid[0:i], m.invalid[i+1:]...) - } - } - for i := range m.unknown { - if m.unknown[i] == k { - m.unknown = append(m.unknown[0:i], m.unknown[i+1:]...) - } - } - } - return admin.ClusterConfigWriteResult{}, nil -} - -func (m *mockAdminAPI) CreateUser(_ context.Context, _, _, _ string) error { - m.log.WithName("CreateUser").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.unavailable { - return &unavailableError{} - } - return nil -} - -func (m *mockAdminAPI) DeleteUser(_ context.Context, _ string) error { - m.log.WithName("DeleteUser").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.unavailable { - return &unavailableError{} - } - return nil -} - -func (m *mockAdminAPI) Clear() { - m.log.WithName("Clear").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - m.config = nil - m.schema = nil - m.patches = nil - m.unavailable = false - m.directValidation = false - m.brokers = nil -} - -func (m *mockAdminAPI) GetFeatures( - _ context.Context, -) (admin.FeaturesResponse, error) { - m.log.WithName("GetFeatures").Info("called") - return admin.FeaturesResponse{ - ClusterVersion: 0, - Features: []admin.Feature{ - { - Name: "central_config", - State: admin.FeatureStateActive, - WasActive: true, - }, - }, - }, nil -} - -func (m *mockAdminAPI) SetLicense(_ context.Context, _ interface{}) error { - m.log.WithName("SetLicense").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.unavailable { - return &unavailableError{} - } - return nil -} - -func (m *mockAdminAPI) GetLicenseInfo(_ context.Context) (admin.License, error) { - m.log.WithName("GetLicenseInfo").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.unavailable { - return admin.License{}, &unavailableError{} - } - return admin.License{}, nil -} - -//nolint:gocritic // It's test API -func (m *mockAdminAPI) RegisterPropertySchema( - name string, metadata admin.ConfigPropertyMetadata, -) { - m.log.WithName("RegisterPropertySchema").WithValues("name", name, "metadata", metadata).Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.schema == nil { - m.schema = make(map[string]admin.ConfigPropertyMetadata) - } - m.schema[name] = metadata -} - -func (m *mockAdminAPI) PropertyGetter(name string) func() interface{} { - m.log.WithName("PropertyGetter").WithValues("name", name).Info("called") - return func() interface{} { - m.monitor.Lock() - defer m.monitor.Unlock() - return m.config[name] - } -} - -func (m *mockAdminAPI) ConfigGetter() func() admin.Config { - m.log.WithName("ConfigGetter").Info("called") - return func() admin.Config { - m.monitor.Lock() - defer m.monitor.Unlock() - var res admin.Config - makeCopy(m.config, &res) - return res - } -} - -func (m *mockAdminAPI) PatchesGetter() func() []configuration.CentralConfigurationPatch { - return func() []configuration.CentralConfigurationPatch { - m.log.WithName("PatchesGetter(func)").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - var res []configuration.CentralConfigurationPatch - makeCopy(m.patches, &res) - return res - } -} - -func (m *mockAdminAPI) NumPatchesGetter() func() int { - return func() int { - m.log.WithName("NumPatchesGetter(func)").Info("called") - return len(m.PatchesGetter()()) - } -} - -func (m *mockAdminAPI) SetProperty(key string, value interface{}) { - m.log.WithName("SetProperty").WithValues("key", key, "value", value).Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - if m.config == nil { - m.config = make(map[string]interface{}) - } - m.config[key] = value -} - -func (m *mockAdminAPI) SetUnavailable(unavailable bool) { - m.log.WithName("SetUnavailable").WithValues("unavailable", unavailable).Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - m.unavailable = unavailable -} - -func (m *mockAdminAPI) GetNodeConfig( - _ context.Context, -) (admin.NodeConfig, error) { - m.log.WithName("GetNodeConfig").Info("called") - return admin.NodeConfig{}, nil -} - -//nolint:goerr113 // test code -func (s *scopedMockAdminAPI) GetNodeConfig( - ctx context.Context, -) (admin.NodeConfig, error) { - brokers, err := s.Brokers(ctx) - if err != nil { - return admin.NodeConfig{}, err - } - if len(brokers) <= int(s.ordinal) { - return admin.NodeConfig{}, fmt.Errorf("broker not registered") - } - return admin.NodeConfig{ - NodeID: brokers[int(s.ordinal)].NodeID, - }, nil -} - -func (m *mockAdminAPI) SetDirectValidationEnabled(directValidation bool) { - m.log.WithName("SetDirectValicationEnabled").WithValues("directValidation", directValidation).Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - m.directValidation = directValidation -} - -func (m *mockAdminAPI) AddBroker(broker admin.Broker) { - m.log.WithName("AddBroker").WithValues("broker", broker).Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - - m.brokers = append(m.brokers, broker) -} - -func (m *mockAdminAPI) RemoveBroker(id int) bool { - m.log.WithName("RemoveBroker").WithValues("id", id).Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - - idx := -1 - for i := range m.brokers { - if m.brokers[i].NodeID == id { - idx = i - break - } - } - if idx < 0 { - return false - } - m.brokers = append(m.brokers[:idx], m.brokers[idx+1:]...) - return true -} - -func (m *mockAdminAPI) Brokers(_ context.Context) ([]admin.Broker, error) { - m.log.WithName("RemoveBroker").Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - - return append([]admin.Broker{}, m.brokers...), nil -} - -func (m *mockAdminAPI) BrokerStatusGetter( - id int, -) func() admin.MembershipStatus { - m.log.WithName("BrokerStatusGetter(func)").WithValues("id", id).Info("called") - return func() admin.MembershipStatus { - m.monitor.Lock() - defer m.monitor.Unlock() - - for i := range m.brokers { - if m.brokers[i].NodeID == id { - return m.brokers[i].MembershipStatus - } - } - return "" - } -} - -func (m *mockAdminAPI) DecommissionBroker(_ context.Context, id int) error { - m.log.WithName("DecommissionBroker").WithValues("id", id).Info("called") - return m.SetBrokerStatus(id, admin.MembershipStatusDraining) -} - -func (m *mockAdminAPI) RecommissionBroker(_ context.Context, id int) error { - m.log.WithName("RecommissionBroker").WithValues("id", id).Info("called") - return m.SetBrokerStatus(id, admin.MembershipStatusActive) -} - -func (m *mockAdminAPI) EnableMaintenanceMode(_ context.Context, _ int) error { - m.log.WithName("EnableMaintenanceMode").Info("called") - return nil -} - -func (m *mockAdminAPI) DisableMaintenanceMode(_ context.Context, _ int) error { - m.log.WithName("DisableMaintenanceMode").Info("called") - return nil -} - -func (m *mockAdminAPI) GetHealthOverview(_ context.Context) (admin.ClusterHealthOverview, error) { - return admin.ClusterHealthOverview{ - IsHealthy: true, - }, nil -} - -//nolint:goerr113 // test code -func (m *mockAdminAPI) SetBrokerStatus( - id int, status admin.MembershipStatus, -) error { - m.log.WithName("SetBrokerStatus").WithValues("id", id, "status", status).Info("called") - m.monitor.Lock() - defer m.monitor.Unlock() - - for i := range m.brokers { - if m.brokers[i].NodeID == id { - m.brokers[i].MembershipStatus = status - return nil - } - } - return fmt.Errorf("unknown broker %d", id) -} - -func makeCopy(input, output interface{}) { - ser, err := json.Marshal(input) - if err != nil { - panic(err) - } - decoder := json.NewDecoder(bytes.NewReader(ser)) - decoder.UseNumber() - err = decoder.Decode(output) - if err != nil { - panic(err) - } -} - -func addAsSet(sliceSet []string, vals ...string) []string { - asSet := make(map[string]bool, len(sliceSet)+len(vals)) - for _, k := range sliceSet { - asSet[k] = true - } - for _, v := range vals { - asSet[v] = true - } - lst := make([]string, 0, len(asSet)) - for k := range asSet { - lst = append(lst, k) - } - sort.Strings(lst) - return lst -} diff --git a/src/go/k8s/pkg/admin/mock_admin.go b/src/go/k8s/pkg/admin/mock_admin.go new file mode 100644 index 000000000000..1d1175965e1d --- /dev/null +++ b/src/go/k8s/pkg/admin/mock_admin.go @@ -0,0 +1,445 @@ +// Copyright 2022 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package admin + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "sort" + "sync" + + "github.com/go-logr/logr" + "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/configuration" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/api/admin" + ctrl "sigs.k8s.io/controller-runtime" +) + +type MockAdminAPI struct { + config admin.Config + schema admin.ConfigSchema + patches []configuration.CentralConfigurationPatch + unavailable bool + invalid []string + unknown []string + directValidation bool + brokers []admin.Broker + monitor sync.Mutex + Log logr.Logger +} + +var _ AdminAPIClient = &MockAdminAPI{Log: ctrl.Log.WithName("AdminAPIClient").WithName("mockAdminAPI")} + +type ScopedMockAdminAPI struct { + *MockAdminAPI + Ordinal int32 +} + +type unavailableError struct{} + +func (*unavailableError) Error() string { + return "unavailable" +} + +func (m *MockAdminAPI) Config(context.Context, bool) (admin.Config, error) { + m.Log.WithName("Config").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.unavailable { + return admin.Config{}, &unavailableError{} + } + var res admin.Config + makeCopy(m.config, &res) + return res, nil +} + +func (m *MockAdminAPI) ClusterConfigStatus( + _ context.Context, _ bool, +) (admin.ConfigStatusResponse, error) { + m.Log.WithName("ClusterConfigStatus").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.unavailable { + return admin.ConfigStatusResponse{}, &unavailableError{} + } + node := admin.ConfigStatus{ + Invalid: append([]string{}, m.invalid...), + Unknown: append([]string{}, m.unknown...), + } + return []admin.ConfigStatus{node}, nil +} + +func (m *MockAdminAPI) ClusterConfigSchema( + _ context.Context, +) (admin.ConfigSchema, error) { + m.Log.WithName("ClusterConfigSchema").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.unavailable { + return admin.ConfigSchema{}, &unavailableError{} + } + var res admin.ConfigSchema + makeCopy(m.schema, &res) + return res, nil +} + +func (m *MockAdminAPI) PatchClusterConfig( + _ context.Context, upsert map[string]interface{}, remove []string, +) (admin.ClusterConfigWriteResult, error) { + m.Log.WithName("PatchClusterConfig").WithValues("upsert", upsert, "remove", remove).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.unavailable { + return admin.ClusterConfigWriteResult{}, &unavailableError{} + } + m.patches = append(m.patches, configuration.CentralConfigurationPatch{ + Upsert: upsert, + Remove: remove, + }) + var newInvalid []string + var newUnknown []string + for k := range upsert { + if meta, ok := m.schema[k]; !ok { + newUnknown = append(newUnknown, k) + } else if meta.Description == "invalid" { + newInvalid = append(newInvalid, k) + } + } + invalidRequest := len(newInvalid)+len(newUnknown) > 0 + if m.directValidation && invalidRequest { + return admin.ClusterConfigWriteResult{}, &admin.HTTPResponseError{ + Method: http.MethodPut, + URL: "/v1/cluster_config", + Response: &http.Response{ + Status: "Bad Request", + StatusCode: 400, + }, + Body: []byte("Mock bad request message"), + } + } + if invalidRequest { + m.invalid = addAsSet(m.invalid, newInvalid...) + m.unknown = addAsSet(m.unknown, newUnknown...) + return admin.ClusterConfigWriteResult{}, nil + } + if m.config == nil { + m.config = make(map[string]interface{}) + } + for k, v := range upsert { + m.config[k] = v + } + for _, k := range remove { + delete(m.config, k) + for i := range m.invalid { + if m.invalid[i] == k { + m.invalid = append(m.invalid[0:i], m.invalid[i+1:]...) + } + } + for i := range m.unknown { + if m.unknown[i] == k { + m.unknown = append(m.unknown[0:i], m.unknown[i+1:]...) + } + } + } + return admin.ClusterConfigWriteResult{}, nil +} + +func (m *MockAdminAPI) CreateUser(_ context.Context, _, _, _ string) error { + m.Log.WithName("CreateUser").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.unavailable { + return &unavailableError{} + } + return nil +} + +func (m *MockAdminAPI) DeleteUser(_ context.Context, _ string) error { + m.Log.WithName("DeleteUser").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.unavailable { + return &unavailableError{} + } + return nil +} + +func (m *MockAdminAPI) Clear() { + m.Log.WithName("Clear").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + m.config = nil + m.schema = nil + m.patches = nil + m.unavailable = false + m.directValidation = false + m.brokers = nil +} + +func (m *MockAdminAPI) GetFeatures( + _ context.Context, +) (admin.FeaturesResponse, error) { + m.Log.WithName("GetFeatures").Info("called") + return admin.FeaturesResponse{ + ClusterVersion: 0, + Features: []admin.Feature{ + { + Name: "central_config", + State: admin.FeatureStateActive, + WasActive: true, + }, + }, + }, nil +} + +func (m *MockAdminAPI) SetLicense(_ context.Context, _ interface{}) error { + m.Log.WithName("SetLicense").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.unavailable { + return &unavailableError{} + } + return nil +} + +func (m *MockAdminAPI) GetLicenseInfo(_ context.Context) (admin.License, error) { + m.Log.WithName("GetLicenseInfo").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.unavailable { + return admin.License{}, &unavailableError{} + } + return admin.License{}, nil +} + +//nolint:gocritic // It's test API +func (m *MockAdminAPI) RegisterPropertySchema( + name string, metadata admin.ConfigPropertyMetadata, +) { + m.Log.WithName("RegisterPropertySchema").WithValues("name", name, "metadata", metadata).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.schema == nil { + m.schema = make(map[string]admin.ConfigPropertyMetadata) + } + m.schema[name] = metadata +} + +func (m *MockAdminAPI) PropertyGetter(name string) func() interface{} { + return func() interface{} { + m.Log.WithName("PropertyGetter").WithValues("name", name).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + return m.config[name] + } +} + +func (m *MockAdminAPI) ConfigGetter() func() admin.Config { + return func() admin.Config { + m.Log.WithName("ConfigGetter").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + var res admin.Config + makeCopy(m.config, &res) + return res + } +} + +func (m *MockAdminAPI) PatchesGetter() func() []configuration.CentralConfigurationPatch { + return func() []configuration.CentralConfigurationPatch { + m.Log.WithName("PatchesGetter(func)").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + var res []configuration.CentralConfigurationPatch + makeCopy(m.patches, &res) + return res + } +} + +func (m *MockAdminAPI) NumPatchesGetter() func() int { + return func() int { + m.Log.WithName("NumPatchesGetter(func)").Info("called") + return len(m.PatchesGetter()()) + } +} + +func (m *MockAdminAPI) SetProperty(key string, value interface{}) { + m.Log.WithName("SetProperty").WithValues("key", key, "value", value).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + if m.config == nil { + m.config = make(map[string]interface{}) + } + m.config[key] = value +} + +func (m *MockAdminAPI) SetUnavailable(unavailable bool) { + m.Log.WithName("SetUnavailable").WithValues("unavailable", unavailable).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + m.unavailable = unavailable +} + +func (m *MockAdminAPI) GetNodeConfig( + _ context.Context, +) (admin.NodeConfig, error) { + m.Log.WithName("GetNodeConfig").Info("called") + return admin.NodeConfig{}, nil +} + +//nolint:goerr113 // test code +func (s *ScopedMockAdminAPI) GetNodeConfig( + ctx context.Context, +) (admin.NodeConfig, error) { + brokers, err := s.Brokers(ctx) + if err != nil { + return admin.NodeConfig{}, err + } + if len(brokers) <= int(s.Ordinal) { + return admin.NodeConfig{}, fmt.Errorf("broker not registered") + } + return admin.NodeConfig{ + NodeID: brokers[int(s.Ordinal)].NodeID, + }, nil +} + +func (m *MockAdminAPI) SetDirectValidationEnabled(directValidation bool) { + m.Log.WithName("SetDirectValicationEnabled").WithValues("directValidation", directValidation).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + m.directValidation = directValidation +} + +func (m *MockAdminAPI) AddBroker(broker admin.Broker) { + m.Log.WithName("AddBroker").WithValues("broker", broker).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + + m.brokers = append(m.brokers, broker) +} + +func (m *MockAdminAPI) RemoveBroker(id int) bool { + m.Log.WithName("RemoveBroker").WithValues("id", id).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + + idx := -1 + for i := range m.brokers { + if m.brokers[i].NodeID == id { + idx = i + break + } + } + if idx < 0 { + return false + } + m.brokers = append(m.brokers[:idx], m.brokers[idx+1:]...) + return true +} + +func (m *MockAdminAPI) Brokers(_ context.Context) ([]admin.Broker, error) { + m.Log.WithName("RemoveBroker").Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + + return append([]admin.Broker{}, m.brokers...), nil +} + +func (m *MockAdminAPI) BrokerStatusGetter( + id int, +) func() admin.MembershipStatus { + return func() admin.MembershipStatus { + m.Log.WithName("BrokerStatusGetter(func)").WithValues("id", id).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + + for i := range m.brokers { + if m.brokers[i].NodeID == id { + return m.brokers[i].MembershipStatus + } + } + return "" + } +} + +func (m *MockAdminAPI) DecommissionBroker(_ context.Context, id int) error { + m.Log.WithName("DecommissionBroker").WithValues("id", id).Info("called") + return m.SetBrokerStatus(id, admin.MembershipStatusDraining) +} + +func (m *MockAdminAPI) RecommissionBroker(_ context.Context, id int) error { + m.Log.WithName("RecommissionBroker").WithValues("id", id).Info("called") + return m.SetBrokerStatus(id, admin.MembershipStatusActive) +} + +func (m *MockAdminAPI) EnableMaintenanceMode(_ context.Context, _ int) error { + m.Log.WithName("EnableMaintenanceMode").Info("called") + return nil +} + +func (m *MockAdminAPI) DisableMaintenanceMode(_ context.Context, _ int) error { + m.Log.WithName("DisableMaintenanceMode").Info("called") + return nil +} + +func (m *MockAdminAPI) GetHealthOverview(_ context.Context) (admin.ClusterHealthOverview, error) { + m.Log.WithName("GetHealthOverview").Info("called") + return admin.ClusterHealthOverview{ + IsHealthy: true, + }, nil +} + +//nolint:goerr113 // test code +func (m *MockAdminAPI) SetBrokerStatus( + id int, status admin.MembershipStatus, +) error { + m.Log.WithName("SetBrokerStatus").WithValues("id", id, "status", status).Info("called") + m.monitor.Lock() + defer m.monitor.Unlock() + + for i := range m.brokers { + if m.brokers[i].NodeID == id { + m.brokers[i].MembershipStatus = status + return nil + } + } + return fmt.Errorf("unknown broker %d", id) +} + +func makeCopy(input, output interface{}) { + ser, err := json.Marshal(input) + if err != nil { + panic(err) + } + decoder := json.NewDecoder(bytes.NewReader(ser)) + decoder.UseNumber() + err = decoder.Decode(output) + if err != nil { + panic(err) + } +} + +func addAsSet(sliceSet []string, vals ...string) []string { + asSet := make(map[string]bool, len(sliceSet)+len(vals)) + for _, k := range sliceSet { + asSet[k] = true + } + for _, v := range vals { + asSet[v] = true + } + lst := make([]string, 0, len(asSet)) + for k := range asSet { + lst = append(lst, k) + } + sort.Strings(lst) + return lst +} diff --git a/src/go/k8s/pkg/resources/statefulset_test.go b/src/go/k8s/pkg/resources/statefulset_test.go index 48dbfef25c28..c60656ed9d81 100644 --- a/src/go/k8s/pkg/resources/statefulset_test.go +++ b/src/go/k8s/pkg/resources/statefulset_test.go @@ -17,6 +17,7 @@ import ( redpandav1alpha1 "github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1" adminutils "github.com/redpanda-data/redpanda/src/go/k8s/pkg/admin" res "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources" + resourcetypes "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/types" "github.com/stretchr/testify/assert" v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -110,7 +111,9 @@ func TestEnsure(t *testing.T) { ImagePullPolicy: "Always", }, func(ctx context.Context) (string, error) { return hash, nil }, - adminutils.NewInternalAdminAPI, + func(ctx context.Context, k8sClient client.Reader, redpandaCluster *redpandav1alpha1.Cluster, fqdn string, adminTLSProvider resourcetypes.AdminTLSConfigProvider, ordinals ...int32) (adminutils.AdminAPIClient, error) { + return &adminutils.MockAdminAPI{Log: ctrl.Log.WithName("testAdminAPI").WithName("mockAdminAPI")}, nil + }, time.Second, ctrl.Log.WithName("test")) From bb10e0a901ac858248aa70a5abd7ce0e5b899438 Mon Sep 17 00:00:00 2001 From: Rafal Korepta Date: Mon, 28 Nov 2022 15:13:53 +0100 Subject: [PATCH 3/5] k8s: Create negative test for upgrade procedure When cluster is unhealthy the upgrade/restarting procedure should not be executed. --- src/go/k8s/controllers/redpanda/suite_test.go | 2 ++ src/go/k8s/pkg/admin/mock_admin.go | 9 +++++- src/go/k8s/pkg/resources/statefulset_test.go | 30 ++++++++++++++----- .../k8s/pkg/resources/statefulset_update.go | 4 +++ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/go/k8s/controllers/redpanda/suite_test.go b/src/go/k8s/controllers/redpanda/suite_test.go index 44f18c248eb4..ca71ee01ab59 100644 --- a/src/go/k8s/controllers/redpanda/suite_test.go +++ b/src/go/k8s/controllers/redpanda/suite_test.go @@ -102,6 +102,8 @@ var _ = BeforeSuite(func(done Done) { } return testAdminAPI, nil } + testAdminAPI.SetClusterHealth(true) + testStore = consolepkg.NewStore(k8sManager.GetClient(), k8sManager.GetScheme()) testKafkaAdmin = &mockKafkaAdmin{} testKafkaAdminFactory = func(context.Context, client.Client, *redpandav1alpha1.Cluster, *consolepkg.Store) (consolepkg.KafkaAdminClient, error) { diff --git a/src/go/k8s/pkg/admin/mock_admin.go b/src/go/k8s/pkg/admin/mock_admin.go index 1d1175965e1d..80f20e1767d2 100644 --- a/src/go/k8s/pkg/admin/mock_admin.go +++ b/src/go/k8s/pkg/admin/mock_admin.go @@ -35,6 +35,7 @@ type MockAdminAPI struct { brokers []admin.Broker monitor sync.Mutex Log logr.Logger + clusterHealth bool } var _ AdminAPIClient = &MockAdminAPI{Log: ctrl.Log.WithName("AdminAPIClient").WithName("mockAdminAPI")} @@ -50,6 +51,12 @@ func (*unavailableError) Error() string { return "unavailable" } +func (m *MockAdminAPI) SetClusterHealth(health bool) { + m.monitor.Lock() + defer m.monitor.Unlock() + m.clusterHealth = health +} + func (m *MockAdminAPI) Config(context.Context, bool) (admin.Config, error) { m.Log.WithName("Config").Info("called") m.monitor.Lock() @@ -394,7 +401,7 @@ func (m *MockAdminAPI) DisableMaintenanceMode(_ context.Context, _ int) error { func (m *MockAdminAPI) GetHealthOverview(_ context.Context) (admin.ClusterHealthOverview, error) { m.Log.WithName("GetHealthOverview").Info("called") return admin.ClusterHealthOverview{ - IsHealthy: true, + IsHealthy: m.clusterHealth, }, nil } diff --git a/src/go/k8s/pkg/resources/statefulset_test.go b/src/go/k8s/pkg/resources/statefulset_test.go index c60656ed9d81..02d0eb16ff3c 100644 --- a/src/go/k8s/pkg/resources/statefulset_test.go +++ b/src/go/k8s/pkg/resources/statefulset_test.go @@ -11,6 +11,7 @@ package resources_test import ( "context" + "errors" "testing" "time" @@ -65,17 +66,25 @@ func TestEnsure(t *testing.T) { // Remove shadow-indexing-cache from the volume claim templates stsWithoutSecondPersistentVolume.Spec.VolumeClaimTemplates = stsWithoutSecondPersistentVolume.Spec.VolumeClaimTemplates[:1] + unhealthyRedpandaCluster := cluster.DeepCopy() + tests := []struct { name string existingObject client.Object pandaCluster *redpandav1alpha1.Cluster expectedObject *v1.StatefulSet + clusterHealth bool + expectedError error }{ - {"none existing", nil, cluster, stsResource}, - {"update resources", stsResource, resourcesUpdatedCluster, resourcesUpdatedSts}, - {"update redpanda resources", stsResource, resourcesUpdatedRedpandaCluster, resourcesUpdatedSts}, - {"disabled sidecar", nil, noSidecarCluster, noSidecarSts}, - {"cluster without shadow index cache dir", stsResource, withoutShadowIndexCacheDirectory, stsWithoutSecondPersistentVolume}, + {"none existing", nil, cluster, stsResource, true, nil}, + {"update resources", stsResource, resourcesUpdatedCluster, resourcesUpdatedSts, true, nil}, + {"update redpanda resources", stsResource, resourcesUpdatedRedpandaCluster, resourcesUpdatedSts, true, nil}, + {"disabled sidecar", nil, noSidecarCluster, noSidecarSts, true, nil}, + {"cluster without shadow index cache dir", stsResource, withoutShadowIndexCacheDirectory, stsWithoutSecondPersistentVolume, true, nil}, + {"update none healthy cluster", stsResource, unhealthyRedpandaCluster, stsResource, false, &res.RequeueAfterError{ + RequeueAfter: res.RequeueDuration, + Msg: "wait for cluster to become healthy (cluster restarting)", + }}, } for _, tt := range tests { @@ -112,13 +121,20 @@ func TestEnsure(t *testing.T) { }, func(ctx context.Context) (string, error) { return hash, nil }, func(ctx context.Context, k8sClient client.Reader, redpandaCluster *redpandav1alpha1.Cluster, fqdn string, adminTLSProvider resourcetypes.AdminTLSConfigProvider, ordinals ...int32) (adminutils.AdminAPIClient, error) { - return &adminutils.MockAdminAPI{Log: ctrl.Log.WithName("testAdminAPI").WithName("mockAdminAPI")}, nil + health := tt.clusterHealth + adminAPI := &adminutils.MockAdminAPI{Log: ctrl.Log.WithName("testAdminAPI").WithName("mockAdminAPI")} + adminAPI.SetClusterHealth(health) + return adminAPI, nil }, time.Second, ctrl.Log.WithName("test")) err = sts.Ensure(context.Background()) - assert.NoError(t, err, tt.name) + if tt.expectedError != nil && errors.Is(err, tt.expectedError) { + assert.Error(t, err) + } else { + assert.NoError(t, err, tt.name) + } actual := &v1.StatefulSet{} err = c.Get(context.Background(), sts.Key(), actual) diff --git a/src/go/k8s/pkg/resources/statefulset_update.go b/src/go/k8s/pkg/resources/statefulset_update.go index 618cf33199a0..c12467d22f32 100644 --- a/src/go/k8s/pkg/resources/statefulset_update.go +++ b/src/go/k8s/pkg/resources/statefulset_update.go @@ -457,6 +457,10 @@ func (e *RequeueAfterError) Error() string { return fmt.Sprintf("RequeueAfterError %s", e.Msg) } +func (e *RequeueAfterError) Is(target error) bool { + return e.Error() == target.Error() +} + // RequeueError error to requeue using default retry backoff. type RequeueError struct { Msg string From c93d9670c8b11e6e7d1ff30418daf1b6113fbe7b Mon Sep 17 00:00:00 2001 From: Rafal Korepta Date: Wed, 30 Nov 2022 00:12:04 +0100 Subject: [PATCH 4/5] k8s: Adjust upgrade end to end tests Before 22.X the cluster health overview is not available. All tests could not upgrade from 21.X as operator could validate the health status. --- .../00-redpanda-cluster.yaml | 2 +- .../centralized-configuration-upgrade/01-assert.yaml | 4 ++-- .../01-first-upgrade.yaml | 2 +- .../02-update-to-central-config-with-prop.yaml | 2 +- .../e2e/update-image-and-node-port/00-assert.yaml | 2 +- .../update-image-and-node-port/00-current-image.yaml | 2 +- .../e2e/update-image-and-node-port/01-assert.yaml | 12 +++++++++--- .../01-new-image-and-pv.yaml | 2 +- .../e2e/update-image-and-node-port/02-assert.yaml | 6 +++--- .../e2e/update-image-and-node-port/02-new-image.yaml | 2 +- .../00-current-image.yaml | 2 +- .../e2e/update-image-tls-client-auth/01-assert.yaml | 10 ++++++++-- .../01-new-image-and-pv.yaml | 2 +- .../e2e/update-image-tls-client-auth/02-assert.yaml | 4 ++-- .../update-image-tls-client-auth/02-new-image.yaml | 2 +- .../tests/e2e/update-image-tls/00-current-image.yaml | 2 +- src/go/k8s/tests/e2e/update-image-tls/01-assert.yaml | 10 ++++++++-- .../e2e/update-image-tls/01-new-image-and-pv.yaml | 2 +- src/go/k8s/tests/e2e/update-image-tls/02-assert.yaml | 4 ++-- .../k8s/tests/e2e/update-image-tls/02-new-image.yaml | 2 +- 20 files changed, 47 insertions(+), 29 deletions(-) diff --git a/src/go/k8s/tests/e2e/centralized-configuration-upgrade/00-redpanda-cluster.yaml b/src/go/k8s/tests/e2e/centralized-configuration-upgrade/00-redpanda-cluster.yaml index 7c2ae09fc36e..958d68218643 100644 --- a/src/go/k8s/tests/e2e/centralized-configuration-upgrade/00-redpanda-cluster.yaml +++ b/src/go/k8s/tests/e2e/centralized-configuration-upgrade/00-redpanda-cluster.yaml @@ -4,7 +4,7 @@ metadata: name: centralized-configuration-upgrade spec: image: "vectorized/redpanda" - version: "v21.11.11" + version: "v22.1.10" replicas: 2 resources: requests: diff --git a/src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-assert.yaml b/src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-assert.yaml index 855fa42d4007..390529ead9cd 100644 --- a/src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-assert.yaml +++ b/src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-assert.yaml @@ -13,7 +13,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v21.11.16" + image: "vectorized/redpanda:v22.1.10" status: phase: "Running" --- @@ -25,7 +25,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v21.11.16" + image: "vectorized/redpanda:v22.1.10" status: phase: "Running" --- diff --git a/src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-first-upgrade.yaml b/src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-first-upgrade.yaml index 1fb941917d5f..958d68218643 100644 --- a/src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-first-upgrade.yaml +++ b/src/go/k8s/tests/e2e/centralized-configuration-upgrade/01-first-upgrade.yaml @@ -4,7 +4,7 @@ metadata: name: centralized-configuration-upgrade spec: image: "vectorized/redpanda" - version: "v21.11.16" + version: "v22.1.10" replicas: 2 resources: requests: diff --git a/src/go/k8s/tests/e2e/centralized-configuration-upgrade/02-update-to-central-config-with-prop.yaml b/src/go/k8s/tests/e2e/centralized-configuration-upgrade/02-update-to-central-config-with-prop.yaml index aa8c1640de50..dd21bcfc91c0 100644 --- a/src/go/k8s/tests/e2e/centralized-configuration-upgrade/02-update-to-central-config-with-prop.yaml +++ b/src/go/k8s/tests/e2e/centralized-configuration-upgrade/02-update-to-central-config-with-prop.yaml @@ -4,7 +4,7 @@ metadata: name: centralized-configuration-upgrade spec: image: "vectorized/redpanda" - version: "v22.2.7" + version: "v22.2.8" replicas: 2 resources: requests: diff --git a/src/go/k8s/tests/e2e/update-image-and-node-port/00-assert.yaml b/src/go/k8s/tests/e2e/update-image-and-node-port/00-assert.yaml index 4e1089e18a37..e2dd56f2c8c2 100644 --- a/src/go/k8s/tests/e2e/update-image-and-node-port/00-assert.yaml +++ b/src/go/k8s/tests/e2e/update-image-and-node-port/00-assert.yaml @@ -58,4 +58,4 @@ kind: Cluster metadata: name: update-image-cluster-and-node-port status: - version: "v21.11.1" + version: "v22.1.10" diff --git a/src/go/k8s/tests/e2e/update-image-and-node-port/00-current-image.yaml b/src/go/k8s/tests/e2e/update-image-and-node-port/00-current-image.yaml index 73e3d73fc378..4eec97f5e13d 100644 --- a/src/go/k8s/tests/e2e/update-image-and-node-port/00-current-image.yaml +++ b/src/go/k8s/tests/e2e/update-image-and-node-port/00-current-image.yaml @@ -23,7 +23,7 @@ metadata: name: update-image-cluster-and-node-port spec: image: "vectorized/redpanda" - version: "v21.11.1" + version: "v22.1.10" replicas: 2 resources: requests: diff --git a/src/go/k8s/tests/e2e/update-image-and-node-port/01-assert.yaml b/src/go/k8s/tests/e2e/update-image-and-node-port/01-assert.yaml index 3c84d64bb11d..f243155bdb99 100644 --- a/src/go/k8s/tests/e2e/update-image-and-node-port/01-assert.yaml +++ b/src/go/k8s/tests/e2e/update-image-and-node-port/01-assert.yaml @@ -14,12 +14,15 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v21.11.12" + image: "vectorized/redpanda:v22.2.8" volumeMounts: - mountPath: /etc/redpanda name: config-dir - mountPath: /scripts name: hook-scripts-dir + - mountPath: /etc/redpanda/.bootstrap.yaml + name: configmap-dir + subPath: .bootstrap.yaml - mountPath: /var/lib/redpanda/data name: datadir - mountPath: /var/lib/shadow-index-cache @@ -38,12 +41,15 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v21.11.12" + image: "vectorized/redpanda:v22.2.8" volumeMounts: - mountPath: /etc/redpanda name: config-dir - mountPath: /scripts name: hook-scripts-dir + - mountPath: /etc/redpanda/.bootstrap.yaml + name: configmap-dir + subPath: .bootstrap.yaml - mountPath: /var/lib/redpanda/data name: datadir - mountPath: /var/lib/shadow-index-cache @@ -90,4 +96,4 @@ kind: Cluster metadata: name: update-image-cluster-and-node-port status: - version: "v21.11.12" + version: "v22.2.8" diff --git a/src/go/k8s/tests/e2e/update-image-and-node-port/01-new-image-and-pv.yaml b/src/go/k8s/tests/e2e/update-image-and-node-port/01-new-image-and-pv.yaml index 31d110804aa8..093656f73fac 100644 --- a/src/go/k8s/tests/e2e/update-image-and-node-port/01-new-image-and-pv.yaml +++ b/src/go/k8s/tests/e2e/update-image-and-node-port/01-new-image-and-pv.yaml @@ -12,7 +12,7 @@ kind: Cluster metadata: name: update-image-cluster-and-node-port spec: - version: "v21.11.12" + version: "v22.2.8" cloudStorage: enabled: true accessKey: XXX diff --git a/src/go/k8s/tests/e2e/update-image-and-node-port/02-assert.yaml b/src/go/k8s/tests/e2e/update-image-and-node-port/02-assert.yaml index 34393f3dc4bb..e4a52d982b61 100644 --- a/src/go/k8s/tests/e2e/update-image-and-node-port/02-assert.yaml +++ b/src/go/k8s/tests/e2e/update-image-and-node-port/02-assert.yaml @@ -14,7 +14,7 @@ metadata: spec: containers: - name: redpanda - image: vectorized/redpanda:v22.2.7 + image: vectorized/redpanda:v22.3.4 status: phase: "Running" @@ -27,7 +27,7 @@ metadata: spec: containers: - name: redpanda - image: vectorized/redpanda:v22.2.7 + image: vectorized/redpanda:v22.3.4 status: phase: "Running" @@ -47,4 +47,4 @@ kind: Cluster metadata: name: update-image-cluster-and-node-port status: - version: v22.2.7 + version: v22.3.4 diff --git a/src/go/k8s/tests/e2e/update-image-and-node-port/02-new-image.yaml b/src/go/k8s/tests/e2e/update-image-and-node-port/02-new-image.yaml index d5cb962c813a..bdce10ff9d65 100644 --- a/src/go/k8s/tests/e2e/update-image-and-node-port/02-new-image.yaml +++ b/src/go/k8s/tests/e2e/update-image-and-node-port/02-new-image.yaml @@ -4,4 +4,4 @@ metadata: name: update-image-cluster-and-node-port spec: image: "vectorized/redpanda" - version: "v22.2.7" + version: "v22.3.4" diff --git a/src/go/k8s/tests/e2e/update-image-tls-client-auth/00-current-image.yaml b/src/go/k8s/tests/e2e/update-image-tls-client-auth/00-current-image.yaml index fbaf9ceb16f6..dd86baf8d7d8 100644 --- a/src/go/k8s/tests/e2e/update-image-tls-client-auth/00-current-image.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls-client-auth/00-current-image.yaml @@ -4,7 +4,7 @@ metadata: name: up-img spec: image: "vectorized/redpanda" - version: "v21.11.1" + version: "v22.1.10" replicas: 2 resources: requests: diff --git a/src/go/k8s/tests/e2e/update-image-tls-client-auth/01-assert.yaml b/src/go/k8s/tests/e2e/update-image-tls-client-auth/01-assert.yaml index 363f40b90bda..6bb158264a7c 100644 --- a/src/go/k8s/tests/e2e/update-image-tls-client-auth/01-assert.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls-client-auth/01-assert.yaml @@ -14,7 +14,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v21.11.12" + image: "vectorized/redpanda:v22.2.8" volumeMounts: - mountPath: /etc/redpanda name: config-dir @@ -24,6 +24,9 @@ spec: name: tlsadmincert - mountPath: /etc/tls/certs/admin/ca name: tlsadminca + - mountPath: /etc/redpanda/.bootstrap.yaml + name: configmap-dir + subPath: .bootstrap.yaml - mountPath: /var/lib/redpanda/data name: datadir - mountPath: /var/lib/shadow-index-cache @@ -42,7 +45,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v21.11.12" + image: "vectorized/redpanda:v22.2.8" volumeMounts: - mountPath: /etc/redpanda name: config-dir @@ -52,6 +55,9 @@ spec: name: tlsadmincert - mountPath: /etc/tls/certs/admin/ca name: tlsadminca + - mountPath: /etc/redpanda/.bootstrap.yaml + name: configmap-dir + subPath: .bootstrap.yaml - mountPath: /var/lib/redpanda/data name: datadir - mountPath: /var/lib/shadow-index-cache diff --git a/src/go/k8s/tests/e2e/update-image-tls-client-auth/01-new-image-and-pv.yaml b/src/go/k8s/tests/e2e/update-image-tls-client-auth/01-new-image-and-pv.yaml index 4b4b31efaaea..01489ade6065 100644 --- a/src/go/k8s/tests/e2e/update-image-tls-client-auth/01-new-image-and-pv.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls-client-auth/01-new-image-and-pv.yaml @@ -12,7 +12,7 @@ kind: Cluster metadata: name: up-img spec: - version: "v21.11.12" + version: "v22.2.8" cloudStorage: enabled: true accessKey: XXX diff --git a/src/go/k8s/tests/e2e/update-image-tls-client-auth/02-assert.yaml b/src/go/k8s/tests/e2e/update-image-tls-client-auth/02-assert.yaml index 7d480ead821c..03c57c52f72b 100644 --- a/src/go/k8s/tests/e2e/update-image-tls-client-auth/02-assert.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls-client-auth/02-assert.yaml @@ -14,7 +14,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v22.2.7" + image: "vectorized/redpanda:v22.3.4" status: phase: "Running" @@ -27,7 +27,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v22.2.7" + image: "vectorized/redpanda:v22.3.4" status: phase: "Running" diff --git a/src/go/k8s/tests/e2e/update-image-tls-client-auth/02-new-image.yaml b/src/go/k8s/tests/e2e/update-image-tls-client-auth/02-new-image.yaml index b703a20991e4..fd14223c4081 100644 --- a/src/go/k8s/tests/e2e/update-image-tls-client-auth/02-new-image.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls-client-auth/02-new-image.yaml @@ -4,4 +4,4 @@ metadata: name: up-img spec: image: "vectorized/redpanda" - version: "v22.2.7" + version: "v22.3.4" diff --git a/src/go/k8s/tests/e2e/update-image-tls/00-current-image.yaml b/src/go/k8s/tests/e2e/update-image-tls/00-current-image.yaml index d9290b6dcb1f..d0005394f00e 100644 --- a/src/go/k8s/tests/e2e/update-image-tls/00-current-image.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls/00-current-image.yaml @@ -4,7 +4,7 @@ metadata: name: up-img spec: image: "vectorized/redpanda" - version: "v21.11.1" + version: "v22.1.10" replicas: 2 resources: requests: diff --git a/src/go/k8s/tests/e2e/update-image-tls/01-assert.yaml b/src/go/k8s/tests/e2e/update-image-tls/01-assert.yaml index 536cd30490f7..73171931ecba 100644 --- a/src/go/k8s/tests/e2e/update-image-tls/01-assert.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls/01-assert.yaml @@ -14,7 +14,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v21.11.12" + image: "vectorized/redpanda:v22.2.8" volumeMounts: - mountPath: /etc/redpanda name: config-dir @@ -22,6 +22,9 @@ spec: name: hook-scripts-dir - mountPath: /etc/tls/certs/admin name: tlsadmincert + - mountPath: /etc/redpanda/.bootstrap.yaml + name: configmap-dir + subPath: .bootstrap.yaml - mountPath: /var/lib/redpanda/data name: datadir - mountPath: /var/lib/shadow-index-cache @@ -40,7 +43,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v21.11.12" + image: "vectorized/redpanda:v22.2.8" volumeMounts: - mountPath: /etc/redpanda name: config-dir @@ -48,6 +51,9 @@ spec: name: hook-scripts-dir - mountPath: /etc/tls/certs/admin name: tlsadmincert + - mountPath: /etc/redpanda/.bootstrap.yaml + name: configmap-dir + subPath: .bootstrap.yaml - mountPath: /var/lib/redpanda/data name: datadir - mountPath: /var/lib/shadow-index-cache diff --git a/src/go/k8s/tests/e2e/update-image-tls/01-new-image-and-pv.yaml b/src/go/k8s/tests/e2e/update-image-tls/01-new-image-and-pv.yaml index 4b4b31efaaea..01489ade6065 100644 --- a/src/go/k8s/tests/e2e/update-image-tls/01-new-image-and-pv.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls/01-new-image-and-pv.yaml @@ -12,7 +12,7 @@ kind: Cluster metadata: name: up-img spec: - version: "v21.11.12" + version: "v22.2.8" cloudStorage: enabled: true accessKey: XXX diff --git a/src/go/k8s/tests/e2e/update-image-tls/02-assert.yaml b/src/go/k8s/tests/e2e/update-image-tls/02-assert.yaml index 7d480ead821c..03c57c52f72b 100644 --- a/src/go/k8s/tests/e2e/update-image-tls/02-assert.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls/02-assert.yaml @@ -14,7 +14,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v22.2.7" + image: "vectorized/redpanda:v22.3.4" status: phase: "Running" @@ -27,7 +27,7 @@ metadata: spec: containers: - name: redpanda - image: "vectorized/redpanda:v22.2.7" + image: "vectorized/redpanda:v22.3.4" status: phase: "Running" diff --git a/src/go/k8s/tests/e2e/update-image-tls/02-new-image.yaml b/src/go/k8s/tests/e2e/update-image-tls/02-new-image.yaml index b703a20991e4..fd14223c4081 100644 --- a/src/go/k8s/tests/e2e/update-image-tls/02-new-image.yaml +++ b/src/go/k8s/tests/e2e/update-image-tls/02-new-image.yaml @@ -4,4 +4,4 @@ metadata: name: up-img spec: image: "vectorized/redpanda" - version: "v22.2.7" + version: "v22.3.4" From 2d775e6c35ae37279be2a5cc18ba29b5ebb78c1e Mon Sep 17 00:00:00 2001 From: Rafal Korepta Date: Thu, 1 Dec 2022 01:53:01 +0100 Subject: [PATCH 5/5] k8s: Remove require client authorization Admin API In the centralized configuration e2e test the cluster health can not be retrieved if required client authorization is removed from Admin API. Nodes that are running with mTLS configuration does not respond to operator get health overview. If first out of N brokers is restarted and stops serving Admin API with mTLS configuration, then rpk adminAPI implementation sends http request to all in sequence get health overview. The problem is with http client and TLS configuration as one out of N doesn not need client certificate. --- .../00-redpanda-cluster.yaml | 1 - .../01-redpanda-cluster-change.yaml | 3 ++- .../tests/e2e/centralized-configuration-tls/02-probe.yaml | 8 +------- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/go/k8s/tests/e2e/centralized-configuration-tls/00-redpanda-cluster.yaml b/src/go/k8s/tests/e2e/centralized-configuration-tls/00-redpanda-cluster.yaml index 4ecd3d0ec6c9..000f8c67dbcd 100644 --- a/src/go/k8s/tests/e2e/centralized-configuration-tls/00-redpanda-cluster.yaml +++ b/src/go/k8s/tests/e2e/centralized-configuration-tls/00-redpanda-cluster.yaml @@ -22,7 +22,6 @@ spec: - port: 9644 tls: enabled: true - requireClientAuth: true pandaproxyApi: - port: 8082 developerMode: true \ No newline at end of file diff --git a/src/go/k8s/tests/e2e/centralized-configuration-tls/01-redpanda-cluster-change.yaml b/src/go/k8s/tests/e2e/centralized-configuration-tls/01-redpanda-cluster-change.yaml index 27f695651b6a..89f21bdf74cc 100644 --- a/src/go/k8s/tests/e2e/centralized-configuration-tls/01-redpanda-cluster-change.yaml +++ b/src/go/k8s/tests/e2e/centralized-configuration-tls/01-redpanda-cluster-change.yaml @@ -22,9 +22,10 @@ spec: - port: 9644 tls: enabled: true - requireClientAuth: true pandaproxyApi: - port: 8082 + tls: + enabled: true developerMode: true additionalConfiguration: redpanda.segment_appender_flush_timeout_ms: "1003" diff --git a/src/go/k8s/tests/e2e/centralized-configuration-tls/02-probe.yaml b/src/go/k8s/tests/e2e/centralized-configuration-tls/02-probe.yaml index aaa67303b284..3c4e6a81fe2f 100644 --- a/src/go/k8s/tests/e2e/centralized-configuration-tls/02-probe.yaml +++ b/src/go/k8s/tests/e2e/centralized-configuration-tls/02-probe.yaml @@ -8,10 +8,6 @@ spec: spec: activeDeadlineSeconds: 90 volumes: - - name: tlsadmin - secret: - defaultMode: 420 - secretName: centralized-configuration-tls-admin-api-client - name: tlsadminca secret: defaultMode: 420 @@ -31,14 +27,12 @@ spec: args: - > url=https://centralized-configuration-tls-0.centralized-configuration-tls.$NAMESPACE.svc.cluster.local:9644/v1/config - res=$(curl --cacert /etc/tls/certs/admin/ca/ca.crt --cert /etc/tls/certs/admin/tls.crt --key /etc/tls/certs/admin/tls.key --silent -L $url | grep -o '\"segment_appender_flush_timeout_ms\":[^,}]*' | grep -o '[^:]*$') && + res=$(curl --cacert /etc/tls/certs/admin/ca/ca.crt --silent -L $url | grep -o '\"segment_appender_flush_timeout_ms\":[^,}]*' | grep -o '[^:]*$') && echo $res > /dev/termination-log && if [[ "$res" != "1003" ]]; then exit 1; fi volumeMounts: - - mountPath: /etc/tls/certs/admin - name: tlsadmin - mountPath: /etc/tls/certs/admin/ca name: tlsadminca restartPolicy: Never