diff --git a/src/go/k8s/controllers/redpanda/cluster_controller.go b/src/go/k8s/controllers/redpanda/cluster_controller.go index 1260e18a14b5..bfd0e44ecda5 100644 --- a/src/go/k8s/controllers/redpanda/cluster_controller.go +++ b/src/go/k8s/controllers/redpanda/cluster_controller.go @@ -150,15 +150,8 @@ func (r *ClusterReconciler) Reconcile( headlessSvc.HeadlessServiceFQDN(r.clusterDomain), headlessSvc.Key().Name, nodeportSvc.Key(), - pki.RedpandaNodeCert(), - pki.RedpandaOperatorClientCert(), - pki.RedpandaAdminCert(), - pki.AdminAPINodeCert(), - pki.AdminAPIClientCert(), - pki.PandaproxyAPINodeCert(), - pki.PandaproxyAPIClientCert(), - pki.SchemaRegistryAPINodeCert(), - pki.SchemaRegistryAPIClientCert(), + pki.StatefulSetVolumeProvider(), + pki.AdminAPIConfigProvider(), sa.Key().Name, r.configuratorSettings, configMapResource.GetNodeConfigHash, @@ -204,7 +197,7 @@ func (r *ClusterReconciler) Reconcile( secrets = append(secrets, schemaRegistrySu.Key()) } - err := r.setInitialSuperUserPassword(ctx, &redpandaCluster, headlessSvc.HeadlessServiceFQDN(r.clusterDomain), pki.AdminAPINodeCert(), pki.AdminAPIClientCert(), secrets) + err := r.setInitialSuperUserPassword(ctx, &redpandaCluster, headlessSvc.HeadlessServiceFQDN(r.clusterDomain), pki.AdminAPIConfigProvider(), secrets) var e *resources.RequeueAfterError if errors.As(err, &e) { @@ -502,11 +495,10 @@ func (r *ClusterReconciler) setInitialSuperUserPassword( ctx context.Context, redpandaCluster *redpandav1alpha1.Cluster, fqdn string, - adminAPINodeCertSecretKey client.ObjectKey, - adminAPIClientCertSecretKey client.ObjectKey, + adminTLSConfigProvider resources.AdminTLSConfigProvider, objs []types.NamespacedName, ) error { - adminAPI, err := r.AdminAPIClientFactory(ctx, r, redpandaCluster, fqdn, adminAPINodeCertSecretKey, adminAPIClientCertSecretKey) + adminAPI, err := r.AdminAPIClientFactory(ctx, r, redpandaCluster, fqdn, adminTLSConfigProvider) if err != nil && errors.Is(err, &adminutils.NoInternalAdminAPI{}) { return nil } else if err != nil { diff --git a/src/go/k8s/controllers/redpanda/cluster_controller_configuration.go b/src/go/k8s/controllers/redpanda/cluster_controller_configuration.go index cef2b9e051a9..c658b2183e64 100644 --- a/src/go/k8s/controllers/redpanda/cluster_controller_configuration.go +++ b/src/go/k8s/controllers/redpanda/cluster_controller_configuration.go @@ -77,7 +77,7 @@ func (r *ClusterReconciler) reconcileConfiguration( } } - adminAPI, err := r.AdminAPIClientFactory(ctx, r, redpandaCluster, fqdn, pki.AdminAPINodeCert(), pki.AdminAPIClientCert()) + adminAPI, err := r.AdminAPIClientFactory(ctx, r, redpandaCluster, fqdn, pki.AdminAPIConfigProvider()) if err != nil { return errorWithContext(err, "error creating the admin API client") } diff --git a/src/go/k8s/controllers/redpanda/cluster_controller_configuration_drift.go b/src/go/k8s/controllers/redpanda/cluster_controller_configuration_drift.go index 6679bfa1eebe..68654fbf8265 100644 --- a/src/go/k8s/controllers/redpanda/cluster_controller_configuration_drift.go +++ b/src/go/k8s/controllers/redpanda/cluster_controller_configuration_drift.go @@ -123,7 +123,7 @@ func (r *ClusterConfigurationDriftReconciler) Reconcile( return ctrl.Result{RequeueAfter: r.getDriftCheckPeriod()}, nil } - adminAPI, err := r.AdminAPIClientFactory(ctx, r, &redpandaCluster, headlessSvc.HeadlessServiceFQDN(r.clusterDomain), pki.AdminAPINodeCert(), pki.AdminAPIClientCert()) + adminAPI, err := r.AdminAPIClientFactory(ctx, r, &redpandaCluster, headlessSvc.HeadlessServiceFQDN(r.clusterDomain), pki.AdminAPIConfigProvider()) if err != nil { return ctrl.Result{}, fmt.Errorf("could not get admin API to check drifts on the cluster: %w", err) } diff --git a/src/go/k8s/controllers/redpanda/suite_test.go b/src/go/k8s/controllers/redpanda/suite_test.go index ae5b77e14022..b3365cfff450 100644 --- a/src/go/k8s/controllers/redpanda/suite_test.go +++ b/src/go/k8s/controllers/redpanda/suite_test.go @@ -90,8 +90,7 @@ var _ = BeforeSuite(func(done Done) { _ client.Reader, _ *redpandav1alpha1.Cluster, _ string, - _ client.ObjectKey, - _ client.ObjectKey, + _ resources.AdminTLSConfigProvider, ) (adminutils.AdminAPIClient, error) { return testAdminAPI, nil } diff --git a/src/go/k8s/pkg/admin/admin.go b/src/go/k8s/pkg/admin/admin.go index 460aa1071391..dff1eb54f8d8 100644 --- a/src/go/k8s/pkg/admin/admin.go +++ b/src/go/k8s/pkg/admin/admin.go @@ -16,6 +16,7 @@ import ( "fmt" redpandav1alpha1 "github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1" + "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources" "github.com/redpanda-data/redpanda/src/go/rpk/pkg/api/admin" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -34,8 +35,7 @@ func NewInternalAdminAPI( k8sClient client.Reader, redpandaCluster *redpandav1alpha1.Cluster, fqdn string, - adminAPINodeCertSecretKey client.ObjectKey, - adminAPIClientCertSecretKey client.ObjectKey, + adminTLSProvider resources.AdminTLSConfigProvider, ) (AdminAPIClient, error) { adminInternal := redpandaCluster.AdminAPIInternal() if adminInternal == nil { @@ -45,7 +45,7 @@ func NewInternalAdminAPI( var tlsConfig *tls.Config if adminInternal.TLS.Enabled { var err error - tlsConfig, err = GetTLSConfig(ctx, k8sClient, redpandaCluster, adminAPINodeCertSecretKey, adminAPIClientCertSecretKey) + tlsConfig, err = adminTLSProvider.GetTLSConfig(ctx, k8sClient) if err != nil { return nil, fmt.Errorf("could not create tls configuration for internal admin API: %w", err) } @@ -89,8 +89,7 @@ type AdminAPIClientFactory func( k8sClient client.Reader, redpandaCluster *redpandav1alpha1.Cluster, fqdn string, - adminAPINodeCertSecretKey client.ObjectKey, - adminAPIClientCertSecretKey client.ObjectKey, + adminTLSProvider resources.AdminTLSConfigProvider, ) (AdminAPIClient, error) var _ AdminAPIClientFactory = NewInternalAdminAPI diff --git a/src/go/k8s/pkg/admin/tls.go b/src/go/k8s/pkg/admin/tls.go deleted file mode 100644 index 92e0a946da8e..000000000000 --- a/src/go/k8s/pkg/admin/tls.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2022 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package admin - -import ( - "context" - "crypto/tls" - "crypto/x509" - - cmetav1 "github.com/jetstack/cert-manager/pkg/apis/meta/v1" - redpandav1alpha1 "github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// GetTLSConfig computes crypto/TLS configuration for certificate used by the operator -// during its client authentication. -func GetTLSConfig( - ctx context.Context, - k8sClient client.Reader, - pandaCluster *redpandav1alpha1.Cluster, - adminAPINodeCertSecretKey client.ObjectKey, - adminAPIClientCertSecretKey client.ObjectKey, -) (*tls.Config, error) { - tlsConfig := tls.Config{MinVersion: tls.VersionTLS12} // TLS12 is min version allowed by gosec. - - var nodeCertSecret corev1.Secret - err := k8sClient.Get(ctx, adminAPINodeCertSecretKey, &nodeCertSecret) - if err != nil { - return nil, err - } - - // Add root CA - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(nodeCertSecret.Data[cmetav1.TLSCAKey]) - tlsConfig.RootCAs = caCertPool - - if pandaCluster.AdminAPITLS() != nil && pandaCluster.AdminAPITLS().TLS.RequireClientAuth { - var clientCertSecret corev1.Secret - err := k8sClient.Get(ctx, adminAPIClientCertSecretKey, &clientCertSecret) - if err != nil { - return nil, err - } - cert, err := tls.X509KeyPair(clientCertSecret.Data[corev1.TLSCertKey], clientCertSecret.Data[corev1.TLSPrivateKeyKey]) - if err != nil { - return nil, err - } - tlsConfig.Certificates = []tls.Certificate{cert} - } - - return &tlsConfig, nil -} diff --git a/src/go/k8s/pkg/resources/certmanager/admin_api.go b/src/go/k8s/pkg/resources/certmanager/admin_api.go index 1f5c883d6aef..18237d2b2666 100644 --- a/src/go/k8s/pkg/resources/certmanager/admin_api.go +++ b/src/go/k8s/pkg/resources/certmanager/admin_api.go @@ -10,20 +10,8 @@ // Package certmanager contains resources for TLS certificate handling using cert-manager package certmanager -import "k8s.io/apimachinery/pkg/types" - const ( adminAPI = "admin" adminAPIClientCert = "admin-api-client" adminAPINodeCert = "admin-api-node" ) - -// AdminAPINodeCert returns the namespaced name for the Admin API certificate used by nodes -func (r *PkiReconciler) AdminAPINodeCert() types.NamespacedName { - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + adminAPINodeCert, Namespace: r.pandaCluster.Namespace} -} - -// AdminAPIClientCert returns the namespaced name for the Admin API certificate used by clients -func (r *PkiReconciler) AdminAPIClientCert() types.NamespacedName { - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + adminAPIClientCert, Namespace: r.pandaCluster.Namespace} -} diff --git a/src/go/k8s/pkg/resources/certmanager/kafka_api.go b/src/go/k8s/pkg/resources/certmanager/kafka_api.go index d16af820f0ac..ffe1b92b9a88 100644 --- a/src/go/k8s/pkg/resources/certmanager/kafka_api.go +++ b/src/go/k8s/pkg/resources/certmanager/kafka_api.go @@ -9,8 +9,6 @@ package certmanager -import "k8s.io/apimachinery/pkg/types" - const ( kafkaAPI = "kafka" // OperatorClientCert cert name - used by kubernetes operator to call KafkaAPI @@ -22,26 +20,3 @@ const ( // RedpandaNodeCert cert name - node certificate RedpandaNodeCert = "redpanda" ) - -// RedpandaOperatorClientCert returns the namespaced name for the client certificate -// used by the Kubernetes operator -func (r *PkiReconciler) RedpandaOperatorClientCert() types.NamespacedName { - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + OperatorClientCert, Namespace: r.pandaCluster.Namespace} -} - -// RedpandaAdminCert returns the namespaced name for the certificate used by an administrator to query the Kafka API -func (r *PkiReconciler) RedpandaAdminCert() types.NamespacedName { - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + OperatorClientCert, Namespace: r.pandaCluster.Namespace} -} - -// RedpandaNodeCert returns the namespaced name for Redpanda's node certificate -func (r *PkiReconciler) RedpandaNodeCert() types.NamespacedName { - tlsListener := r.pandaCluster.KafkaTLSListener() - if tlsListener != nil && tlsListener.TLS.NodeSecretRef != nil { - return types.NamespacedName{ - Name: tlsListener.TLS.NodeSecretRef.Name, - Namespace: r.pandaCluster.Namespace, - } - } - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + RedpandaNodeCert, Namespace: r.pandaCluster.Namespace} -} diff --git a/src/go/k8s/pkg/resources/certmanager/pandaproxy_api.go b/src/go/k8s/pkg/resources/certmanager/pandaproxy_api.go index 895adcc7318a..508c47f0d943 100644 --- a/src/go/k8s/pkg/resources/certmanager/pandaproxy_api.go +++ b/src/go/k8s/pkg/resources/certmanager/pandaproxy_api.go @@ -10,20 +10,8 @@ // Package certmanager contains resources for TLS certificate handling using cert-manager package certmanager -import "k8s.io/apimachinery/pkg/types" - const ( pandaproxyAPI = "proxy" pandaproxyAPIClientCert = "proxy-api-client" pandaproxyAPINodeCert = "proxy-api-node" ) - -// PandaproxyAPINodeCert returns the namespaced name for the Pandaproxy API certificate used by nodes -func (r *PkiReconciler) PandaproxyAPINodeCert() types.NamespacedName { - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + pandaproxyAPINodeCert, Namespace: r.pandaCluster.Namespace} -} - -// PandaproxyAPIClientCert returns the namespaced name for the Pandaproxy API certificate used by clients -func (r *PkiReconciler) PandaproxyAPIClientCert() types.NamespacedName { - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + pandaproxyAPIClientCert, Namespace: r.pandaCluster.Namespace} -} diff --git a/src/go/k8s/pkg/resources/certmanager/pki.go b/src/go/k8s/pkg/resources/certmanager/pki.go index ff94374b9279..826348173bb0 100644 --- a/src/go/k8s/pkg/resources/certmanager/pki.go +++ b/src/go/k8s/pkg/resources/certmanager/pki.go @@ -15,19 +15,15 @@ import ( "fmt" "github.com/go-logr/logr" - cmmetav1 "github.com/jetstack/cert-manager/pkg/apis/meta/v1" redpandav1alpha1 "github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1" "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) var ( - _ resources.Reconciler = &PkiReconciler{} - errNoDNSNames = fmt.Errorf("failed to generate node TLS certificate") + _ resources.Reconciler = &PkiReconciler{} ) // RootCert cert name @@ -42,6 +38,8 @@ type PkiReconciler struct { internalFQDN string clusterFQDN string logger logr.Logger + + clusterCertificates *ClusterCertificates } // NewPki creates PkiReconciler @@ -55,84 +53,26 @@ func NewPki( ) *PkiReconciler { return &PkiReconciler{ client, scheme, pandaCluster, fqdn, clusterFQDN, logger.WithValues("Reconciler", "pki"), + NewClusterCertificates(pandaCluster, keyStoreKey(pandaCluster), client, fqdn, clusterFQDN, scheme, logger), } } -func (r *PkiReconciler) prepareRoot( - prefix string, -) ([]resources.Resource, *cmmetav1.ObjectReference) { - toApply := []resources.Resource{} - selfSignedIssuer := NewIssuer(r.Client, - r.scheme, - r.pandaCluster, - r.issuerNamespacedName(prefix+"-"+"selfsigned-issuer"), - "", - r.logger) - - rootCn := NewCommonName(r.pandaCluster.Name, prefix+"-root-certificate") - rootKey := types.NamespacedName{Name: string(rootCn), Namespace: r.pandaCluster.Namespace} - rootCertificate := NewCACertificate(r.Client, - r.scheme, - r.pandaCluster, - rootKey, - selfSignedIssuer.objRef(), - rootCn, - nil, - r.logger) - - leafIssuer := NewIssuer(r.Client, - r.scheme, - r.pandaCluster, - r.issuerNamespacedName(prefix+"-"+"root-issuer"), - rootCertificate.Key().Name, - r.logger) - - leafIssuerRef := leafIssuer.objRef() - - toApply = append(toApply, selfSignedIssuer, rootCertificate, leafIssuer) - return toApply, leafIssuerRef +func keyStoreKey(pandaCluster *redpandav1alpha1.Cluster) types.NamespacedName { + return types.NamespacedName{Name: keystoreName(pandaCluster.Name), Namespace: pandaCluster.Namespace} } // Ensure will manage PKI for redpanda.vectorized.io custom resource func (r *PkiReconciler) Ensure(ctx context.Context) error { toApply := []resources.Resource{} - keystoreKey := types.NamespacedName{Name: keystoreName(r.pandaCluster.Name), Namespace: r.pandaCluster.Namespace} - keystoreSecret := NewKeystoreSecretResource(r.Client, r.scheme, r.pandaCluster, keystoreKey, r.logger) + keystoreSecret := NewKeystoreSecretResource(r.Client, r.scheme, r.pandaCluster, keyStoreKey(r.pandaCluster), r.logger) toApply = append(toApply, keystoreSecret) - - if kafkaListeners := kafkaAPIListeners(r.pandaCluster); len(kafkaListeners) > 0 { - toApplyAPI, err := r.prepareAPI(ctx, kafkaAPI, RedpandaNodeCert, []string{OperatorClientCert, UserClientCert, AdminClientCert}, kafkaListeners, &keystoreKey) - if err != nil { - return err - } - toApply = append(toApply, toApplyAPI...) - } - - if adminListeners := adminAPIListeners(r.pandaCluster); len(adminListeners) > 0 { - toApplyAPI, err := r.prepareAPI(ctx, adminAPI, adminAPINodeCert, []string{adminAPIClientCert}, adminListeners, &keystoreKey) - if err != nil { - return err - } - toApply = append(toApply, toApplyAPI...) - } - - if pandaProxyListeners := pandaProxyAPIListeners(r.pandaCluster); len(pandaProxyListeners) > 0 { - toApplyAPI, err := r.prepareAPI(ctx, pandaproxyAPI, pandaproxyAPINodeCert, []string{pandaproxyAPIClientCert}, pandaProxyListeners, &keystoreKey) - if err != nil { - return err - } - toApply = append(toApply, toApplyAPI...) - } - - if schemaRegistryListeners := schemaRegistryAPIListeners(r.pandaCluster); len(schemaRegistryListeners) > 0 { - toApplyAPI, err := r.prepareAPI(ctx, schemaRegistryAPI, schemaRegistryAPINodeCert, []string{schemaRegistryAPIClientCert}, schemaRegistryListeners, &keystoreKey) - if err != nil { - return err - } - toApply = append(toApply, toApplyAPI...) + res, err := r.clusterCertificates.Resources(ctx) + if err != nil { + return fmt.Errorf("creating resources %w", err) } + toApply = append(toApply, res...) for _, res := range toApply { err := res.Ensure(ctx) @@ -144,112 +84,12 @@ func (r *PkiReconciler) Ensure(ctx context.Context) error { return nil } -func (r *PkiReconciler) issuerNamespacedName(name string) types.NamespacedName { - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + name, Namespace: r.pandaCluster.Namespace} +// StatefulSetVolumeProvider returns volume provider for all TLS certificates +func (r *PkiReconciler) StatefulSetVolumeProvider() resources.StatefulsetTLSVolumeProvider { + return r.clusterCertificates } -func (r *PkiReconciler) prepareAPI( - ctx context.Context, - rootCertSuffix string, - nodeCertSuffix string, - clientCerts []string, - listeners []APIListener, - keystoreSecret *types.NamespacedName, -) ([]resources.Resource, error) { - var ( - tlsListener = getTLSListener(listeners) - toApply = []resources.Resource{} - externalTLSListener = getExternalTLSListener(listeners) - internalTLSListener = getInternalTLSListener(listeners) - // Issuer for the nodes - nodeIssuerRef *cmmetav1.ObjectReference - ) - - if tlsListener == nil || tlsListener.GetTLS() == nil || !tlsListener.GetTLS().Enabled { - return []resources.Resource{}, nil - } - - // TODO(#3550): Do not create rootIssuer if nodeSecretRef is passed and mTLS is disabled - toApplyRoot, rootIssuerRef := r.prepareRoot(rootCertSuffix) - toApply = append(toApply, toApplyRoot...) - nodeIssuerRef = rootIssuerRef - - if tlsListener.GetTLS().IssuerRef != nil { - // if external issuer is provided, we will use it to generate node certificates - nodeIssuerRef = tlsListener.GetTLS().IssuerRef - } - - nodeSecretRef := tlsListener.GetTLS().NodeSecretRef - if nodeSecretRef == nil || nodeSecretRef.Name == "" { - certName := NewCertName(r.pandaCluster.Name, nodeCertSuffix) - certsKey := types.NamespacedName{Name: string(certName), Namespace: r.pandaCluster.Namespace} - dnsNames := []string{} - - if internalTLSListener != nil { - dnsNames = append(dnsNames, r.clusterFQDN, r.internalFQDN) - } - // TODO(#2256): Add support for external listener + TLS certs for IPs - if externalTLSListener != nil && externalTLSListener.GetExternal().Subdomain != "" { - dnsNames = append(dnsNames, externalTLSListener.GetExternal().Subdomain) - } - - if len(dnsNames) == 0 { - return nil, fmt.Errorf("failed to generate node TLS certificate %s. If external is enabled, please add a subdomain: %w", certName, errNoDNSNames) - } - - nodeCert := NewNodeCertificate(r.Client, r.scheme, r.pandaCluster, certsKey, nodeIssuerRef, dnsNames, EmptyCommonName, keystoreSecret, r.logger) - toApply = append(toApply, nodeCert) - } - - if nodeSecretRef != nil && nodeSecretRef.Name != "" && nodeSecretRef.Namespace != r.pandaCluster.Namespace { - if err := r.copyNodeSecretToLocalNamespace(ctx, nodeSecretRef); err != nil { - return nil, fmt.Errorf("copy node secret for %s cert group to namespace %s: %w", rootCertSuffix, nodeSecretRef.Namespace, err) - } - } - - if tlsListener.GetTLS().RequireClientAuth { - for _, clientCertName := range clientCerts { - clientCn := NewCommonName(r.pandaCluster.Name, clientCertName) - clientKey := types.NamespacedName{Name: string(clientCn), Namespace: r.pandaCluster.Namespace} - clientCert := NewCertificate(r.Client, r.scheme, r.pandaCluster, clientKey, rootIssuerRef, clientCn, false, keystoreSecret, r.logger) - toApply = append(toApply, clientCert) - } - } - - return toApply, nil -} - -// Creates copy of secret in Redpanda cluster's namespace -func (r *PkiReconciler) copyNodeSecretToLocalNamespace( - ctx context.Context, secretRef *corev1.ObjectReference, -) error { - var secret corev1.Secret - err := r.Get(ctx, types.NamespacedName{Name: secretRef.Name, Namespace: secretRef.Namespace}, &secret) - if err != nil { - return err - } - - tlsKey := secret.Data[corev1.TLSPrivateKeyKey] - tlsCrt := secret.Data[corev1.TLSCertKey] - caCrt := secret.Data[cmmetav1.TLSCAKey] - - caSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secret.Name, - Namespace: r.pandaCluster.Namespace, - Labels: secret.Labels, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - Type: secret.Type, - Data: map[string][]byte{ - cmmetav1.TLSCAKey: caCrt, - corev1.TLSCertKey: tlsCrt, - corev1.TLSPrivateKeyKey: tlsKey, - }, - } - _, err = resources.CreateIfNotExists(ctx, r, caSecret, r.logger) - return err +// AdminAPIConfigProvider returns provider of admin TLS configuration +func (r *PkiReconciler) AdminAPIConfigProvider() resources.AdminTLSConfigProvider { + return r.clusterCertificates } diff --git a/src/go/k8s/pkg/resources/certmanager/pki_test.go b/src/go/k8s/pkg/resources/certmanager/pki_test.go deleted file mode 100644 index b497a9c417b7..000000000000 --- a/src/go/k8s/pkg/resources/certmanager/pki_test.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2021 Redpanda Data, Inc. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.md -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0 - -package certmanager_test - -import ( - "testing" - - "github.com/go-logr/logr" - cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1" - "github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1" - "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/certmanager" - "github.com/stretchr/testify/require" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestSchemaRegistryCerts(t *testing.T) { - testcases := []struct { - name string - cluster v1alpha1.Cluster - expectedClientCertName string - expectedNodeCertName string - }{ - { - name: "SchemaRegistry: When NodeSecretRef is used", - cluster: v1alpha1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "redpanda", - Name: "fake-cluster", - }, - Spec: v1alpha1.ClusterSpec{ - Configuration: v1alpha1.RedpandaConfig{ - SchemaRegistry: &v1alpha1.SchemaRegistryAPI{ - TLS: &v1alpha1.SchemaRegistryAPITLS{ - Enabled: true, - NodeSecretRef: &v1.ObjectReference{ - Namespace: "other-namespace", - Name: "custom-secret-ref", - }, - }, - }, - }, - }, - }, - expectedClientCertName: "fake-cluster-schema-registry-client", - expectedNodeCertName: "custom-secret-ref", - }, - { - name: "SchemaRegistry: When IssuerRef is used", - cluster: v1alpha1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "redpanda", - Name: "fake-cluster", - }, - Spec: v1alpha1.ClusterSpec{ - Configuration: v1alpha1.RedpandaConfig{ - SchemaRegistry: &v1alpha1.SchemaRegistryAPI{ - TLS: &v1alpha1.SchemaRegistryAPITLS{ - Enabled: true, - IssuerRef: &cmmeta.ObjectReference{ - Group: "certmanager", - Kind: "ClusterIssuer", - Name: "cluster-issuer", - }, - }, - }, - }, - }, - }, - expectedClientCertName: "fake-cluster-schema-registry-client", - expectedNodeCertName: "fake-cluster-schema-registry-node", - }, - { - // TODO(#3550): refactor how methods used to get APIs node and client certificates behiave when - // TLS ot mTLS are disabled. Currently they always return a NamespacedName pointing to the a - // default certificate but that certificate doesn't exist. - // A better approach might be to return a certificate or secret object which could be nil in case - // TLS or mTLS are disabled. - name: "SchemaRegistry: When TLS and mTLS are disabled", - cluster: v1alpha1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "redpanda", - Name: "fake-cluster-with-no-mtls", - }, - Spec: v1alpha1.ClusterSpec{ - Configuration: v1alpha1.RedpandaConfig{ - SchemaRegistry: &v1alpha1.SchemaRegistryAPI{ - TLS: &v1alpha1.SchemaRegistryAPITLS{ - RequireClientAuth: false, - Enabled: false, - }, - }, - }, - }, - }, - expectedClientCertName: "fake-cluster-with-no-mtls-schema-registry-client", - expectedNodeCertName: "fake-cluster-with-no-mtls-schema-registry-node", - }, - } - - for _, tt := range testcases { - t.Run(tt.name, func(t *testing.T) { - pki := certmanager.NewPki(nil, &tt.cluster, "FQDN", "clusterFQDN", nil, logr.Discard()) - // Client cert - clientCert := pki.SchemaRegistryAPIClientCert() - require.NotNil(t, clientCert) - require.Equal(t, tt.expectedClientCertName, clientCert.Name) - require.Equal(t, tt.cluster.ObjectMeta.Namespace, clientCert.Namespace) - - // Node cert - nodeCert := pki.SchemaRegistryAPINodeCert() - require.NotNil(t, nodeCert) - require.Equal(t, tt.expectedNodeCertName, nodeCert.Name) - require.Equal(t, tt.cluster.ObjectMeta.Namespace, nodeCert.Namespace) - }) - } -} diff --git a/src/go/k8s/pkg/resources/certmanager/schemaregistry_api.go b/src/go/k8s/pkg/resources/certmanager/schemaregistry_api.go index 2e3a44272175..181f314a5659 100644 --- a/src/go/k8s/pkg/resources/certmanager/schemaregistry_api.go +++ b/src/go/k8s/pkg/resources/certmanager/schemaregistry_api.go @@ -10,27 +10,8 @@ // Package certmanager contains resources for TLS certificate handling using cert-manager package certmanager -import "k8s.io/apimachinery/pkg/types" - const ( schemaRegistryAPI = "schema-registry" schemaRegistryAPIClientCert = "schema-registry-client" schemaRegistryAPINodeCert = "schema-registry-node" ) - -// SchemaRegistryAPINodeCert returns the namespaced name for the SchemaRegistry API certificate used by nodes -func (r *PkiReconciler) SchemaRegistryAPINodeCert() types.NamespacedName { - schemaRegistryTLSListener := getTLSListener(schemaRegistryAPIListeners(r.pandaCluster)) - if schemaRegistryTLSListener != nil && schemaRegistryTLSListener.GetTLS() != nil && schemaRegistryTLSListener.GetTLS().NodeSecretRef != nil { - return types.NamespacedName{ - Name: schemaRegistryTLSListener.GetTLS().NodeSecretRef.Name, - Namespace: r.pandaCluster.Namespace, - } - } - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + schemaRegistryAPINodeCert, Namespace: r.pandaCluster.Namespace} -} - -// SchemaRegistryAPIClientCert returns the namespaced name for the SchemaRegistry API certificate used by clients -func (r *PkiReconciler) SchemaRegistryAPIClientCert() types.NamespacedName { - return types.NamespacedName{Name: r.pandaCluster.Name + "-" + schemaRegistryAPIClientCert, Namespace: r.pandaCluster.Namespace} -} diff --git a/src/go/k8s/pkg/resources/certmanager/type_helpers.go b/src/go/k8s/pkg/resources/certmanager/type_helpers.go index 36edcf385c6b..bbd44289644f 100644 --- a/src/go/k8s/pkg/resources/certmanager/type_helpers.go +++ b/src/go/k8s/pkg/resources/certmanager/type_helpers.go @@ -9,7 +9,34 @@ package certmanager -import redpandav1alpha1 "github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1" +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + + "github.com/go-logr/logr" + cmmetav1 "github.com/jetstack/cert-manager/pkg/apis/meta/v1" + redpandav1alpha1 "github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1" + "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + redpandaCertVolName = "tlscert" + redpandaCAVolName = "tlsca" + adminAPICertVolName = "tlsadmincert" + adminAPICAVolName = "tlsadminca" + pandaProxyCertVolName = "tlspandaproxycert" + pandaProxyCAVolName = "tlspandaproxyca" + schemaRegistryCertVolName = "tlsschemaregistrycert" + schemaRegistryCAVolName = "tlsschemaregistryca" +) // Helper functions and types for Listeners @@ -18,6 +45,8 @@ var ( _ APIListener = redpandav1alpha1.AdminAPI{} _ APIListener = redpandav1alpha1.PandaproxyAPI{} _ APIListener = redpandav1alpha1.SchemaRegistryAPI{} + + errNoTLSError = errors.New("no TLS enabled for admin API") ) // APIListener is a generic API Listener @@ -87,10 +116,450 @@ func getInternalTLSListener(listeners []APIListener) APIListener { func getTLSListener(listeners []APIListener) APIListener { for _, el := range listeners { - tls := el.GetTLS() - if tls != nil && tls.Enabled { + tlsConfig := el.GetTLS() + if tlsConfig != nil && tlsConfig.Enabled { return el } } return nil } + +// apiCertificates is a collection of certificate resources per single API. It +// contains node and client certificates (if mutual TLS is enabled) +type apiCertificates struct { + nodeCertificate resources.Resource + clientCertificates []resources.Resource + rootResources []resources.Resource + tlsEnabled bool + + // CR allows to specify node certificate, if not provided this will be nil + externalNodeCertificate *corev1.ObjectReference + + // all certificates need to exist in this namespace for mounting of secrets to work + clusterNamespace string +} + +func tlsDisabledAPICertificates() *apiCertificates { + return &apiCertificates{ + tlsEnabled: false, + } +} + +func tlsEnabledAPICertificates(namespace string) *apiCertificates { + return &apiCertificates{ + tlsEnabled: true, + clusterNamespace: namespace, + } +} + +// ClusterCertificates contains definition for all resources needed to be +// created to support TLS on all redpanda APIs where TLS is enabled +type ClusterCertificates struct { + // certificates to be created for these APIs. We currently create different + // set of node and client certificates per API + kafkaAPI *apiCertificates + schemaRegistryAPI *apiCertificates + adminAPI *apiCertificates + pandaProxyAPI *apiCertificates + + client client.Client + scheme *runtime.Scheme + pandaCluster *redpandav1alpha1.Cluster + internalFQDN string + clusterFQDN string + logger logr.Logger +} + +// NewClusterCertificates creates new cluster tls certificates resources +func NewClusterCertificates( + cluster *redpandav1alpha1.Cluster, + keystoreSecret types.NamespacedName, + k8sClient client.Client, + fqdn string, + clusterFQDN string, + scheme *runtime.Scheme, + logger logr.Logger, +) *ClusterCertificates { + cc := &ClusterCertificates{ + pandaCluster: cluster, + client: k8sClient, + scheme: scheme, + internalFQDN: fqdn, + clusterFQDN: clusterFQDN, + logger: logger, + + kafkaAPI: tlsDisabledAPICertificates(), + schemaRegistryAPI: tlsDisabledAPICertificates(), + adminAPI: tlsDisabledAPICertificates(), + pandaProxyAPI: tlsDisabledAPICertificates(), + } + if kafkaListeners := kafkaAPIListeners(cluster); len(kafkaListeners) > 0 { + cc.kafkaAPI = cc.prepareAPI(kafkaAPI, RedpandaNodeCert, []string{OperatorClientCert, UserClientCert, AdminClientCert}, kafkaListeners, &keystoreSecret) + } + + if adminListeners := adminAPIListeners(cluster); len(adminListeners) > 0 { + cc.adminAPI = cc.prepareAPI(adminAPI, adminAPINodeCert, []string{adminAPIClientCert}, adminListeners, &keystoreSecret) + } + + if pandaProxyListeners := pandaProxyAPIListeners(cluster); len(pandaProxyListeners) > 0 { + cc.pandaProxyAPI = cc.prepareAPI(pandaproxyAPI, pandaproxyAPINodeCert, []string{pandaproxyAPIClientCert}, pandaProxyListeners, &keystoreSecret) + } + + if schemaRegistryListeners := schemaRegistryAPIListeners(cluster); len(schemaRegistryListeners) > 0 { + cc.schemaRegistryAPI = cc.prepareAPI(schemaRegistryAPI, schemaRegistryAPINodeCert, []string{schemaRegistryAPIClientCert}, schemaRegistryListeners, &keystoreSecret) + } + + return cc +} + +func (cc *ClusterCertificates) prepareAPI( + rootCertSuffix string, + nodeCertSuffix string, + clientCerts []string, + listeners []APIListener, + keystoreSecret *types.NamespacedName, +) *apiCertificates { + tlsListener := getTLSListener(listeners) + externalTLSListener := getExternalTLSListener(listeners) + internalTLSListener := getInternalTLSListener(listeners) + + if tlsListener == nil || tlsListener.GetTLS() == nil || !tlsListener.GetTLS().Enabled { + return tlsDisabledAPICertificates() + } + result := tlsEnabledAPICertificates(cc.pandaCluster.Namespace) + + // TODO(#3550): Do not create rootIssuer if nodeSecretRef is passed and mTLS is disabled + toApplyRoot, rootIssuerRef := prepareRoot(rootCertSuffix, cc.client, cc.pandaCluster, cc.scheme, cc.logger) + result.rootResources = toApplyRoot + nodeIssuerRef := rootIssuerRef + + if tlsListener.GetTLS().IssuerRef != nil { + // if external issuer is provided, we will use it to generate node certificates + nodeIssuerRef = tlsListener.GetTLS().IssuerRef + } + + nodeSecretRef := tlsListener.GetTLS().NodeSecretRef + result.externalNodeCertificate = nodeSecretRef + if nodeSecretRef == nil || nodeSecretRef.Name == "" { + certName := NewCertName(cc.pandaCluster.Name, nodeCertSuffix) + certsKey := types.NamespacedName{Name: string(certName), Namespace: cc.pandaCluster.Namespace} + dnsNames := []string{} + + if internalTLSListener != nil { + dnsNames = append(dnsNames, cc.clusterFQDN, cc.internalFQDN) + } + // TODO(#2256): Add support for external listener + TLS certs for IPs + if externalTLSListener != nil && externalTLSListener.GetExternal().Subdomain != "" { + dnsNames = append(dnsNames, externalTLSListener.GetExternal().Subdomain) + } + + nodeCert := NewNodeCertificate( + cc.client, + cc.scheme, + cc.pandaCluster, + certsKey, + nodeIssuerRef, + dnsNames, + EmptyCommonName, + keystoreSecret, + cc.logger) + result.nodeCertificate = nodeCert + } + + if tlsListener.GetTLS().RequireClientAuth { + for _, clientCertName := range clientCerts { + clientCn := NewCommonName(cc.pandaCluster.Name, clientCertName) + clientKey := types.NamespacedName{Name: string(clientCn), Namespace: cc.pandaCluster.Namespace} + clientCert := NewCertificate(cc.client, cc.scheme, cc.pandaCluster, clientKey, rootIssuerRef, clientCn, false, keystoreSecret, cc.logger) + result.clientCertificates = append(result.clientCertificates, clientCert) + } + } + + return result +} + +func prepareRoot( + prefix string, + k8sClient client.Client, + pandaCluster *redpandav1alpha1.Cluster, + scheme *runtime.Scheme, + logger logr.Logger, +) ([]resources.Resource, *cmmetav1.ObjectReference) { + toApply := []resources.Resource{} + selfSignedIssuer := NewIssuer(k8sClient, + scheme, + pandaCluster, + issuerNamespacedName(pandaCluster.Name, pandaCluster.Namespace, prefix+"-"+"selfsigned-issuer"), + "", + logger) + + rootCn := NewCommonName(pandaCluster.Name, prefix+"-root-certificate") + rootKey := types.NamespacedName{Name: string(rootCn), Namespace: pandaCluster.Namespace} + rootCertificate := NewCACertificate(k8sClient, + scheme, + pandaCluster, + rootKey, + selfSignedIssuer.objRef(), + rootCn, + nil, + logger) + + leafIssuer := NewIssuer(k8sClient, + scheme, + pandaCluster, + issuerNamespacedName(pandaCluster.Name, pandaCluster.Namespace, prefix+"-"+"root-issuer"), + rootCertificate.Key().Name, + logger) + + leafIssuerRef := leafIssuer.objRef() + + toApply = append(toApply, selfSignedIssuer, rootCertificate, leafIssuer) + return toApply, leafIssuerRef +} + +func issuerNamespacedName( + pandaClusterName, pandaClusterNamespace, name string, +) types.NamespacedName { + return types.NamespacedName{Name: pandaClusterName + "-" + name, Namespace: pandaClusterNamespace} +} + +func (ac *apiCertificates) resources( + ctx context.Context, k8sClient client.Client, logger logr.Logger, +) ([]resources.Resource, error) { + if !ac.tlsEnabled { + return []resources.Resource{}, nil + } + nodeSecretRef := ac.externalNodeCertificate + if nodeSecretRef != nil && nodeSecretRef.Name != "" && nodeSecretRef.Namespace != ac.clusterNamespace { + if err := copyNodeSecretToLocalNamespace(ctx, nodeSecretRef, ac.clusterNamespace, k8sClient, logger); err != nil { + return nil, fmt.Errorf("copy node secret for %s cert group: %w", ac.nodeCertificateName().Name, err) + } + } + + res := []resources.Resource{} + res = append(res, ac.rootResources...) + if ac.nodeCertificate != nil { + res = append(res, ac.nodeCertificate) + } + res = append(res, ac.clientCertificates...) + return res, nil +} + +// Creates copy of secret in Redpanda cluster's namespace +func copyNodeSecretToLocalNamespace( + ctx context.Context, + secretRef *corev1.ObjectReference, + namespace string, + k8sClient client.Client, + logger logr.Logger, +) error { + var secret corev1.Secret + err := k8sClient.Get(ctx, types.NamespacedName{Name: secretRef.Name, Namespace: secretRef.Namespace}, &secret) + if err != nil { + return err + } + + tlsKey := secret.Data[corev1.TLSPrivateKeyKey] + tlsCrt := secret.Data[corev1.TLSCertKey] + caCrt := secret.Data[cmmetav1.TLSCAKey] + + caSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secret.Name, + Namespace: namespace, + Labels: secret.Labels, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + Type: secret.Type, + Data: map[string][]byte{ + cmmetav1.TLSCAKey: caCrt, + corev1.TLSCertKey: tlsCrt, + corev1.TLSPrivateKeyKey: tlsKey, + }, + } + _, err = resources.CreateIfNotExists(ctx, k8sClient, caSecret, logger) + return err +} + +func (ac *apiCertificates) nodeCertificateName() *types.NamespacedName { + if ac.externalNodeCertificate != nil { + return &types.NamespacedName{ + Name: ac.externalNodeCertificate.Name, + Namespace: ac.externalNodeCertificate.Namespace, + } + } + if ac.nodeCertificate != nil { + name := ac.nodeCertificate.Key() + return &name + } + return nil +} + +func (ac *apiCertificates) clientCertificateNames() []types.NamespacedName { + names := []types.NamespacedName{} + for _, c := range ac.clientCertificates { + names = append(names, c.Key()) + } + return names +} + +// Resources returns all resources that need to exist in the cluster to support +// TLS on all redpanda APIs where TLS is enabled +func (cc *ClusterCertificates) Resources( + ctx context.Context, +) ([]resources.Resource, error) { + res := []resources.Resource{} + kafkaResources, err := cc.kafkaAPI.resources(ctx, cc.client, cc.logger) + if err != nil { + return nil, fmt.Errorf("retrieving kafkaapi resources %w", err) + } + adminResources, err := cc.adminAPI.resources(ctx, cc.client, cc.logger) + if err != nil { + return nil, fmt.Errorf("retrieving adminapi resources %w", err) + } + pandaProxyResources, err := cc.pandaProxyAPI.resources(ctx, cc.client, cc.logger) + if err != nil { + return nil, fmt.Errorf("retrieving pandaproxyapi resources %w", err) + } + schemaRegistryResources, err := cc.schemaRegistryAPI.resources(ctx, cc.client, cc.logger) + if err != nil { + return nil, fmt.Errorf("retrieving schemaRegistryapi resources %w", err) + } + + res = append(res, kafkaResources...) + res = append(res, adminResources...) + res = append(res, pandaProxyResources...) + res = append(res, schemaRegistryResources...) + return res, nil +} + +// Volumes returns volumes and mounts that statefulset has to define to have +// access to all TLS certificates redpanda has enabled +func (cc *ClusterCertificates) Volumes() ( + []corev1.Volume, + []corev1.VolumeMount, +) { + var vols []corev1.Volume + var mounts []corev1.VolumeMount + mountPoints := resources.GetTLSMountPoints() + + vol, mount := secretVolumesForTLS(cc.kafkaAPI.nodeCertificateName(), cc.kafkaAPI.clientCertificates, redpandaCertVolName, redpandaCAVolName, mountPoints.KafkaAPI.NodeCertMountDir, mountPoints.KafkaAPI.ClientCAMountDir) + vols = append(vols, vol...) + mounts = append(mounts, mount...) + + vol, mount = secretVolumesForTLS(cc.adminAPI.nodeCertificateName(), cc.adminAPI.clientCertificates, adminAPICertVolName, adminAPICAVolName, mountPoints.AdminAPI.NodeCertMountDir, mountPoints.AdminAPI.ClientCAMountDir) + vols = append(vols, vol...) + mounts = append(mounts, mount...) + + vol, mount = secretVolumesForTLS(cc.pandaProxyAPI.nodeCertificateName(), cc.pandaProxyAPI.clientCertificates, pandaProxyCertVolName, pandaProxyCAVolName, mountPoints.PandaProxyAPI.NodeCertMountDir, mountPoints.PandaProxyAPI.ClientCAMountDir) + vols = append(vols, vol...) + mounts = append(mounts, mount...) + + vol, mount = secretVolumesForTLS(cc.schemaRegistryAPI.nodeCertificateName(), cc.schemaRegistryAPI.clientCertificates, schemaRegistryCertVolName, schemaRegistryCAVolName, mountPoints.SchemaRegistryAPI.NodeCertMountDir, mountPoints.SchemaRegistryAPI.ClientCAMountDir) + vols = append(vols, vol...) + mounts = append(mounts, mount...) + + return vols, mounts +} + +func secretVolumesForTLS( + nodeCertificate *types.NamespacedName, + clientCertificates []resources.Resource, + volumeName, caVolumeName, mountDir, caMountDir string, +) ([]corev1.Volume, []corev1.VolumeMount) { + var vols []corev1.Volume + var mounts []corev1.VolumeMount + if nodeCertificate == nil { + return vols, mounts + } + + // mount node certificate's private key + vols = append(vols, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: nodeCertificate.Name, + Items: []corev1.KeyToPath{ + { + Key: corev1.TLSPrivateKeyKey, + Path: corev1.TLSPrivateKeyKey, + }, + { + Key: corev1.TLSCertKey, + Path: corev1.TLSCertKey, + }, + }, + }, + }, + }) + mounts = append(mounts, corev1.VolumeMount{ + Name: volumeName, + MountPath: mountDir, + }) + + // if mutual TLS is enabled, mount also client cerificate CA to be able to + // verify client certificates + if len(clientCertificates) > 0 { + vols = append(vols, corev1.Volume{ + Name: caVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: clientCertificates[0].Key().Name, + Items: []corev1.KeyToPath{ + { + Key: cmmetav1.TLSCAKey, + Path: cmmetav1.TLSCAKey, + }, + }, + }, + }, + }) + mounts = append(mounts, corev1.VolumeMount{ + Name: caVolumeName, + MountPath: caMountDir, + }) + } + + return vols, mounts +} + +// GetTLSConfig returns TLS config for adminAPI that can then be used to connect +// to the admin API of the current cluster +func (cc *ClusterCertificates) GetTLSConfig( + ctx context.Context, k8sClient client.Reader, +) (*tls.Config, error) { + nodeCertificateName := cc.adminAPI.nodeCertificateName() + if nodeCertificateName == nil { + return nil, errNoTLSError + } + tlsConfig := tls.Config{MinVersion: tls.VersionTLS12} // TLS12 is min version allowed by gosec. + + var nodeCertSecret corev1.Secret + err := k8sClient.Get(ctx, *nodeCertificateName, &nodeCertSecret) + if err != nil { + return nil, err + } + + // Add root CA + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(nodeCertSecret.Data[cmmetav1.TLSCAKey]) + tlsConfig.RootCAs = caCertPool + + if len(cc.adminAPI.clientCertificates) > 0 { + var clientCertSecret corev1.Secret + err := k8sClient.Get(ctx, cc.adminAPI.clientCertificateNames()[0], &clientCertSecret) + if err != nil { + return nil, err + } + cert, err := tls.X509KeyPair(clientCertSecret.Data[corev1.TLSCertKey], clientCertSecret.Data[corev1.TLSPrivateKeyKey]) + if err != nil { + return nil, err + } + tlsConfig.Certificates = []tls.Certificate{cert} + } + + return &tlsConfig, nil +} diff --git a/src/go/k8s/pkg/resources/certmanager/type_helpers_test.go b/src/go/k8s/pkg/resources/certmanager/type_helpers_test.go new file mode 100644 index 000000000000..28addae8ce82 --- /dev/null +++ b/src/go/k8s/pkg/resources/certmanager/type_helpers_test.go @@ -0,0 +1,290 @@ +// Copyright 2021 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package certmanager_test + +import ( + "context" + "fmt" + "testing" + + "github.com/go-logr/logr" + cmmetav1 "github.com/jetstack/cert-manager/pkg/apis/meta/v1" + "github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1" + "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/certmanager" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// nolint:funlen // the subtests might causes linter to complain +func TestClusterCertificates(t *testing.T) { + secret := corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: "cluster-tls-secret-node-certificate", + Namespace: "cert-manager", + }, + Data: map[string][]byte{ + "tls.crt": []byte("XXX"), + "tls.key": []byte("XXX"), + "ca.crt": []byte("XXX"), + }, + } + tests := []struct { + name string + pandaCluster *v1alpha1.Cluster + expectedNames []string + volumesCount int + }{ + {"kafka tls disabled", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + KafkaAPI: []v1alpha1.KafkaAPI{ + { + TLS: v1alpha1.KafkaAPITLS{ + Enabled: false, + }, + }, + }, + }, + }, + }, []string{}, 0}, + {"kafka tls", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + KafkaAPI: []v1alpha1.KafkaAPI{ + { + TLS: v1alpha1.KafkaAPITLS{ + Enabled: true, + }, + }, + }, + }, + }, + }, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda"}, 1}, + {"kafka tls with external node issuer", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + KafkaAPI: []v1alpha1.KafkaAPI{ + { + TLS: v1alpha1.KafkaAPITLS{ + Enabled: true, + IssuerRef: &cmmetav1.ObjectReference{ + Name: "issuer", + }, + }, + }, + }, + }, + }, + }, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda"}, 1}, + {"kafka mutual tls", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + KafkaAPI: []v1alpha1.KafkaAPI{ + { + TLS: v1alpha1.KafkaAPITLS{ + Enabled: true, + RequireClientAuth: true, + }, + }, + }, + }, + }, + }, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda", "test-operator-client", "test-user-client", "test-admin-client"}, 2}, + {"admin api tls disabled", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + AdminAPI: []v1alpha1.AdminAPI{ + { + TLS: v1alpha1.AdminAPITLS{ + Enabled: false, + }, + }, + }, + }, + }, + }, []string{}, 0}, + {"admin api tls", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + AdminAPI: []v1alpha1.AdminAPI{ + { + TLS: v1alpha1.AdminAPITLS{ + Enabled: true, + }, + }, + }, + }, + }, + }, []string{"test-admin-selfsigned-issuer", "test-admin-root-certificate", "test-admin-root-issuer", "test-admin-api-node"}, 1}, + {"admin api mutual tls", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + AdminAPI: []v1alpha1.AdminAPI{ + { + TLS: v1alpha1.AdminAPITLS{ + Enabled: true, + RequireClientAuth: true, + }, + }, + }, + }, + }, + }, []string{"test-admin-selfsigned-issuer", "test-admin-root-certificate", "test-admin-root-issuer", "test-admin-api-node", "test-admin-api-client"}, 2}, + {"pandaproxy api tls disabled", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + PandaproxyAPI: []v1alpha1.PandaproxyAPI{ + { + TLS: v1alpha1.PandaproxyAPITLS{ + Enabled: false, + }, + }, + }, + }, + }, + }, []string{}, 0}, + {"pandaproxy api tls", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + PandaproxyAPI: []v1alpha1.PandaproxyAPI{ + { + TLS: v1alpha1.PandaproxyAPITLS{ + Enabled: true, + }, + }, + }, + }, + }, + }, []string{"test-proxy-selfsigned-issuer", "test-proxy-root-certificate", "test-proxy-root-issuer", "test-proxy-api-node"}, 1}, + {"pandaproxy api mutual tls", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + PandaproxyAPI: []v1alpha1.PandaproxyAPI{ + { + TLS: v1alpha1.PandaproxyAPITLS{ + Enabled: true, + RequireClientAuth: true, + }, + }, + }, + }, + }, + }, []string{"test-proxy-selfsigned-issuer", "test-proxy-root-certificate", "test-proxy-root-issuer", "test-proxy-api-node", "test-proxy-api-client"}, 2}, + {"schematregistry api tls disabled", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + SchemaRegistry: &v1alpha1.SchemaRegistryAPI{ + TLS: &v1alpha1.SchemaRegistryAPITLS{ + Enabled: false, + }, + }, + }, + }, + }, []string{}, 0}, + {"schematregistry api tls", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + SchemaRegistry: &v1alpha1.SchemaRegistryAPI{ + TLS: &v1alpha1.SchemaRegistryAPITLS{ + Enabled: true, + }, + }, + }, + }, + }, + []string{"test-schema-registry-selfsigned-issuer", "test-schema-registry-root-certificate", "test-schema-registry-root-issuer", "test-schema-registry-node"}, 1}, + {"schematregistry api mutual tls", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + SchemaRegistry: &v1alpha1.SchemaRegistryAPI{ + TLS: &v1alpha1.SchemaRegistryAPITLS{ + Enabled: true, + RequireClientAuth: true, + }, + }, + }, + }, + }, + []string{"test-schema-registry-selfsigned-issuer", "test-schema-registry-root-certificate", "test-schema-registry-root-issuer", "test-schema-registry-node", "test-schema-registry-client"}, 2}, + {"kafka and schematregistry with nodesecretref", &v1alpha1.Cluster{ + ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: v1alpha1.ClusterSpec{ + Configuration: v1alpha1.RedpandaConfig{ + SchemaRegistry: &v1alpha1.SchemaRegistryAPI{ + TLS: &v1alpha1.SchemaRegistryAPITLS{ + Enabled: true, + RequireClientAuth: true, + NodeSecretRef: &corev1.ObjectReference{ + Name: secret.Name, + Namespace: secret.Namespace, + }, + }, + }, + KafkaAPI: []v1alpha1.KafkaAPI{ + { + TLS: v1alpha1.KafkaAPITLS{ + Enabled: true, + RequireClientAuth: true, + NodeSecretRef: &corev1.ObjectReference{ + Name: secret.Name, + Namespace: secret.Namespace, + }, + }, + }, + }, + }, + }, + }, + []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-operator-client", "test-user-client", "test-admin-client", "test-schema-registry-selfsigned-issuer", "test-schema-registry-root-certificate", "test-schema-registry-root-issuer", "test-schema-registry-client"}, 4}, + } + for _, tt := range tests { + cc := certmanager.NewClusterCertificates(tt.pandaCluster, + types.NamespacedName{ + Name: "test", + Namespace: "test", + }, fake.NewClientBuilder().WithRuntimeObjects(&secret).Build(), "cluster.local", "cluster2.local", scheme.Scheme, logr.DiscardLogger{}) + resources, err := cc.Resources(context.TODO()) + require.NoError(t, err) + for _, r := range resources { + fmt.Println(r.Key().Name) + } + require.Equal(t, len(tt.expectedNames), len(resources)) + for _, n := range tt.expectedNames { + found := false + for _, r := range resources { + if r.Key().Name == n { + found = true + break + } + } + require.True(t, found, fmt.Sprintf("name %s not found in resources", n)) + } + v, vm := cc.Volumes() + require.Equal(t, tt.volumesCount, len(v), fmt.Sprintf("%s: volumes count don't match", tt.name)) + require.Equal(t, tt.volumesCount, len(vm), fmt.Sprintf("%s: volume mounts count don't match", tt.name)) + } +} diff --git a/src/go/k8s/pkg/resources/configmap.go b/src/go/k8s/pkg/resources/configmap.go index 79404024c5ac..7e3e69b85882 100644 --- a/src/go/k8s/pkg/resources/configmap.go +++ b/src/go/k8s/pkg/resources/configmap.go @@ -40,22 +40,6 @@ const ( dataDirectory = "/var/lib/redpanda/data" archivalCacheIndexDirectory = "/var/lib/shadow-index-cache" - // We need 2 secrets and 2 mount points for each API endpoint that supports TLS and mTLS: - // 1. The Node certs used by the API endpoint to sign requests - // 2. The CA used to sign mTLS client certs which will be use by the endpoint to validate mTLS client certs - // - // Node certs might be signed by a provided Issuer (i.e. when using Letsencrypt), in which case - // the operator won't generate a CA to sign the node certs. But, if at the same time mTLS is enabled - // a CA will be created to sign the mTLS client certs, and it will be stored in a separate secret. - tlsKafkaAPIDir = "/etc/tls/certs" - tlsKafkaAPIDirCA = "/etc/tls/certs/ca" - tlsAdminAPIDir = "/etc/tls/certs/admin" - tlsAdminAPIDirCA = "/etc/tls/certs/admin/ca" - tlsPandaproxyAPIDir = "/etc/tls/certs/pandaproxy" - tlsPandaproxyAPIDirCA = "/etc/tls/certs/pandaproxy/ca" - tlsSchemaRegistryDir = "/etc/tls/certs/schema-registry" - tlsSchemaRegistryDirCA = "/etc/tls/certs/schema-registry/ca" - superusersConfigurationKey = "superusers" oneMB = 1024 * 1024 @@ -232,6 +216,7 @@ func (r *ConfigMapResource) CreateConfiguration( ) (*configuration.GlobalConfiguration, error) { cfg := configuration.For(r.pandaCluster.Spec.Version) cfg.NodeConfiguration = *config.Default() + mountPoints := GetTLSMountPoints() c := r.pandaCluster.Spec.Configuration cr := &cfg.NodeConfiguration.Redpanda @@ -287,13 +272,13 @@ func (r *ConfigMapResource) CreateConfiguration( } tls := config.ServerTLS{ Name: name, - KeyFile: fmt.Sprintf("%s/%s", tlsKafkaAPIDir, corev1.TLSPrivateKeyKey), // tls.key - CertFile: fmt.Sprintf("%s/%s", tlsKafkaAPIDir, corev1.TLSCertKey), // tls.crt + KeyFile: fmt.Sprintf("%s/%s", mountPoints.KafkaAPI.NodeCertMountDir, corev1.TLSPrivateKeyKey), // tls.key + CertFile: fmt.Sprintf("%s/%s", mountPoints.KafkaAPI.NodeCertMountDir, corev1.TLSCertKey), // tls.crt Enabled: true, RequireClientAuth: tlsListener.TLS.RequireClientAuth, } if tlsListener.TLS.RequireClientAuth { - tls.TruststoreFile = fmt.Sprintf("%s/%s", tlsKafkaAPIDirCA, cmetav1.TLSCAKey) + tls.TruststoreFile = fmt.Sprintf("%s/%s", mountPoints.KafkaAPI.ClientCAMountDir, cmetav1.TLSCAKey) } cr.KafkaAPITLS = []config.ServerTLS{ tls, @@ -309,13 +294,13 @@ func (r *ConfigMapResource) CreateConfiguration( } adminTLS := config.ServerTLS{ Name: name, - KeyFile: fmt.Sprintf("%s/%s", tlsAdminAPIDir, corev1.TLSPrivateKeyKey), - CertFile: fmt.Sprintf("%s/%s", tlsAdminAPIDir, corev1.TLSCertKey), + KeyFile: fmt.Sprintf("%s/%s", mountPoints.AdminAPI.NodeCertMountDir, corev1.TLSPrivateKeyKey), + CertFile: fmt.Sprintf("%s/%s", mountPoints.AdminAPI.NodeCertMountDir, corev1.TLSCertKey), Enabled: true, RequireClientAuth: adminAPITLSListener.TLS.RequireClientAuth, } if adminAPITLSListener.TLS.RequireClientAuth { - adminTLS.TruststoreFile = fmt.Sprintf("%s/%s", tlsAdminAPIDirCA, cmetav1.TLSCAKey) + adminTLS.TruststoreFile = fmt.Sprintf("%s/%s", mountPoints.AdminAPI.ClientCAMountDir, cmetav1.TLSCAKey) } cr.AdminAPITLS = append(cr.AdminAPITLS, adminTLS) } @@ -374,7 +359,7 @@ func (r *ConfigMapResource) CreateConfiguration( } r.preparePandaproxy(&cfg.NodeConfiguration) - r.preparePandaproxyTLS(&cfg.NodeConfiguration) + r.preparePandaproxyTLS(&cfg.NodeConfiguration, mountPoints) err := r.preparePandaproxyClient(ctx, cfg) if err != nil { return nil, err @@ -391,7 +376,7 @@ func (r *ConfigMapResource) CreateConfiguration( }, } } - r.prepareSchemaRegistryTLS(&cfg.NodeConfiguration) + r.prepareSchemaRegistryTLS(&cfg.NodeConfiguration, mountPoints) err = r.prepareSchemaRegistryClient(ctx, cfg) if err != nil { return nil, err @@ -562,7 +547,9 @@ func (r *ConfigMapResource) prepareSchemaRegistryClient( return cfg.AppendToAdditionalRedpandaProperty(superusersConfigurationKey, username) } -func (r *ConfigMapResource) preparePandaproxyTLS(cfgRpk *config.Config) { +func (r *ConfigMapResource) preparePandaproxyTLS( + cfgRpk *config.Config, mountPoints *TLSMountPoints, +) { tlsListener := r.pandaCluster.PandaproxyAPITLS() if tlsListener != nil { // Only one TLS listener is supported (restricted by the webhook). @@ -573,32 +560,34 @@ func (r *ConfigMapResource) preparePandaproxyTLS(cfgRpk *config.Config) { } tls := config.ServerTLS{ Name: name, - KeyFile: fmt.Sprintf("%s/%s", tlsPandaproxyAPIDir, corev1.TLSPrivateKeyKey), // tls.key - CertFile: fmt.Sprintf("%s/%s", tlsPandaproxyAPIDir, corev1.TLSCertKey), // tls.crt + KeyFile: fmt.Sprintf("%s/%s", mountPoints.PandaProxyAPI.NodeCertMountDir, corev1.TLSPrivateKeyKey), // tls.key + CertFile: fmt.Sprintf("%s/%s", mountPoints.PandaProxyAPI.NodeCertMountDir, corev1.TLSCertKey), // tls.crt Enabled: true, RequireClientAuth: tlsListener.TLS.RequireClientAuth, } if tlsListener.TLS.RequireClientAuth { - tls.TruststoreFile = fmt.Sprintf("%s/%s", tlsPandaproxyAPIDirCA, cmetav1.TLSCAKey) + tls.TruststoreFile = fmt.Sprintf("%s/%s", mountPoints.PandaProxyAPI.ClientCAMountDir, cmetav1.TLSCAKey) } cfgRpk.Pandaproxy.PandaproxyAPITLS = []config.ServerTLS{tls} } } -func (r *ConfigMapResource) prepareSchemaRegistryTLS(cfgRpk *config.Config) { +func (r *ConfigMapResource) prepareSchemaRegistryTLS( + cfgRpk *config.Config, mountPoints *TLSMountPoints, +) { if r.pandaCluster.Spec.Configuration.SchemaRegistry != nil && r.pandaCluster.Spec.Configuration.SchemaRegistry.TLS != nil { name := SchemaRegistryPortName tls := config.ServerTLS{ Name: name, - KeyFile: fmt.Sprintf("%s/%s", tlsSchemaRegistryDir, corev1.TLSPrivateKeyKey), // tls.key - CertFile: fmt.Sprintf("%s/%s", tlsSchemaRegistryDir, corev1.TLSCertKey), // tls.crt + KeyFile: fmt.Sprintf("%s/%s", mountPoints.SchemaRegistryAPI.NodeCertMountDir, corev1.TLSPrivateKeyKey), // tls.key + CertFile: fmt.Sprintf("%s/%s", mountPoints.SchemaRegistryAPI.NodeCertMountDir, corev1.TLSCertKey), // tls.crt Enabled: true, RequireClientAuth: r.pandaCluster.Spec.Configuration.SchemaRegistry.TLS.RequireClientAuth, } if r.pandaCluster.Spec.Configuration.SchemaRegistry.TLS.RequireClientAuth { - tls.TruststoreFile = fmt.Sprintf("%s/%s", tlsSchemaRegistryDirCA, cmetav1.TLSCAKey) + tls.TruststoreFile = fmt.Sprintf("%s/%s", mountPoints.SchemaRegistryAPI.ClientCAMountDir, cmetav1.TLSCAKey) } cfgRpk.SchemaRegistry.SchemaRegistryAPITLS = []config.ServerTLS{tls} } diff --git a/src/go/k8s/pkg/resources/resource_integration_test.go b/src/go/k8s/pkg/resources/resource_integration_test.go index 133f33d8d9e5..f47e213c09bf 100644 --- a/src/go/k8s/pkg/resources/resource_integration_test.go +++ b/src/go/k8s/pkg/resources/resource_integration_test.go @@ -11,6 +11,7 @@ package resources_test import ( "context" + "crypto/tls" "log" "os" "path/filepath" @@ -83,15 +84,8 @@ func TestEnsure_StatefulSet(t *testing.T) { "cluster.local", "servicename", types.NamespacedName{Name: "test", Namespace: "test"}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, + TestStatefulsetTLSVolumeProvider{}, + TestAdminTLSConfigProvider{}, "", res.ConfiguratorSettings{ ConfiguratorBaseImage: "vectorized/configurator", @@ -435,3 +429,20 @@ func TestEnsure_LoadbalancerService(t *testing.T) { assert.Equal(t, "kafka-external-bootstrap", actual.Spec.Ports[0].Name) }) } + +type TestStatefulsetTLSVolumeProvider struct{} + +func (TestStatefulsetTLSVolumeProvider) Volumes() ( + []corev1.Volume, + []corev1.VolumeMount, +) { + return []corev1.Volume{}, []corev1.VolumeMount{} +} + +type TestAdminTLSConfigProvider struct{} + +func (TestAdminTLSConfigProvider) GetTLSConfig( + ctx context.Context, k8sClient client.Reader, +) (*tls.Config, error) { + return nil, nil +} diff --git a/src/go/k8s/pkg/resources/statefulset.go b/src/go/k8s/pkg/resources/statefulset.go index d7c1677459dc..3e1892924ba4 100644 --- a/src/go/k8s/pkg/resources/statefulset.go +++ b/src/go/k8s/pkg/resources/statefulset.go @@ -19,7 +19,6 @@ import ( "strings" "github.com/go-logr/logr" - cmetav1 "github.com/jetstack/cert-manager/pkg/apis/meta/v1" redpandav1alpha1 "github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1" "github.com/redpanda-data/redpanda/src/go/k8s/pkg/labels" "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/featuregates" @@ -55,15 +54,6 @@ const ( datadirName = "datadir" archivalCacheIndexAnchorName = "shadow-index-cache" defaultDatadirCapacity = "100Gi" - - redpandaCertVolName = "tlscert" - redpandaCAVolName = "tlsca" - adminAPICertVolName = "tlsadmincert" - adminAPICAVolName = "tlsadminca" - pandaProxyCertVolName = "tlspandaproxycert" - pandaProxyCAVolName = "tlspandaproxyca" - schemaRegistryCertVolName = "tlsschemaregistrycert" - schemaRegistryCAVolName = "tlsschemaregistryca" ) var ( @@ -88,23 +78,16 @@ type ConfiguratorSettings struct { // focusing on the management of redpanda cluster type StatefulSetResource struct { k8sclient.Client - scheme *runtime.Scheme - pandaCluster *redpandav1alpha1.Cluster - serviceFQDN string - serviceName string - nodePortName types.NamespacedName - nodePortSvc corev1.Service - redpandaCertSecretKey types.NamespacedName - internalClientCertSecretKey types.NamespacedName - adminCertSecretKey types.NamespacedName - adminAPINodeCertSecretKey types.NamespacedName - adminAPIClientCertSecretKey types.NamespacedName - pandaproxyAPINodeCertSecretKey types.NamespacedName - pandaproxyClientCertSecretKey types.NamespacedName - schemaRegistryAPINodeCertSecretKey types.NamespacedName - schemaRegistryClientCertSecretKey types.NamespacedName - serviceAccountName string - configuratorSettings ConfiguratorSettings + scheme *runtime.Scheme + pandaCluster *redpandav1alpha1.Cluster + serviceFQDN string + serviceName string + nodePortName types.NamespacedName + nodePortSvc corev1.Service + volumeProvider StatefulsetTLSVolumeProvider + adminTLSConfigProvider AdminTLSConfigProvider + serviceAccountName string + configuratorSettings ConfiguratorSettings // hash of configmap containing configuration for redpanda (node config only), it's injected to // annotation to ensure the pods get restarted when configuration changes // this has to be retrieved lazily to achieve the correct order of resources @@ -123,15 +106,8 @@ func NewStatefulSet( serviceFQDN string, serviceName string, nodePortName types.NamespacedName, - redpandaCertSecretKey types.NamespacedName, - internalClientCertSecretKey types.NamespacedName, - adminCertSecretKey types.NamespacedName, - adminAPINodeCertSecretKey types.NamespacedName, - adminAPIClientCertSecretKey types.NamespacedName, - pandaproxyAPINodeCertSecretKey types.NamespacedName, - pandaproxyClientCertSecretKey types.NamespacedName, - schemaRegistryAPINodeCertSecretKey types.NamespacedName, - schemaRegistryClientCertSecretKey types.NamespacedName, + volumeProvider StatefulsetTLSVolumeProvider, + adminTLSConfigProvider AdminTLSConfigProvider, serviceAccountName string, configuratorSettings ConfiguratorSettings, nodeConfigMapHashGetter func(context.Context) (string, error), @@ -145,15 +121,8 @@ func NewStatefulSet( serviceName, nodePortName, corev1.Service{}, - redpandaCertSecretKey, - internalClientCertSecretKey, - adminCertSecretKey, - adminAPINodeCertSecretKey, - adminAPIClientCertSecretKey, - pandaproxyAPINodeCertSecretKey, - pandaproxyClientCertSecretKey, - schemaRegistryAPINodeCertSecretKey, - schemaRegistryClientCertSecretKey, + volumeProvider, + adminTLSConfigProvider, serviceAccountName, configuratorSettings, nodeConfigMapHashGetter, @@ -299,7 +268,7 @@ func (r *StatefulSetResource) obj( externalSubdomain = externalListener.External.Subdomain externalAddressType = externalListener.External.PreferredAddressType } - + tlsVolumes, tlsVolumeMounts := r.volumeProvider.Volumes() ss := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: r.Key().Namespace, @@ -347,7 +316,7 @@ func (r *StatefulSetResource) obj( EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, - }, r.secretVolumes()...), + }, tlsVolumes...), TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, InitContainers: []corev1.Container{ { @@ -478,7 +447,7 @@ func (r *StatefulSetResource) obj( Name: "config-dir", MountPath: configDestinationDir, }, - }, r.secretVolumeMounts()...), + }, tlsVolumeMounts...), }, }, Tolerations: tolerations, @@ -535,7 +504,7 @@ func (r *StatefulSetResource) obj( setCloudStorage(ss, r.pandaCluster) - rpkStatusContainer := r.rpkStatusContainer() + rpkStatusContainer := r.rpkStatusContainer(tlsVolumeMounts) if rpkStatusContainer != nil { ss.Spec.Template.Spec.Containers = append(ss.Spec.Template.Spec.Containers, *rpkStatusContainer) } @@ -669,7 +638,9 @@ func setCloudStorage( } } -func (r *StatefulSetResource) rpkStatusContainer() *corev1.Container { +func (r *StatefulSetResource) rpkStatusContainer( + tlsVolumeMounts []corev1.VolumeMount, +) *corev1.Container { if r.pandaCluster.Spec.Sidecars.RpkStatus == nil || !r.pandaCluster.Spec.Sidecars.RpkStatus.Enabled { return nil } @@ -689,7 +660,7 @@ func (r *StatefulSetResource) rpkStatusContainer() *corev1.Container { Name: "config-dir", MountPath: configDestinationDir, }, - }, r.secretVolumeMounts()...), + }, tlsVolumeMounts...), } } @@ -756,120 +727,6 @@ func (r *StatefulSetResource) pandaproxyEnvVars() []corev1.EnvVar { return envs } -func (r *StatefulSetResource) secretVolumeMounts() []corev1.VolumeMount { - var mounts []corev1.VolumeMount - if kafkaListener := r.pandaCluster.KafkaTLSListener(); kafkaListener != nil { - mounts = append(mounts, r.secretVolumeMountForTLS(kafkaListener.GetTLS(), redpandaCertVolName, tlsKafkaAPIDir, redpandaCAVolName, tlsKafkaAPIDirCA)...) - } - if adminAPI := r.pandaCluster.AdminAPITLS(); adminAPI != nil { - mounts = append(mounts, r.secretVolumeMountForTLS(adminAPI.GetTLS(), adminAPICertVolName, tlsAdminAPIDir, adminAPICAVolName, tlsAdminAPIDirCA)...) - } - if pandaProxy := r.pandaCluster.PandaproxyAPITLS(); pandaProxy != nil { - mounts = append(mounts, r.secretVolumeMountForTLS(pandaProxy.GetTLS(), pandaProxyCertVolName, tlsPandaproxyAPIDir, pandaProxyCAVolName, tlsPandaproxyAPIDirCA)...) - } - if schemaRegistry := r.pandaCluster.SchemaRegistryAPITLS(); schemaRegistry != nil { - mounts = append(mounts, r.secretVolumeMountForTLS(schemaRegistry.GetTLS(), schemaRegistryCertVolName, tlsSchemaRegistryDir, schemaRegistryCAVolName, tlsSchemaRegistryDirCA)...) - } - - return mounts -} - -func (r *StatefulSetResource) secretVolumeMountForTLS( - tlsConfig *redpandav1alpha1.TLSConfig, - tlsVolName, tlsMoundDir, caVolName, caMountDir string, -) []corev1.VolumeMount { - var mounts []corev1.VolumeMount - if tlsConfig == nil || !tlsConfig.Enabled { - return mounts - } - mounts = append(mounts, corev1.VolumeMount{ - Name: tlsVolName, - MountPath: tlsMoundDir, - }) - - if tlsConfig.RequireClientAuth { - mounts = append(mounts, corev1.VolumeMount{ - Name: caVolName, - MountPath: caMountDir, - }) - } - - return mounts -} - -// The controller should have more focused feature -// oriented functions. E.g. each TLS volume could be managed by -// one function along side with root certificate, issuer, certificate and -// volumesMount in statefulset. -func (r *StatefulSetResource) secretVolumes() []corev1.Volume { - var vols []corev1.Volume - if kafkaListener := r.pandaCluster.KafkaTLSListener(); kafkaListener != nil { - vols = append(vols, r.secretVolumesForTLS(kafkaListener.GetTLS(), redpandaCertVolName, r.redpandaCertSecretKey, redpandaCAVolName, r.internalClientCertSecretKey)...) - } - if adminAPI := r.pandaCluster.AdminAPITLS(); adminAPI != nil { - vols = append(vols, r.secretVolumesForTLS(adminAPI.GetTLS(), adminAPICertVolName, r.adminAPINodeCertSecretKey, adminAPICAVolName, r.adminAPIClientCertSecretKey)...) - } - if pandaProxy := r.pandaCluster.PandaproxyAPITLS(); pandaProxy != nil { - vols = append(vols, r.secretVolumesForTLS(pandaProxy.GetTLS(), pandaProxyCertVolName, r.pandaproxyAPINodeCertSecretKey, pandaProxyCAVolName, r.pandaproxyClientCertSecretKey)...) - } - if schemaRegistry := r.pandaCluster.SchemaRegistryAPITLS(); schemaRegistry != nil { - vols = append(vols, r.secretVolumesForTLS(schemaRegistry.GetTLS(), schemaRegistryCertVolName, r.schemaRegistryAPINodeCertSecretKey, schemaRegistryCAVolName, r.schemaRegistryClientCertSecretKey)...) - } - - return vols -} - -func (r *StatefulSetResource) secretVolumesForTLS( - tlsConfig *redpandav1alpha1.TLSConfig, - tlsVolName string, - tlsSecretRef types.NamespacedName, - caVolName string, - mutualTLSSecretRef types.NamespacedName, -) []corev1.Volume { - var vols []corev1.Volume - if tlsConfig == nil || !tlsConfig.Enabled { - return vols - } - - vols = append(vols, corev1.Volume{ - Name: tlsVolName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: tlsSecretRef.Name, - Items: []corev1.KeyToPath{ - { - Key: corev1.TLSPrivateKeyKey, - Path: corev1.TLSPrivateKeyKey, - }, - { - Key: corev1.TLSCertKey, - Path: corev1.TLSCertKey, - }, - }, - }, - }, - }) - - if tlsConfig.RequireClientAuth { - vols = append(vols, corev1.Volume{ - Name: caVolName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: mutualTLSSecretRef.Name, - Items: []corev1.KeyToPath{ - { - Key: cmetav1.TLSCAKey, - Path: cmetav1.TLSCAKey, - }, - }, - }, - }, - }) - } - - return vols -} - func (r *StatefulSetResource) getNodePort(name string) string { for _, port := range r.nodePortSvc.Spec.Ports { if port.Name == name { diff --git a/src/go/k8s/pkg/resources/statefulset_test.go b/src/go/k8s/pkg/resources/statefulset_test.go index dbbacf9b00a9..96e24fc66abf 100644 --- a/src/go/k8s/pkg/resources/statefulset_test.go +++ b/src/go/k8s/pkg/resources/statefulset_test.go @@ -107,15 +107,8 @@ func TestEnsure(t *testing.T) { "cluster.local", "servicename", types.NamespacedName{Name: "test", Namespace: "test"}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, - types.NamespacedName{}, + TestStatefulsetTLSVolumeProvider{}, + TestAdminTLSConfigProvider{}, "", res.ConfiguratorSettings{ ConfiguratorBaseImage: "vectorized/configurator", diff --git a/src/go/k8s/pkg/resources/statefulset_update.go b/src/go/k8s/pkg/resources/statefulset_update.go index bbb03c018290..c5cd048bd35d 100644 --- a/src/go/k8s/pkg/resources/statefulset_update.go +++ b/src/go/k8s/pkg/resources/statefulset_update.go @@ -22,7 +22,6 @@ import ( "time" "github.com/banzaicloud/k8s-objectmatcher/patch" - adminutils "github.com/redpanda-data/redpanda/src/go/k8s/pkg/admin" "github.com/redpanda-data/redpanda/src/go/k8s/pkg/labels" "github.com/redpanda-data/redpanda/src/go/k8s/pkg/utils" appsv1 "k8s.io/api/apps/v1" @@ -381,7 +380,7 @@ func (r *StatefulSetResource) queryRedpandaStatus( // will be fixed by https://github.com/redpanda-data/redpanda/issues/1084 if r.pandaCluster.AdminAPITLS() != nil && r.pandaCluster.AdminAPIExternal() == nil { - tlsConfig, err := adminutils.GetTLSConfig(ctx, r, r.pandaCluster, r.adminAPINodeCertSecretKey, r.adminAPIClientCertSecretKey) + tlsConfig, err := r.adminTLSConfigProvider.GetTLSConfig(ctx, r) if err != nil { return err } diff --git a/src/go/k8s/pkg/resources/tls_types.go b/src/go/k8s/pkg/resources/tls_types.go new file mode 100644 index 000000000000..23fcb0ea68d0 --- /dev/null +++ b/src/go/k8s/pkg/resources/tls_types.go @@ -0,0 +1,74 @@ +// Copyright 2021 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package resources + +import ( + "context" + "crypto/tls" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// StatefulsetTLSVolumeProvider returns volumes and volume mounts that +// tatefulset needs to be able to support TLS config specified by cluster custom +// resource +type StatefulsetTLSVolumeProvider interface { + Volumes() ([]corev1.Volume, []corev1.VolumeMount) +} + +// AdminTLSConfigProvider returns TLS config for admin API +type AdminTLSConfigProvider interface { + GetTLSConfig(ctx context.Context, k8sClient client.Reader) (*tls.Config, error) +} + +// TLSMountPoint defines paths to be mounted +// We need 2 secrets and 2 mount points for each API endpoint that supports TLS and mTLS: +// 1. The Node certs used by the API endpoint to sign requests +// 2. The CA used to sign mTLS client certs which will be use by the endpoint to validate mTLS client certs +// +// Node certs might be signed by a provided Issuer (i.e. when using Letsencrypt), in which case +// the operator won't generate a CA to sign the node certs. But, if at the same time mTLS is enabled +// a CA will be created to sign the mTLS client certs, and it will be stored in a separate secret. +type TLSMountPoint struct { + NodeCertMountDir string + ClientCAMountDir string +} + +// TLSMountPoints are mount points per API +type TLSMountPoints struct { + KafkaAPI *TLSMountPoint + AdminAPI *TLSMountPoint + PandaProxyAPI *TLSMountPoint + SchemaRegistryAPI *TLSMountPoint +} + +// GetTLSMountPoints returns configuration for all TLS mount paths for all +// redpanda APIs +func GetTLSMountPoints() *TLSMountPoints { + return &TLSMountPoints{ + KafkaAPI: &TLSMountPoint{ + NodeCertMountDir: "/etc/tls/certs", + ClientCAMountDir: "/etc/tls/certs/ca", + }, + AdminAPI: &TLSMountPoint{ + NodeCertMountDir: "/etc/tls/certs/admin", + ClientCAMountDir: "/etc/tls/certs/admin/ca", + }, + PandaProxyAPI: &TLSMountPoint{ + NodeCertMountDir: "/etc/tls/certs/pandaproxy", + ClientCAMountDir: "/etc/tls/certs/pandaproxy/ca", + }, + SchemaRegistryAPI: &TLSMountPoint{ + NodeCertMountDir: "/etc/tls/certs/schema-registry", + ClientCAMountDir: "/etc/tls/certs/schema-registry/ca", + }, + } +}