Cluster Agent Auth Lifecycle Improvment (#740)

* Adding conversion of DCA token

* Adding MD5 hash for DCA token

* Adding hash to components' annotations

apis/datadoghq/v1alpha1/datadogagent_conversion.go

@@ -59,7 +59,11 @@ func convertSpec(src *DatadogAgentSpec, dst *v2alpha1.DatadogAgent) error {
 	// Convert credentials
 	if src.Credentials != nil {
 		if dstCred := convertCredentials(&src.Credentials.DatadogCredentials); dstCred != nil {
-			getV2GlobalConfig(dst).Credentials = dstCred
+			dstCredGlobal := getV2GlobalConfig(dst)
+			dstCredGlobal.Credentials = dstCred
+			if src.Credentials.Token != "" {
+				dstCredGlobal.ClusterAgentToken = &src.Credentials.Token
+			}
 		}
 	}
 

apis/datadoghq/v1alpha1/testdata/all.expected.yaml

@@ -71,6 +71,7 @@ spec:
       enabled: true
   global:
     clusterName: foo
+    clusterAgentToken: "foo-bar-baz"
     credentials:
       apiKey: api-key-inline
       apiSecret:

apis/datadoghq/v1alpha1/testdata/all.yaml

@@ -16,7 +16,7 @@ spec:
     appSecret:
       secretName: datadog-secret
       keyName: app-key
-    token: "foo-bar-baz" # ignored by conversion
+    token: "foo-bar-baz"
     useSecretBackend: true
   features:
     orchestratorExplorer:

controllers/datadogagent/feature/enabledefault/feature.go

@@ -17,9 +17,12 @@ import (
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/component/agent"
 	componentdca "github.com/DataDog/datadog-operator/controllers/datadogagent/component/clusteragent"
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/feature"
+	"github.com/DataDog/datadog-operator/controllers/datadogagent/object"
+	"github.com/DataDog/datadog-operator/pkg/controller/utils/comparison"
 	"github.com/DataDog/datadog-operator/pkg/kubernetes"
 	"github.com/DataDog/datadog-operator/pkg/version"
 
+	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/errors"
@@ -46,6 +49,10 @@ func buildDefaultFeature(options *feature.Options) feature.Feature {
 		},
 	}
 
+	if options != nil {
+		dF.logger = options.Logger
+	}
+
 	return dF
 }
 
@@ -57,6 +64,10 @@ type defaultFeature struct {
 	clusterAgent        clusterAgentConfig
 	agent               agentConfig
 	clusterChecksRunner clusterChecksRunnerConfig
+	logger              logr.Logger
+
+	customConfigAnnotationKey   string
+	customConfigAnnotationValue string
 }
 
 type credentialsInfo struct {
@@ -156,6 +167,14 @@ func (f *defaultFeature) Configure(dda *v2alpha1.DatadogAgent) feature.RequiredC
 				f.dcaTokenInfo.secretCreation.data[apicommon.DefaultTokenKey] = dda.Status.ClusterAgent.GeneratedToken
 			}
 		}
+		hash, err := comparison.GenerateMD5ForSpec(f.dcaTokenInfo.secretCreation.data)
+		if err != nil {
+			f.logger.Error(err, "couldn't generate hash for Cluster Agent token hash")
+		} else {
+			f.logger.V(2).Info("built Cluster Agent token hash", "hash", hash)
+		}
+		f.customConfigAnnotationValue = hash
+		f.customConfigAnnotationKey = object.GetChecksumAnnotationKey(string(feature.DefaultIDType))
 	}
 
 	return feature.RequiredComponents{
@@ -245,6 +264,11 @@ func (f *defaultFeature) ManageDependencies(managers feature.ResourceManagers, c
 				errs = append(errs, err)
 			}
 		}
+		// Adding Annotation containing data hash to secret.
+		if err := managers.SecretManager().AddAnnotations(f.logger, f.owner.GetNamespace(), f.dcaTokenInfo.secretCreation.name, map[string]string{f.customConfigAnnotationKey: f.customConfigAnnotationValue}); err != nil {
+			errs = append(errs, err)
+		}
+
 	}
 
 	// Create install-info configmap
@@ -349,21 +373,30 @@ func (f *defaultFeature) clusterChecksRunnerDependencies(managers feature.Resour
 // It should do nothing if the feature doesn't need to configure it.
 func (f *defaultFeature) ManageClusterAgent(managers feature.PodTemplateManagers) error {
 	f.addDefaultCommonEnvs(managers)
-
+	if f.customConfigAnnotationKey != "" && f.customConfigAnnotationValue != "" {
+		managers.Annotation().AddAnnotation(f.customConfigAnnotationKey, f.customConfigAnnotationValue)
+	}
 	return nil
 }
 
 // ManageNodeAgent allows a feature to configure the Node Agent's corev1.PodTemplateSpec
 // It should do nothing if the feature doesn't need to configure it.
 func (f *defaultFeature) ManageNodeAgent(managers feature.PodTemplateManagers) error {
 	f.addDefaultCommonEnvs(managers)
+	if f.customConfigAnnotationKey != "" && f.customConfigAnnotationValue != "" {
+		managers.Annotation().AddAnnotation(f.customConfigAnnotationKey, f.customConfigAnnotationValue)
+	}
+
 	return nil
 }
 
 // ManageClusterChecksRunner allows a feature to configure the ClusterChecksRunnerAgent's corev1.PodTemplateSpec
 // It should do nothing if the feature doesn't need to configure it.
 func (f *defaultFeature) ManageClusterChecksRunner(managers feature.PodTemplateManagers) error {
 	f.addDefaultCommonEnvs(managers)
+	if f.customConfigAnnotationKey != "" && f.customConfigAnnotationValue != "" {
+		managers.Annotation().AddAnnotation(f.customConfigAnnotationKey, f.customConfigAnnotationValue)
+	}
 
 	return nil
 }

controllers/datadogagent/feature/externalmetrics/feature.go

@@ -19,6 +19,7 @@ import (
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/feature"
 	cilium "github.com/DataDog/datadog-operator/pkg/cilium/v1"
 	"github.com/DataDog/datadog-operator/pkg/kubernetes/rbac"
+	"github.com/go-logr/logr"
 
 	corev1 "k8s.io/api/core/v1"
 	netv1 "k8s.io/api/networking/v1"
@@ -37,6 +38,10 @@ func init() {
 func buildExternalMetricsFeature(options *feature.Options) feature.Feature {
 	externalMetricsFeat := &externalMetricsFeature{}
 
+	if options != nil {
+		externalMetricsFeat.logger = options.Logger
+	}
+
 	return externalMetricsFeat
 }
 
@@ -48,6 +53,7 @@ type externalMetricsFeature struct {
 	keySecret          map[string]secret
 	serviceAccountName string
 	owner              metav1.Object
+	logger             logr.Logger
 
 	createKubernetesNetworkPolicy bool
 	createCiliumNetworkPolicy     bool

controllers/datadogagent/merger/secret.go

@@ -11,12 +11,15 @@ import (
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/dependencies"
+	"github.com/DataDog/datadog-operator/controllers/datadogagent/object"
 	"github.com/DataDog/datadog-operator/pkg/kubernetes"
+	"github.com/go-logr/logr"
 )
 
 // SecretManager Kubernetes Secret Manager interface
 type SecretManager interface {
 	AddSecret(secretNamespace, secretName, key, value string) error
+	AddAnnotations(logger logr.Logger, secretNamespace, secretName string, extraAnnotations map[string]string) error
 }
 
 // NewSecretManager return new SecretManager instance
@@ -45,3 +48,18 @@ func (m *secretManagerImpl) AddSecret(secretNamespace, secretName, key, value st
 
 	return m.store.AddOrUpdate(kubernetes.SecretsKind, secret)
 }
+
+func (m *secretManagerImpl) AddAnnotations(logger logr.Logger, secretNamespace, secretName string, extraAnnotations map[string]string) error {
+	obj, _ := m.store.GetOrCreate(kubernetes.SecretsKind, secretNamespace, secretName)
+	secret, ok := obj.(*corev1.Secret)
+	if !ok {
+		return fmt.Errorf("unable to get the Secret %s/%s from the store", secretNamespace, secretName)
+	}
+
+	if len(extraAnnotations) > 0 {
+		annotations := object.MergeAnnotationsLabels(logger, secret.GetAnnotations(), extraAnnotations, "*")
+		secret.SetAnnotations(annotations)
+	}
+
+	return m.store.AddOrUpdate(kubernetes.SecretsKind, secret)
+}

controllers/datadogagent/merger/secret_test.go

@@ -10,17 +10,22 @@ import (
 
 	"github.com/DataDog/datadog-operator/apis/datadoghq/v2alpha1"
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/dependencies"
+	"github.com/DataDog/datadog-operator/controllers/datadogagent/object"
 	"github.com/DataDog/datadog-operator/pkg/kubernetes"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 func Test_secretManagerImpl_AddSecret(t *testing.T) {
+	logger := logf.Log.WithName(t.Name())
 	secretNs := "foo"
 	secretName := "bar"
-
+	secretAnnotations := map[string]string{
+		"checksum/default-custom-config": "0fe60b5fsweqe3224werwer",
+	}
 	owner := &v2alpha1.DatadogAgent{
 		ObjectMeta: v1.ObjectMeta{
 			Namespace: secretNs,
@@ -36,8 +41,9 @@ func Test_secretManagerImpl_AddSecret(t *testing.T) {
 
 	secret1 := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      secretName,
-			Namespace: secretNs,
+			Name:        secretName,
+			Namespace:   secretNs,
+			Annotations: secretAnnotations,
 		},
 		Data: map[string][]byte{
 			"key1": []byte("defaultvalue"),
@@ -98,6 +104,9 @@ func Test_secretManagerImpl_AddSecret(t *testing.T) {
 				if _, ok := secret.Data["key"]; !ok {
 					t.Errorf("key not found in Secret %s/%s", secretNs, secretName)
 				}
+				if _, ok := secret.Annotations[object.GetChecksumAnnotationKey("default")]; !ok {
+					t.Errorf("missing extraMetadata in Secret %s/%s", secretNs, secretName)
+				}
 			},
 		},
 	}
@@ -109,6 +118,9 @@ func Test_secretManagerImpl_AddSecret(t *testing.T) {
 			if err := m.AddSecret(tt.args.secretNamespace, tt.args.secretName, tt.args.key, tt.args.value); (err != nil) != tt.wantErr {
 				t.Errorf("secretManagerImpl.AddSecret() error = %v, wantErr %v", err, tt.wantErr)
 			}
+			if err := m.AddAnnotations(logger, tt.args.secretNamespace, tt.args.secretName, secretAnnotations); (err != nil) != tt.wantErr {
+				t.Errorf("secretManagerImpl.AddAnnotations() error = %v, wantErr %v", err, tt.wantErr)
+			}
 			if tt.validateFunc != nil {
 				tt.validateFunc(t, tt.store)
 			}