Add support for unbundled events in event collection (#1295)

* Add support for unbundled events in event collection

* doc update

---------

Co-authored-by: Celene <celene@datadoghq.com>

apis/datadoghq/common/const.go

@@ -72,6 +72,8 @@ const (
 	DefaultKubeStateMetricsCoreConf string = "kube-state-metrics-core-config"
 	// DefaultOrchestratorExplorerConf default orchestrator explorer ConfigMap name
 	DefaultOrchestratorExplorerConf string = "orchestrator-explorer-config"
+	// DefaultKubeAPIServerConf default Kubernetes APIServer ConfigMap name
+	DefaultKubeAPIServerConf string = "kube-apiserver-config"
 	// DefaultSystemProbeSocketPath default System Probe socket path
 	DefaultSystemProbeSocketPath string = "/var/run/sysprobe/sysprobe.sock"
 	// DefaultCSPMConf default CSPM ConfigMap name
@@ -301,7 +303,8 @@ const (
 	ClusterAgentCustomConfigVolumePath    = "/etc/datadog-agent/datadog-cluster.yaml"
 	ClusterAgentCustomConfigVolumeSubPath = "datadog-cluster.yaml"
 
-	HelmCheckConfigVolumeName = "helm-check-config"
+	HelmCheckConfigVolumeName                = "helm-check-config"
+	KubernetesAPIServerCheckConfigVolumeName = "kubernetes-apiserver-check-config"
 
 	FIPSProxyCustomConfigVolumeName = "fips-proxy-cfg"
 	FIPSProxyCustomConfigFileName   = "datadog-fips-proxy.cfg"

apis/datadoghq/v2alpha1/datadogagent_default.go

@@ -402,6 +402,22 @@ func defaultFeaturesConfig(ddaSpec *DatadogAgentSpec) {
 		ddaSpec.Features.EventCollection = &EventCollectionFeatureConfig{}
 	}
 	apiutils.DefaultBooleanIfUnset(&ddaSpec.Features.EventCollection.CollectKubernetesEvents, defaultCollectKubernetesEvents)
+	if apiutils.BoolValue(ddaSpec.Features.EventCollection.UnbundleEvents) && ddaSpec.Features.EventCollection.CollectedEventTypes == nil {
+		ddaSpec.Features.EventCollection.CollectedEventTypes = []EventTypes{
+			{
+				Kind:    "Pod",
+				Reasons: []string{"Failed", "BackOff", "Unhealthy", "FailedScheduling", "FailedMount", "FailedAttachVolume"},
+			},
+			{
+				Kind:    "Node",
+				Reasons: []string{"TerminatingEvictedPod", "NodeNotReady", "Rebooted", "HostPortConflict"},
+			},
+			{
+				Kind:    "CronJob",
+				Reasons: []string{"SawCompletedJob"},
+			},
+		}
+	}
 
 	// OrchestratorExplorer check Feature
 	if ddaSpec.Features.OrchestratorExplorer == nil {

apis/datadoghq/v2alpha1/datadogagent_types.go

@@ -581,6 +581,32 @@ type EventCollectionFeatureConfig struct {
 	// CollectKubernetesEvents enables Kubernetes event collection.
 	// Default: true
 	CollectKubernetesEvents *bool `json:"collectKubernetesEvents,omitempty"`
+
+	// UnbundleEvents enables collection of Kubernetes events as individual events.
+	// Default: false
+	// +optional
+	UnbundleEvents *bool `json:"unbundleEvents,omitempty"`
+
+	// CollectedEventTypes defines the list of events to collect when UnbundleEvents is enabled.
+	// Default:
+	// [
+	// {"kind":"Pod","reasons":["Failed","BackOff","Unhealthy","FailedScheduling","FailedMount","FailedAttachVolume"]},
+	// {"kind":"Node","reasons":["TerminatingEvictedPod","NodeNotReady","Rebooted","HostPortConflict"]},
+	// {"kind":"CronJob","reasons":["SawCompletedJob"]}
+	// ]
+	// +optional
+	// +listType=atomic
+	CollectedEventTypes []EventTypes `json:"collectedEventTypes,omitempty"`
+}
+
+// EventTypes defines the kind and reasons of events to collect.
+type EventTypes struct {
+	// Kind is the kind of event to collect. (ex: Pod, Node, CronJob)
+	Kind string `json:"kind"`
+
+	// Reasons is a list of event reasons to collect. (ex: Failed, BackOff, Unhealthy)
+	// +listType=atomic
+	Reasons []string `json:"reasons"`
 }
 
 // OrchestratorExplorerFeatureConfig contains the Orchestrator Explorer check feature configuration.

apis/datadoghq/v2alpha1/test/builder.go

@@ -218,10 +218,9 @@ func (builder *DatadogAgentBuilder) WithAdmissionControllerRegistry(name string)
 	return builder
 }
 
-//sidecar Injection
-
+// sidecar Injection
 func (builder *DatadogAgentBuilder) WithSidecarInjectionEnabled(enabled bool) *DatadogAgentBuilder {
-	//builder.initAdmissionController()
+	// builder.initAdmissionController()
 	builder.initSidecarInjection()
 	builder.datadogAgent.Spec.Features.AdmissionController.AgentSidecarInjection.Enabled = apiutils.NewBoolPointer(enabled)
 	if enabled {
@@ -391,6 +390,14 @@ func (builder *DatadogAgentBuilder) WithEventCollectionKubernetesEvents(enabled
 	return builder
 }
 
+func (builder *DatadogAgentBuilder) WithEventCollectionUnbundleEvents(enabled bool, eventTypes []v2alpha1.EventTypes) *DatadogAgentBuilder {
+	builder.initEventCollection()
+	builder.datadogAgent.Spec.Features.EventCollection.UnbundleEvents = apiutils.NewBoolPointer(enabled)
+	builder.datadogAgent.Spec.Features.EventCollection.CollectedEventTypes = eventTypes
+
+	return builder
+}
+
 // Remote Config
 func (builder *DatadogAgentBuilder) initRemoteConfig() {
 	if builder.datadogAgent.Spec.Features.RemoteConfiguration == nil {
@@ -577,7 +584,6 @@ func (builder *DatadogAgentBuilder) WithASMEnabled(threats, sca, iast bool) *Dat
 		},
 	}
 	return builder
-
 }
 
 // OTLP

config/crd/bases/v1/datadoghq.com_datadogagents.yaml

@@ -866,6 +866,38 @@ spec:
                             CollectKubernetesEvents enables Kubernetes event collection.
                             Default: true
                           type: boolean
+                        collectedEventTypes:
+                          description: |-
+                            CollectedEventTypes defines the list of events to collect when UnbundleEvents is enabled.
+                            Default:
+                            [
+                            {"kind":"Pod","reasons":["Failed","BackOff","Unhealthy","FailedScheduling","FailedMount","FailedAttachVolume"]},
+                            {"kind":"Node","reasons":["TerminatingEvictedPod","NodeNotReady","Rebooted","HostPortConflict"]},
+                            {"kind":"CronJob","reasons":["SawCompletedJob"]}
+                            ]
+                          items:
+                            description: EventTypes defines the kind and reasons of events to collect.
+                            properties:
+                              kind:
+                                description: 'Kind is the kind of event to collect. (ex: Pod, Node, CronJob)'
+                                type: string
+                              reasons:
+                                description: 'Reasons is a list of event reasons to collect. (ex: Failed, BackOff, Unhealthy)'
+                                items:
+                                  type: string
+                                type: array
+                                x-kubernetes-list-type: atomic
+                            required:
+                              - kind
+                              - reasons
+                            type: object
+                          type: array
+                          x-kubernetes-list-type: atomic
+                        unbundleEvents:
+                          description: |-
+                            UnbundleEvents enables collection of Kubernetes events as individual events.
+                            Default: false
+                          type: boolean
                       type: object
                     externalMetricsServer:
                       description: ExternalMetricsServer configuration.
@@ -6687,6 +6719,38 @@ spec:
                                 CollectKubernetesEvents enables Kubernetes event collection.
                                 Default: true
                               type: boolean
+                            collectedEventTypes:
+                              description: |-
+                                CollectedEventTypes defines the list of events to collect when UnbundleEvents is enabled.
+                                Default:
+                                [
+                                {"kind":"Pod","reasons":["Failed","BackOff","Unhealthy","FailedScheduling","FailedMount","FailedAttachVolume"]},
+                                {"kind":"Node","reasons":["TerminatingEvictedPod","NodeNotReady","Rebooted","HostPortConflict"]},
+                                {"kind":"CronJob","reasons":["SawCompletedJob"]}
+                                ]
+                              items:
+                                description: EventTypes defines the kind and reasons of events to collect.
+                                properties:
+                                  kind:
+                                    description: 'Kind is the kind of event to collect. (ex: Pod, Node, CronJob)'
+                                    type: string
+                                  reasons:
+                                    description: 'Reasons is a list of event reasons to collect. (ex: Failed, BackOff, Unhealthy)'
+                                    items:
+                                      type: string
+                                    type: array
+                                    x-kubernetes-list-type: atomic
+                                required:
+                                  - kind
+                                  - reasons
+                                type: object
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            unbundleEvents:
+                              description: |-
+                                UnbundleEvents enables collection of Kubernetes events as individual events.
+                                Default: false
+                              type: boolean
                           type: object
                         externalMetricsServer:
                           description: ExternalMetricsServer configuration.

config/rbac/role.yaml

@@ -12,13 +12,6 @@ rules:
   - /metrics/slis
   verbs:
   - get
-- apiGroups:
-  - ""
-  resources:
-  - '*/scale'
-  verbs:
-  - get
-  - update
 - apiGroups:
   - ""
   resources:
@@ -195,6 +188,13 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - '*'
+  resources:
+  - '*/scale'
+  verbs:
+  - get
+  - update
 - apiGroups:
   - admissionregistration.k8s.io
   resources:

controllers/datadogagent/controller.go

@@ -35,6 +35,7 @@ import (
 	_ "github.com/DataDog/datadog-operator/controllers/datadogagent/feature/admissioncontroller"
 	_ "github.com/DataDog/datadog-operator/controllers/datadogagent/feature/apm"
 	_ "github.com/DataDog/datadog-operator/controllers/datadogagent/feature/asm"
+	_ "github.com/DataDog/datadog-operator/controllers/datadogagent/feature/autoscaling"
 	_ "github.com/DataDog/datadog-operator/controllers/datadogagent/feature/clusterchecks"
 	_ "github.com/DataDog/datadog-operator/controllers/datadogagent/feature/cspm"
 	_ "github.com/DataDog/datadog-operator/controllers/datadogagent/feature/cws"
@@ -91,7 +92,8 @@ type Reconciler struct {
 
 // NewReconciler returns a reconciler for DatadogAgent
 func NewReconciler(options ReconcilerOptions, client client.Client, versionInfo *version.Info, platformInfo kubernetes.PlatformInfo,
-	scheme *runtime.Scheme, log logr.Logger, recorder record.EventRecorder, metricForwarder datadog.MetricForwardersManager) (*Reconciler, error) {
+	scheme *runtime.Scheme, log logr.Logger, recorder record.EventRecorder, metricForwarder datadog.MetricForwardersManager,
+) (*Reconciler, error) {
 	return &Reconciler{
 		options:      options,
 		client:       client,

controllers/datadogagent/feature/eventcollection/configmap.go

@@ -0,0 +1,45 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+package eventcollection
+
+import (
+	"github.com/DataDog/datadog-operator/apis/datadoghq/v2alpha1"
+	"gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func buildDefaultConfigMap(namespace, name string, unbundleEvents bool, collectedEventTypes []v2alpha1.EventTypes) (*corev1.ConfigMap, error) {
+	content, err := kubeAPIServerCheckConfig(unbundleEvents, collectedEventTypes)
+	if err != nil {
+		return nil, err
+	}
+
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Data: map[string]string{
+			kubeAPIServerConfigFileName: content,
+		},
+	}, nil
+}
+
+func kubeAPIServerCheckConfig(unbundleEvents bool, collectedEventTypes []v2alpha1.EventTypes) (string, error) {
+	cm := map[string]any{
+		"init_config": nil,
+		"instances": []map[string]any{
+			{
+				"unbundle_events":       unbundleEvents,
+				"collected_event_types": collectedEventTypes,
+			},
+		},
+	}
+
+	b, err := yaml.Marshal(cm)
+	return string(b), err
+}

controllers/datadogagent/feature/eventcollection/const.go

@@ -12,7 +12,9 @@ import (
 )
 
 const (
-	eventCollectionRBACPrefix = "event"
+	kubeAPIServerConfigFileName   = "kubernetes_apiserver.yaml"
+	kubeAPIServerConfigFolderName = "kubernetes_apiserver.d"
+	eventCollectionRBACPrefix     = "event"
 )
 
 // getRBACResourceName return the RBAC resources name