feat: add apiserver_admission_webhook_request_total metric (#17690)

Co-authored-by: Steven Blumenthal <blumenthal.steven.m@gmail.com>

kube_apiserver_metrics/changelog.d/17690.added

@@ -0,0 +1 @@
+Add the apiserver_admission_webhook_request_total

kube_apiserver_metrics/datadog_checks/kube_apiserver_metrics/kube_apiserver_metrics.py

@@ -117,6 +117,8 @@ def __init__(self, name, init_config, instances=None):
             # For Kubernetes >= 1.24
             # https://github.com/kubernetes/kubernetes/pull/107171
             'apiserver_admission_webhook_fail_open_count': self.apiserver_admission_webhook_fail_open_count,
+            # https://github.com/kubernetes/kubernetes/pull/103162
+            'apiserver_admission_webhook_request_total': self.apiserver_admission_webhook_request_total,
         }
         self.kube_apiserver_config = None
 
@@ -242,3 +244,6 @@ def apiserver_request_terminations_total(self, metric, scraper_config):
 
     def apiserver_admission_webhook_fail_open_count(self, metric, scraper_config):
         self.submit_metric('.apiserver_admission_webhook_fail_open_count', metric, scraper_config)
+
+    def apiserver_admission_webhook_request_total(self, metric, scraper_config):
+        self.submit_metric('.apiserver_admission_webhook_request_total', metric, scraper_config)

kube_apiserver_metrics/metadata.csv

@@ -12,6 +12,8 @@ kube_apiserver.admission_webhook_admission_latencies_seconds.sum,gauge,,second,,
 kube_apiserver.aggregator_unavailable_apiservice,gauge,,,,Gauge of APIServices which are marked as unavailable broken down by APIService name (alpha; Kubernetes 1.14+),0,kubernetes_api_server_metrics,unavailable apiservice,
 kube_apiserver.apiserver_admission_webhook_fail_open_count,gauge,,,,"Admission webhook fail open count, identified by name and broken out for each admission type (validating or mutating).",0,kubernetes_api_server_metrics,webhook fail open count,
 kube_apiserver.apiserver_admission_webhook_fail_open_count.count,count,,,,"Admission webhook fail open count, identified by name and broken out for each admission type (validating or mutating).",0,kubernetes_api_server_metrics,webhook fail open count,
+kube_apiserver.apiserver_admission_webhook_request_total,gauge,,,,"Admission webhook request total, identified by name and broken out for each admission type (alpha; Kubernetes 1.23+)",0,kubernetes_api_server_metrics,webhook request total,
+kube_apiserver.apiserver_admission_webhook_request_total.count,count,,,,"Admission webhook request total, identified by name and broken out for each admission type (alpha; Kubernetes 1.23+)",0,kubernetes_api_server_metrics,webhook request total,
 kube_apiserver.apiserver_dropped_requests_total,gauge,,request,,The accumulated number of requests dropped with 'Try again later' response,1,kubernetes_api_server_metrics,accumulated number of dropped requests,
 kube_apiserver.apiserver_dropped_requests_total.count,count,,request,,The monotonic count of requests dropped with 'Try again later' response,0,kubernetes_api_server_metrics,count of dropped requests,
 kube_apiserver.apiserver_request_count,gauge,,request,,The accumulated number of apiserver requests broken out for each verb API resource client and HTTP response contentType and code (deprecated in Kubernetes 1.15),1,kubernetes_api_server_metrics,accumulated number of requests,

kube_apiserver_metrics/tests/fixtures/metrics_1.23.0.txt

@@ -554,6 +554,11 @@ apiserver_admission_step_admission_duration_seconds_summary{operation="UPDATE",r
 apiserver_admission_step_admission_duration_seconds_summary{operation="UPDATE",rejected="false",type="validate",quantile="0.99"} 0.0005589
 apiserver_admission_step_admission_duration_seconds_summary_sum{operation="UPDATE",rejected="false",type="validate"} 1.734694500000006
 apiserver_admission_step_admission_duration_seconds_summary_count{operation="UPDATE",rejected="false",type="validate"} 7427
+# HELP apiserver_admission_webhook_request_total [ALPHA] Admission webhook request total, identified by name and broken out for each admission type (validating or mutating) and operation. Additional labels specify whether the request was rejected or not and an HTTP status code. Codes greater than 600 are truncated to 600, to keep the metrics cardinality bounded.
+# TYPE apiserver_admission_webhook_request_total counter
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.auto.instrumentation",operation="CREATE",rejected="false",type="admit"} 17
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.config",operation="CREATE",rejected="false",type="admit"} 18
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.tags",operation="CREATE",rejected="false",type="admit"} 17
 # HELP apiserver_audit_event_total [ALPHA] Counter of audit events generated and sent to the audit backend.
 # TYPE apiserver_audit_event_total counter
 apiserver_audit_event_total 0
@@ -19216,4 +19221,4 @@ workqueue_work_duration_seconds_bucket{name="priority_and_fairness_config_queue"
 workqueue_work_duration_seconds_bucket{name="priority_and_fairness_config_queue",le="10"} 33
 workqueue_work_duration_seconds_bucket{name="priority_and_fairness_config_queue",le="+Inf"} 33
 workqueue_work_duration_seconds_sum{name="priority_and_fairness_config_queue"} 0.1424415
-workqueue_work_duration_seconds_count{name="priority_and_fairness_config_queue"} 33
+workqueue_work_duration_seconds_count{name="priority_and_fairness_config_queue"} 33

kube_apiserver_metrics/tests/fixtures/metrics_1.26.0.txt

@@ -583,6 +583,11 @@ apiserver_admission_step_admission_duration_seconds_summary{operation="UPDATE",r
 apiserver_admission_step_admission_duration_seconds_summary{operation="UPDATE",rejected="false",type="validate",quantile="0.99"} 0.000180532
 apiserver_admission_step_admission_duration_seconds_summary_sum{operation="UPDATE",rejected="false",type="validate"} 0.03223122800000003
 apiserver_admission_step_admission_duration_seconds_summary_count{operation="UPDATE",rejected="false",type="validate"} 686
+# HELP apiserver_admission_webhook_request_total [ALPHA] Admission webhook request total, identified by name and broken out for each admission type (validating or mutating) and operation. Additional labels specify whether the request was rejected or not and an HTTP status code. Codes greater than 600 are truncated to 600, to keep the metrics cardinality bounded.
+# TYPE apiserver_admission_webhook_request_total counter
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.auto.instrumentation",operation="CREATE",rejected="false",type="admit"} 17
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.config",operation="CREATE",rejected="false",type="admit"} 18
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.tags",operation="CREATE",rejected="false",type="admit"} 17
 # HELP apiserver_audit_event_total [ALPHA] Counter of audit events generated and sent to the audit backend.
 # TYPE apiserver_audit_event_total counter
 apiserver_audit_event_total 0

kube_apiserver_metrics/tests/fixtures/metrics_1.27.0.txt

@@ -635,6 +635,11 @@ apiserver_admission_step_admission_duration_seconds_summary{operation="UPDATE",r
 apiserver_admission_step_admission_duration_seconds_summary{operation="UPDATE",rejected="false",type="validate",quantile="0.99"} 0.000158667
 apiserver_admission_step_admission_duration_seconds_summary_sum{operation="UPDATE",rejected="false",type="validate"} 0.38836563199999974
 apiserver_admission_step_admission_duration_seconds_summary_count{operation="UPDATE",rejected="false",type="validate"} 19075
+# HELP apiserver_admission_webhook_request_total [ALPHA] Admission webhook request total, identified by name and broken out for each admission type (validating or mutating) and operation. Additional labels specify whether the request was rejected or not and an HTTP status code. Codes greater than 600 are truncated to 600, to keep the metrics cardinality bounded.
+# TYPE apiserver_admission_webhook_request_total counter
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.auto.instrumentation",operation="CREATE",rejected="false",type="admit"} 17
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.config",operation="CREATE",rejected="false",type="admit"} 18
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.tags",operation="CREATE",rejected="false",type="admit"} 17
 # HELP apiserver_audit_event_total [ALPHA] Counter of audit events generated and sent to the audit backend.
 # TYPE apiserver_audit_event_total counter
 apiserver_audit_event_total 0

kube_apiserver_metrics/tests/fixtures/metrics_1.28.0.txt

@@ -616,6 +616,11 @@ apiserver_admission_step_admission_duration_seconds_summary{operation="UPDATE",r
 apiserver_admission_step_admission_duration_seconds_summary{operation="UPDATE",rejected="false",type="validate",quantile="0.99"} 0.000148958
 apiserver_admission_step_admission_duration_seconds_summary_sum{operation="UPDATE",rejected="false",type="validate"} 0.43113972400000006
 apiserver_admission_step_admission_duration_seconds_summary_count{operation="UPDATE",rejected="false",type="validate"} 19367
+# HELP apiserver_admission_webhook_request_total [ALPHA] Admission webhook request total, identified by name and broken out for each admission type (validating or mutating) and operation. Additional labels specify whether the request was rejected or not and an HTTP status code. Codes greater than 600 are truncated to 600, to keep the metrics cardinality bounded.
+# TYPE apiserver_admission_webhook_request_total counter
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.auto.instrumentation",operation="CREATE",rejected="false",type="admit"} 17
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.config",operation="CREATE",rejected="false",type="admit"} 18
+apiserver_admission_webhook_request_total{code="200",name="datadog.webhook.tags",operation="CREATE",rejected="false",type="admit"} 17
 # HELP apiserver_audit_event_total [ALPHA] Counter of audit events generated and sent to the audit backend.
 # TYPE apiserver_audit_event_total counter
 apiserver_audit_event_total 0

kube_apiserver_metrics/tests/test_kube_apiserver_metrics_1_23.py

@@ -38,6 +38,7 @@ class TestKubeAPIServerMetrics:
         'grpc_client_started_total',
         'rest_client_request_latency_seconds.sum',
         'rest_client_request_latency_seconds.count',
+        'apiserver_admission_webhook_request_total',
         'admission_step_admission_latencies_seconds.sum',
         'admission_step_admission_latencies_seconds.count',
         'admission_step_admission_latencies_seconds_summary.sum',
@@ -74,6 +75,7 @@ class TestKubeAPIServerMetrics:
         'audit_event.count',
         'rest_client_requests_total.count',
         'authenticated_user_requests.count',
+        'apiserver_admission_webhook_request_total.count',
         'apiserver_request_total.count',
         'apiserver_request_terminations_total.count',
     ]

kube_apiserver_metrics/tests/test_kube_apiserver_metrics_1_24.py

@@ -63,6 +63,7 @@ class TestKubeAPIServerMetrics:
         'storage_list_returned_objects_total',
         'requested_deprecated_apis',
         'apiserver_admission_webhook_fail_open_count',
+        'apiserver_admission_webhook_request_total',
         'admission_webhook_admission_latencies_seconds.sum',
         'admission_webhook_admission_latencies_seconds.count',
         'aggregator_unavailable_apiservice',
@@ -79,6 +80,7 @@ class TestKubeAPIServerMetrics:
         'apiserver_request_total.count',
         'apiserver_request_terminations_total.count',
         'apiserver_admission_webhook_fail_open_count.count',
+        'apiserver_admission_webhook_request_total.count',
     ]
 
     def test_check(self, dd_run_check, aggregator, mock_http_response):

kube_apiserver_metrics/tests/test_kube_apiserver_metrics_1_25.py

@@ -82,6 +82,7 @@ class TestKubeAPIServerMetrics:
         NAMESPACE + '.storage_list_returned_objects_total',
         NAMESPACE + '.requested_deprecated_apis',
         NAMESPACE + '.apiserver_admission_webhook_fail_open_count',
+        NAMESPACE + '.apiserver_admission_webhook_request_total',
         NAMESPACE + '.admission_webhook_admission_latencies_seconds.sum',
         NAMESPACE + '.admission_webhook_admission_latencies_seconds.count',
         NAMESPACE + '.kubernetes_feature_enabled',
@@ -100,6 +101,7 @@ class TestKubeAPIServerMetrics:
         NAMESPACE + '.apiserver_request_total.count',
         NAMESPACE + '.apiserver_request_terminations_total.count',
         NAMESPACE + '.apiserver_admission_webhook_fail_open_count.count',
+        NAMESPACE + '.apiserver_admission_webhook_request_total.count',
     ]
 
     def test_check(self, dd_run_check, aggregator, mock_get):

kube_apiserver_metrics/tests/test_kube_apiserver_metrics_1_26.py

@@ -29,6 +29,7 @@ class TestKubeAPIServerMetrics:
         'go_goroutines',
         'rest_client_requests_total',
         'authenticated_user_requests',
+        'apiserver_admission_webhook_request_total',
         'apiserver_request_total',
         'apiserver_request_terminations_total',
         'grpc_client_handled_total',
@@ -76,6 +77,7 @@ class TestKubeAPIServerMetrics:
         'authenticated_user_requests.count',
         'apiserver_request_total.count',
         'apiserver_request_terminations_total.count',
+        'apiserver_admission_webhook_request_total.count',
     ]
 
     def test_check(self, dd_run_check, aggregator, mock_http_response):

kube_apiserver_metrics/tests/test_kube_apiserver_metrics_1_27.py

@@ -37,6 +37,7 @@ class TestKubeAPIServerMetrics:
         'grpc_client_started_total',
         'rest_client_request_latency_seconds.sum',
         'rest_client_request_latency_seconds.count',
+        'apiserver_admission_webhook_request_total',
         'admission_step_admission_latencies_seconds.sum',
         'admission_step_admission_latencies_seconds.count',
         'admission_step_admission_latencies_seconds_summary.sum',
@@ -74,6 +75,7 @@ class TestKubeAPIServerMetrics:
         'audit_event.count',
         'rest_client_requests_total.count',
         'authenticated_user_requests.count',
+        'apiserver_admission_webhook_request_total.count',
         'apiserver_request_total.count',
         'apiserver_request_terminations_total.count',
     ]

kube_apiserver_metrics/tests/test_kube_apiserver_metrics_1_28.py

@@ -37,6 +37,7 @@ class TestKubeAPIServerMetrics:
         'grpc_client_started_total',
         'rest_client_request_latency_seconds.sum',
         'rest_client_request_latency_seconds.count',
+        'apiserver_admission_webhook_request_total',
         'admission_step_admission_latencies_seconds.sum',
         'admission_step_admission_latencies_seconds.count',
         'admission_step_admission_latencies_seconds_summary.sum',
@@ -76,6 +77,7 @@ class TestKubeAPIServerMetrics:
         'audit_event.count',
         'rest_client_requests_total.count',
         'authenticated_user_requests.count',
+        'apiserver_admission_webhook_request_total.count',
         'apiserver_request_total.count',
         'apiserver_request_terminations_total.count',
     ]