diff --git a/kubelet/assets/configuration/spec.yaml b/kubelet/assets/configuration/spec.yaml index 387da3bd379b7..a6ade122776a0 100644 --- a/kubelet/assets/configuration/spec.yaml +++ b/kubelet/assets/configuration/spec.yaml @@ -20,6 +20,11 @@ files: URL of the kubelet metrics prometheus endpoint Pass an empty string to disable kubelet metrics collection. example: http://10.8.0.1:10255/metrics + - name: probes_metrics_endpoint + description: | + URL of the probe metrics prometheus endpoint + Pass an empty string to disable probe metrics collection. + example: http://10.8.0.1:10255/metrics/probes - name: cadvisor_port description: | Metric collection for legacy (< 1.7.6) clusters via the kubelet's cadvisor port. diff --git a/kubelet/datadog_checks/kubelet/common.py b/kubelet/datadog_checks/kubelet/common.py index ee8f68dc5d630..e71ef0d660a27 100644 --- a/kubelet/datadog_checks/kubelet/common.py +++ b/kubelet/datadog_checks/kubelet/common.py @@ -91,6 +91,17 @@ def replace_container_rt_prefix(cid): return cid +def get_container_label(labels, l_name): + """ + Iter on all labels to find the label.name equal to the l_name + :param labels: list of labels + :param l_name: str + :return: str or None + """ + if l_name in labels: + return labels[l_name] + + class PodListUtils(object): """ Queries the podlist and the agent6's filtering logic to determine whether to @@ -208,3 +219,22 @@ def is_namespace_excluded(self, namespace): excluded = c_is_excluded('', '', namespace) self.cache_namespace_exclusion[namespace] = excluded return excluded + + def get_cid_by_labels(self, labels): + """ + Should only be called on a container-scoped metric + It gets the container id from the podlist using the metrics labels + + :param labels + :return str or None + """ + namespace = get_container_label(labels, "namespace") + # k8s >= 1.16 + pod_name = get_container_label(labels, "pod") + container_name = get_container_label(labels, "container") + # k8s < 1.16 + if not pod_name: + pod_name = get_container_label(labels, "pod_name") + if not container_name: + container_name = get_container_label(labels, "container_name") + return self.get_cid_by_name_tuple((namespace, pod_name, container_name)) diff --git a/kubelet/datadog_checks/kubelet/data/conf.yaml.example b/kubelet/datadog_checks/kubelet/data/conf.yaml.example index 85673877709c3..69e0e3ab21e0e 100644 --- a/kubelet/datadog_checks/kubelet/data/conf.yaml.example +++ b/kubelet/datadog_checks/kubelet/data/conf.yaml.example @@ -59,6 +59,11 @@ instances: # # kubelet_metrics_endpoint: http://10.8.0.1:10255/metrics + ## URL of the probe metrics prometheus endpoint + ## Pass an empty string to disable probe metrics collection. + # + # probes_metrics_endpoint: http://10.8.0.1:10255/metrics/probes + ## Metric collection for legacy (< 1.7.6) clusters via the kubelet's cadvisor port. ## This port is closed by default on k8s 1.7+ and OpenShift, enable it ## via the `--cadvisor-port=4194` kubelet option. diff --git a/kubelet/datadog_checks/kubelet/kubelet.py b/kubelet/datadog_checks/kubelet/kubelet.py index 2164c41810624..9db585ee5b12b 100644 --- a/kubelet/datadog_checks/kubelet/kubelet.py +++ b/kubelet/datadog_checks/kubelet/kubelet.py @@ -20,6 +20,7 @@ from .cadvisor import CadvisorScraper from .common import CADVISOR_DEFAULT_PORT, PodListUtils, replace_container_rt_prefix +from .probes import ProbesPrometheusScraperMixin from .prometheus import CadvisorPrometheusScraperMixin from .summary import SummaryScraperMixin @@ -29,6 +30,7 @@ CADVISOR_METRICS_PATH = '/metrics/cadvisor' KUBELET_METRICS_PATH = '/metrics' STATS_PATH = '/stats/summary/' +PROBES_METRICS_PATH = '/metrics/probes' # Suffixes per # https://github.com/kubernetes/kubernetes/blob/8fd414537b5143ab039cb910590237cabf4af783/pkg/api/resource/suffix.go#L108 @@ -128,6 +130,7 @@ class KubeletCheck( OpenMetricsBaseCheck, CadvisorScraper, SummaryScraperMixin, + ProbesPrometheusScraperMixin, KubeletBase, ): """ @@ -173,7 +176,8 @@ def __init__(self, name, init_config, instances): self.pod_level_metrics = ["{0}.{1}".format(self.NAMESPACE, x) for x in pod_level_metrics] kubelet_instance = self._create_kubelet_prometheus_instance(inst) - generic_instances = [cadvisor_instance, kubelet_instance] + probes_instance = self._create_probes_prometheus_instance(inst) + generic_instances = [cadvisor_instance, kubelet_instance, probes_instance] super(KubeletCheck, self).__init__(name, init_config, generic_instances) self.cadvisor_legacy_port = inst.get('cadvisor_port', CADVISOR_DEFAULT_PORT) @@ -189,6 +193,8 @@ def __init__(self, name, init_config, instances): self.kubelet_scraper_config = self.get_scraper_config(kubelet_instance) + self.probes_scraper_config = self.get_scraper_config(probes_instance) + counter_transformers = {k: self.send_always_counter for k in self.COUNTER_METRICS} histogram_transformers = { @@ -199,6 +205,7 @@ def __init__(self, name, init_config, instances): self.transformers = {} for d in [ + self.PROBES_METRIC_TRANSFORMERS, self.CADVISOR_METRIC_TRANSFORMERS, counter_transformers, histogram_transformers, @@ -320,9 +327,14 @@ def check(self, instance): 'kubelet_metrics_endpoint', urljoin(endpoint, KUBELET_METRICS_PATH) ) + self.probes_scraper_config['prometheus_url'] = instance.get( + 'probes_metrics_endpoint', urljoin(endpoint, PROBES_METRICS_PATH) + ) + # Kubelet credentials handling self.kubelet_credentials.configure_scraper(self.cadvisor_scraper_config) self.kubelet_credentials.configure_scraper(self.kubelet_scraper_config) + self.kubelet_credentials.configure_scraper(self.probes_scraper_config) # Legacy cadvisor support try: @@ -356,6 +368,10 @@ def check(self, instance): self.log.debug('processing kubelet metrics') self.process(self.kubelet_scraper_config, metric_transformers=self.transformers) + if self.probes_scraper_config['prometheus_url']: + self.log.debug('processing probe metrics') + self.process(self.probes_scraper_config, metric_transformers=self.transformers) + self.first_run = False # Free up memory diff --git a/kubelet/datadog_checks/kubelet/probes.py b/kubelet/datadog_checks/kubelet/probes.py new file mode 100644 index 0000000000000..55b18212e4632 --- /dev/null +++ b/kubelet/datadog_checks/kubelet/probes.py @@ -0,0 +1,94 @@ +# (C) Datadog, Inc. 2018-present +# All rights reserved +# Licensed under Simplified BSD License (see LICENSE) +from __future__ import division + +from copy import deepcopy + +from datadog_checks.base.checks.openmetrics import OpenMetricsBaseCheck +from datadog_checks.base.utils.tagging import tagger + +from .common import get_container_label, replace_container_rt_prefix, tags_for_docker + + +class ProbesPrometheusScraperMixin(object): + """ + This class scrapes metrics for the kubelet "/metrics/probes" prometheus endpoint and submits + them on behalf of a check. + """ + + def __init__(self, *args, **kwargs): + super(ProbesPrometheusScraperMixin, self).__init__(*args, **kwargs) + + self.PROBES_METRIC_TRANSFORMERS = { + 'prober_probe_total': self.prober_probe_total, + } + + def _create_probes_prometheus_instance(self, instance): + """ + Create a copy of the instance and set default values. + This is so the base class can create a scraper_config with the proper values. + """ + probes_instance = deepcopy(instance) + probes_instance.update( + { + 'namespace': self.NAMESPACE, + # We need to specify a prometheus_url so the base class can use it as the key for our config_map, + # we specify a dummy url that will be replaced in the `check()` function. We append it with "probes" + # so the key is different than the rest of the kubelet scrapers. + 'prometheus_url': instance.get('probes_metrics_endpoint', 'dummy_url/probes'), + } + ) + return probes_instance + + def prober_probe_total(self, metric, scraper_config): + for sample in metric.samples: + metric_name_suffix = '' + labels = sample[OpenMetricsBaseCheck.SAMPLE_LABELS] + + probe_type = labels.get('probe_type') + if probe_type == 'Liveness': + metric_name_suffix = '.liveness_probe' + elif probe_type == 'Readiness': + metric_name_suffix = '.readiness_probe' + else: + self.log.debug("Unsupported probe type %s", probe_type) + continue + + result = labels.get('result') + if result == 'successful': + metric_name_suffix = metric_name_suffix + '.success.total' + elif result == 'failed': + metric_name_suffix = metric_name_suffix + '.failure.total' + elif result == 'unknown': + metric_name_suffix = metric_name_suffix + '.unknown.total' + else: + self.log.debug("Unsupported probe result %s", result) + continue + + metric_name = scraper_config['namespace'] + metric_name_suffix + + container_id = self.pod_list_utils.get_cid_by_labels(labels) + if container_id is None: + self.log.debug( + "Container id not found from /pods for container: %s/%s/%s - no metrics will be sent", + get_container_label(labels, 'namespace'), + get_container_label(labels, 'pod'), + get_container_label(labels, 'container'), + ) + continue + + if self.pod_list_utils.is_excluded(container_id): + continue + + container_tags = tags_for_docker(replace_container_rt_prefix(container_id), tagger.HIGH, True) + if not container_tags: + self.log.debug( + "Tags not found for container: %s/%s/%s:%s - no metrics will be sent", + get_container_label(labels, 'namespace'), + get_container_label(labels, 'pod'), + get_container_label(labels, 'container'), + container_id, + ) + + self.count(metric_name, sample[self.SAMPLE_VALUE], container_tags + self.instance_tags) diff --git a/kubelet/datadog_checks/kubelet/prometheus.py b/kubelet/datadog_checks/kubelet/prometheus.py index da62c4832132a..a6536f12bdf57 100644 --- a/kubelet/datadog_checks/kubelet/prometheus.py +++ b/kubelet/datadog_checks/kubelet/prometheus.py @@ -10,7 +10,7 @@ from datadog_checks.base.checks.openmetrics import OpenMetricsBaseCheck from datadog_checks.base.utils.tagging import tagger -from .common import get_pod_by_uid, is_static_pending_pod, replace_container_rt_prefix +from .common import get_container_label, get_pod_by_uid, is_static_pending_pod, replace_container_rt_prefix METRIC_TYPES = ['counter', 'gauge', 'summary'] @@ -146,36 +146,6 @@ def _is_pod_metric(labels): return True return False - @staticmethod - def _get_container_label(labels, l_name): - """ - Iter on all labels to find the label.name equal to the l_name - :param labels: list of labels - :param l_name: str - :return: str or None - """ - if l_name in labels: - return labels[l_name] - - def _get_container_id(self, labels): - """ - Should only be called on a container-scoped metric - It gets the container id from the podlist using the metrics labels - - :param labels - :return str or None - """ - namespace = CadvisorPrometheusScraperMixin._get_container_label(labels, "namespace") - # k8s >= 1.16 - pod_name = CadvisorPrometheusScraperMixin._get_container_label(labels, "pod") - container_name = CadvisorPrometheusScraperMixin._get_container_label(labels, "container") - # k8s < 1.16 - if not pod_name: - pod_name = CadvisorPrometheusScraperMixin._get_container_label(labels, "pod_name") - if not container_name: - container_name = CadvisorPrometheusScraperMixin._get_container_label(labels, "container_name") - return self.pod_list_utils.get_cid_by_name_tuple((namespace, pod_name, container_name)) - def _get_entity_id_if_container_metric(self, labels): """ Checks the labels indicate a container metric, @@ -190,7 +160,7 @@ def _get_entity_id_if_container_metric(self, labels): # If the pod is static, ContainerStatus is unavailable. # Return the pod UID so that we can collect metrics from it later on. return self._get_pod_uid(labels) - return self._get_container_id(labels) + return self.pod_list_utils.get_cid_by_labels(labels) def _get_pod_uid(self, labels): """ @@ -198,11 +168,11 @@ def _get_pod_uid(self, labels): :param labels: :return: str or None """ - namespace = CadvisorPrometheusScraperMixin._get_container_label(labels, "namespace") - pod_name = CadvisorPrometheusScraperMixin._get_container_label(labels, "pod") + namespace = get_container_label(labels, "namespace") + pod_name = get_container_label(labels, "pod") # k8s < 1.16 if not pod_name: - pod_name = CadvisorPrometheusScraperMixin._get_container_label(labels, "pod_name") + pod_name = get_container_label(labels, "pod_name") return self.pod_list_utils.get_uid_by_name_tuple((namespace, pod_name)) def _get_pod_uid_if_pod_metric(self, labels): @@ -246,10 +216,10 @@ def _get_kube_container_name(labels): :param labels: metric labels: iterable :return: list """ - container_name = CadvisorPrometheusScraperMixin._get_container_label(labels, "container") + container_name = get_container_label(labels, "container") # k8s < 1.16 if not container_name: - container_name = CadvisorPrometheusScraperMixin._get_container_label(labels, "container_name") + container_name = get_container_label(labels, "container_name") if container_name: return ["kube_container_name:%s" % container_name] return [] @@ -376,7 +346,7 @@ def _process_usage_metric(self, m_name, metric, cache, scraper_config, labels=No samples = self._sum_values_by_context(metric, self._get_entity_id_if_container_metric) for c_id, sample in iteritems(samples): - c_name = self._get_container_label(sample[self.SAMPLE_LABELS], 'name') + c_name = get_container_label(sample[self.SAMPLE_LABELS], 'name') if not c_name: continue pod_uid = self._get_pod_uid(sample[self.SAMPLE_LABELS]) @@ -436,7 +406,7 @@ def _process_limit_metric(self, m_name, metric, cache, scraper_config, pct_m_nam self.gauge(m_name, limit, tags) if pct_m_name and limit > 0: - c_name = self._get_container_label(sample[self.SAMPLE_LABELS], 'name') + c_name = get_container_label(sample[self.SAMPLE_LABELS], 'name') if not c_name: continue usage, tags = cache.get(c_name, (None, None)) diff --git a/kubelet/metadata.csv b/kubelet/metadata.csv index f6643ca009079..bb77ac7341421 100644 --- a/kubelet/metadata.csv +++ b/kubelet/metadata.csv @@ -74,3 +74,7 @@ kubernetes.kubelet.docker.operations.duration.sum,gauge,,operation,,The sum of d kubernetes.kubelet.docker.operations.duration.count,gauge,,,,The count of docker operations,0,kubernetes,k8s.docker.duration, kubernetes.go_threads,gauge,,,,Number of OS threads created,0,kubernetes,k8s.go.threads, kubernetes.go_goroutines,gauge,,,,Number of goroutines that currently exist,0,kubernetes,k8s.go.goroutines, +kubernetes.liveness_probe.success.total,count,,,,Cumulative number of successful liveness probe for a container (ALPHA in kubernetes v1.15),-1,kubernetes,k8s.liveness_probe.success.total, +kubernetes.liveness_probe.failure.total,count,,,,Cumulative number of failed liveness probe for a container (ALPHA in kubernetes v1.15),-1,kubernetes,k8s.liveness_probe.failure.total, +kubernetes.readiness_probe.success.total,count,,,,Cumulative number of successful readiness probe for a container (ALPHA in kubernetes v1.15),-1,kubernetes,k8s.liveness_probe.success.total, +kubernetes.readiness_probe.failure.total,count,,,,Cumulative number of failed readiness probe for a container (ALPHA in kubernetes v1.15),-1,kubernetes,k8s.liveness_probe.failure.total, diff --git a/kubelet/tests/fixtures/pod_list_probes.json b/kubelet/tests/fixtures/pod_list_probes.json new file mode 100644 index 0000000000000..320c9cffc6c6c --- /dev/null +++ b/kubelet/tests/fixtures/pod_list_probes.json @@ -0,0 +1,3235 @@ +{ + "kind": "PodList", + "apiVersion": "v1", + "metadata": {}, + "items": [ + { + "metadata": { + "name": "calico-typha-horizontal-autoscaler-f5888d4fd-44x5s", + "generateName": "calico-typha-horizontal-autoscaler-f5888d4fd-", + "namespace": "kube-system", + "selfLink": "/api/v1/namespaces/kube-system/pods/calico-typha-horizontal-autoscaler-f5888d4fd-44x5s", + "uid": "035421ef-ecba-4c70-9331-7f1df5272109", + "resourceVersion": "365696172", + "creationTimestamp": "2021-09-02T05:06:26Z", + "labels": { + "k8s-app": "calico-typha-autoscaler", + "pod-template-hash": "f5888d4fd" + }, + "annotations": { + "cni.projectcalico.org/podIP": "10.0.7.2/32", + "components.gke.io/component-name": "networkpolicy-calico", + "components.gke.io/component-version": "1.0.12", + "kubernetes.io/config.seen": "2021-09-02T05:06:26.609077882Z", + "kubernetes.io/config.source": "api" + }, + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "calico-typha-horizontal-autoscaler-f5888d4fd", + "uid": "019a0a4d-ee4e-49a9-9996-52a5f969e407", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "volumes": [ + { + "name": "typha-cpha-token-4xt2g", + "secret": { + "secretName": "typha-cpha-token-4xt2g", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "autoscaler", + "image": "gke.gcr.io/cluster-proportional-autoscaler-amd64:1.8.1-gke.0", + "command": [ + "/cluster-proportional-autoscaler", + "--namespace=kube-system", + "--configmap=calico-typha-horizontal-autoscaler", + "--target=deployment/calico-typha", + "--logtostderr=true", + "--v=2" + ], + "resources": { + "requests": { + "cpu": "10m" + } + }, + "volumeMounts": [ + { + "name": "typha-cpha-token-4xt2g", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "drop": [ + "all" + ] + }, + "allowPrivilegeEscalation": false + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "serviceAccountName": "typha-cpha", + "serviceAccount": "typha-cpha", + "nodeName": "gke-ahmed-default-pool-4658d5d4-q4j7", + "securityContext": { + "runAsUser": 1000, + "runAsGroup": 1000, + "supplementalGroups": [ + 65534 + ], + "fsGroup": 65534 + }, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "key": "components.gke.io/gke-managed-components", + "operator": "Exists" + }, + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + } + ], + "priorityClassName": "system-cluster-critical", + "priority": 2000000000, + "enableServiceLinks": true + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:26Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:31Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:31Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:26Z" + } + ], + "hostIP": "10.132.0.60", + "podIP": "10.0.7.2", + "podIPs": [ + { + "ip": "10.0.7.2" + } + ], + "startTime": "2021-09-02T05:06:26Z", + "containerStatuses": [ + { + "name": "autoscaler", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:30Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/cluster-proportional-autoscaler-amd64:1.8.1-gke.0", + "imageID": "docker-pullable://gke.gcr.io/cluster-proportional-autoscaler-amd64@sha256:3b600d2d5eafa6d5fcedbd783b28694af3f7dd470f3580e83200244fab32b35e", + "containerID": "docker://0189d6ba85f4b30889b1e0fdb4186abde101649bbe3c4f6a4e2462c4bdbb080a", + "started": true + } + ], + "qosClass": "Burstable" + } + }, + { + "metadata": { + "name": "gke-metrics-agent-8rw7w", + "generateName": "gke-metrics-agent-", + "namespace": "kube-system", + "selfLink": "/api/v1/namespaces/kube-system/pods/gke-metrics-agent-8rw7w", + "uid": "38f2cee8-82bc-4fc5-ae7a-6668496f13f3", + "resourceVersion": "452031504", + "creationTimestamp": "2022-02-11T05:15:06Z", + "labels": { + "component": "gke-metrics-agent", + "controller-revision-hash": "78f7c4f68b", + "k8s-app": "gke-metrics-agent", + "pod-template-generation": "8" + }, + "annotations": { + "components.gke.io/component-name": "gke-metrics-agent", + "components.gke.io/component-version": "0.51.0", + "configHash": "XdLNLINPXkYWpNcLitrLyttGblTNhbflhxgg0vV+NWg=", + "kubernetes.io/config.seen": "2022-02-11T05:15:06.155412393Z", + "kubernetes.io/config.source": "api" + }, + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "name": "gke-metrics-agent", + "uid": "e6af5655-db0a-48c5-bd3e-97d9b31e4004", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "volumes": [ + { + "name": "gke-metrics-agent-config-vol", + "configMap": { + "name": "gke-metrics-agent-conf", + "items": [ + { + "key": "gke-metrics-agent-config", + "path": "gke-metrics-agent-config.yaml" + } + ], + "defaultMode": 420 + } + }, + { + "name": "ssl-certs", + "hostPath": { + "path": "/etc/ssl/certs", + "type": "" + } + }, + { + "name": "gke-metrics-agent-token-pbbxz", + "secret": { + "secretName": "gke-metrics-agent-token-pbbxz", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "gke-metrics-agent", + "image": "gke.gcr.io/gke-metrics-agent:0.3.5-gke.0", + "command": [ + "/otelsvc", + "--config=/conf/gke-metrics-agent-config.yaml", + "--metrics-level=NONE", + "--legacy-metrics=true", + "--new-metrics=false", + "--metrics-prefix=" + ], + "env": [ + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "POD_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.name" + } + } + }, + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "KUBELET_HOST", + "value": "127.0.0.1" + }, + { + "name": "ARG1", + "value": "${1}" + }, + { + "name": "ARG2", + "value": "${2}" + }, + { + "name": "WINDOWS_JOB_ACTION", + "value": "drop" + } + ], + "resources": { + "limits": { + "memory": "50Mi" + }, + "requests": { + "cpu": "3m", + "memory": "50Mi" + } + }, + "volumeMounts": [ + { + "name": "gke-metrics-agent-config-vol", + "mountPath": "/conf" + }, + { + "name": "ssl-certs", + "readOnly": true, + "mountPath": "/etc/ssl/certs" + }, + { + "name": "gke-metrics-agent-token-pbbxz", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "drop": [ + "all" + ] + }, + "runAsUser": 1000, + "runAsGroup": 1000, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "nodeSelector": { + "kubernetes.io/os": "linux" + }, + "serviceAccountName": "gke-metrics-agent", + "serviceAccount": "gke-metrics-agent", + "nodeName": "gke-ahmed-default-pool-4658d5d4-q4j7", + "hostNetwork": true, + "securityContext": {}, + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-ahmed-default-pool-4658d5d4-q4j7" + ] + } + ] + } + ] + } + } + }, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "operator": "Exists", + "effect": "NoExecute" + }, + { + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "components.gke.io/gke-managed-components", + "operator": "Exists" + }, + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists", + "effect": "NoSchedule" + } + ], + "priorityClassName": "system-node-critical", + "priority": 2000001000, + "enableServiceLinks": true + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-02-11T05:15:06Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-02-11T05:15:07Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-02-11T05:15:07Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-02-11T05:15:06Z" + } + ], + "hostIP": "10.132.0.60", + "podIP": "10.132.0.60", + "podIPs": [ + { + "ip": "10.132.0.60" + } + ], + "startTime": "2022-02-11T05:15:06Z", + "containerStatuses": [ + { + "name": "gke-metrics-agent", + "state": { + "running": { + "startedAt": "2022-02-11T05:15:06Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/gke-metrics-agent:0.3.5-gke.0", + "imageID": "docker-pullable://gke.gcr.io/gke-metrics-agent@sha256:aabb1efa428778b876014d99399826319d81d96ee269e459deb81df7e78d7a57", + "containerID": "docker://a8285e7b1a38d3808c59caea80e3d7a14b425143f74af39895a995d963847071", + "started": true + } + ], + "qosClass": "Burstable" + } + }, + { + "metadata": { + "name": "kube-proxy-gke-ahmed-default-pool-4658d5d4-q4j7", + "namespace": "kube-system", + "selfLink": "/api/v1/namespaces/kube-system/pods/kube-proxy-gke-ahmed-default-pool-4658d5d4-q4j7", + "uid": "73e382c16da2abad565679de829d6c2f", + "creationTimestamp": null, + "labels": { + "component": "kube-proxy", + "tier": "node" + }, + "annotations": { + "kubernetes.io/config.hash": "73e382c16da2abad565679de829d6c2f", + "kubernetes.io/config.seen": "2021-09-02T05:05:45.692543046Z", + "kubernetes.io/config.source": "file" + } + }, + "spec": { + "volumes": [ + { + "name": "usr-ca-certs", + "hostPath": { + "path": "/usr/share/ca-certificates", + "type": "" + } + }, + { + "name": "etc-ssl-certs", + "hostPath": { + "path": "/etc/ssl/certs", + "type": "" + } + }, + { + "name": "kubeconfig", + "hostPath": { + "path": "/var/lib/kube-proxy/kubeconfig", + "type": "FileOrCreate" + } + }, + { + "name": "varlog", + "hostPath": { + "path": "/var/log", + "type": "" + } + }, + { + "name": "iptableslock", + "hostPath": { + "path": "/run/xtables.lock", + "type": "FileOrCreate" + } + }, + { + "name": "lib-modules", + "hostPath": { + "path": "/lib/modules", + "type": "" + } + } + ], + "containers": [ + { + "name": "kube-proxy", + "image": "gke.gcr.io/kube-proxy-amd64:v1.18.20-gke.901", + "command": [ + "/bin/sh", + "-c", + "exec kube-proxy --master=https://35.187.58.48 --kubeconfig=/var/lib/kube-proxy/kubeconfig --cluster-cidr=10.0.0.0/14 --oom-score-adj=-998 --v=2 --feature-gates=DynamicKubeletConfig=false,RotateKubeletServerCertificate=true --iptables-sync-period=1m --iptables-min-sync-period=10s --ipvs-sync-period=1m --ipvs-min-sync-period=10s --detect-local-mode=NodeCIDR 1\u003e\u003e/var/log/kube-proxy.log 2\u003e\u00261" + ], + "resources": { + "requests": { + "cpu": "100m" + } + }, + "volumeMounts": [ + { + "name": "etc-ssl-certs", + "readOnly": true, + "mountPath": "/etc/ssl/certs" + }, + { + "name": "usr-ca-certs", + "readOnly": true, + "mountPath": "/usr/share/ca-certificates" + }, + { + "name": "varlog", + "mountPath": "/var/log" + }, + { + "name": "kubeconfig", + "mountPath": "/var/lib/kube-proxy/kubeconfig" + }, + { + "name": "iptableslock", + "mountPath": "/run/xtables.lock" + }, + { + "name": "lib-modules", + "readOnly": true, + "mountPath": "/lib/modules" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "privileged": true + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "nodeName": "gke-ahmed-default-pool-4658d5d4-q4j7", + "hostNetwork": true, + "securityContext": {}, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "operator": "Exists", + "effect": "NoExecute" + }, + { + "operator": "Exists", + "effect": "NoSchedule" + } + ], + "priorityClassName": "system-node-critical", + "priority": 2000001000, + "enableServiceLinks": true + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:05:46Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:05:49Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:05:49Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:05:46Z" + } + ], + "hostIP": "10.132.0.60", + "podIP": "10.132.0.60", + "podIPs": [ + { + "ip": "10.132.0.60" + } + ], + "startTime": "2021-09-02T05:05:46Z", + "containerStatuses": [ + { + "name": "kube-proxy", + "state": { + "running": { + "startedAt": "2021-09-02T05:05:48Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/kube-proxy-amd64:v1.18.20-gke.901", + "imageID": "docker://sha256:77e1fa12f59b01e7e23de95ae01aacc6f09027575ec23b340bb2d6004945f8d4", + "containerID": "docker://fd6dc524098c6bc2bb7ae52c829d9a2fd263b76323ff1c006cf346339728377f", + "started": true + } + ], + "qosClass": "Burstable" + } + }, + { + "metadata": { + "name": "datadog-t9f28", + "generateName": "datadog-", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/datadog-t9f28", + "uid": "2e4619c2-15e1-4a2f-8b27-f1bb9d57b957", + "resourceVersion": "469639411", + "creationTimestamp": "2022-03-16T09:16:25Z", + "labels": { + "app": "datadog", + "app.kubernetes.io/component": "agent", + "app.kubernetes.io/instance": "datadog", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "datadog", + "controller-revision-hash": "67d57c74d7", + "pod-template-generation": "1" + }, + "annotations": { + "checksum/api_key": "8627e17d7513ba8d949a4806d0c175b1964f3b2547fe0d40082d463cdb8847e3", + "checksum/autoconf-config": "74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b", + "checksum/checksd-config": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "checksum/clusteragent_token": "df93ae3ed5cf9bc598a6b801b95aa10ca56b95300f92bd0c08a54dd3349e174b", + "checksum/confd-config": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "checksum/install_info": "c0246bdce957ff0547bb2815bd5d78737e2235af7eda8c588d0b7c1096d4ea12", + "cni.projectcalico.org/podIP": "10.0.7.128/32", + "kubernetes.io/config.seen": "2022-03-16T09:16:25.377025346Z", + "kubernetes.io/config.source": "api", + "kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu request for container agent; cpu request for container process-agent; cpu request for init container init-volume; cpu request for init container init-config" + }, + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "name": "datadog", + "uid": "58d3c436-a025-4a8e-ba45-09dc33a03389", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "volumes": [ + { + "name": "installinfo", + "configMap": { + "name": "datadog-installinfo", + "defaultMode": 420 + } + }, + { + "name": "config", + "emptyDir": {} + }, + { + "name": "logdatadog", + "emptyDir": {} + }, + { + "name": "tmpdir", + "emptyDir": {} + }, + { + "name": "procdir", + "hostPath": { + "path": "/proc", + "type": "" + } + }, + { + "name": "cgroups", + "hostPath": { + "path": "/sys/fs/cgroup", + "type": "" + } + }, + { + "name": "etc-os-release", + "hostPath": { + "path": "/etc/os-release", + "type": "" + } + }, + { + "name": "etc-redhat-release", + "hostPath": { + "path": "/etc/redhat-release", + "type": "" + } + }, + { + "name": "etc-fedora-release", + "hostPath": { + "path": "/etc/fedora-release", + "type": "" + } + }, + { + "name": "etc-lsb-release", + "hostPath": { + "path": "/etc/lsb-release", + "type": "" + } + }, + { + "name": "dsdsocket", + "hostPath": { + "path": "/var/run/datadog/", + "type": "DirectoryOrCreate" + } + }, + { + "name": "s6-run", + "emptyDir": {} + }, + { + "name": "passwd", + "hostPath": { + "path": "/etc/passwd", + "type": "" + } + }, + { + "name": "pointerdir", + "hostPath": { + "path": "/var/lib/datadog-agent/logs", + "type": "" + } + }, + { + "name": "logpodpath", + "hostPath": { + "path": "/var/log/pods", + "type": "" + } + }, + { + "name": "logscontainerspath", + "hostPath": { + "path": "/var/log/containers", + "type": "" + } + }, + { + "name": "logdockercontainerpath", + "hostPath": { + "path": "/var/lib/docker/containers", + "type": "" + } + }, + { + "name": "runtimesocketdir", + "hostPath": { + "path": "/var/run", + "type": "" + } + }, + { + "name": "datadog-token-c2vxr", + "secret": { + "secretName": "datadog-token-c2vxr", + "defaultMode": 420 + } + } + ], + "initContainers": [ + { + "name": "init-volume", + "image": "gcr.io/datadoghq/agent:7.35.0-rc.4", + "command": [ + "bash", + "-c" + ], + "args": [ + "cp -r /etc/datadog-agent /opt" + ], + "resources": { + "requests": { + "cpu": "100m" + } + }, + "volumeMounts": [ + { + "name": "config", + "mountPath": "/opt/datadog-agent" + }, + { + "name": "datadog-token-c2vxr", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "Always" + }, + { + "name": "init-config", + "image": "gcr.io/datadoghq/agent:7.35.0-rc.4", + "command": [ + "bash", + "-c" + ], + "args": [ + "for script in $(find /etc/cont-init.d/ -type f -name '*.sh' | sort) ; do bash $script ; done" + ], + "env": [ + { + "name": "GODEBUG", + "value": "x509ignoreCN=0" + }, + { + "name": "DD_API_KEY", + "valueFrom": { + "secretKeyRef": { + "name": "datadog", + "key": "api-key" + } + } + }, + { + "name": "DD_KUBERNETES_KUBELET_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "DD_KUBELET_TLS_VERIFY", + "value": "false" + }, + { + "name": "KUBERNETES", + "value": "yes" + } + ], + "resources": { + "requests": { + "cpu": "100m" + } + }, + "volumeMounts": [ + { + "name": "logdatadog", + "mountPath": "/var/log/datadog" + }, + { + "name": "config", + "mountPath": "/etc/datadog-agent" + }, + { + "name": "procdir", + "readOnly": true, + "mountPath": "/host/proc", + "mountPropagation": "None" + }, + { + "name": "runtimesocketdir", + "readOnly": true, + "mountPath": "/host/var/run", + "mountPropagation": "None" + }, + { + "name": "datadog-token-c2vxr", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "Always" + } + ], + "containers": [ + { + "name": "agent", + "image": "gcr.io/datadoghq/agent:7.35.0-rc.4", + "command": [ + "agent", + "run" + ], + "ports": [ + { + "name": "dogstatsdport", + "containerPort": 8125, + "protocol": "UDP" + } + ], + "env": [ + { + "name": "GODEBUG", + "value": "x509ignoreCN=0" + }, + { + "name": "DD_API_KEY", + "valueFrom": { + "secretKeyRef": { + "name": "datadog", + "key": "api-key" + } + } + }, + { + "name": "DD_KUBERNETES_KUBELET_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "DD_KUBELET_TLS_VERIFY", + "value": "false" + }, + { + "name": "KUBERNETES", + "value": "yes" + }, + { + "name": "DD_LOG_LEVEL", + "value": "INFO" + }, + { + "name": "DD_DOGSTATSD_PORT", + "value": "8125" + }, + { + "name": "DD_DOGSTATSD_NON_LOCAL_TRAFFIC", + "value": "true" + }, + { + "name": "DD_CLUSTER_AGENT_ENABLED", + "value": "true" + }, + { + "name": "DD_CLUSTER_AGENT_KUBERNETES_SERVICE_NAME", + "value": "datadog-cluster-agent" + }, + { + "name": "DD_CLUSTER_AGENT_AUTH_TOKEN", + "valueFrom": { + "secretKeyRef": { + "name": "datadog-cluster-agent", + "key": "token" + } + } + }, + { + "name": "DD_APM_ENABLED", + "value": "false" + }, + { + "name": "DD_LOGS_ENABLED", + "value": "true" + }, + { + "name": "DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL", + "value": "true" + }, + { + "name": "DD_LOGS_CONFIG_K8S_CONTAINER_USE_FILE", + "value": "true" + }, + { + "name": "DD_HEALTH_PORT", + "value": "5555" + }, + { + "name": "DD_DOGSTATSD_SOCKET", + "value": "/var/run/datadog/dsd.socket" + }, + { + "name": "DD_IGNORE_AUTOCONF", + "value": "kubernetes_state" + }, + { + "name": "DD_EXPVAR_PORT", + "value": "6000" + } + ], + "resources": { + "requests": { + "cpu": "100m" + } + }, + "volumeMounts": [ + { + "name": "installinfo", + "readOnly": true, + "mountPath": "/etc/datadog-agent/install_info", + "subPath": "install_info" + }, + { + "name": "logdatadog", + "mountPath": "/var/log/datadog" + }, + { + "name": "tmpdir", + "mountPath": "/tmp" + }, + { + "name": "config", + "mountPath": "/etc/datadog-agent" + }, + { + "name": "runtimesocketdir", + "readOnly": true, + "mountPath": "/host/var/run", + "mountPropagation": "None" + }, + { + "name": "dsdsocket", + "mountPath": "/var/run/datadog" + }, + { + "name": "procdir", + "readOnly": true, + "mountPath": "/host/proc", + "mountPropagation": "None" + }, + { + "name": "cgroups", + "readOnly": true, + "mountPath": "/host/sys/fs/cgroup", + "mountPropagation": "None" + }, + { + "name": "pointerdir", + "mountPath": "/opt/datadog-agent/run", + "mountPropagation": "None" + }, + { + "name": "logpodpath", + "readOnly": true, + "mountPath": "/var/log/pods", + "mountPropagation": "None" + }, + { + "name": "logscontainerspath", + "readOnly": true, + "mountPath": "/var/log/containers", + "mountPropagation": "None" + }, + { + "name": "logdockercontainerpath", + "readOnly": true, + "mountPath": "/var/lib/docker/containers", + "mountPropagation": "None" + }, + { + "name": "datadog-token-c2vxr", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "livenessProbe": { + "httpGet": { + "path": "/live", + "port": 5555, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "timeoutSeconds": 5, + "periodSeconds": 15, + "successThreshold": 1, + "failureThreshold": 6 + }, + "readinessProbe": { + "httpGet": { + "path": "/ready", + "port": 5555, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "timeoutSeconds": 5, + "periodSeconds": 15, + "successThreshold": 1, + "failureThreshold": 6 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "Always" + }, + { + "name": "process-agent", + "image": "gcr.io/datadoghq/agent:7.35.0-rc.4", + "command": [ + "process-agent", + "-config=/etc/datadog-agent/datadog.yaml" + ], + "env": [ + { + "name": "GODEBUG", + "value": "x509ignoreCN=0" + }, + { + "name": "DD_API_KEY", + "valueFrom": { + "secretKeyRef": { + "name": "datadog", + "key": "api-key" + } + } + }, + { + "name": "DD_KUBERNETES_KUBELET_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "DD_KUBELET_TLS_VERIFY", + "value": "false" + }, + { + "name": "KUBERNETES", + "value": "yes" + }, + { + "name": "DD_CLUSTER_AGENT_ENABLED", + "value": "true" + }, + { + "name": "DD_CLUSTER_AGENT_KUBERNETES_SERVICE_NAME", + "value": "datadog-cluster-agent" + }, + { + "name": "DD_CLUSTER_AGENT_AUTH_TOKEN", + "valueFrom": { + "secretKeyRef": { + "name": "datadog-cluster-agent", + "key": "token" + } + } + }, + { + "name": "DD_PROCESS_AGENT_DISCOVERY_ENABLED", + "value": "false" + }, + { + "name": "DD_LOG_LEVEL", + "value": "INFO" + }, + { + "name": "DD_SYSTEM_PROBE_ENABLED", + "value": "false" + }, + { + "name": "DD_DOGSTATSD_SOCKET", + "value": "/var/run/datadog/dsd.socket" + }, + { + "name": "DD_ORCHESTRATOR_EXPLORER_ENABLED", + "value": "true" + } + ], + "resources": { + "requests": { + "cpu": "100m" + } + }, + "volumeMounts": [ + { + "name": "config", + "mountPath": "/etc/datadog-agent" + }, + { + "name": "logdatadog", + "mountPath": "/var/log/datadog" + }, + { + "name": "tmpdir", + "mountPath": "/tmp" + }, + { + "name": "runtimesocketdir", + "readOnly": true, + "mountPath": "/host/var/run", + "mountPropagation": "None" + }, + { + "name": "cgroups", + "readOnly": true, + "mountPath": "/host/sys/fs/cgroup", + "mountPropagation": "None" + }, + { + "name": "passwd", + "readOnly": true, + "mountPath": "/etc/passwd" + }, + { + "name": "procdir", + "readOnly": true, + "mountPath": "/host/proc", + "mountPropagation": "None" + }, + { + "name": "dsdsocket", + "readOnly": true, + "mountPath": "/var/run/datadog" + }, + { + "name": "datadog-token-c2vxr", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "Always" + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "nodeSelector": { + "kubernetes.io/os": "linux" + }, + "serviceAccountName": "datadog", + "serviceAccount": "datadog", + "nodeName": "gke-ahmed-default-pool-4658d5d4-q4j7", + "securityContext": {}, + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-ahmed-default-pool-4658d5d4-q4j7" + ] + } + ] + } + ] + } + } + }, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists", + "effect": "NoSchedule" + } + ], + "priority": 0, + "enableServiceLinks": true + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-03-16T09:16:29Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-03-16T09:16:49Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-03-16T09:16:49Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-03-16T09:16:25Z" + } + ], + "hostIP": "10.132.0.60", + "podIP": "10.0.7.128", + "podIPs": [ + { + "ip": "10.0.7.128" + } + ], + "startTime": "2022-03-16T09:16:25Z", + "initContainerStatuses": [ + { + "name": "init-volume", + "state": { + "terminated": { + "exitCode": 0, + "reason": "Completed", + "startedAt": "2022-03-16T09:16:27Z", + "finishedAt": "2022-03-16T09:16:27Z", + "containerID": "docker://094e720ed0e2ff3e165dad8ed2e4ed49cd45f3bde7f567d0c20768e0f0ff8ec5" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gcr.io/datadoghq/agent:7.35.0-rc.4", + "imageID": "docker-pullable://gcr.io/datadoghq/agent@sha256:a65ca0ff6415ec63565d52d06e097b646faf392abe32518e36a5b81fd2bc8ca8", + "containerID": "docker://094e720ed0e2ff3e165dad8ed2e4ed49cd45f3bde7f567d0c20768e0f0ff8ec5" + }, + { + "name": "init-config", + "state": { + "terminated": { + "exitCode": 0, + "reason": "Completed", + "startedAt": "2022-03-16T09:16:29Z", + "finishedAt": "2022-03-16T09:16:29Z", + "containerID": "docker://fbeb65e16c9c98bad4abab849698d0d38a6acb04d248e32cd2f3a20246239d34" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gcr.io/datadoghq/agent:7.35.0-rc.4", + "imageID": "docker-pullable://gcr.io/datadoghq/agent@sha256:a65ca0ff6415ec63565d52d06e097b646faf392abe32518e36a5b81fd2bc8ca8", + "containerID": "docker://fbeb65e16c9c98bad4abab849698d0d38a6acb04d248e32cd2f3a20246239d34" + } + ], + "containerStatuses": [ + { + "name": "agent", + "state": { + "running": { + "startedAt": "2022-03-16T09:16:30Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gcr.io/datadoghq/agent:7.35.0-rc.4", + "imageID": "docker-pullable://gcr.io/datadoghq/agent@sha256:a65ca0ff6415ec63565d52d06e097b646faf392abe32518e36a5b81fd2bc8ca8", + "containerID": "docker://c81dfc25dd24b538a880bfd0f807ba9ec1ff4541e8b8eb49a8d1afcdecc5ef59", + "started": true + }, + { + "name": "process-agent", + "state": { + "running": { + "startedAt": "2022-03-16T09:16:31Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gcr.io/datadoghq/agent:7.35.0-rc.4", + "imageID": "docker-pullable://gcr.io/datadoghq/agent@sha256:a65ca0ff6415ec63565d52d06e097b646faf392abe32518e36a5b81fd2bc8ca8", + "containerID": "docker://a670bc474cfeb082ae1fdccf8039d58986938a30aa5be89d6c0fe3d82639734e", + "started": true + } + ], + "qosClass": "Burstable" + } + }, + { + "metadata": { + "name": "fluentbit-gke-45gvm", + "generateName": "fluentbit-gke-", + "namespace": "kube-system", + "selfLink": "/api/v1/namespaces/kube-system/pods/fluentbit-gke-45gvm", + "uid": "0310ae03-63ae-468e-b49d-7b78a7c5b0e9", + "resourceVersion": "365695845", + "creationTimestamp": "2021-09-02T05:05:46Z", + "labels": { + "component": "fluentbit-gke", + "controller-revision-hash": "5cc5f7866", + "k8s-app": "fluentbit-gke", + "kubernetes.io/cluster-service": "true", + "pod-template-generation": "8" + }, + "annotations": { + "EnableNodeJournal": "false", + "EnablePodSecurityPolicy": "false", + "SystemOnlyLogging": "false", + "components.gke.io/component-name": "fluentbit", + "components.gke.io/component-version": "1.4.4", + "kubernetes.io/config.seen": "2021-09-02T05:05:46.889702563Z", + "kubernetes.io/config.source": "api", + "monitoring.gke.io/path": "/api/v1/metrics/prometheus" + }, + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "name": "fluentbit-gke", + "uid": "60d147c0-5308-464b-a40f-79fbd1b38093", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "volumes": [ + { + "name": "varrun", + "hostPath": { + "path": "/var/run/google-fluentbit/pos-files", + "type": "" + } + }, + { + "name": "varlog", + "hostPath": { + "path": "/var/log", + "type": "" + } + }, + { + "name": "varlibkubeletpods", + "hostPath": { + "path": "/var/lib/kubelet/pods", + "type": "" + } + }, + { + "name": "varlibdockercontainers", + "hostPath": { + "path": "/var/lib/docker/containers", + "type": "" + } + }, + { + "name": "config-volume", + "configMap": { + "name": "fluentbit-gke-config-v1.0.6", + "defaultMode": 420 + } + }, + { + "name": "fluentbit-gke-token-qs6v5", + "secret": { + "secretName": "fluentbit-gke-token-qs6v5", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "fluentbit", + "image": "gke.gcr.io/fluent-bit:v1.5.7-gke.1", + "ports": [ + { + "name": "metrics", + "hostPort": 2020, + "containerPort": 2020, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "250Mi" + }, + "requests": { + "cpu": "50m", + "memory": "100Mi" + } + }, + "volumeMounts": [ + { + "name": "varrun", + "mountPath": "/var/run/google-fluentbit/pos-files" + }, + { + "name": "varlog", + "mountPath": "/var/log" + }, + { + "name": "varlibkubeletpods", + "mountPath": "/var/lib/kubelet/pods" + }, + { + "name": "varlibdockercontainers", + "readOnly": true, + "mountPath": "/var/lib/docker/containers" + }, + { + "name": "config-volume", + "mountPath": "/fluent-bit/etc/" + }, + { + "name": "fluentbit-gke-token-qs6v5", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "livenessProbe": { + "httpGet": { + "path": "/", + "port": 2020, + "scheme": "HTTP" + }, + "initialDelaySeconds": 120, + "timeoutSeconds": 1, + "periodSeconds": 60, + "successThreshold": 1, + "failureThreshold": 3 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent" + }, + { + "name": "fluentbit-gke", + "image": "gke.gcr.io/fluent-bit-gke-exporter:v0.16.2-gke.0", + "command": [ + "/fluent-bit-gke-exporter", + "--kubernetes-separator=_", + "--stackdriver-resource-model=k8s", + "--enable-pod-label-discovery", + "--pod-label-dot-replacement=_", + "--split-stdout-stderr", + "--logtostderr" + ], + "ports": [ + { + "name": "metrics", + "hostPort": 2021, + "containerPort": 2021, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "250Mi" + }, + "requests": { + "cpu": "50m", + "memory": "100Mi" + } + }, + "volumeMounts": [ + { + "name": "fluentbit-gke-token-qs6v5", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "livenessProbe": { + "httpGet": { + "path": "/healthz", + "port": 2021, + "scheme": "HTTP" + }, + "initialDelaySeconds": 120, + "timeoutSeconds": 1, + "periodSeconds": 60, + "successThreshold": 1, + "failureThreshold": 3 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "drop": [ + "all" + ] + }, + "runAsUser": 1000, + "runAsGroup": 1000, + "allowPrivilegeEscalation": false + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 120, + "dnsPolicy": "Default", + "nodeSelector": { + "kubernetes.io/os": "linux" + }, + "serviceAccountName": "fluentbit-gke", + "serviceAccount": "fluentbit-gke", + "nodeName": "gke-ahmed-default-pool-4658d5d4-q4j7", + "hostNetwork": true, + "securityContext": {}, + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-ahmed-default-pool-4658d5d4-q4j7" + ] + } + ] + } + ] + } + } + }, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "operator": "Exists", + "effect": "NoExecute" + }, + { + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "components.gke.io/gke-managed-components", + "operator": "Exists" + }, + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists", + "effect": "NoSchedule" + } + ], + "priorityClassName": "system-node-critical", + "priority": 2000001000, + "enableServiceLinks": true + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:05:46Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:12Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:12Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:05:46Z" + } + ], + "hostIP": "10.132.0.60", + "podIP": "10.132.0.60", + "podIPs": [ + { + "ip": "10.132.0.60" + } + ], + "startTime": "2021-09-02T05:05:46Z", + "containerStatuses": [ + { + "name": "fluentbit", + "state": { + "running": { + "startedAt": "2021-09-02T05:05:56Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/fluent-bit:v1.5.7-gke.1", + "imageID": "docker-pullable://gke.gcr.io/fluent-bit@sha256:7ee887d550fb472cda882212112dcf3cca313fffa318fc762032415e190a10f1", + "containerID": "docker://efa5b57cc110de6d2ca3b4a0e12c0a378090530e5e2d0ba0882dffe9c5846067", + "started": true + }, + { + "name": "fluentbit-gke", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:11Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/fluent-bit-gke-exporter:v0.16.2-gke.0", + "imageID": "docker-pullable://gke.gcr.io/fluent-bit-gke-exporter@sha256:31efae697729afb1f4c2fbf4de07961232c9b54668b4977b960a27f1ef9aafb1", + "containerID": "docker://3102f0d9499c5cd0875225208e3d048e3a9d829f5cdd74758b2d79a429a579fa", + "started": true + } + ], + "qosClass": "Burstable" + } + }, + { + "metadata": { + "name": "ip-masq-agent-27zzz", + "generateName": "ip-masq-agent-", + "namespace": "kube-system", + "selfLink": "/api/v1/namespaces/kube-system/pods/ip-masq-agent-27zzz", + "uid": "554cd8ec-67ac-489d-8496-9783c867f4d3", + "resourceVersion": "365695864", + "creationTimestamp": "2021-09-02T05:05:47Z", + "labels": { + "controller-revision-hash": "6fbc47b7b6", + "k8s-app": "ip-masq-agent", + "pod-template-generation": "2" + }, + "annotations": { + "kubernetes.io/config.seen": "2021-09-02T05:05:47.320682888Z", + "kubernetes.io/config.source": "api" + }, + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "name": "ip-masq-agent", + "uid": "5bd1679a-b7e7-4634-906d-3dc4b0d5279a", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "volumes": [ + { + "name": "config", + "configMap": { + "name": "ip-masq-agent", + "items": [ + { + "key": "config", + "path": "ip-masq-agent" + } + ], + "defaultMode": 420, + "optional": true + } + }, + { + "name": "ip-masq-agent-token-sm9h7", + "secret": { + "secretName": "ip-masq-agent-token-sm9h7", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "ip-masq-agent", + "image": "k8s.gcr.io/ip-masq-agent-amd64:v2.4.1", + "args": [ + "--masq-chain=IP-MASQ", + "--nomasq-all-reserved-ranges" + ], + "resources": { + "requests": { + "cpu": "10m", + "memory": "16Mi" + } + }, + "volumeMounts": [ + { + "name": "config", + "mountPath": "/etc/config" + }, + { + "name": "ip-masq-agent-token-sm9h7", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "privileged": true + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "nodeSelector": { + "beta.kubernetes.io/os": "linux", + "node.kubernetes.io/masq-agent-ds-ready": "true" + }, + "serviceAccountName": "ip-masq-agent", + "serviceAccount": "ip-masq-agent", + "nodeName": "gke-ahmed-default-pool-4658d5d4-q4j7", + "hostNetwork": true, + "securityContext": {}, + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-ahmed-default-pool-4658d5d4-q4j7" + ] + } + ] + } + ] + } + } + }, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "CriticalAddonsOnly", + "operator": "Exists" + }, + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists", + "effect": "NoSchedule" + } + ], + "priorityClassName": "system-node-critical", + "priority": 2000001000, + "enableServiceLinks": true + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:05:47Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:02Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:02Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:05:47Z" + } + ], + "hostIP": "10.132.0.60", + "podIP": "10.132.0.60", + "podIPs": [ + { + "ip": "10.132.0.60" + } + ], + "startTime": "2021-09-02T05:05:47Z", + "containerStatuses": [ + { + "name": "ip-masq-agent", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:01Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "k8s.gcr.io/ip-masq-agent-amd64:v2.4.1", + "imageID": "docker-pullable://k8s.gcr.io/ip-masq-agent-amd64@sha256:eb43b4cdc43a50260b82fdd63208e0b2d401ec830b486d0c7a570a6d73854451", + "containerID": "docker://294650160ef864d861b38b84d7d8394fbe349ae7bb6151b5df99f8a6c4b3304f", + "started": true + } + ], + "qosClass": "Burstable" + } + }, + { + "metadata": { + "name": "calico-node-9qkw7", + "generateName": "calico-node-", + "namespace": "kube-system", + "selfLink": "/api/v1/namespaces/kube-system/pods/calico-node-9qkw7", + "uid": "6daeeb11-1f06-4d07-ac39-7df7aad8ae86", + "resourceVersion": "365695866", + "creationTimestamp": "2021-09-02T05:05:47Z", + "labels": { + "controller-revision-hash": "c596fbd5c", + "k8s-app": "calico-node", + "pod-template-generation": "6" + }, + "annotations": { + "components.gke.io/component-name": "networkpolicy-calico", + "components.gke.io/component-version": "1.0.12", + "kubernetes.io/config.seen": "2021-09-02T05:05:47.334036841Z", + "kubernetes.io/config.source": "api", + "scheduler.alpha.kubernetes.io/critical-pod": "" + }, + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "name": "calico-node", + "uid": "ba375172-ac7a-4457-a470-5b362d27bd84", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "volumes": [ + { + "name": "lib-modules", + "hostPath": { + "path": "/lib/modules", + "type": "" + } + }, + { + "name": "cni-bin-dir", + "hostPath": { + "path": "/home/kubernetes/bin", + "type": "" + } + }, + { + "name": "cni-net-dir", + "hostPath": { + "path": "/etc/cni/net.d", + "type": "" + } + }, + { + "name": "var-run-calico", + "hostPath": { + "path": "/var/run/calico", + "type": "" + } + }, + { + "name": "var-lib-calico", + "hostPath": { + "path": "/var/lib/calico", + "type": "" + } + }, + { + "name": "xtables-lock", + "hostPath": { + "path": "/run/xtables.lock", + "type": "FileOrCreate" + } + }, + { + "name": "calico-sa-token-qtjrx", + "secret": { + "secretName": "calico-sa-token-qtjrx", + "defaultMode": 420 + } + } + ], + "initContainers": [ + { + "name": "install-cni", + "image": "gke.gcr.io/calico/cni:v3.8.8-1-gke.2-amd64", + "command": [ + "/install-cni.sh" + ], + "env": [ + { + "name": "CNI_NETWORK_CONFIG", + "value": "{\n \"name\": \"k8s-pod-network\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": [\n {\n \"type\": \"calico\",\n \"mtu\": 1460,\n \"log_level\": \"warning\",\n \"datastore_type\": \"kubernetes\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n \"ipam\": {\n \"type\": \"host-local\",\n \"subnet\": \"usePodCidr\"\n },\n \"policy\": {\n \"type\": \"k8s\"\n },\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n }\n },\n {\n \"type\": \"portmap\",\n \"capabilities\": {\"portMappings\": true},\n \"snat\": true\n },\n {\n \"type\": \"bandwidth\",\n \"capabilities\": {\"bandwidth\": true}\n }\n ]\n}" + }, + { + "name": "CNI_CONF_NAME", + "value": "10-calico.conflist" + }, + { + "name": "KUBERNETES_NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "SLEEP", + "value": "false" + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "cni-bin-dir", + "mountPath": "/host/opt/cni/bin" + }, + { + "name": "cni-net-dir", + "mountPath": "/host/etc/cni/net.d" + }, + { + "name": "calico-sa-token-qtjrx", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent" + } + ], + "containers": [ + { + "name": "calico-node", + "image": "gke.gcr.io/calico/node:v3.8.8-1-gke.2-amd64", + "env": [ + { + "name": "CALICO_MANAGE_CNI", + "value": "true" + }, + { + "name": "CALICO_DISABLE_FILE_LOGGING", + "value": "true" + }, + { + "name": "CALICO_NETWORKING_BACKEND", + "value": "none" + }, + { + "name": "DATASTORE_TYPE", + "value": "kubernetes" + }, + { + "name": "FELIX_DEFAULTENDPOINTTOHOSTACTION", + "value": "ACCEPT" + }, + { + "name": "FELIX_HEALTHENABLED", + "value": "true" + }, + { + "name": "FELIX_IGNORELOOSERPF", + "value": "false" + }, + { + "name": "FELIX_IPTABLESMANGLEALLOWACTION", + "value": "RETURN" + }, + { + "name": "FELIX_IPV6SUPPORT", + "value": "false" + }, + { + "name": "FELIX_LOGFILEPATH", + "value": "none" + }, + { + "name": "FELIX_LOGSEVERITYSYS", + "value": "none" + }, + { + "name": "FELIX_LOGSEVERITYSCREEN", + "value": "warning" + }, + { + "name": "FELIX_PROMETHEUSMETRICSENABLED", + "value": "true" + }, + { + "name": "FELIX_REPORTINGINTERVALSECS", + "value": "0" + }, + { + "name": "FELIX_TYPHAK8SSERVICENAME", + "value": "calico-typha" + }, + { + "name": "USE_POD_CIDR", + "value": "true" + }, + { + "name": "IP", + "value": "autodetect" + }, + { + "name": "NO_DEFAULT_POOLS", + "value": "true" + }, + { + "name": "NODENAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "WAIT_FOR_DATASTORE", + "value": "true" + } + ], + "resources": { + "requests": { + "cpu": "100m" + } + }, + "volumeMounts": [ + { + "name": "cni-net-dir", + "mountPath": "/host/etc/cni/net.d" + }, + { + "name": "lib-modules", + "readOnly": true, + "mountPath": "/lib/modules" + }, + { + "name": "var-run-calico", + "mountPath": "/var/run/calico" + }, + { + "name": "var-lib-calico", + "mountPath": "/var/lib/calico" + }, + { + "name": "xtables-lock", + "mountPath": "/run/xtables.lock" + }, + { + "name": "calico-sa-token-qtjrx", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "livenessProbe": { + "httpGet": { + "path": "/liveness", + "port": 9099, + "host": "127.0.0.1", + "scheme": "HTTP" + }, + "initialDelaySeconds": 10, + "timeoutSeconds": 1, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 6 + }, + "readinessProbe": { + "httpGet": { + "path": "/readiness", + "port": 9099, + "host": "127.0.0.1", + "scheme": "HTTP" + }, + "timeoutSeconds": 1, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 3 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "privileged": true + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 0, + "dnsPolicy": "ClusterFirst", + "nodeSelector": { + "beta.kubernetes.io/os": "linux", + "projectcalico.org/ds-ready": "true" + }, + "serviceAccountName": "calico-sa", + "serviceAccount": "calico-sa", + "nodeName": "gke-ahmed-default-pool-4658d5d4-q4j7", + "hostNetwork": true, + "securityContext": {}, + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-ahmed-default-pool-4658d5d4-q4j7" + ] + } + ] + } + ] + } + } + }, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "CriticalAddonsOnly", + "operator": "Exists" + }, + { + "key": "components.gke.io/gke-managed-components", + "operator": "Exists" + }, + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute" + }, + { + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists", + "effect": "NoSchedule" + }, + { + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists", + "effect": "NoSchedule" + } + ], + "priorityClassName": "system-node-critical", + "priority": 2000001000, + "enableServiceLinks": true + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:11Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-02-17T19:05:42Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2022-02-17T19:05:42Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:05:47Z" + } + ], + "hostIP": "10.132.0.60", + "podIP": "10.132.0.60", + "podIPs": [ + { + "ip": "10.132.0.60" + } + ], + "startTime": "2021-09-02T05:05:47Z", + "initContainerStatuses": [ + { + "name": "install-cni", + "state": { + "terminated": { + "exitCode": 0, + "reason": "Completed", + "startedAt": "2021-09-02T05:06:10Z", + "finishedAt": "2021-09-02T05:06:10Z", + "containerID": "docker://2f6c4dc19b2d35835eda8ae9e9c3557ab42abf8f258ea7cb541b48c06f1188c1" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/calico/cni:v3.8.8-1-gke.2-amd64", + "imageID": "docker-pullable://gke.gcr.io/calico/cni@sha256:06ff571b3d08181a8184a621851b5151736e34d2462503518d73123ed8a3e547", + "containerID": "docker://2f6c4dc19b2d35835eda8ae9e9c3557ab42abf8f258ea7cb541b48c06f1188c1" + } + ], + "containerStatuses": [ + { + "name": "calico-node", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:18Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/calico/node:v3.8.8-1-gke.2-amd64", + "imageID": "docker-pullable://gke.gcr.io/calico/node@sha256:ea105837c7d19dd79fee246e867a8edd8cac3d7836450da611eb3ce4bd52055f", + "containerID": "docker://1669a6277ebb44aedd2790ba8bce83d21899ba85b1afde4330caf4a4161eee26", + "started": true + } + ], + "qosClass": "Burstable" + } + }, + { + "metadata": { + "name": "event-exporter-gke-666b7ffbf7-8vfcx", + "generateName": "event-exporter-gke-666b7ffbf7-", + "namespace": "kube-system", + "selfLink": "/api/v1/namespaces/kube-system/pods/event-exporter-gke-666b7ffbf7-8vfcx", + "uid": "1e620d75-a8f8-4a3a-b21f-c09f7f61e53b", + "resourceVersion": "365696177", + "creationTimestamp": "2021-09-02T05:06:26Z", + "labels": { + "k8s-app": "event-exporter", + "pod-template-hash": "666b7ffbf7", + "version": "v0.3.4" + }, + "annotations": { + "cni.projectcalico.org/podIP": "10.0.7.4/32", + "components.gke.io/component-name": "event-exporter", + "components.gke.io/component-version": "1.0.9", + "kubernetes.io/config.seen": "2021-09-02T05:06:26.634076900Z", + "kubernetes.io/config.source": "api" + }, + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "event-exporter-gke-666b7ffbf7", + "uid": "8bb72421-8578-43f6-bbff-838728d4e5ac", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "volumes": [ + { + "name": "ssl-certs", + "hostPath": { + "path": "/etc/ssl/certs", + "type": "" + } + }, + { + "name": "event-exporter-sa-token-m9qx6", + "secret": { + "secretName": "event-exporter-sa-token-m9qx6", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "event-exporter", + "image": "gke.gcr.io/event-exporter:v0.3.4-gke.0", + "command": [ + "/event-exporter", + "-sink-opts=-stackdriver-resource-model=new -endpoint=https://logging.googleapis.com", + "-prometheus-endpoint=:8080" + ], + "resources": {}, + "volumeMounts": [ + { + "name": "event-exporter-sa-token-m9qx6", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "drop": [ + "all" + ] + }, + "allowPrivilegeEscalation": false + } + }, + { + "name": "prometheus-to-sd-exporter", + "image": "gke.gcr.io/prometheus-to-sd:v0.10.0-gke.0", + "command": [ + "/monitor", + "--stackdriver-prefix=container.googleapis.com/internal/addons", + "--api-override=https://monitoring.googleapis.com/", + "--source=event_exporter:http://localhost:8080?whitelisted=stackdriver_sink_received_entry_count,stackdriver_sink_request_count,stackdriver_sink_successfully_sent_entry_count", + "--pod-id=$(POD_NAME)", + "--namespace-id=$(POD_NAMESPACE)", + "--node-name=$(NODE_NAME)" + ], + "env": [ + { + "name": "POD_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.name" + } + } + }, + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "event-exporter-sa-token-m9qx6", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "drop": [ + "all" + ] + }, + "allowPrivilegeEscalation": false + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 120, + "dnsPolicy": "ClusterFirst", + "nodeSelector": { + "kubernetes.io/os": "linux" + }, + "serviceAccountName": "event-exporter-sa", + "serviceAccount": "event-exporter-sa", + "nodeName": "gke-ahmed-default-pool-4658d5d4-q4j7", + "securityContext": { + "runAsUser": 1000, + "runAsGroup": 1000 + }, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "key": "components.gke.io/gke-managed-components", + "operator": "Exists" + }, + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + } + ], + "priority": 0, + "enableServiceLinks": true + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:26Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:40Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:40Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:26Z" + } + ], + "hostIP": "10.132.0.60", + "podIP": "10.0.7.4", + "podIPs": [ + { + "ip": "10.0.7.4" + } + ], + "startTime": "2021-09-02T05:06:26Z", + "containerStatuses": [ + { + "name": "event-exporter", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:34Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/event-exporter:v0.3.4-gke.0", + "imageID": "docker-pullable://gke.gcr.io/event-exporter@sha256:34b034af960047020f4674151bbb2b55878f82a0b76dabca81b518e9fe82180d", + "containerID": "docker://422f6ae9162cd2f771954c474cca743869a1c332e95a1bbcf6afd12c53e27f41", + "started": true + }, + { + "name": "prometheus-to-sd-exporter", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:40Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/prometheus-to-sd:v0.10.0-gke.0", + "imageID": "docker-pullable://gke.gcr.io/prometheus-to-sd@sha256:c5e12680a431990d5e39ed249dcb43a7672e99f7ef94a9928be40cf2f418f62f", + "containerID": "docker://2dde62ce74adf0cc04ef2c4489255a9695f5097e730b9d3fb2d2e52ff02f8c95", + "started": true + } + ], + "qosClass": "BestEffort" + } + }, + { + "metadata": { + "name": "kube-dns-c598bd956-wgf4n", + "generateName": "kube-dns-c598bd956-", + "namespace": "kube-system", + "selfLink": "/api/v1/namespaces/kube-system/pods/kube-dns-c598bd956-wgf4n", + "uid": "03bc86a4-c87a-4537-9b0f-17e52cfd3162", + "resourceVersion": "365696195", + "creationTimestamp": "2021-09-02T05:06:26Z", + "labels": { + "k8s-app": "kube-dns", + "pod-template-hash": "c598bd956" + }, + "annotations": { + "cni.projectcalico.org/podIP": "10.0.7.3/32", + "kubernetes.io/config.seen": "2021-09-02T05:06:26.821244728Z", + "kubernetes.io/config.source": "api", + "scheduler.alpha.kubernetes.io/critical-pod": "", + "seccomp.security.alpha.kubernetes.io/pod": "runtime/default" + }, + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "kube-dns-c598bd956", + "uid": "1755736f-32d6-4458-a380-6fa34400261c", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "volumes": [ + { + "name": "kube-dns-config", + "configMap": { + "name": "kube-dns", + "defaultMode": 420, + "optional": true + } + }, + { + "name": "kube-dns-token-nspgq", + "secret": { + "secretName": "kube-dns-token-nspgq", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "kubedns", + "image": "gke.gcr.io/k8s-dns-kube-dns-amd64:1.17.3-gke.0", + "args": [ + "--domain=cluster.local.", + "--dns-port=10053", + "--config-dir=/kube-dns-config", + "--v=2" + ], + "ports": [ + { + "name": "dns-local", + "containerPort": 10053, + "protocol": "UDP" + }, + { + "name": "dns-tcp-local", + "containerPort": 10053, + "protocol": "TCP" + }, + { + "name": "metrics", + "containerPort": 10055, + "protocol": "TCP" + } + ], + "env": [ + { + "name": "PROMETHEUS_PORT", + "value": "10055" + } + ], + "resources": { + "limits": { + "memory": "210Mi" + }, + "requests": { + "cpu": "100m", + "memory": "70Mi" + } + }, + "volumeMounts": [ + { + "name": "kube-dns-config", + "mountPath": "/kube-dns-config" + }, + { + "name": "kube-dns-token-nspgq", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "livenessProbe": { + "httpGet": { + "path": "/healthcheck/kubedns", + "port": 10054, + "scheme": "HTTP" + }, + "initialDelaySeconds": 60, + "timeoutSeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 5 + }, + "readinessProbe": { + "httpGet": { + "path": "/readiness", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 3, + "timeoutSeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 3 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "runAsUser": 1001, + "runAsGroup": 1001, + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false + } + }, + { + "name": "dnsmasq", + "image": "gke.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.17.3-gke.0", + "args": [ + "-v=2", + "-logtostderr", + "-configDir=/etc/k8s/dns/dnsmasq-nanny", + "-restartDnsmasq=true", + "--", + "-k", + "--cache-size=1000", + "--no-negcache", + "--dns-forward-max=1500", + "--log-facility=-", + "--server=/cluster.local/127.0.0.1#10053", + "--server=/in-addr.arpa/127.0.0.1#10053", + "--server=/ip6.arpa/127.0.0.1#10053" + ], + "ports": [ + { + "name": "dns", + "containerPort": 53, + "protocol": "UDP" + }, + { + "name": "dns-tcp", + "containerPort": 53, + "protocol": "TCP" + } + ], + "resources": { + "requests": { + "cpu": "150m", + "memory": "20Mi" + } + }, + "volumeMounts": [ + { + "name": "kube-dns-config", + "mountPath": "/etc/k8s/dns/dnsmasq-nanny" + }, + { + "name": "kube-dns-token-nspgq", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "livenessProbe": { + "httpGet": { + "path": "/healthcheck/dnsmasq", + "port": 10054, + "scheme": "HTTP" + }, + "initialDelaySeconds": 60, + "timeoutSeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 5 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "add": [ + "NET_BIND_SERVICE", + "SETGID" + ], + "drop": [ + "all" + ] + } + } + }, + { + "name": "sidecar", + "image": "gke.gcr.io/k8s-dns-sidecar-amd64:1.17.3-gke.0", + "args": [ + "--v=2", + "--logtostderr", + "--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,SRV", + "--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,SRV" + ], + "ports": [ + { + "name": "metrics", + "containerPort": 10054, + "protocol": "TCP" + } + ], + "resources": { + "requests": { + "cpu": "10m", + "memory": "20Mi" + } + }, + "volumeMounts": [ + { + "name": "kube-dns-token-nspgq", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "livenessProbe": { + "httpGet": { + "path": "/metrics", + "port": 10054, + "scheme": "HTTP" + }, + "initialDelaySeconds": 60, + "timeoutSeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 5 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "runAsUser": 1001, + "runAsGroup": 1001, + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false + } + }, + { + "name": "prometheus-to-sd", + "image": "gke.gcr.io/prometheus-to-sd:v0.4.2", + "command": [ + "/monitor", + "--source=kubedns:http://localhost:10054?whitelisted=probe_kubedns_latency_ms,probe_kubedns_errors,dnsmasq_misses,dnsmasq_hits", + "--stackdriver-prefix=container.googleapis.com/internal/addons", + "--api-override=https://monitoring.googleapis.com/", + "--pod-id=$(POD_NAME)", + "--namespace-id=$(POD_NAMESPACE)", + "--v=2" + ], + "env": [ + { + "name": "POD_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.name" + } + } + }, + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "kube-dns-token-nspgq", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "runAsUser": 1001, + "runAsGroup": 1001, + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "Default", + "nodeSelector": { + "kubernetes.io/os": "linux" + }, + "serviceAccountName": "kube-dns", + "serviceAccount": "kube-dns", + "nodeName": "gke-ahmed-default-pool-4658d5d4-q4j7", + "securityContext": { + "supplementalGroups": [ + 65534 + ], + "fsGroup": 65534 + }, + "affinity": { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "weight": 100, + "podAffinityTerm": { + "labelSelector": { + "matchExpressions": [ + { + "key": "k8s-app", + "operator": "In", + "values": [ + "kube-dns" + ] + } + ] + }, + "topologyKey": "kubernetes.io/hostname" + } + } + ] + } + }, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "key": "CriticalAddonsOnly", + "operator": "Exists" + }, + { + "key": "components.gke.io/gke-managed-components", + "operator": "Exists" + }, + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + } + ], + "priorityClassName": "system-cluster-critical", + "priority": 2000000000, + "enableServiceLinks": true + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:26Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:57Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:57Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2021-09-02T05:06:26Z" + } + ], + "hostIP": "10.132.0.60", + "podIP": "10.0.7.3", + "podIPs": [ + { + "ip": "10.0.7.3" + } + ], + "startTime": "2021-09-02T05:06:26Z", + "containerStatuses": [ + { + "name": "dnsmasq", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:42Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.17.3-gke.0", + "imageID": "docker-pullable://gke.gcr.io/k8s-dns-dnsmasq-nanny-amd64@sha256:6f59f610ddc1e0279b83705d545b668f77733a9bb74db1afdf5b575fe862f5ab", + "containerID": "docker://0d8eea0b23688a4c3fbc29828b455734b323d6aac085c88f8f112e296cd5b521", + "started": true + }, + { + "name": "kubedns", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:38Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/k8s-dns-kube-dns-amd64:1.17.3-gke.0", + "imageID": "docker-pullable://gke.gcr.io/k8s-dns-kube-dns-amd64@sha256:f5210cf47c3d04c72835499fdade27f176ebedb7316172c560251d3dbd5180fb", + "containerID": "docker://b13f7638c80c98946900bdeabec06be564d203330f5bb706a40e6fa7466a674d", + "started": true + }, + { + "name": "prometheus-to-sd", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:47Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/prometheus-to-sd:v0.4.2", + "imageID": "docker-pullable://gke.gcr.io/prometheus-to-sd@sha256:aca8ef83a7fae83f1f8583e978dd4d1ff655b9f2ca0a76bda5edce6d8965bdf2", + "containerID": "docker://1d964407d20a1881f43f97d59755241225ea7180289022b169753fa3c828a502", + "started": true + }, + { + "name": "sidecar", + "state": { + "running": { + "startedAt": "2021-09-02T05:06:45Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "gke.gcr.io/k8s-dns-sidecar-amd64:1.17.3-gke.0", + "imageID": "docker-pullable://gke.gcr.io/k8s-dns-sidecar-amd64@sha256:4a0754f097979fd6fc957fd3e6371be82e278a3dc1643eec69a1b48b2d34c874", + "containerID": "docker://2c3f5608164033a850c9acbbfdb7fffa6ce1f68feedb1b8dad99777373c35b16", + "started": true + } + ], + "qosClass": "Burstable" + } + } + ] +} \ No newline at end of file diff --git a/kubelet/tests/fixtures/probes.txt b/kubelet/tests/fixtures/probes.txt new file mode 100644 index 0000000000000..8db7c7f8aa656 --- /dev/null +++ b/kubelet/tests/fixtures/probes.txt @@ -0,0 +1,17 @@ +# HELP prober_probe_total [ALPHA] Cumulative number of a liveness, readiness or startup probe for a container by result. +# TYPE prober_probe_total counter +prober_probe_total{container="agent",namespace="default",pod="datadog-t9f28",pod_uid="2e4619c2-15e1-4a2f-8b27-f1bb9d57b957",probe_type="Liveness",result="successful"} 3 +prober_probe_total{container="agent",namespace="default",pod="datadog-t9f28",pod_uid="2e4619c2-15e1-4a2f-8b27-f1bb9d57b957",probe_type="Readiness",result="successful"} 3 +prober_probe_total{container="calico-node",namespace="kube-system",pod="calico-node-9qkw7",pod_uid="6daeeb11-1f06-4d07-ac39-7df7aad8ae86",probe_type="Liveness",result="successful"} 1.686306e+06 +prober_probe_total{container="calico-node",namespace="kube-system",pod="calico-node-9qkw7",pod_uid="6daeeb11-1f06-4d07-ac39-7df7aad8ae86",probe_type="Readiness",result="failed"} 180 +prober_probe_total{container="calico-node",namespace="kube-system",pod="calico-node-9qkw7",pod_uid="6daeeb11-1f06-4d07-ac39-7df7aad8ae86",probe_type="Liveness",result="failed"} 100 +prober_probe_total{container="calico-node",namespace="kube-system",pod="calico-node-9qkw7",pod_uid="6daeeb11-1f06-4d07-ac39-7df7aad8ae86",probe_type="Readiness",result="successful"} 1.686127e+06 +prober_probe_total{container="dnsmasq",namespace="kube-system",pod="kube-dns-c598bd956-wgf4n",pod_uid="03bc86a4-c87a-4537-9b0f-17e52cfd3162",probe_type="Liveness",result="successful"} 1.686298e+06 +prober_probe_total{container="fluentbit",namespace="kube-system",pod="fluentbit-gke-45gvm",pod_uid="0310ae03-63ae-468e-b49d-7b78a7c5b0e9",probe_type="Liveness",result="successful"} 281049 +prober_probe_total{container="fluentbit-gke",namespace="kube-system",pod="fluentbit-gke-45gvm",pod_uid="0310ae03-63ae-468e-b49d-7b78a7c5b0e9",probe_type="Liveness",result="successful"} 281049 +prober_probe_total{container="kubedns",namespace="kube-system",pod="kube-dns-c598bd956-wgf4n",pod_uid="03bc86a4-c87a-4537-9b0f-17e52cfd3162",probe_type="Liveness",result="successful"} 1.686298e+06 +prober_probe_total{container="kubedns",namespace="kube-system",pod="kube-dns-c598bd956-wgf4n",pod_uid="03bc86a4-c87a-4537-9b0f-17e52cfd3162",probe_type="Readiness",result="successful"} 1.686303e+06 +prober_probe_total{container="sidecar",namespace="kube-system",pod="kube-dns-c598bd956-wgf4n",pod_uid="03bc86a4-c87a-4537-9b0f-17e52cfd3162",probe_type="Liveness",result="successful"} 1.686298e+06 +# HELP process_start_time_seconds [ALPHA] Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.63055914421e+09 diff --git a/kubelet/tests/test_common.py b/kubelet/tests/test_common.py index f4c5a9608f3b2..6c578d7fc81d1 100644 --- a/kubelet/tests/test_common.py +++ b/kubelet/tests/test_common.py @@ -11,6 +11,7 @@ from datadog_checks.base.checks.kubelet_base.base import KubeletCredentials, urljoin from datadog_checks.base.checks.openmetrics import OpenMetricsBaseCheck from datadog_checks.kubelet import PodListUtils, get_pod_by_uid, is_static_pending_pod +from datadog_checks.kubelet.common import get_container_label from .test_kubelet import mock_from_file @@ -215,3 +216,26 @@ def test_credentials_token_noverify(): assert scraper_config['ssl_cert'] is None assert scraper_config['ssl_private_key'] is None assert scraper_config['extra_headers'] == {} + + +def test_get_container_label(): + labels = {"container_name": "POD", "id": "/kubepods/burstable/pod531c80d9-9fc4-11e7-ba8b-42010af002bb"} + assert get_container_label(labels, "container_name") == "POD" + assert get_container_label({}, "not-in") is None + + +def test_get_cid_by_labels(): + pods = json.loads(mock_from_file('podlist_containerd.json')) + pod_list_utils = PodListUtils(pods) + + # k8s >= 1.16 + labels = {"container": "datadog-agent", "namespace": "default", "pod": "datadog-agent-pbqt2"} + container_id = pod_list_utils.get_cid_by_labels(labels) + assert container_id == "containerd://51cba2ca229069039575750d44ed3a67e9b5ead651312ba7ff218dd9202fde64" + + # k8s < 1.16 + labels = {"container_name": "datadog-agent", "namespace": "default", "pod_name": "datadog-agent-pbqt2"} + container_id = pod_list_utils.get_cid_by_labels(labels) + assert container_id == "containerd://51cba2ca229069039575750d44ed3a67e9b5ead651312ba7ff218dd9202fde64" + + assert pod_list_utils.get_cid_by_labels([]) is None diff --git a/kubelet/tests/test_kubelet.py b/kubelet/tests/test_kubelet.py index a7e8db6a4bc21..3b0c34b7eb2f0 100644 --- a/kubelet/tests/test_kubelet.py +++ b/kubelet/tests/test_kubelet.py @@ -212,6 +212,44 @@ ], } +PROBE_TAGS = { + 'container_id://2c3f5608164033a850c9acbbfdb7fffa6ce1f68feedb1b8dad99777373c35b16': [ + 'kube_namespace:kube-system', + 'pod_name:kube-dns-c598bd956-wgf4n', + 'kube_container_name:sidecar', + ], + 'container_id://b13f7638c80c98946900bdeabec06be564d203330f5bb706a40e6fa7466a674d': [ + 'kube_namespace:kube-system', + 'pod_name:kube-dns-c598bd956-wgf4n', + 'kube_container_name:kubedns', + ], + 'container_id://3102f0d9499c5cd0875225208e3d048e3a9d829f5cdd74758b2d79a429a579fa': [ + 'kube_namespace:kube-system', + 'pod_name:fluentbit-gke-45gvm', + 'kube_container_name:fluentbit-gke', + ], + 'container_id://efa5b57cc110de6d2ca3b4a0e12c0a378090530e5e2d0ba0882dffe9c5846067': [ + 'kube_namespace:kube-system', + 'pod_name:fluentbit-gke-45gvm', + 'kube_container_name:fluentbit', + ], + 'container_id://0d8eea0b23688a4c3fbc29828b455734b323d6aac085c88f8f112e296cd5b521': [ + 'kube_namespace:kube-system', + 'pod_name:kube-dns-c598bd956-wgf4n', + 'kube_container_name:dnsmasq', + ], + 'container_id://1669a6277ebb44aedd2790ba8bce83d21899ba85b1afde4330caf4a4161eee26': [ + 'kube_namespace:kube-system', + 'pod_name:calico-node-9qkw7', + 'kube_container_name:calico-node', + ], + 'container_id://c81dfc25dd24b538a880bfd0f807ba9ec1ff4541e8b8eb49a8d1afcdecc5ef59': [ + 'kube_namespace:default', + 'pod_name:datadog-t9f28', + 'kube_container_name:agent', + ], +} + METRICS_WITH_DEVICE_TAG = { 'kubernetes.filesystem.usage': '/dev/sda1', 'kubernetes.io.read_bytes': '/dev/sda', @@ -237,13 +275,13 @@ def tagger(): return tagger -def mock_kubelet_check(monkeypatch, instances, kube_version=KUBE_1_14, stats_summary_fail=False): +def mock_kubelet_check(monkeypatch, instances, kube_version=KUBE_1_14, stats_summary_fail=False, pod_list='pods.json'): """ Returns a check that uses mocked data for responses from prometheus endpoints, pod list, and node spec. """ check = KubeletCheck('kubelet', {}, instances) - monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) + monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file(pod_list)))) mock_resp = mock.Mock(status_code=200, raise_for_status=mock.Mock(), json=mock.Mock(return_value=NODE_SPEC)) monkeypatch.setattr(check, '_retrieve_node_spec', mock.Mock(return_value=mock_resp)) if stats_summary_fail: @@ -266,6 +304,9 @@ def _mocked_poll(*args, **kwargs): elif prometheus_url.endswith('/metrics'): # Mock response for "/metrics" content = mock_from_file(kubelet_response) + elif prometheus_url.endswith('/metrics/probes'): + # Mock response for "/metrics/probes" + content = mock_from_file('probes.txt') else: raise Exception("Must be a valid endpoint") @@ -327,9 +368,11 @@ def test_kubelet_default_options(): check = KubeletCheck('kubelet', {}, [{}]) assert check.cadvisor_scraper_config['namespace'] == 'kubernetes' assert check.kubelet_scraper_config['namespace'] == 'kubernetes' + assert check.probes_scraper_config['namespace'] == 'kubernetes' assert isinstance(check.cadvisor_scraper_config, dict) assert isinstance(check.kubelet_scraper_config, dict) + assert isinstance(check.probes_scraper_config, dict) def test_kubelet_check_prometheus_instance_tags(monkeypatch, aggregator, tagger): @@ -1151,3 +1194,84 @@ def test_ignore_namespace_for_volume_metrics(monkeypatch): for metric in volume_metrics: assert metric in metrics_reported + + +def test_probe_metrics(monkeypatch, aggregator, tagger): + tagger.reset() + tagger.set_tags(PROBE_TAGS) + + check = mock_kubelet_check(monkeypatch, [{}], pod_list='pod_list_probes.json') + check.check({'cadvisor_metrics_endpoint': '', 'kubelet_metrics_endpoint': ''}) + check._perform_kubelet_check.assert_called_once() + + aggregator.assert_metric( + 'kubernetes.liveness_probe.success.total', + 3, + ['kube_namespace:default', 'pod_name:datadog-t9f28', 'kube_container_name:agent'], + ) + + aggregator.assert_metric( + 'kubernetes.readiness_probe.success.total', + 3, + ['kube_namespace:default', 'pod_name:datadog-t9f28', 'kube_container_name:agent'], + ) + + aggregator.assert_metric( + 'kubernetes.liveness_probe.success.total', + 281049, + ['kube_container_name:fluentbit', 'kube_namespace:kube-system', 'pod_name:fluentbit-gke-45gvm'], + ) + + aggregator.assert_metric( + 'kubernetes.liveness_probe.success.total', + 281049, + ['kube_container_name:fluentbit-gke', 'kube_namespace:kube-system', 'pod_name:fluentbit-gke-45gvm'], + ) + + aggregator.assert_metric( + 'kubernetes.liveness_probe.success.total', + 1686298, + ['kube_container_name:kubedns', 'kube_namespace:kube-system', 'pod_name:kube-dns-c598bd956-wgf4n'], + ) + + aggregator.assert_metric( + 'kubernetes.readiness_probe.success.total', + 1686303, + ['kube_container_name:kubedns', 'kube_namespace:kube-system', 'pod_name:kube-dns-c598bd956-wgf4n'], + ) + + aggregator.assert_metric( + 'kubernetes.readiness_probe.failure.total', + 180, + ['kube_container_name:calico-node', 'kube_namespace:kube-system', 'pod_name:calico-node-9qkw7'], + ) + + aggregator.assert_metric( + 'kubernetes.liveness_probe.failure.total', + 100, + ['kube_container_name:calico-node', 'kube_namespace:kube-system', 'pod_name:calico-node-9qkw7'], + ) + + aggregator.assert_metric( + 'kubernetes.liveness_probe.success.total', + 1686306, + ['kube_container_name:calico-node', 'kube_namespace:kube-system', 'pod_name:calico-node-9qkw7'], + ) + + aggregator.assert_metric( + 'kubernetes.readiness_probe.success.total', + 1686127, + ['kube_container_name:calico-node', 'kube_namespace:kube-system', 'pod_name:calico-node-9qkw7'], + ) + + aggregator.assert_metric( + 'kubernetes.liveness_probe.success.total', + 1686298, + ['kube_container_name:sidecar', 'kube_namespace:kube-system', 'pod_name:kube-dns-c598bd956-wgf4n'], + ) + + aggregator.assert_metric( + 'kubernetes.liveness_probe.success.total', + 1686298, + ['kube_container_name:dnsmasq', 'kube_namespace:kube-system', 'pod_name:kube-dns-c598bd956-wgf4n'], + ) diff --git a/kubelet/tests/test_prometheus.py b/kubelet/tests/test_prometheus.py index c53fb744ea647..a2a3fc853964a 100644 --- a/kubelet/tests/test_prometheus.py +++ b/kubelet/tests/test_prometheus.py @@ -142,26 +142,6 @@ def test_is_pod_metric(): assert CadvisorPrometheusScraperMixin._is_pod_metric(metric.label) is True -def test_get_container_label(): - labels = {"container_name": "POD", "id": "/kubepods/burstable/pod531c80d9-9fc4-11e7-ba8b-42010af002bb"} - assert CadvisorPrometheusScraperMixin._get_container_label(labels, "container_name") == "POD" - assert CadvisorPrometheusScraperMixin._get_container_label({}, "not-in") is None - - -def test_get_container_id(cadvisor_scraper): - # k8s >= 1.16 - labels = {"container": "datadog-agent", "namespace": "default", "pod": "datadog-agent-pbqt2"} - container_id = cadvisor_scraper._get_container_id(labels) - assert container_id == "containerd://51cba2ca229069039575750d44ed3a67e9b5ead651312ba7ff218dd9202fde64" - - # k8s < 1.16 - labels = {"container_name": "datadog-agent", "namespace": "default", "pod_name": "datadog-agent-pbqt2"} - container_id = cadvisor_scraper._get_container_id(labels) - assert container_id == "containerd://51cba2ca229069039575750d44ed3a67e9b5ead651312ba7ff218dd9202fde64" - - assert cadvisor_scraper._get_container_id([]) is None - - def test_get_entity_id_if_container_metric(cadvisor_scraper): # k8s >= 1.16 static_pod_1_16 = MockMetric(