Add restart and container state metrics to kubelet

kubelet/datadog_checks/kubelet/kubelet.py

@@ -50,6 +50,13 @@
     'Ei': 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
 }
 
+
+WHITELISTED_CONTAINER_STATE_REASONS = {
+    'waiting': ['errimagepull', 'imagepullbackoff', 'crashloopbackoff', 'containercreating'],
+    'terminated': ['oomkilled', 'containercannotrun', 'error']
+}
+
+
 log = logging.getLogger('collector')
 
 
@@ -158,6 +165,7 @@ def check(self, instance):
         self._report_node_metrics(self.instance_tags)
         self._report_pods_running(self.pod_list, self.instance_tags)
         self._report_container_spec_metrics(self.pod_list, self.instance_tags)
+        self._report_container_state_metrics(self.pod_list, self.instance_tags)
 
         if self.cadvisor_legacy_url:  # Legacy cAdvisor
             self.log.debug('processing legacy cadvisor metrics')
@@ -344,6 +352,51 @@ def _report_container_spec_metrics(self, pod_list, instance_tags):
                 except (KeyError, AttributeError) as e:
                     self.log.debug("Unable to retrieve container limits for %s: %s", c_name, e)
 
+    def _report_container_state_metrics(self, pod_list, instance_tags):
+        """Reports container state & reasons by looking at container statuses"""
+        for pod in pod_list['items']:
+            pod_name = pod.get('metadata', {}).get('name')
+            pod_uid = pod.get('metadata', {}).get('uid')
+
+            if not pod_name or not pod_uid:
+                continue
+
+            for ctr_status in pod['status'].get('containerStatuses', []):
+                c_name = ctr_status.get('name')
+                cid = ctr_status.get('containerID')
+                if not c_name or not cid:
+                    continue
+
+                if self.pod_list_utils.is_excluded(cid, pod_uid):
+                    continue
+
+                tags = get_tags('%s' % cid, True) + instance_tags
+
+                restart_count = ctr_status.get('restartCount', 0)
+                self.gauge(self.NAMESPACE + '.containers.restarts', restart_count, tags)
+
+                for (metric_name, field_name) in [('state', 'state'), ('last_state', 'lastState')]:
+                    c_state = ctr_status.get(field_name, {})
+
+                    for state_name in ['terminated', 'waiting']:
+                        state_reasons = WHITELISTED_CONTAINER_STATE_REASONS.get(state_name, [])
+                        self._submit_container_state_metric(metric_name, state_name, c_state, state_reasons, tags)
+
+    def _submit_container_state_metric(self, metric_name, state_name, c_state, state_reasons, tags):
+        reason_tags = []
+
+        state_value = c_state.get(state_name)
+        if state_value:
+            reason = state_value.get('reason', '')
+
+            if reason.lower() in state_reasons:
+                reason_tags.append('reason:%s' % (reason))
+            else:
+                return
+
+            gauge_name = '{}.containers.{}.{}'.format(self.NAMESPACE, metric_name, state_name)
+            self.gauge(gauge_name, 1, tags + reason_tags)
+
     @staticmethod
     def parse_quantity(string):
         """

kubelet/metadata.csv

@@ -1,4 +1,8 @@
 metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name
+kubernetes.containers.last_state.terminated,gauge,,,,The number of containers that were previously terminated,0,kubelet,k8s.containers.last_state.terminated
+kubernetes.containers.restarts,gauge,,,,The number of times the container has been restarted,1,kubelet,k8s.containers.restarts
+kubernetes.containers.state.terminated,gauge,,,,The number of currently terminated containers,0,kubelet,k8s.containers.state.terminated
+kubernetes.containers.state.waiting,gauge,,,,The number of currently waiting containers,0,kubelet,k8s.containers.state.waiting
 kubernetes.cpu.load.10s.avg,gauge,,,,Container cpu load average over the last 10 seconds,0,kubelet,k8s.cpu.load.10s
 kubernetes.cpu.system.total,rate,,core,,The number of cores used for system time,0,kubelet,k8s.cpu.system
 kubernetes.cpu.user.total,rate,,core,,The number of cores used for user time,0,kubelet,k8s.cpu.user