Support for gathering metrics from prometheus endpoint for the kubele…

…t itself. (#1581)

* Add support for scraping metrics from prometheus endpoint for kubelet metrics.

* Test cleanup.

* Code review changes.

* Fix failing test.

* Log warning for deprecated config option.

* Remove etcd metrics from kubelet prometheus endpoint.

datadog_checks_base/datadog_checks/checks/prometheus/__init__.py

@@ -14,12 +14,12 @@
     from .prometheus_base import PrometheusCheck
     from .base_check import GenericPrometheusCheck
 
-from .base_check import Scraper
+from .base_check import PrometheusScraper
 
 __all__ = [
     'PrometheusFormat',
     'UnknownFormatError',
     'PrometheusCheck',
     'GenericPrometheusCheck',
-    'Scraper'
+    'PrometheusScraper',
 ]

datadog_checks_base/datadog_checks/checks/prometheus/base_check.py

@@ -2,14 +2,20 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
-from .mixins import PrometheusScraper
+from .mixins import PrometheusScraperMixin
 
 from .. import AgentCheck
 from ...errors import CheckException
 
-class Scraper(PrometheusScraper):
+
+class PrometheusScraper(PrometheusScraperMixin):
+    """
+    This class scrapes a prometheus endpoint and submits the metrics on behalf of a check. This class
+    is used by checks that scrape more than one prometheus endpoint.
+    """
+
     def __init__(self, check):
-        super(Scraper, self).__init__()
+        super(PrometheusScraper, self).__init__()
         self.check = check
 
     def _submit_rate(self, metric_name, val, metric, custom_tags=None, hostname=None):
@@ -65,7 +71,7 @@ def _submit_service_check(self, *args, **kwargs):
 
 class GenericPrometheusCheck(AgentCheck):
     """
-    GenericPrometheusCheck is a class that helps instanciating PrometheusCheck only
+    GenericPrometheusCheck is a class that helps instantiating PrometheusCheck only
     with YAML configurations. As each check has it own states it maintains a map
     of all checks so that the one corresponding to the instance is executed
 
@@ -127,7 +133,7 @@ def get_scraper(self, instance):
             return self.scrapers_map[endpoint]
 
         # Otherwise we create the scraper
-        scraper = Scraper(self)
+        scraper = PrometheusScraper(self)
         scraper.NAMESPACE = namespace
         # Metrics are preprocessed if no mapping
         metrics_mapper = {}

datadog_checks_base/datadog_checks/checks/prometheus/mixins.py

@@ -30,7 +30,7 @@ class UnknownFormatError(TypeError):
     pass
 
 
-class PrometheusScraper(object):
+class PrometheusScraperMixin(object):
     # pylint: disable=E1101
     # This class is not supposed to be used by itself, it provides scraping behavior but
     # need to be within a check in the end
@@ -39,7 +39,7 @@ class PrometheusScraper(object):
     REQUESTS_CHUNK_SIZE = 1024 * 10  # use 10kb as chunk size when using the Stream feature in requests.get
 
     def __init__(self, *args, **kwargs):
-        super(PrometheusScraper, self).__init__(*args, **kwargs)
+        super(PrometheusScraperMixin, self).__init__(*args, **kwargs)
 
         # The scraper needs its own logger
         self.log = logging.getLogger(__name__)
@@ -228,10 +228,10 @@ def get_metric_value_by_labels(messages, _metric, _m, metric_suffix):
         """
         metric_name = '{}_{}'.format(_m, metric_suffix)
         expected_labels = set([(k, v) for k, v in _metric["labels"].iteritems()
-                               if k not in PrometheusScraper.UNWANTED_LABELS])
+                               if k not in PrometheusScraperMixin.UNWANTED_LABELS])
         for elt in messages[metric_name]:
             current_labels = set([(k, v) for k, v in elt["labels"].iteritems()
-                                  if k not in PrometheusScraper.UNWANTED_LABELS])
+                                  if k not in PrometheusScraperMixin.UNWANTED_LABELS])
             # As we have two hashable objects we can compare them without any side effects
             if current_labels == expected_labels:
                 return float(elt["value"])

datadog_checks_base/datadog_checks/checks/prometheus/prometheus_base.py

@@ -2,15 +2,15 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
-from .mixins import PrometheusScraper
+from .mixins import PrometheusScraperMixin
 from .. import AgentCheck
 
 # Prometheus check is a parent class providing a structure and some helpers
 # to collect metrics, events and service checks exposed via Prometheus.
 #
 # It must be noted that if the check implementing this class is not officially
 # supported
-# its metrics will count as cutom metrics and WILL impact billing.
+# its metrics will count as custom metrics and WILL impact billing.
 #
 # Minimal config for checks based on this class include:
 #   - implementing the check method
@@ -21,7 +21,7 @@
 #     it will be called in `process_metric`
 #
 
-class PrometheusCheck(PrometheusScraper, AgentCheck):
+class PrometheusCheck(PrometheusScraperMixin, AgentCheck):
     def __init__(self, name, init_config, agentConfig, instances=None):
         super(PrometheusCheck, self).__init__(name, init_config, agentConfig, instances)
 

istio/datadog_checks/istio/istio.py

@@ -2,7 +2,7 @@
 # All rights reserved
 # Licensed under Simplified BSD License (see LICENSE)
 
-from datadog_checks.checks.prometheus import Scraper, PrometheusCheck
+from datadog_checks.checks.prometheus import PrometheusScraper, PrometheusCheck
 
 
 class Istio(PrometheusCheck):
@@ -57,7 +57,7 @@ def _get_istio_mesh_scraper(self, instance):
         if self._scrapers.get(endpoint, None):
             return self._scrapers.get(endpoint)
 
-        scraper = Scraper(self)
+        scraper = PrometheusScraper(self)
         self._scrapers[endpoint] = scraper
         scraper.NAMESPACE = self.MESH_NAMESPACE
         scraper.metrics_mapper = {
@@ -81,7 +81,7 @@ def _get_mixer_scraper(self, instance):
         if self._scrapers.get(endpoint, None):
             return self._scrapers.get(endpoint)
 
-        scraper = Scraper(self)
+        scraper = PrometheusScraper(self)
         self._scrapers[endpoint] = scraper
         scraper.NAMESPACE = self.MIXER_NAMESPACE
         scraper.metrics_mapper = {

istio/tests/test_check.py

@@ -11,7 +11,7 @@
 
 # project
 from datadog_checks.istio import Istio
-from datadog_checks.checks.prometheus import Scraper
+from datadog_checks.checks.prometheus import PrometheusScraper
 
 
 MESH_METRICS = ['istio.mesh.request.count',
@@ -183,7 +183,7 @@ def mesh_mixture_fixture():
     with open(mixer_file_path, 'rb') as f:
         responses.append(f.read())
 
-    p = mock.patch('datadog_checks.checks.prometheus.Scraper.poll',
+    p = mock.patch('datadog_checks.checks.prometheus.PrometheusScraper.poll',
                    return_value=MockResponse(responses, 'text/plain'),
                    __name__="poll")
     yield p.start()
@@ -229,8 +229,8 @@ def test_scraper_creator():
     assert istio_mesh_scraper == istio_mesh_scraper_dict
     assert mixer_scraper == mixer_scraper_dict
 
-    assert isinstance(istio_mesh_scraper, Scraper)
-    assert isinstance(mixer_scraper, Scraper)
+    assert isinstance(istio_mesh_scraper, PrometheusScraper)
+    assert isinstance(mixer_scraper, PrometheusScraper)
 
     assert istio_mesh_scraper.NAMESPACE == 'istio.mesh'
     assert mixer_scraper.NAMESPACE == 'istio.mixer'

kubelet/conf.yaml.example

@@ -13,11 +13,18 @@ instances:
     ### This is the default setting. See next section for legacy clusters.
     ###
     #
-    # url of the kubelet metrics endpoint
-    # Pass an empty string, or set the cadvisor_port option to disable
-    # prometheus metrics collection
+    # DEPRECATED: please specify `cadvisor_metrics_endpoint` instead
     # metrics_endpoint: http://10.8.0.1:10255/metrics/cadvisor
     #
+    # url of the cadvisor metrics prometheus endpoint
+    # Pass an empty string, or set the cadvisor_port option to disable
+    # cadvisor metrics collection
+    # cadvisor_metrics_endpoint: http://10.8.0.1:10255/metrics/cadvisor
+    #
+    # url of the kubelet metrics prometheus endpoint
+    # Pass an empty string to disable kubelet metrics collection
+    # kubelet_metrics_endpoint: http://10.8.0.1:10255/metrics
+    #
     # The histogram buckets can be noisy and generate a lot of tags.
     # send_histograms_buckets controls whether or not you want to pull them.
     #