Add prometheus metrics access for keda operator (#118)

Co-authored-by: Tom Kerkhove <kerkhove.tom@gmail.com>
kedacore · Feb 8, 2021 · c4d7ea7 · c4d7ea7
1 parent e87878e
commit c4d7ea7
Show file tree

Hide file tree

Showing 6 changed files with 128 additions and 61 deletions.
diff --git a/keda/README.md b/keda/README.md
@@ -54,42 +54,58 @@ The command removes all the Kubernetes components associated with the chart and
 The following table lists the configurable parameters of the Promitor chart and
 their default values.
 
-| Parameter                              | Description                               | Default                                         |
-|:---------------------------------------|:------------------------------------------|:------------------------------------------------|
-| `image.keda.repository`                | Image name of KEDA operator               | `docker.io/kedacore/keda`                 |
-| `image.keda.tag`                       | Image tag of KEDA operator. Optional, given app version of Helm chart is used by default | ``                 |
-| `image.metricsApiServer.repository`    | Image name of KEDA Metrics API Server        | `docker.io/kedacore/keda-metrics-apiserver` |
-| `image.metricsApiServer.tag`           | Image tag of KEDA Metrics API Server. Optional, given app version of Helm chart is used by default | ``                 |
-| `watchNamespace`                       | Defines Kubernetes namespaces to watch to scale their workloads. Default watches all namespaces | `` |
-| `operator.name`                        | Name of the KEDA operator | `keda-operator` |
-| `imagePullSecrets`                     | Name of secret to use to pull images to use to pull Docker images | `[]` |
-| `additionalLabels`                     | Additional labels to apply to KEDA workloads | `` |
-| `podAnnotations.keda`                  | Pod annotations for KEDA operator | `` |
-| `podAnnotations.metricsAdapter`        | Pod annotations for KEDA Metrics Adapter | `` |
-| `podDisruptionBudget`                  | Capability to configure [Pod Disruption Budget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/)       | `{}` |
-| `rbac.create`                          | Specifies whether RBAC should be used | `true`                                        |
-| `serviceAccount.create`                | Specifies whether a service account should be created       | `true`                                        |
-| `serviceAccount.name`                  | The name of the service account to use. If not set and create is true, a name is generated.      | `keda-operator` |
-| `serviceAccount.annotations`           | Annotations to add to the service account | `{}` |
-| `podIdentity.activeDirectory.identity` | Identity in Azure Active Directory to use for Azure pod identity | `` |
-| `grpcTLSCertsSecret`                   | Name of the secret that will be mounted to the /grpccerts path on the Pod to communicate over TLS with external scaler(s) (recommended).  | ``|
-| `hashiCorpVaultTLS`                    | Name of the secret that will be mounted to the /vault path on the Pod to communicate over TLS with HashiCorp Vault (recommended). | `` |
-| `logging.operator.level`               | Logging level for KEDA Operator. Allowed values are 'debug', 'info' & 'error'. | `info`                                        |
-| `logging.operator.timeFormat`          | Logging time format for KEDA Operator. Allowed values are 'epoch', 'millis', 'nano', or 'iso8601'. | `epoch` |
-| `logging.metricServer.level`           | Logging level for Metrics Server.Policy to use to pull Docker images. Allowed values are '0' for info, '4' for debug, or an integer value greater than 0, specified as string | `0` |
-| `securityContext`                      | Security context of the pod ([docs](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)) | `{}` |
-| `podSecurityContext`                   | Pod security context of the pod ([docs](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)) | `{}` |
-| `resources`                            | Manage resource request & limits of KEDA workload ([docs](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/)) | `{}` |
-| `nodeSelector`                         | Node selector for pod scheduling ([docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/)) | `{}` |
-| `tolerations`                          | Tolerations for pod scheduling ([docs](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)) | `{}` |
-| `affinity`                             | Affinity for pod scheduling ([docs](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/)) | `{}` |
-| `priorityClassName`                    | Pod priority for KEDA Operator and Metrics Adapter ([docs](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/)) | `` |
-| `env`                                  | Additional environment variables that will be passed onto KEDA operator and metrics api service | `` |
-| `service.annotations`                  | Annotations to add the KEDA Metric Server service | `{}`                                        |
-| `service.portHttp`                     | Service HTTP port for KEDA Metric Server service | `80`                                        |
-| `service.portHttpTarget`               | Service HTTP port for KEDA Metric Server container | `8080`                                        |
-| `service.portHttps`                    | HTTPS port for KEDA Metric Server service | `443`                                        |
-| `service.portHttpsTarget`              | HTTPS port for KEDA Metric Server container | `6443`                                        |
+| Parameter                                                  | Description                               | Default                                         |
+|:-----------------------------------------------------------|:------------------------------------------|:------------------------------------------------|
+| `image.keda.repository`                                    | Image name of KEDA operator               | `docker.io/kedacore/keda`                 |
+| `image.keda.tag`                                           | Image tag of KEDA operator. Optional, given app version of Helm chart is used by default | ``                 |
+| `image.metricsApiServer.repository`                        | Image name of KEDA Metrics API Server        | `docker.io/kedacore/keda-metrics-apiserver` |
+| `image.metricsApiServer.tag`                               | Image tag of KEDA Metrics API Server. Optional, given app version of Helm chart is used by default | ``                 |
+| `watchNamespace`                                           | Defines Kubernetes namespaces to watch to scale their workloads. Default watches all namespaces | `` |
+| `operator.name`                                            | Name of the KEDA operator | `keda-operator` |
+| `imagePullSecrets`                                         | Name of secret to use to pull images to use to pull Docker images | `[]` |
+| `additionalLabels`                                         | Additional labels to apply to KEDA workloads | `` |
+| `podAnnotations.keda`                                      | Pod annotations for KEDA operator | `` |
+| `podAnnotations.metricsAdapter`                            | Pod annotations for KEDA Metrics Adapter | `` |
+| `podDisruptionBudget`                                      | Capability to configure [Pod Disruption Budget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/)       | `{}` |
+| `rbac.create`                                              | Specifies whether RBAC should be used | `true`                                        |
+| `serviceAccount.create`                                    | Specifies whether a service account should be created       | `true`                                        |
+| `serviceAccount.name`                                      | The name of the service account to use. If not set and create is true, a name is generated.      | `keda-operator` |
+| `serviceAccount.annotations`                               | Annotations to add to the service account | `{}` |
+| `podIdentity.activeDirectory.identity`                     | Identity in Azure Active Directory to use for Azure pod identity | `` |
+| `grpcTLSCertsSecret`                                       | Name of the secret that will be mounted to the /grpccerts path on the Pod to communicate over TLS with external scaler(s) (recommended).  | ``|
+| `hashiCorpVaultTLS`                                        | Name of the secret that will be mounted to the /vault path on the Pod to communicate over TLS with HashiCorp Vault (recommended). | `` |
+| `logging.operator.level`                                   | Logging level for KEDA Operator. Allowed values are 'debug', 'info' & 'error'. | `info`                                        |
+| `logging.operator.timeFormat`                              | Logging time format for KEDA Operator. Allowed values are 'epoch', 'millis', 'nano', or 'iso8601'. | `epoch` |
+| `logging.metricServer.level`                               | Logging level for Metrics Server.Policy to use to pull Docker images. Allowed values are '0' for info, '4' for debug, or an integer value greater than 0, specified as string | `0` |
+| `securityContext`                                          | Security context of the pod ([docs](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)) | `{}` |
+| `podSecurityContext`                                       | Pod security context of the pod ([docs](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)) | `{}` |
+| `resources`                                                | Manage resource request & limits of KEDA workload ([docs](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/)) | `{}` |
+| `nodeSelector`                                             | Node selector for pod scheduling ([docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/)) | `{}` |
+| `tolerations`                                              | Tolerations for pod scheduling ([docs](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)) | `{}` |
+| `affinity`                                                 | Affinity for pod scheduling ([docs](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/)) | `{}` |
+| `priorityClassName`                                        | Pod priority for KEDA Operator and Metrics Adapter ([docs](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/)) | `` |
+| `env`                                                      | Additional environment variables that will be passed onto KEDA operator and metrics api service | `` |
+| `service.annotations`                                      | Annotations to add the KEDA Metric Server service | `{}`                                        |
+| `service.portHttp`                                         | Service HTTP port for KEDA Metric Server service | `80`                                        |
+| `service.portHttpTarget`                                   | Service HTTP port for KEDA Metric Server container | `8080`                                        |
+| `service.portHttps`                                        | HTTPS port for KEDA Metric Server service | `443`                                        |
+| `service.portHttpsTarget`                                  | HTTPS port for KEDA Metric Server container | `6443`                                        |
+| `prometheus.metricServer.enabled`                          | Enable metric server prometheus metrics expose | `false`
+| `prometheus.metricServer.port`                             | HTTP port used for exposing metrics server prometheus metrics | `9022`
+| `prometheus.metricServer.path`                             | Path used for exposing metric server prometheus metrics | `/metrics`
+| `prometheus.metricServer.podMonitor.enabled`               | Enable monitoring for metric server using podMonitor crd (prometheus operator) | `false`
+| `prometheus.metricServer.podMonitor.interval`              | Scraping interval for metric server using podMonitor crd (prometheus operator) | ``
+| `prometheus.metricServer.podMonitor.scrapeTimeout`         | Scraping timeout for metric server using podMonitor crd (prometheus operator) | ``
+| `prometheus.metricServer.podMonitor.namespace`             | Scraping namespace for metric server using podMonitor crd (prometheus operator) | ``
+| `prometheus.metricServer.podMonitor.additionalLabels`      | Additional labels to add for metric server using podMonitor crd (prometheus operator) | `{}`
+| `prometheus.operator.enabled`                              | Enable keda operator prometheus metrics expose | `false`
+| `prometheus.operator.port`                                 | HTTP port used for exposing keda operator prometheus metrics | `9022`
+| `prometheus.operator.path`                                 | Path used for exposing keda operator prometheus metrics | `/metrics`
+| `prometheus.operator.podMonitor.enabled`                   | Enable monitoring for keda operator using podMonitor crd (prometheus operator) | `false`
+| `prometheus.operator.podMonitor.interval`                  | Scraping interval for keda operator using podMonitor crd (prometheus operator) | ``
+| `prometheus.operator.podMonitor.scrapeTimeout`             | Scraping timeout for keda operator using podMonitor crd (prometheus operator) | ``
+| `prometheus.operator.podMonitor.namespace`                 | Scraping namespace for keda operator using podMonitor crd (prometheus operator) | ``
+| `prometheus.operator.podMonitor.additionalLabels`          | Additional labels to add for keda operator using podMonitor crd (prometheus operator) | `{}`
 
 Specify each parameter using the `--set key=value[,key=value]` argument to
 `helm install`. For example:
@@ -105,4 +121,4 @@ be provided while installing the chart. For example,
 
 ```console
 helm install keda kedacore/keda --namespace keda -f values.yaml
-```
+```
diff --git a/keda/templates/12-keda-deployment.yaml b/keda/templates/12-keda-deployment.yaml
@@ -62,6 +62,10 @@ spec:
               path: /readyz
               port: 8081
             initialDelaySeconds: 20
+          ports:
+          - containerPort: 8080
+            name: http
+            protocol: TCP
           env:
             - name: WATCH_NAMESPACE
               value: {{ .Values.watchNamespace | quote }}

diff --git a/keda/templates/14-keda-podmonitor.yaml b/keda/templates/14-keda-podmonitor.yaml
@@ -0,0 +1,35 @@
+{{- if and .Values.prometheus.operator.enabled .Values.prometheus.operator.podMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ .Values.operator.name }}
+  labels:
+    app.kubernetes.io/name: {{ .Values.operator.name }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    app.kubernetes.io/part-of: {{ .Values.operator.name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- range $key, $value := .Values.prometheus.operator.podMonitor.additionalLabels }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+  {{- with .Values.prometheus.operator.podMonitor.namespace }}
+  namespace: {{ . }}
+  {{- end }}
+spec:
+  podMetricsEndpoints:
+  - port: http
+    path: {{ .Values.prometheus.operator.path }}
+    honorLabels: true
+    {{- with .Values.prometheus.operator.podMonitor.interval }}
+    interval: {{ . }}
+    {{- end }}
+    {{- with .Values.prometheus.operator.podMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ . }}
+    {{- end }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app: {{ .Values.operator.name }}
+{{- end }}
diff --git a/keda/templates/22-metrics-deployment.yaml b/keda/templates/22-metrics-deployment.yaml
@@ -23,10 +23,10 @@ spec:
         aadpodidbinding: {{ .Values.podIdentity.activeDirectory.identity }}
         {{- end }}
       annotations:
-      {{- if .Values.prometheus.enabled }}
+      {{- if .Values.prometheus.metricServer.enabled }}
         prometheus.io/scrape: "true"
-        prometheus.io/port: {{ .Values.prometheus.port | quote }}
-        prometheus.io/path: {{ .Values.prometheus.path }}
+        prometheus.io/port: {{ .Values.prometheus.metricServer.port | quote }}
+        prometheus.io/path: {{ .Values.prometheus.metricServer.path }}
       {{- end }}
       {{- if .Values.podAnnotations.metricsAdapter }}
       {{- toYaml .Values.podAnnotations.metricsAdapter | nindent 8}}
@@ -70,9 +70,9 @@ spec:
           - /usr/local/bin/keda-adapter
           - --secure-port={{ .Values.service.portHttpsTarget }}
           - --logtostderr=true
-          {{ if .Values.prometheus.enabled }}
-          - --metrics-port={{ .Values.prometheus.port }}
-          - --metrics-path={{ .Values.prometheus.path }}
+          {{ if .Values.prometheus.metricServer.enabled }}
+          - --metrics-port={{ .Values.prometheus.metricServer.port }}
+          - --metrics-path={{ .Values.prometheus.metricServer.path }}
           {{ end }}
           - --v={{ .Values.logging.metricServer.level }}
           ports:
@@ -82,8 +82,8 @@ spec:
             - containerPort: {{ .Values.service.portHttpTarget }}
               name: http
               protocol: TCP
-            {{ if .Values.prometheus.enabled }}
-            - containerPort: {{ .Values.prometheus.port }}
+            {{ if .Values.prometheus.metricServer.enabled }}
+            - containerPort: {{ .Values.prometheus.metricServer.port }}
               name: metrics
               protocol: TCP
             {{ end }}
@@ -120,4 +120,4 @@ spec:
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+      {{- end }}
diff --git a/keda/templates/26-metrics-podmonitor.yaml b/keda/templates/26-metrics-podmonitor.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.prometheus.enabled .Values.prometheus.podMonitor.enabled }}
+{{- if and .Values.prometheus.metricServer.enabled .Values.prometheus.metricServer.podMonitor.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
@@ -9,21 +9,21 @@ metadata:
     app.kubernetes.io/part-of: {{ .Values.operator.name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
-    {{- range $key, $value := .Values.prometheus.podMonitor.additionalLabels }}
+    {{- range $key, $value := .Values.prometheus.metricServer.podMonitor.additionalLabels }}
     {{ $key }}: {{ $value | quote }}
     {{- end }}
-  {{- with .Values.prometheus.podMonitor.namespace }}
+  {{- with .Values.prometheus.metricServer.podMonitor.namespace }}
   namespace: {{ . }}
   {{- end }}
 spec:
   podMetricsEndpoints:
   - port: metrics
-    path: {{ .Values.prometheus.path }}
+    path: {{ .Values.prometheus.metricServer.path }}
     honorLabels: true
-    {{- with .Values.prometheus.podMonitor.interval }}
+    {{- with .Values.prometheus.metricServer.podMonitor.interval }}
     interval: {{ . }}
     {{- end }}
-    {{- with .Values.prometheus.podMonitor.scrapeTimeout }}
+    {{- with .Values.prometheus.metricServer.podMonitor.scrapeTimeout }}
     scrapeTimeout: {{ . }}
     {{- end }}
   namespaceSelector:

diff --git a/keda/values.yaml b/keda/values.yaml
@@ -127,13 +127,25 @@ env:
 #   value: 'ENV-VALUE'
 
 prometheus:
-  enabled: false
-  port: 9022
-  path: /metrics
-  podMonitor:
-    # Enables PodMonitor creation for the Prometheus Operator
+  metricServer:
+    enabled: false
+    port: 9022
+    path: /metrics
+    podMonitor:
+      # Enables PodMonitor creation for the Prometheus Operator
+      enabled: false
+      interval:
+      scrapeTimeout:
+      namespace:
+      additionalLabels: {}
+  operator:
     enabled: false
-    interval:
-    scrapeTimeout:
-    namespace:
-    additionalLabels: {}
+    port: 8080
+    path: /metrics
+    podMonitor:
+      # Enables PodMonitor creation for the Prometheus Operator
+      enabled: false
+      interval:
+      scrapeTimeout:
+      namespace:
+      additionalLabels: {}