From 0582f418ae776a2562109860cd30683d7d2e90aa Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 12 Jan 2023 18:01:20 +0100 Subject: [PATCH] Use job name to differentiate beacon and validator metrics (#4952) Grafana dashboards now use the "job" default label instead of "scrape_location" custom label to differentiate between beacon and validator metrics. The prometheus config files are updated to use the expected job name. --- dashboards/lodestar_block_processor.json | 4 ++-- dashboards/lodestar_summary.json | 20 ++++++++++---------- dashboards/lodestar_validator_client.json | 10 +++++----- dashboards/lodestar_validator_monitor.json | 6 ++---- dashboards/lodestar_vm_host.json | 10 +++++----- docker/prometheus/prometheus.yml | 18 ++++++++++++------ prometheus.yml | 2 +- 7 files changed, 37 insertions(+), 33 deletions(-) diff --git a/dashboards/lodestar_block_processor.json b/dashboards/lodestar_block_processor.json index 2f8c29484170..92a703e81402 100644 --- a/dashboards/lodestar_block_processor.json +++ b/dashboards/lodestar_block_processor.json @@ -2757,7 +2757,7 @@ { "matcher": { "id": "byName", - "options": "beacon_fork_choice_errors_total{group=\"beta\", instance=\"contabo-13\", job=\"beacon\", scrape_location=\"beacon\"}" + "options": "beacon_fork_choice_errors_total{group=\"beta\", instance=\"contabo-13\", job=\"beacon\"}" }, "properties": [ { @@ -2993,7 +2993,7 @@ { "matcher": { "id": "byName", - "options": "{instance=\"beacon_node:8008\", job=\"Lodestar\"}" + "options": "{job=\"beacon\"}" }, "properties": [ { diff --git a/dashboards/lodestar_summary.json b/dashboards/lodestar_summary.json index d1f32481f1f3..28fcb37529ec 100644 --- a/dashboards/lodestar_summary.json +++ b/dashboards/lodestar_summary.json @@ -575,7 +575,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(process_cpu_user_seconds_total{scrape_location=\"beacon\"} [1m])", + "expr": "rate(process_cpu_user_seconds_total{job=\"beacon\"} [1m])", "interval": "", "legendFormat": "", "refId": "A" @@ -629,7 +629,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "process_heap_bytes{scrape_location=\"beacon\"}", + "expr": "process_heap_bytes{job=\"beacon\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -687,7 +687,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "process_start_time_seconds{scrape_location=\"beacon\"}*1000", + "expr": "process_start_time_seconds{job=\"beacon\"}*1000", "instant": false, "interval": "", "legendFormat": "", @@ -815,7 +815,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "lodestar_version{scrape_location=\"beacon\"}", + "expr": "lodestar_version{job=\"beacon\"}", "format": "time_series", "instant": true, "interval": "", @@ -869,7 +869,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "lodestar_version{scrape_location=\"beacon\"}", + "expr": "lodestar_version{job=\"beacon\"}", "format": "time_series", "instant": true, "interval": "", @@ -932,7 +932,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "nodejs_version_info{scrape_location=\"beacon\"}", + "expr": "nodejs_version_info{job=\"beacon\"}", "instant": true, "interval": "", "legendFormat": "{{version}}", @@ -1553,7 +1553,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum(rate(nodejs_gc_duration_seconds_sum{scrape_location=\"beacon\"} [$rate_interval])) by (instance)", + "expr": "sum(rate(nodejs_gc_duration_seconds_sum{job=\"beacon\"} [$rate_interval])) by (instance)", "hide": false, "interval": "", "legendFormat": "gc_duration_sum", @@ -1565,7 +1565,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum(rate(nodejs_gc_pause_seconds_total{scrape_location=\"beacon\"} [$rate_interval])) by (instance)", + "expr": "sum(rate(nodejs_gc_pause_seconds_total{job=\"beacon\"} [$rate_interval])) by (instance)", "hide": false, "interval": "", "legendFormat": "gc_pause_sum", @@ -1687,7 +1687,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "avg_over_time( scrape_duration_seconds{scrape_location=\"beacon\"} [$rate_interval])", + "expr": "avg_over_time( scrape_duration_seconds{job=\"beacon\"} [$rate_interval])", "hide": false, "interval": "", "legendFormat": "prometheus_beacon_scrape_roundtrip", @@ -1699,7 +1699,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "avg_over_time( scrape_duration_seconds{scrape_location=\"validator\"} [$rate_interval])", + "expr": "avg_over_time( scrape_duration_seconds{job=\"validator\"} [$rate_interval])", "hide": false, "interval": "", "legendFormat": "prometheus_validator_scrape_roundtrip", diff --git a/dashboards/lodestar_validator_client.json b/dashboards/lodestar_validator_client.json index 23a765d4c3f4..373f240f15e0 100644 --- a/dashboards/lodestar_validator_client.json +++ b/dashboards/lodestar_validator_client.json @@ -261,7 +261,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "lodestar_version{scrape_location=\"validator\"}", + "expr": "lodestar_version{job=\"validator\"}", "format": "time_series", "instant": true, "interval": "", @@ -328,7 +328,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "nodejs_version_info{scrape_location=\"validator\"}", + "expr": "nodejs_version_info{job=\"validator\"}", "instant": true, "interval": "", "legendFormat": "{{version}}", @@ -393,7 +393,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "lodestar_version{scrape_location=\"validator\"}", + "expr": "lodestar_version{job=\"validator\"}", "format": "time_series", "instant": true, "interval": "", @@ -522,7 +522,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "process_heap_bytes{scrape_location=\"validator\"}", + "expr": "process_heap_bytes{job=\"validator\"}", "interval": "", "legendFormat": "", "refId": "A" @@ -586,7 +586,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "nodejs_heap_size_used_bytes{scrape_location=\"validator\"}", + "expr": "nodejs_heap_size_used_bytes{job=\"validator\"}", "interval": "", "legendFormat": "", "refId": "A" diff --git a/dashboards/lodestar_validator_monitor.json b/dashboards/lodestar_validator_monitor.json index 0864709f7e94..e01a3a2558fb 100644 --- a/dashboards/lodestar_validator_monitor.json +++ b/dashboards/lodestar_validator_monitor.json @@ -220,8 +220,7 @@ "group": false, "instance": true, "job": true, - "network": false, - "scrape_location": true + "network": false }, "indexByName": { "Time": 1, @@ -231,8 +230,7 @@ "index": 0, "instance": 4, "job": 5, - "network": 6, - "scrape_location": 7 + "network": 6 }, "renameByName": {} } diff --git a/dashboards/lodestar_vm_host.json b/dashboards/lodestar_vm_host.json index a96bcee9968b..c065917bda41 100644 --- a/dashboards/lodestar_vm_host.json +++ b/dashboards/lodestar_vm_host.json @@ -641,7 +641,7 @@ { "matcher": { "id": "byName", - "options": "{instance=\"beacon_node:8008\", job=\"Lodestar\"}" + "options": "{job=\"beacon\"}" }, "properties": [ { @@ -2474,7 +2474,7 @@ "expr": "avg_over_time(scrape_duration_seconds[$rate_interval])", "interval": "", "intervalFactor": 1, - "legendFormat": "{{scrape_location}}", + "legendFormat": "{{job}}", "refId": "A" }, { @@ -2486,7 +2486,7 @@ "expr": "rate(lodestar_metrics_scrape_seconds_sum[$rate_interval])\n/\nrate(lodestar_metrics_scrape_seconds_count[$rate_interval])", "hide": false, "interval": "", - "legendFormat": "{{scrape_location}} internal", + "legendFormat": "{{job}} internal", "refId": "B" } ], @@ -2562,7 +2562,7 @@ { "matcher": { "id": "byName", - "options": "{instance=\"beacon_node:8008\", job=\"Lodestar\"}" + "options": "{job=\"beacon\"}" }, "properties": [ { @@ -2605,7 +2605,7 @@ "exemplar": false, "expr": "1-up", "interval": "", - "legendFormat": "{{scrape_location}}", + "legendFormat": "{{job}}", "refId": "A" } ], diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 39b364fb1c6b..f496b9811409 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -1,9 +1,15 @@ +global: + scrape_interval: 20s + scrape_timeout: 20s + +# Tags (values starting with #) have to be replaced in the Dockerfile with sed +# Modified datasource to work with a network_mode: host scrape_configs: - - job_name: Lodestar - scrape_interval: 20s - scrape_timeout: 20s + - job_name: beacon metrics_path: /metrics static_configs: - # This tag is to be replaced in the Dockerfile with sed - # Modified datasource to work with a network_mode: host - - targets: ["#BEACON_URL", "#VC_URL"] + - targets: ["#BEACON_URL"] + - job_name: validator + metrics_path: /metrics + static_configs: + - targets: ["#VC_URL"] diff --git a/prometheus.yml b/prometheus.yml index 7b80a025a6f0..10da941d2085 100644 --- a/prometheus.yml +++ b/prometheus.yml @@ -1,5 +1,5 @@ scrape_configs: - - job_name: Lodestar + - job_name: beacon scrape_interval: 5s metrics_path: /metrics static_configs: