From 8e5765c7c321a6edd7a753b110072d9de1679ce1 Mon Sep 17 00:00:00 2001
From: Ben Pope <ben@redpanda.com>
Date: Mon, 6 Jun 2022 22:29:09 +0100
Subject: [PATCH] metrics: storage: default aggregations

Aggregate over shard, partition by default

Signed-off-by: Ben Pope <ben@redpanda.com>
---
 src/v/storage/probe.cc | 70 ++++++++++++++++++++++++++++++------------
 1 file changed, 51 insertions(+), 19 deletions(-)

diff --git a/src/v/storage/probe.cc b/src/v/storage/probe.cc
index 49727667da843..4747823819c34 100644
--- a/src/v/storage/probe.cc
+++ b/src/v/storage/probe.cc
@@ -70,6 +70,10 @@ void probe::setup_metrics(const model::ntp& ntp) {
       topic_label(ntp.tp.topic()),
       partition_label(ntp.tp.partition()),
     };
+    auto aggregate_labels
+      = config::shard_local_cfg().aggregate_metrics()
+          ? std::vector<sm::label>{sm::shard_label, partition_label}
+          : std::vector<sm::label>{};
 
     _metrics.add_group(
       prometheus_sanitize::metrics_name("storage:log"),
@@ -78,78 +82,97 @@ void probe::setup_metrics(const model::ntp& ntp) {
           "written_bytes",
           [this] { return _bytes_written; },
           sm::description("Total number of bytes written"),
-          labels),
+          labels,
+          sm::impl::shard(),
+          aggregate_labels),
         sm::make_counter(
           "batches_written",
           [this] { return _batches_written; },
           sm::description("Total number of batches written"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_total_bytes(
           "read_bytes",
           [this] { return _bytes_read; },
           sm::description("Total number of bytes read"),
-          labels),
+          labels,
+          sm::impl::shard(),
+          aggregate_labels),
         sm::make_total_bytes(
           "cached_read_bytes",
           [this] { return _cached_bytes_read; },
           sm::description("Total number of cached bytes read"),
-          labels),
+          labels,
+          sm::impl::shard(),
+          aggregate_labels),
         sm::make_counter(
           "batches_read",
           [this] { return _batches_read; },
           sm::description("Total number of batches read"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "cached_batches_read",
           [this] { return _cached_batches_read; },
           sm::description("Total number of cached batches read"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "log_segments_created",
           [this] { return _log_segments_created; },
           sm::description("Number of created log segments"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "log_segments_removed",
           [this] { return _log_segments_removed; },
           sm::description("Number of removed log segments"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "log_segments_active",
           [this] { return _log_segments_active; },
           sm::description("Number of active log segments"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "batch_parse_errors",
           [this] { return _batch_parse_errors; },
           sm::description("Number of batch parsing (reading) errors"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "batch_write_errors",
           [this] { return _batch_write_errors; },
           sm::description("Number of batch write errors"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "corrupted_compaction_indices",
           [this] { return _corrupted_compaction_index; },
           sm::description("Number of times we had to re-construct the "
                           ".compaction index on a segment"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "compacted_segment",
           [this] { return _segment_compacted; },
           sm::description("Number of compacted segments"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_gauge(
           "partition_size",
           [this] { return _partition_bytes; },
           sm::description("Current size of partition in bytes"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_total_bytes(
           "compaction_ratio",
           [this] { return _compaction_ratio; },
           sm::description("Average segment compaction ratio"),
-          labels),
+          labels,
+          sm::impl::shard(),
+          aggregate_labels),
       });
 }
 
@@ -168,6 +191,11 @@ void readers_cache_probe::setup_metrics(const model::ntp& ntp) {
     auto ns_label = sm::label("namespace");
     auto topic_label = sm::label("topic");
     auto partition_label = sm::label("partition");
+    auto aggregate_labels
+      = config::shard_local_cfg().aggregate_metrics()
+          ? std::vector<sm::label>{sm::shard_label, partition_label}
+          : std::vector<sm::label>{};
+
     const std::vector<sm::label_instance> labels = {
       ns_label(ntp.ns()),
       topic_label(ntp.tp.topic()),
@@ -181,22 +209,26 @@ void readers_cache_probe::setup_metrics(const model::ntp& ntp) {
           "readers_added",
           [this] { return _readers_added; },
           sm::description("Number of readers added to cache"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "readers_evicted",
           [this] { return _readers_evicted; },
           sm::description("Number of readers evicted from cache"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "cache_hits",
           [this] { return _cache_hits; },
           sm::description("Reader cache hits"),
-          labels),
+          labels,
+          aggregate_labels),
         sm::make_counter(
           "cache_misses",
           [this] { return _cache_misses; },
           sm::description("Reader cache misses"),
-          labels),
+          labels,
+          aggregate_labels),
       });
 }
 } // namespace storage