From 57f55f820582c843e978575c4c9fe08fd3ea9b15 Mon Sep 17 00:00:00 2001 From: BenPope Date: Mon, 6 Jun 2022 22:34:37 +0100 Subject: [PATCH] metrics: Allow aggregating counter This is super-hacky at the moment. Signed-off-by: BenPope --- include/seastar/core/metrics.hh | 16 +++--- src/core/prometheus.cc | 92 +++++++++++++++++++++------------ 2 files changed, 68 insertions(+), 40 deletions(-) diff --git a/include/seastar/core/metrics.hh b/include/seastar/core/metrics.hh index d55ac9c1de6..afd5575d32e 100644 --- a/include/seastar/core/metrics.hh +++ b/include/seastar/core/metrics.hh @@ -435,8 +435,8 @@ extern label shard_label; */ template impl::metric_definition_impl make_gauge(metric_name_type name, - T&& val, description d=description(), std::vector labels = {}) { - return {name, {impl::data_type::GAUGE, "gauge"}, make_function(std::forward(val), impl::data_type::GAUGE), d, labels}; + T&& val, description d=description(), std::vector labels = {}, std::vector aggregate_labels = {}) { + return {name, {impl::data_type::GAUGE, "gauge"}, make_function(std::forward(val), impl::data_type::GAUGE), d, std::move(labels), std::move(aggregate_labels)}; } /*! @@ -521,9 +521,9 @@ impl::metric_definition_impl make_derive(metric_name_type name, description d, s */ template impl::metric_definition_impl make_counter(metric_name_type name, - T&& val, description d=description(), std::vector labels = {}) { + T&& val, description d=description(), std::vector labels = {}, std::vector aggregate_labels = {}) { auto type = impl::counter_type_traits>::type; - return {name, {type, "counter"}, make_function(std::forward(val), type), d, labels}; + return {name, {type, "counter"}, make_function(std::forward(val), type), d, std::move(labels), std::move(aggregate_labels)}; } /*! @@ -627,8 +627,8 @@ impl::metric_definition_impl make_summary(metric_name_type name, template impl::metric_definition_impl make_total_bytes(metric_name_type name, T&& val, description d=description(), std::vector labels = {}, - instance_id_type instance = impl::shard()) { - return make_counter(name, std::forward(val), d, labels).set_type("total_bytes"); + instance_id_type instance = impl::shard(), std::vector aggregate_labels = {}) { + return make_counter(name, std::forward(val), d, std::move(labels), std::move(aggregate_labels)).set_type("total_bytes"); } /*! @@ -641,8 +641,8 @@ impl::metric_definition_impl make_total_bytes(metric_name_type name, template impl::metric_definition_impl make_current_bytes(metric_name_type name, T&& val, description d=description(), std::vector labels = {}, - instance_id_type instance = impl::shard()) { - return make_gauge(name, std::forward(val), d, labels).set_type("bytes"); + instance_id_type instance = impl::shard(), std::vector aggregate_labels = {}) { + return make_gauge(name, std::forward(val), d, std::move(labels), std::move(aggregate_labels)).set_type("bytes"); } diff --git a/src/core/prometheus.cc b/src/core/prometheus.cc index d1861efd304..7dba4a3fe9b 100644 --- a/src/core/prometheus.cc +++ b/src/core/prometheus.cc @@ -19,6 +19,7 @@ * Copyright (C) 2016 ScyllaDB */ +#include #include #include @@ -481,6 +482,24 @@ metric_family_range get_range(const metrics_families_per_shard& mf, const sstrin } +void write_counter(std::stringstream& s, const config& ctx, const sstring& name, const mi::metric_value& value, std::map labels) { + add_name(s, name, labels, ctx); + std::string value_str; + try { + value_str = to_str(value); + } catch (const std::range_error& e) { + seastar_logger.debug("prometheus: write_text_representation: {}: {}", s.str(), e.what()); + value_str = "NaN"; + } catch (...) { + auto ex = std::current_exception(); + // print this error as it's ignored later on by `connection::start_response` + seastar_logger.error("prometheus: write_text_representation: {}: {}", s.str(), ex); + std::rethrow_exception(std::move(ex)); + } + s << value_str; + s << "\n"; +} + void write_histogram(std::stringstream& s, const config& ctx, const sstring& name, const seastar::metrics::histogram& h, std::map labels) { add_name(s, name + "_sum", labels, ctx); s << h.sample_sum; @@ -545,6 +564,28 @@ void histogram_aggregator::add_histogram(const seastar::metrics::histogram& h, s _histograms[labels] += h; } +class counter_aggregator { + std::vector _remove_labels; + std::unordered_map, mi::metric_value> _counters; +public: + explicit counter_aggregator(std::vector labels) : _remove_labels(std::move(labels)) { + } + void add_counter(const mi::metric_value& value, std::map labels); + const std::unordered_map, mi::metric_value>& get_counters() const { + return _counters; + } + bool empty() const { + return _counters.empty(); + } +}; + +void counter_aggregator::add_counter(const mi::metric_value& value, std::map labels) { + for (auto&& l : _remove_labels) { + labels.erase(l); + } + _counters[labels] += value; +} + future<> write_text_representation(output_stream& out, const config& ctx, const metric_family_range& m) { return seastar::async([&ctx, &out, &m] () mutable { bool found = false; @@ -552,12 +593,10 @@ future<> write_text_representation(output_stream& out, const config& ctx, auto name = ctx.prefix + "_" + metric_family.name(); found = false; histogram_aggregator histograms(metric_family.metadata().aggregate_labels); + counter_aggregator counters(metric_family.metadata().aggregate_labels); bool should_aggregate = !metric_family.metadata().aggregate_labels.empty(); - metric_family.foreach_metric([&out, &ctx, &found, &name, &metric_family, &histograms, should_aggregate](auto value, auto value_info) mutable { + metric_family.foreach_metric([&out, &ctx, &found, &name, &metric_family, &histograms, &counters, should_aggregate](auto value, auto value_info) mutable { std::stringstream s; - if (value.is_empty()) { - return; - } if (!found) { if (metric_family.metadata().d.str() != "") { s << "# HELP " << name << " " << metric_family.metadata().d.str() << "\n"; @@ -574,39 +613,28 @@ future<> write_text_representation(output_stream& out, const config& ctx, write_histogram(s, ctx, name, value.get_histogram(), value_info.id.labels()); } } else { - add_name(s, name, value_info.id.labels(), ctx); - std::string value_str; - try { - value_str = to_str(value); - } catch (const std::range_error& e) { - seastar_logger.debug("prometheus: write_text_representation: {}: {}", s.str(), e.what()); - value_str = "NaN"; - } catch (...) { - auto ex = std::current_exception(); - // print this error as it's ignored later on by `connection::start_response` - seastar_logger.error("prometheus: write_text_representation: {}: {}", s.str(), ex); - std::rethrow_exception(std::move(ex)); + if (should_aggregate) { + counters.add_counter(value, value_info.id.labels()); + } else { + write_counter(s, ctx, name, value, value_info.id.labels()); } - s << value_str; - s << "\n"; } out.write(s.str()).get(); thread::maybe_yield(); }); - if (!histograms.empty()) { - auto name = ctx.prefix + "_" + metric_family.name(); - std::stringstream name_help; - if (metric_family.metadata().d.str() != "") { - name_help << "# HELP " << name << " " << metric_family.metadata().d.str() << "\n"; - } - name_help << "# TYPE " << name << " histogram" << "\n"; - out.write(name_help.str()).get(); - for (auto&& h : histograms.get_histograms()) { - std::stringstream s; - write_histogram(s, ctx, name, h.second, h.first); - out.write(s.str()).get(); - thread::maybe_yield(); - } + + for (auto&& h : histograms.get_histograms()) { + std::stringstream s; + write_histogram(s, ctx, name, h.second, h.first); + out.write(s.str()).get(); + thread::maybe_yield(); + } + + for (auto&& h : counters.get_counters()) { + std::stringstream s; + write_counter(s, ctx, name, h.second, h.first); + out.write(s.str()).get(); + thread::maybe_yield(); } } });