application: public metrics as a sharded service

This patch wraps the metric_groups object owned by the application into
a sharded service. This change allows us to register metrics on specific
shards where required. For instance,
redpanda_application_uptime_seconds_total is only registered on one
shard, while redpanda_cpu_busy_seconds_total is registered on all
shards.

src/v/redpanda/application.cc

@@ -335,27 +335,41 @@ void application::setup_public_metrics() {
        {"memory_free_memory", ssx::metrics::public_metrics_handle}})
       .get();
 
-    _public_metrics.add_group(
-      "application",
-      {
-        sm::make_gauge(
-          "uptime_seconds_total",
-          [] {
-              return std::chrono::duration<double>(ss::engine().uptime())
-                .count();
-          },
-          sm::description("Redpanda uptime in seconds"))
-          .aggregate({sm::shard_label}),
-        sm::make_gauge(
-          "busy_seconds_total",
-          [] {
-              return std::chrono::duration<double>(
-                       ss::engine().total_busy_time())
-                .count();
-          },
-          sm::description("Total CPU busy time in seconds"))
-          .aggregate({sm::shard_label}),
-      });
+    _public_metrics.start().get();
+
+    _public_metrics
+      .invoke_on(
+        ss::this_shard_id(),
+        [](auto& public_metrics) {
+            public_metrics.groups.add_group(
+              "application",
+              {sm::make_gauge(
+                 "uptime_seconds_total",
+                 [] {
+                     return std::chrono::duration<double>(ss::engine().uptime())
+                       .count();
+                 },
+                 sm::description("Redpanda uptime in seconds"))
+                 .aggregate({sm::shard_label})});
+        })
+      .get();
+
+    _public_metrics.invoke_on_all([](auto& public_metrics) {
+        public_metrics.groups.add_group(
+          "cpu",
+          {sm::make_gauge(
+            "busy_seconds_total",
+            [] {
+                return std::chrono::duration<double>(
+                         ss::engine().total_busy_time())
+                  .count();
+            },
+            sm::description("Total CPU busy time in seconds"))});
+    }).get();
+
+    _deferred.emplace_back([this] {
+        _public_metrics.stop().get();
+    });
 }
 
 void application::setup_internal_metrics() {

src/v/redpanda/application.h

@@ -179,8 +179,7 @@ class application {
     ss::sharded<archival::upload_controller> _archival_upload_controller;
 
     ss::metrics::metric_groups _metrics;
-    ss::metrics::metric_groups _public_metrics{
-      ssx::metrics::public_metrics_handle};
+    ss::sharded<ssx::metrics::public_metrics_group> _public_metrics;
     std::unique_ptr<kafka::rm_group_proxy_impl> _rm_group_proxy;
     // run these first on destruction
     deferred_actions _deferred;