storage: add metrics clear to reader cache probe

We recently ran into a double registration issue where a reader cache
was still alive even though stop() was called on it. This leads to a
double registration situation because a new reader cache will
attempt to register metrics again.

Fixes #5938

src/v/storage/readers_cache.cc

@@ -171,6 +171,12 @@ ss::future<> readers_cache::stop() {
     _readers.clear_and_dispose([](entry* e) {
         delete e; // NOLINT
     });
+    /**
+     * Stop and clear metrics as well or risk a double registrion on partition
+     * movements. For details see https://github.com/redpanda-data/redpanda/issues/5938
+     */
+    _probe.clear();
+
 }
 
 ss::future<readers_cache::range_lock_holder>

src/v/storage/readers_cache_probe.h

@@ -14,6 +14,7 @@ class readers_cache_probe {
     void reader_evicted() { _readers_evicted++; }
     void cache_hit() { _cache_hits++; }
     void cache_miss() { _cache_misses++; }
+    void clear() { _metrics.clear(); }
 
     void setup_metrics(const model::ntp& ntp);