From cc1c20f4f9b5df28f3c1d362abd70a4294f29495 Mon Sep 17 00:00:00 2001
From: Filip Petkovski <filip.petkovsky@gmail.com>
Date: Mon, 29 Aug 2022 16:23:54 +0200
Subject: [PATCH] Fix creation of already pruned tenants

When a tenant is pruned from a Receiver, its metrics will remain in
the shipper's Prometheus registry. This leads to a panic when the same
tenant is created again in the Receiver since multiTSDB will attempt to
the same metrics again.

The current solution for the TSDB metrics is to wrap the registry in
an UnRegisterer which can deregister metrics with the same name before
registering new metrics. This commit applies the same solution to the
shipper metrics.

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
---
 CHANGELOG.md                  |  1 +
 pkg/receive/multitsdb.go      |  4 +++-
 pkg/receive/multitsdb_test.go | 25 +++++++++++++++++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2ff6eaa2cc..1b63875698 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#5554](https://github.com/thanos-io/thanos/pull/5554) Query/Receiver: Fix querying exemplars from multi-tenant receivers.
 - [#5583](https://github.com/thanos-io/thanos/pull/5583) Query: fix data race between Respond() and query/queryRange functions. Fixes [#5410](https://github.com/thanos-io/thanos/pull/5410).
 - [#5642](https://github.com/thanos-io/thanos/pull/5642) Receive: Log labels correctly in writer debug messages.
+- [#5655](https://github.com/thanos-io/thanos/pull/5655) Receive: Fix recreating already pruned tenants.
 
 ### Added
 
diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go
index 39e065033d..50615439f6 100644
--- a/pkg/receive/multitsdb.go
+++ b/pkg/receive/multitsdb.go
@@ -438,6 +438,8 @@ func (t *MultiTSDB) TenantStats(statsByLabelName string, tenantIDs ...string) []
 
 func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant) error {
 	reg := prometheus.WrapRegistererWith(prometheus.Labels{"tenant": tenantID}, t.reg)
+	reg = &UnRegisterer{Registerer: reg}
+
 	lset := labelpb.ExtendSortedLabels(t.labels, labels.FromStrings(t.tenantLabelName, tenantID))
 	dataDir := t.defaultTenantDataDir(tenantID)
 
@@ -446,7 +448,7 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant
 	s, err := tsdb.Open(
 		dataDir,
 		logger,
-		&UnRegisterer{Registerer: reg},
+		reg,
 		&opts,
 		nil,
 	)
diff --git a/pkg/receive/multitsdb_test.go b/pkg/receive/multitsdb_test.go
index 3869dc5c78..dc1bce1124 100644
--- a/pkg/receive/multitsdb_test.go
+++ b/pkg/receive/multitsdb_test.go
@@ -472,6 +472,31 @@ func TestMultiTSDBPrune(t *testing.T) {
 	}
 }
 
+func TestMultiTSDBRecreatePrunedTenant(t *testing.T) {
+	dir := t.TempDir()
+
+	m := NewMultiTSDB(dir, log.NewNopLogger(), prometheus.NewRegistry(),
+		&tsdb.Options{
+			MinBlockDuration:  (2 * time.Hour).Milliseconds(),
+			MaxBlockDuration:  (2 * time.Hour).Milliseconds(),
+			RetentionDuration: (6 * time.Hour).Milliseconds(),
+		},
+		labels.FromStrings("replica", "test"),
+		"tenant_id",
+		objstore.NewInMemBucket(),
+		false,
+		metadata.NoneFunc,
+	)
+	defer func() { testutil.Ok(t, m.Close()) }()
+
+	testutil.Ok(t, appendSample(m, "foo", time.UnixMilli(int64(10))))
+	testutil.Ok(t, m.Prune(context.Background()))
+	testutil.Equals(t, 0, len(m.TSDBStores()))
+
+	testutil.Ok(t, appendSample(m, "foo", time.UnixMilli(int64(10))))
+	testutil.Equals(t, 1, len(m.TSDBStores()))
+}
+
 func TestMultiTSDBStats(t *testing.T) {
 	tests := []struct {
 		name          string