Fix creation of already pruned tenants (thanos-io#5655)

When a tenant is pruned from a Receiver, its metrics will remain in
the shipper's Prometheus registry. This leads to a panic when the same
tenant is created again in the Receiver since multiTSDB will attempt to
the same metrics again.

The current solution for the TSDB metrics is to wrap the registry in
an UnRegisterer which can deregister metrics with the same name before
registering new metrics. This commit applies the same solution to the
shipper metrics.

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>

Signed-off-by: Filip Petkovski <filip.petkovsky@gmail.com>
Signed-off-by: Prakul Jain <prakul.jain@udaan.com>

CHANGELOG.md

@@ -18,6 +18,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#5554](https://github.com/thanos-io/thanos/pull/5554) Query/Receiver: Fix querying exemplars from multi-tenant receivers.
 - [#5583](https://github.com/thanos-io/thanos/pull/5583) Query: fix data race between Respond() and query/queryRange functions. Fixes [#5410](https://github.com/thanos-io/thanos/pull/5410).
 - [#5642](https://github.com/thanos-io/thanos/pull/5642) Receive: Log labels correctly in writer debug messages.
+- [#5655](https://github.com/thanos-io/thanos/pull/5655) Receive: Fix recreating already pruned tenants.
 
 ### Added
 

pkg/receive/multitsdb.go

@@ -438,6 +438,8 @@ func (t *MultiTSDB) TenantStats(statsByLabelName string, tenantIDs ...string) []
 
 func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant) error {
 	reg := prometheus.WrapRegistererWith(prometheus.Labels{"tenant": tenantID}, t.reg)
+	reg = &UnRegisterer{Registerer: reg}
+
 	lset := labelpb.ExtendSortedLabels(t.labels, labels.FromStrings(t.tenantLabelName, tenantID))
 	dataDir := t.defaultTenantDataDir(tenantID)
 
@@ -446,7 +448,7 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant
 	s, err := tsdb.Open(
 		dataDir,
 		logger,
-		&UnRegisterer{Registerer: reg},
+		reg,
 		&opts,
 		nil,
 	)

pkg/receive/multitsdb_test.go

@@ -472,6 +472,31 @@ func TestMultiTSDBPrune(t *testing.T) {
 	}
 }
 
+func TestMultiTSDBRecreatePrunedTenant(t *testing.T) {
+	dir := t.TempDir()
+
+	m := NewMultiTSDB(dir, log.NewNopLogger(), prometheus.NewRegistry(),
+		&tsdb.Options{
+			MinBlockDuration:  (2 * time.Hour).Milliseconds(),
+			MaxBlockDuration:  (2 * time.Hour).Milliseconds(),
+			RetentionDuration: (6 * time.Hour).Milliseconds(),
+		},
+		labels.FromStrings("replica", "test"),
+		"tenant_id",
+		objstore.NewInMemBucket(),
+		false,
+		metadata.NoneFunc,
+	)
+	defer func() { testutil.Ok(t, m.Close()) }()
+
+	testutil.Ok(t, appendSample(m, "foo", time.UnixMilli(int64(10))))
+	testutil.Ok(t, m.Prune(context.Background()))
+	testutil.Equals(t, 0, len(m.TSDBStores()))
+
+	testutil.Ok(t, appendSample(m, "foo", time.UnixMilli(int64(10))))
+	testutil.Equals(t, 1, len(m.TSDBStores()))
+}
+
 func TestMultiTSDBStats(t *testing.T) {
 	tests := []struct {
 		name          string