From 42d9f100a1c1f9bf9bc9153cd56bfb40a8be9354 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Tue, 31 Jan 2023 02:28:41 -0800 Subject: [PATCH 1/2] Fix: Failure to close BlockSeriesClient cause store-gateway deadlock (#6086) * Fix: Failure to close BlockSeriesClient cause store-gateway deadlock Signed-off-by: Alan Protasio * Adding tests Signed-off-by: Alan Protasio * reverting the change on get series Signed-off-by: Alan Protasio * fix lint Signed-off-by: Alan Protasio --------- Signed-off-by: Alan Protasio --- pkg/store/bucket.go | 5 +++++ pkg/store/bucket_e2e_test.go | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index b1fbd898a2..e662b6bf05 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -1208,6 +1208,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie } shardMatcher := req.ShardInfo.Matcher(&s.buffers) + blockClient := newBlockSeriesClient( srv.Context(), s.logger, @@ -1220,9 +1221,11 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie s.seriesBatchSize, s.metrics.chunkFetchDuration, ) + defer blockClient.Close() g.Go(func() error { + span, _ := tracing.StartSpan(gctx, "bucket_store_block_series", tracing.Tags{ "block.id": blk.meta.ULID, "block.mint": blk.meta.MinTime, @@ -1464,6 +1467,7 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq SkipChunks: true, } blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize, s.metrics.chunkFetchDuration) + defer blockClient.Close() if err := blockClient.ExpandPostings( reqSeriesMatchersNoExtLabels, @@ -1638,6 +1642,7 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR SkipChunks: true, } blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize, s.metrics.chunkFetchDuration) + defer blockClient.Close() if err := blockClient.ExpandPostings( reqSeriesMatchersNoExtLabels, diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index 3588c5f061..e24a83ac3e 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -765,6 +766,10 @@ func TestBucketStore_LabelNames_e2e(t *testing.T) { } { t.Run(name, func(t *testing.T) { vals, err := s.store.LabelNames(ctx, tc.req) + for _, b := range s.store.blocks { + waitTimeout(t, &b.pendingReaders, 5*time.Second) + } + testutil.Ok(t, err) testutil.Equals(t, tc.expected, vals.Names) @@ -868,6 +873,10 @@ func TestBucketStore_LabelValues_e2e(t *testing.T) { } { t.Run(name, func(t *testing.T) { vals, err := s.store.LabelValues(ctx, tc.req) + for _, b := range s.store.blocks { + waitTimeout(t, &b.pendingReaders, 5*time.Second) + } + testutil.Ok(t, err) testutil.Equals(t, tc.expected, emptyToNil(vals.Values)) @@ -882,3 +891,17 @@ func emptyToNil(values []string) []string { } return values } + +func waitTimeout(t *testing.T, wg *sync.WaitGroup, timeout time.Duration) { + c := make(chan struct{}) + go func() { + defer close(c) + wg.Wait() + }() + select { + case <-c: + return + case <-time.After(timeout): + t.Fatalf("timeout waiting wg for %v", timeout) + } +} From 62b6af2db0d614d4483e5f3b5f425d8e019307de Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 31 Jan 2023 09:32:56 -0800 Subject: [PATCH 2/2] update changelog Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cf7485d58..6aa386e4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed - [#6066](https://github.com/thanos-io/thanos/pull/6066) Tracing: fixed panic because of nil sampler +- [#6086](https://github.com/thanos-io/thanos/pull/6086) Store Gateway: Fix store-gateway deadlock due to not close BlockSeriesClient ## [v0.30.1](https://github.com/thanos-io/thanos/tree/release-0.30) - 4.01.2023