diff --git a/CHANGELOG.md b/CHANGELOG.md index 68bffdec484..dc702a6349b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ * [BUGFIX] Querier: return actual error rather than `attempted to read series at index XXX from stream, but the stream has already been exhausted` (or even no error at all) when streaming chunks from ingesters or store-gateways is enabled and an error occurs while streaming chunks. #6346 * [BUGFIX] Querier: reduce log volume when querying ingesters with zone-awareness enabled and one or more instances in a single zone unavailable. #6381 * [BUGFIX] Querier: don't try to query further ingesters if ingester query request minimization is enabled and a query limit is reached as a result of the responses from the initial set of ingesters. #6402 +* [BUGFIX] Ingester: Don't cache context cancellation error when querying. #6446 ### Mixin diff --git a/go.mod b/go.mod index de139487e8d..d6b86633628 100644 --- a/go.mod +++ b/go.mod @@ -248,7 +248,7 @@ require ( ) // Using a fork of Prometheus with Mimir-specific changes. -replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20231019100715-97933fb7d38e +replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20231020144705-d0d6240125f9 // Replace memberlist with our fork which includes some fixes that haven't been // merged upstream yet: diff --git a/go.sum b/go.sum index 714a1b9abd9..d6abee30d65 100644 --- a/go.sum +++ b/go.sum @@ -546,8 +546,8 @@ github.com/grafana/gomemcache v0.0.0-20230914135007-70d78eaabfe1 h1:MLYY2R60/74h github.com/grafana/gomemcache v0.0.0-20230914135007-70d78eaabfe1/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe h1:yIXAAbLswn7VNWBIvM71O2QsgfgW9fRXZNR0DXe6pDU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/grafana/mimir-prometheus v0.0.0-20231019100715-97933fb7d38e h1:tlVYEJDGD9e5Wk7ZYtjXtEJZXuWJqEF3Mb/+SqFiZ18= -github.com/grafana/mimir-prometheus v0.0.0-20231019100715-97933fb7d38e/go.mod h1:bHUBXcO5vIkqWBAy86JlejQPQltETv9Cv5whKCeF2FM= +github.com/grafana/mimir-prometheus v0.0.0-20231020144705-d0d6240125f9 h1:KVelZKI3FNeGvH9V3QYuXc7EGyc1AD7zsKanLqEdvaI= +github.com/grafana/mimir-prometheus v0.0.0-20231020144705-d0d6240125f9/go.mod h1:bHUBXcO5vIkqWBAy86JlejQPQltETv9Cv5whKCeF2FM= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 h1:em1oddjXL8c1tL0iFdtVtPloq2hRPen2MJQKoAWpxu0= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 h1:A3dhViTeFDSQcGOXuUi6ukCQSMyDtDISBp2z6OOo2YM= diff --git a/vendor/github.com/prometheus/prometheus/tsdb/postings_for_matchers_cache.go b/vendor/github.com/prometheus/prometheus/tsdb/postings_for_matchers_cache.go index aea96ca3dcf..865224f7f31 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/postings_for_matchers_cache.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/postings_for_matchers_cache.go @@ -78,37 +78,53 @@ func (c *PostingsForMatchersCache) PostingsForMatchers(ctx context.Context, ix I return c.postingsForMatchers(ctx, ix, ms...) } c.expire() - return c.postingsForMatchersPromise(ctx, ix, ms)() + return c.postingsForMatchersPromise(ix, ms)(ctx) } type postingsForMatcherPromise struct { - sync.WaitGroup + done chan struct{} cloner *index.PostingsCloner err error } -func (p *postingsForMatcherPromise) result() (index.Postings, error) { - p.Wait() - if p.err != nil { - return nil, p.err +func (p *postingsForMatcherPromise) result(ctx context.Context) (index.Postings, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-p.done: + // Checking context error is necessary for deterministic tests, + // as channel selection order is random + if ctx.Err() != nil { + return nil, ctx.Err() + } + if p.err != nil { + return nil, p.err + } + return p.cloner.Clone(), nil } - return p.cloner.Clone(), nil } -func (c *PostingsForMatchersCache) postingsForMatchersPromise(ctx context.Context, ix IndexPostingsReader, ms []*labels.Matcher) func() (index.Postings, error) { +func (c *PostingsForMatchersCache) postingsForMatchersPromise(ix IndexPostingsReader, ms []*labels.Matcher) func(context.Context) (index.Postings, error) { promise := new(postingsForMatcherPromise) - promise.Add(1) + promise.done = make(chan struct{}) key := matchersKey(ms) oldPromise, loaded := c.calls.LoadOrStore(key, promise) if loaded { - promise = oldPromise.(*postingsForMatcherPromise) - return promise.result + // promise was not stored, we return a previously stored promise, that's possibly being fulfilled in another goroutine + close(promise.done) + return oldPromise.(*postingsForMatcherPromise).result } - defer promise.Done() - if postings, err := c.postingsForMatchers(ctx, ix, ms...); err != nil { + // promise was stored, close its channel after fulfilment + defer close(promise.done) + + // Don't let context cancellation fail the promise, since it may be used by multiple goroutines, each with + // its own context. Also, keep the call independent of this particular context, since the promise will be reused. + // FIXME: do we need to cancel the call to postingsForMatchers if all the callers waiting for the result have + // cancelled their context? + if postings, err := c.postingsForMatchers(context.Background(), ix, ms...); err != nil { promise.err = err } else { promise.cloner = index.NewPostingsCloner(postings) diff --git a/vendor/modules.txt b/vendor/modules.txt index 1e42e3d559b..4a3fa2afb8c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -884,7 +884,7 @@ github.com/prometheus/exporter-toolkit/web github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20231019100715-97933fb7d38e +# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20231020144705-d0d6240125f9 ## explicit; go 1.20 github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery @@ -1466,7 +1466,7 @@ sigs.k8s.io/kustomize/kyaml/yaml/walk # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20231019100715-97933fb7d38e +# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20231020144705-d0d6240125f9 # github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe # gopkg.in/yaml.v3 => github.com/colega/go-yaml-yaml v0.0.0-20220720105220-255a8d16d094 # github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6