Query ingesters in PENDING state

CHANGELOG.md

@@ -103,6 +103,7 @@
 * [BUGFIX] Querier: fix issue where queries fail with "context canceled" error when an ingester or store-gateway fails healthcheck while the query is in progress. #6550
 * [BUGFIX] Tracing: When creating an OpenTelemetry tracing span, add it to the context for later retrieval. #6614
 * [BUGFIX] Querier: always report query results to query-frontends, even when cancelled, to ensure query-frontends don't wait for results that will otherwise never arrive. #6703
+* [BUGFIX] Querier: attempt to query ingesters in PENDING state, to reduce the likelihood that scaling up the number of ingesters in multiple zones simultaneously causes a read outage. #6726
 
 ### Mixin
 

pkg/distributor/query.go

@@ -34,7 +34,9 @@ import (
 var (
 	// readNoExtend is a ring.Operation that only selects instances marked as ring.ACTIVE.
 	// This should mirror the operation used when choosing ingesters to write series to (ring.WriteNoExtend).
-	readNoExtend = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, nil)
+	// We include ring.PENDING instances as well to ensure we don't miss any instances that have
+	// recently started and we may not have observed in the ring.ACTIVE state yet.
+	readNoExtend = ring.NewOp([]ring.InstanceState{ring.ACTIVE, ring.PENDING}, nil)
 )
 
 func (d *Distributor) QueryExemplars(ctx context.Context, from, to model.Time, matchers ...[]*labels.Matcher) (*ingester_client.ExemplarQueryResponse, error) {