query-scheduler: instrument number of cancelled requests (#3696)

To help determine the impact of issue 3695 I am adding a counter to
measure the number of requests that end up being discarded because they
are cancelled, but still occupy capacity in the queue.

CHANGELOG.md

@@ -10,6 +10,7 @@
 * [ENHANCEMENT] Distributor: remove labels with empty values #2439
 * [ENHANCEMENT] Query-frontend: track query HTTP requests in the Activity Tracker. #3561
 * [ENHANCEMENT] Store-gateway: Add experimental alternate implementation of index-header reader that does not use mmap. This reader is expected to improve stability of the store-gateway. This implementation can be enabled with the flag `-blocks-storage.bucket-store.index-header.stream-reader-enabled`. #3639
+* [ENHANCEMENT] Query-scheduler: add `cortex_query_scheduler_cancelled_requests_total` metric to track the number of requests that are already cancelled when dequeued. #3696
 * [BUGFIX] Log the names of services that are not yet running rather than `unsupported value type` when calling `/ready` and some services are not running. #3625
 * [BUGFIX] Alertmanager: Fix template spurious deletion with relative data dir. #3604
 

pkg/scheduler/scheduler.go

@@ -68,6 +68,7 @@ type Scheduler struct {
 	// Metrics.
 	queueLength              *prometheus.GaugeVec
 	discardedRequests        *prometheus.CounterVec
+	cancelledRequests        *prometheus.CounterVec
 	connectedQuerierClients  prometheus.GaugeFunc
 	connectedFrontendClients prometheus.GaugeFunc
 	queueDuration            prometheus.Histogram
@@ -125,6 +126,10 @@ func NewScheduler(cfg Config, limits Limits, log log.Logger, registerer promethe
 		Help: "Number of queries in the queue.",
 	}, []string{"user"})
 
+	s.cancelledRequests = promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
+		Name: "cortex_query_scheduler_cancelled_requests_total",
+		Help: "Total number of query requests that were cancelled after enqueuing.",
+	}, []string{"user"})
 	s.discardedRequests = promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
 		Name: "cortex_query_scheduler_discarded_requests_total",
 		Help: "Total number of query requests discarded.",
@@ -462,6 +467,7 @@ func (s *Scheduler) forwardRequestToQuerier(querier schedulerpb.SchedulerForQuer
 		// If the upstream request is cancelled (eg. frontend issued CANCEL or closed connection),
 		// we need to cancel the downstream req. Only way we can do that is to close the stream (by returning error here).
 		// Querier is expecting this semantics.
+		s.cancelledRequests.WithLabelValues(req.userID).Inc()
 		return req.ctx.Err()
 
 	case err := <-errCh:
@@ -565,6 +571,7 @@ func (s *Scheduler) stopping(_ error) error {
 func (s *Scheduler) cleanupMetricsForInactiveUser(user string) {
 	s.queueLength.DeleteLabelValues(user)
 	s.discardedRequests.DeleteLabelValues(user)
+	s.cancelledRequests.DeleteLabelValues(user)
 }
 
 func (s *Scheduler) getConnectedFrontendClientsMetric() float64 {