From 423e955634cf5ccdb2ee46689f1107c1e00539ab Mon Sep 17 00:00:00 2001 From: francoposa Date: Mon, 18 Dec 2023 10:19:03 -0800 Subject: [PATCH 1/2] add additional queue dimensions to query scheduler queue duration histogram --- pkg/scheduler/scheduler.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 6b8c41254df..df6202ae336 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -10,6 +10,7 @@ import ( "flag" "io" "net/http" + "strings" "sync" "time" @@ -72,7 +73,7 @@ type Scheduler struct { cancelledRequests *prometheus.CounterVec connectedQuerierClients prometheus.GaugeFunc connectedFrontendClients prometheus.GaugeFunc - queueDuration prometheus.Histogram + queueDuration *prometheus.HistogramVec inflightRequests prometheus.Summary } @@ -145,11 +146,11 @@ func NewScheduler(cfg Config, limits Limits, log log.Logger, registerer promethe }) s.requestQueue = queue.NewRequestQueue(s.log, cfg.MaxOutstandingPerTenant, cfg.AdditionalQueryQueueDimensionsEnabled, cfg.QuerierForgetDelay, s.queueLength, s.discardedRequests, enqueueDuration) - s.queueDuration = promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ + s.queueDuration = promauto.With(registerer).NewHistogramVec(prometheus.HistogramOpts{ Name: "cortex_query_scheduler_queue_duration_seconds", Help: "Time spent by requests in queue before getting picked up by a querier.", Buckets: prometheus.DefBuckets, - }) + }, []string{"user", "additional_queue_dimensions"}) s.connectedQuerierClients = promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ Name: "cortex_query_scheduler_connected_querier_clients", Help: "Number of querier worker clients currently connected to the query-scheduler.", @@ -399,7 +400,8 @@ func (s *Scheduler) QuerierLoop(querier schedulerpb.SchedulerForQuerier_QuerierL r := req.(*queue.SchedulerRequest) queueTime := time.Since(r.EnqueueTime) - s.queueDuration.Observe(queueTime.Seconds()) + additionalQueueDimensionLabels := strings.Join(r.AdditionalQueueDimensions, ":") + s.queueDuration.WithLabelValues(r.UserID, additionalQueueDimensionLabels).Observe(queueTime.Seconds()) r.QueueSpan.Finish() /* From b64cdcaa509761c0fb8c37d5ce7411f42edcac4e Mon Sep 17 00:00:00 2001 From: francoposa Date: Mon, 18 Dec 2023 11:21:26 -0800 Subject: [PATCH 2/2] better logging of unsupported request type --- pkg/frontend/v2/frontend_scheduler_adapter.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/frontend/v2/frontend_scheduler_adapter.go b/pkg/frontend/v2/frontend_scheduler_adapter.go index f67b7926fee..d71f2fb6f22 100644 --- a/pkg/frontend/v2/frontend_scheduler_adapter.go +++ b/pkg/frontend/v2/frontend_scheduler_adapter.go @@ -4,6 +4,7 @@ package v2 import ( "context" + "fmt" "time" "github.com/go-kit/log" @@ -89,7 +90,7 @@ func (a *frontendToSchedulerAdapter) extractAdditionalQueueDimensions( return []string{ShouldQueryIngestersQueueDimension}, nil default: // no query time params to parse; cannot infer query component - level.Warn(a.log).Log("msg", "unsupported request type", "query", httpRequest) + level.Warn(a.log).Log("msg", "unsupported request type", "query", fmt.Sprintf("%+v", httpRequest)) return nil, nil } }