From 8e77bff4dca05b7b49247975a479ec3d98d4695a Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 17 Sep 2024 21:37:04 -0700 Subject: [PATCH] improve validation metrics for discarded samples and exemplars Signed-off-by: Ben Ye --- pkg/distributor/distributor.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 351187ba83..4acbc76cd4 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -874,7 +874,13 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write d.validateMetrics.DiscardedSamples.WithLabelValues( validation.DroppedByRelabelConfiguration, userID, - ).Add(float64(len(ts.Samples))) + ).Add(float64(len(ts.Samples) + len(ts.Histograms))) + + // all labels are gone, exemplars will be discarded + d.validateMetrics.DiscardedExemplars.WithLabelValues( + validation.DroppedByUserConfigurationOverride, + userID, + ).Add(float64(len(ts.Exemplars))) continue } ts.Labels = cortexpb.FromLabelsToLabelAdapters(l) @@ -892,11 +898,15 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write } if len(ts.Labels) == 0 { - d.validateMetrics.DiscardedExemplars.WithLabelValues( + d.validateMetrics.DiscardedSamples.WithLabelValues( validation.DroppedByUserConfigurationOverride, userID, - ).Add(float64(len(ts.Samples))) + ).Add(float64(len(ts.Samples) + len(ts.Histograms))) + d.validateMetrics.DiscardedExemplars.WithLabelValues( + validation.DroppedByUserConfigurationOverride, + userID, + ).Add(float64(len(ts.Exemplars))) continue }