diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 54d64c89583..a51ede4e8a7 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -63,6 +63,7 @@ import ( "github.com/grafana/mimir/pkg/storage/tsdb/block" "github.com/grafana/mimir/pkg/usagestats" "github.com/grafana/mimir/pkg/util" + util_log "github.com/grafana/mimir/pkg/util/log" util_math "github.com/grafana/mimir/pkg/util/math" "github.com/grafana/mimir/pkg/util/push" util_test "github.com/grafana/mimir/pkg/util/test" @@ -5621,7 +5622,8 @@ func TestIngester_inflightPushRequests(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.InstanceLimitsFn = func() *InstanceLimits { return &limits } - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + reg := prometheus.NewPedanticRegistry() + i, err := prepareIngesterWithBlocksStorage(t, cfg, reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -5688,10 +5690,21 @@ func TestIngester_inflightPushRequests(t *testing.T) { _, err := i.Push(ctx, req) require.Equal(t, errMaxInflightRequestsReached, err) + require.ErrorAs(t, err, &util_log.DoNotLogError{}) return nil }) require.NoError(t, g.Wait()) + + // Ensure the rejected request has been tracked in a metric. + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_ingester_instance_rejected_requests_total Requests rejected for hitting per-instance limits + # TYPE cortex_ingester_instance_rejected_requests_total counter + cortex_ingester_instance_rejected_requests_total{reason="ingester_max_inflight_push_requests"} 1 + cortex_ingester_instance_rejected_requests_total{reason="ingester_max_ingestion_rate"} 0 + cortex_ingester_instance_rejected_requests_total{reason="ingester_max_series"} 0 + cortex_ingester_instance_rejected_requests_total{reason="ingester_max_tenants"} 0 + `), "cortex_ingester_instance_rejected_requests_total")) } func generateSamplesForLabel(baseLabels labels.Labels, series, samples int) *mimirpb.WriteRequest {