diff --git a/CHANGELOG.md b/CHANGELOG.md index cdef174ba0..bd2f15ae02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Added +- [#5814](https://github.com/thanos-io/thanos/pull/5814) - Add metric `thanos_bucket_store_postings_size_bytes` that shows the distribution of how many postings (in bytes) were needed for each Series() call in Thanos Store. Useful for determining limits. + ### Changed ## [v0.29.0](https://github.com/thanos-io/thanos/tree/release-0.29) - Release in progress diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 1fd7494e01..9e32d2d615 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -118,6 +118,7 @@ type bucketStoreMetrics struct { seriesMergeDuration prometheus.Histogram resultSeriesCount prometheus.Summary chunkSizeBytes prometheus.Histogram + postingsSizeBytes prometheus.Histogram queriesDropped *prometheus.CounterVec seriesRefetches prometheus.Counter emptyPostingCount prometheus.Counter @@ -205,6 +206,14 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { }, }) + m.postingsSizeBytes = promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ + Name: "thanos_bucket_store_postings_size_bytes", + Help: "Size in bytes of the postings for a single series call.", + Buckets: []float64{ + 32, 256, 512, 1024, 32 * 1024, 256 * 1024, 512 * 1024, 1024 * 1024, 32 * 1024 * 1024, 256 * 1024 * 1024, 512 * 1024 * 1024, + }, + }) + m.queriesDropped = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Name: "thanos_bucket_store_queries_dropped_total", Help: "Number of queries that were dropped due to the limit.", @@ -1137,6 +1146,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie s.metrics.cachedPostingsCompressionTimeSeconds.WithLabelValues(labelDecode).Add(stats.CachedPostingsDecompressionTimeSum.Seconds()) s.metrics.cachedPostingsOriginalSizeBytes.Add(float64(stats.CachedPostingsOriginalSizeSum)) s.metrics.cachedPostingsCompressedSizeBytes.Add(float64(stats.CachedPostingsCompressedSizeSum)) + s.metrics.postingsSizeBytes.Observe(float64(int(stats.PostingsFetchedSizeSum) + int(stats.PostingsTouchedSizeSum))) level.Debug(s.logger).Log("msg", "stats query processed", "request", req,