From 82f4ad8ad223c73917f398461504c3bc1753a5fd Mon Sep 17 00:00:00 2001 From: Jonathan Halterman Date: Tue, 13 Feb 2024 10:41:31 -0800 Subject: [PATCH] Add timely head compaction support (#7372) * Add timely head compaction support --- CHANGELOG.md | 1 + cmd/mimir/config-descriptor.json | 11 +++++++++++ cmd/mimir/help-all.txt.tmpl | 2 ++ docs/sources/mimir/configure/about-versioning.md | 1 + .../mimir/configure/configuration-parameters/index.md | 6 ++++++ go.mod | 2 +- go.sum | 4 ++-- pkg/ingester/ingester.go | 1 + pkg/storage/tsdb/config.go | 5 +++++ vendor/github.com/prometheus/prometheus/tsdb/db.go | 5 +++++ vendor/github.com/prometheus/prometheus/tsdb/head.go | 11 ++++++++++- vendor/modules.txt | 4 ++-- 12 files changed, 47 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24b46851b67..9aeefb24a24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ * [FEATURE] Alertmanager API: added `-alertmanager.grafana-alertmanager-compatibility-enabled` CLI flag (and respective YAML config option) to enable an experimental API endpoints that support the migration of the Grafana Alertmanager. #7057 * [FEATURE] Alertmanager: Added `-alertmanager.utf8-strict-mode-enabled` to control support for any UTF-8 character as part of Alertmanager configuration/API matchers and labels. It's default value is set to `false`. #6898 * [FEATURE] Querier: added `histogram_avg()` function support to PromQL. #7293 +* [FEATURE] Ingester: added `-blocks-storage.tsdb.timely-head-compaction` flag, which enables more timely head compaction, and defaults to `false`. #7372 * [ENHANCEMENT] Vault: add lifecycle manager for token used to authenticate to Vault. This ensures the client token is always valid. Includes a gauge (`cortex_vault_token_lease_renewal_active`) to check whether token renewal is active, and the counters `cortex_vault_token_lease_renewal_success_total` and `cortex_vault_auth_success_total` to see the total number of successful lease renewals / authentications. #7337 * [ENHANCEMENT] Store-gateway: add no-compact details column on store-gateway tenants admin UI. #6848 * [ENHANCEMENT] PromQL: ignore small errors for bucketQuantile #6766 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 9464b2a37db..5390e8ca00f 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -8781,6 +8781,17 @@ "fieldFlag": "blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage", "fieldType": "int", "fieldCategory": "experimental" + }, + { + "kind": "field", + "name": "timely_head_compaction_enabled", + "required": false, + "desc": "Allows head compaction to happen when the min block range can no longer be appended, without requiring 1.5x the chunk range worth of data in the head.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "blocks-storage.tsdb.timely-head-compaction-enabled", + "fieldType": "boolean", + "fieldCategory": "experimental" } ], "fieldValue": null, diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 8ec5d141bf4..19c6522c429 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -809,6 +809,8 @@ Usage of ./cmd/mimir/mimir: How frequently the TSDB blocks are scanned and new ones are shipped to the storage. 0 means shipping is disabled. (default 1m0s) -blocks-storage.tsdb.stripe-size int The number of shards of series to use in TSDB (must be a power of 2). Reducing this will decrease memory footprint, but can negatively impact performance. (default 16384) + -blocks-storage.tsdb.timely-head-compaction-enabled + [experimental] Allows head compaction to happen when the min block range can no longer be appended, without requiring 1.5x the chunk range worth of data in the head. -blocks-storage.tsdb.wal-compression-enabled True to enable TSDB WAL compression. -blocks-storage.tsdb.wal-replay-concurrency int diff --git a/docs/sources/mimir/configure/about-versioning.md b/docs/sources/mimir/configure/about-versioning.md index 0a2e9b20a40..61e3d1630c5 100644 --- a/docs/sources/mimir/configure/about-versioning.md +++ b/docs/sources/mimir/configure/about-versioning.md @@ -106,6 +106,7 @@ The following features are currently experimental: - Early TSDB Head compaction to reduce in-memory series: - `-blocks-storage.tsdb.early-head-compaction-min-in-memory-series` - `-blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage` + - Timely head compaction (`-blocks-storage.tsdb.timely-head-compaction-enabled`) - Ingester client - Per-ingester circuit breaking based on requests timing out or hitting per-instance limits - `-ingester.client.circuit-breaker.enabled` diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index db08fb9602d..ea94a1f5c01 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -3932,6 +3932,12 @@ tsdb: # percentage (0-100). # CLI flag: -blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage [early_head_compaction_min_estimated_series_reduction_percentage: | default = 15] + + # (experimental) Allows head compaction to happen when the min block range can + # no longer be appended, without requiring 1.5x the chunk range worth of data + # in the head. + # CLI flag: -blocks-storage.tsdb.timely-head-compaction-enabled + [timely_head_compaction_enabled: | default = false] ``` ### compactor diff --git a/go.mod b/go.mod index 7fab0cffacc..159335d8494 100644 --- a/go.mod +++ b/go.mod @@ -256,7 +256,7 @@ require ( ) // Using a fork of Prometheus with Mimir-specific changes. -replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240205112357-84eae046431c +replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240213153929-d32b69f03433 // Replace memberlist with our fork which includes some fixes that haven't been // merged upstream yet: diff --git a/go.sum b/go.sum index 9155f02bbe3..b253b5699a2 100644 --- a/go.sum +++ b/go.sum @@ -505,8 +505,8 @@ github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586 h1:/of8Z8taCPft github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe h1:yIXAAbLswn7VNWBIvM71O2QsgfgW9fRXZNR0DXe6pDU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/grafana/mimir-prometheus v0.0.0-20240205112357-84eae046431c h1:jrkiLy8SjrLKbCeF1SCq6bKfdM2bR5/1hlnx+wwxAPQ= -github.com/grafana/mimir-prometheus v0.0.0-20240205112357-84eae046431c/go.mod h1:IMaHPfxuCuOjlEoyUE0AG7FGZLEwCg6WHXHLNQwPrJQ= +github.com/grafana/mimir-prometheus v0.0.0-20240213153929-d32b69f03433 h1:mW1Cet0WSipL2fBpfWnUyIlS7cZqvg2/5j6IP6lbogA= +github.com/grafana/mimir-prometheus v0.0.0-20240213153929-d32b69f03433/go.mod h1:IMaHPfxuCuOjlEoyUE0AG7FGZLEwCg6WHXHLNQwPrJQ= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 h1:em1oddjXL8c1tL0iFdtVtPloq2hRPen2MJQKoAWpxu0= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo= diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 8ba9d762c9b..c5ee809a4a0 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -2440,6 +2440,7 @@ func (i *Ingester) createTSDB(userID string, walReplayConcurrency int) (*userTSD EnableSharding: true, // Always enable query sharding support. OutOfOrderTimeWindow: oooTW.Milliseconds(), // The unit must be same as our timestamps. OutOfOrderCapMax: int64(i.cfg.BlocksStorageConfig.TSDB.OutOfOrderCapacityMax), + TimelyCompaction: i.cfg.BlocksStorageConfig.TSDB.TimelyHeadCompaction, HeadPostingsForMatchersCacheTTL: i.cfg.BlocksStorageConfig.TSDB.HeadPostingsForMatchersCacheTTL, HeadPostingsForMatchersCacheMaxItems: i.cfg.BlocksStorageConfig.TSDB.HeadPostingsForMatchersCacheMaxItems, HeadPostingsForMatchersCacheMaxBytes: i.cfg.BlocksStorageConfig.TSDB.HeadPostingsForMatchersCacheMaxBytes, diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 8c3135b5dfe..044a581b342 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -264,6 +264,10 @@ type TSDBConfig struct { // HeadCompactionIntervalJitterEnabled is enabled by default, but allows to disable it in tests. HeadCompactionIntervalJitterEnabled bool `yaml:"-"` + + // TimelyHeadCompaction allows head compaction to happen when min block range can no longer be appended, + // without requiring 1.5x the chunk range worth of data in the head. + TimelyHeadCompaction bool `yaml:"timely_head_compaction_enabled" category:"experimental"` } // RegisterFlags registers the TSDBConfig flags. @@ -308,6 +312,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.BlockPostingsForMatchersCacheForce, "blocks-storage.tsdb.block-postings-for-matchers-cache-force", tsdb.DefaultPostingsForMatchersCacheForce, "Force the cache to be used for postings for matchers in compacted blocks, even if it's not a concurrent (query-sharding) call.") f.Int64Var(&cfg.EarlyHeadCompactionMinInMemorySeries, "blocks-storage.tsdb.early-head-compaction-min-in-memory-series", 0, fmt.Sprintf("When the number of in-memory series in the ingester is equal to or greater than this setting, the ingester tries to compact the TSDB Head. The early compaction removes from the memory all samples and inactive series up until -%s time ago. After an early compaction, the ingester will not accept any sample with a timestamp older than -%s time ago (unless out of order ingestion is enabled). The ingester checks every -%s whether an early compaction is required. Use 0 to disable it.", activeseries.IdleTimeoutFlag, activeseries.IdleTimeoutFlag, headCompactionIntervalFlag)) f.IntVar(&cfg.EarlyHeadCompactionMinEstimatedSeriesReductionPercentage, "blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage", 15, "When the early compaction is enabled, the early compaction is triggered only if the estimated series reduction is at least the configured percentage (0-100).") + f.BoolVar(&cfg.TimelyHeadCompaction, "blocks-storage.tsdb.timely-head-compaction-enabled", false, "Allows head compaction to happen when the min block range can no longer be appended, without requiring 1.5x the chunk range worth of data in the head.") cfg.HeadCompactionIntervalJitterEnabled = true } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/db.go b/vendor/github.com/prometheus/prometheus/tsdb/db.go index fc4a57ff387..31c8dfa468d 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/db.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/db.go @@ -211,6 +211,10 @@ type Options struct { // EnableSharding enables query sharding support in TSDB. EnableSharding bool + // Timely compaction allows head compaction to happen when min block range can no longer be appended, + // without requiring 1.5x the chunk range worth of data in the head. + TimelyCompaction bool + // HeadPostingsForMatchersCacheTTL is the TTL of the postings for matchers cache in the Head. // If it's 0, the cache will only deduplicate in-flight requests, deleting the results once the first request has finished. HeadPostingsForMatchersCacheTTL time.Duration @@ -939,6 +943,7 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs headOpts.OutOfOrderTimeWindow.Store(opts.OutOfOrderTimeWindow) headOpts.OutOfOrderCapMax.Store(opts.OutOfOrderCapMax) headOpts.EnableSharding = opts.EnableSharding + headOpts.TimelyCompaction = opts.TimelyCompaction headOpts.PostingsForMatchersCacheTTL = opts.HeadPostingsForMatchersCacheTTL headOpts.PostingsForMatchersCacheMaxItems = opts.HeadPostingsForMatchersCacheMaxItems headOpts.PostingsForMatchersCacheMaxBytes = opts.HeadPostingsForMatchersCacheMaxBytes diff --git a/vendor/github.com/prometheus/prometheus/tsdb/head.go b/vendor/github.com/prometheus/prometheus/tsdb/head.go index fc9c64c595f..4f3ccb211e6 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/head.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/head.go @@ -197,6 +197,10 @@ type HeadOptions struct { // EnableSharding enables ShardedPostings() support in the Head. EnableSharding bool + // Timely compaction allows head compaction to happen when min block range can no longer be appended, + // without requiring 1.5x the chunk range worth of data in the head. + TimelyCompaction bool + PostingsForMatchersCacheTTL time.Duration PostingsForMatchersCacheMaxItems int PostingsForMatchersCacheMaxBytes int64 @@ -1653,9 +1657,14 @@ func (h *Head) MaxOOOTime() int64 { } // compactable returns whether the head has a compactable range. -// The head has a compactable range when the head time range is 1.5 times the chunk range. +// When the TimelyCompaction option is enabled, the head is compactable when the min block end is .5 times the chunk range in the past. +// Else the head has a compactable range when the head time range is 1.5 times the chunk range. // The 0.5 acts as a buffer of the appendable window. func (h *Head) compactable() bool { + if h.opts.TimelyCompaction { + minBlockEnd := rangeForTimestamp(h.MinTime(), h.chunkRange.Load()) + return minBlockEnd < h.appendableMinValidTime() + } return h.MaxTime()-h.MinTime() > h.chunkRange.Load()/2*3 } diff --git a/vendor/modules.txt b/vendor/modules.txt index 8dc49251731..e98da32e43e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -906,7 +906,7 @@ github.com/prometheus/exporter-toolkit/web github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20240205112357-84eae046431c +# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20240213153929-d32b69f03433 ## explicit; go 1.20 github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery @@ -1522,7 +1522,7 @@ sigs.k8s.io/kustomize/kyaml/yaml/walk sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 sigs.k8s.io/yaml/goyaml.v3 -# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240205112357-84eae046431c +# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240213153929-d32b69f03433 # github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe # gopkg.in/yaml.v3 => github.com/colega/go-yaml-yaml v0.0.0-20220720105220-255a8d16d094 # github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6