From 8f40dda3366fbaaa84649c2e767b54c349950ea7 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Tue, 12 Sep 2023 12:15:07 +0200 Subject: [PATCH 1/4] store-gateway: enable lazy loading concurrency by default Lazy loading concurrency has shows good results with slower disks internally at GL, so this PR is enabling it by default. Signed-off-by: Dimitar Dimitrov --- cmd/mimir/config-descriptor.json | 2 +- cmd/mimir/help-all.txt.tmpl | 2 +- docs/sources/mimir/references/configuration-parameters/index.md | 2 +- pkg/storegateway/indexheader/header.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 7bcba3b021f..25c3ba67ad4 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -7710,7 +7710,7 @@ "required": false, "desc": "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.", "fieldValue": null, - "fieldDefaultValue": 0, + "fieldDefaultValue": 4, "fieldFlag": "blocks-storage.bucket-store.index-header.lazy-loading-concurrency", "fieldType": "int", "fieldCategory": "experimental" diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index a6a30b52bf1..29ebc025d4d 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -502,7 +502,7 @@ Usage of ./cmd/mimir/mimir: -blocks-storage.bucket-store.index-header.eager-loading-startup-enabled [experimental] If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled. -blocks-storage.bucket-store.index-header.lazy-loading-concurrency int - [experimental] Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited. + [experimental] Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited. (default 4) -blocks-storage.bucket-store.index-header.lazy-loading-enabled If enabled, store-gateway will lazy load an index-header only once required by a query. (default true) -blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout duration diff --git a/docs/sources/mimir/references/configuration-parameters/index.md b/docs/sources/mimir/references/configuration-parameters/index.md index 186dc4f2d71..c933a56b15e 100644 --- a/docs/sources/mimir/references/configuration-parameters/index.md +++ b/docs/sources/mimir/references/configuration-parameters/index.md @@ -3487,7 +3487,7 @@ bucket_store: # (experimental) Maximum number of concurrent index header loads across all # tenants. If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-header.lazy-loading-concurrency - [lazy_loading_concurrency: | default = 0] + [lazy_loading_concurrency: | default = 4] # (experimental) If enabled, store-gateway will persist a sparse version of # the index-header to disk on construction and load sparse index-headers diff --git a/pkg/storegateway/indexheader/header.go b/pkg/storegateway/indexheader/header.go index 43a6cfdb57a..d5c7dbe3c4b 100644 --- a/pkg/storegateway/indexheader/header.go +++ b/pkg/storegateway/indexheader/header.go @@ -81,7 +81,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { f.UintVar(&cfg.MaxIdleFileHandles, prefix+"max-idle-file-handles", 1, "Maximum number of idle file handles the store-gateway keeps open for each index-header file.") f.BoolVar(&cfg.LazyLoadingEnabled, prefix+"lazy-loading-enabled", DefaultIndexHeaderLazyLoadingEnabled, "If enabled, store-gateway will lazy load an index-header only once required by a query.") f.DurationVar(&cfg.LazyLoadingIdleTimeout, prefix+"lazy-loading-idle-timeout", DefaultIndexHeaderLazyLoadingIdleTimeout, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") - f.IntVar(&cfg.LazyLoadingConcurrency, prefix+"lazy-loading-concurrency", 0, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.") + f.IntVar(&cfg.LazyLoadingConcurrency, prefix+"lazy-loading-concurrency", 4, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.") f.BoolVar(&cfg.EagerLoadingStartupEnabled, prefix+"eager-loading-startup-enabled", false, "If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.") f.BoolVar(&cfg.SparsePersistenceEnabled, prefix+"sparse-persistence-enabled", false, "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.") f.BoolVar(&cfg.VerifyOnLoad, prefix+"verify-on-load", false, "If true, verify the checksum of index headers upon loading them (either on startup or lazily when lazy loading is enabled). Setting to true helps detect disk corruption at the cost of slowing down index header loading.") From 9cd2e88d77603eb172852b18dd09aecf08ce4ce5 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Tue, 12 Sep 2023 12:18:29 +0200 Subject: [PATCH 2/4] Add CHANGELOG.md entry Signed-off-by: Dimitar Dimitrov --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7fb16698ea..25f3489efb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Grafana Mimir +* [CHANGE] Store-gateway: lazy-loading concurrency limit default value is now 4. #6004 * [ENHANCEMENT] Query-scheduler: add `cortex_query_scheduler_enqueue_duration_seconds` metric that records the time taken to enqueue or reject a query request. #5879 * [ENHANCEMENT] Query-frontend: add `cortex_query_frontend_enqueue_duration_seconds` metric that records the time taken to enqueue or reject a query request when not using the query-scheduler. #5879 * [ENHANCEMENT] Expose `/sync/mutex/wait/total:seconds` Go runtime metric as `go_sync_mutex_wait_total_seconds_total` from all components. #5879 From f731fb22bea33e5d6e29cc4482b8b7c7c6f4fbcf Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Tue, 12 Sep 2023 13:44:10 +0200 Subject: [PATCH 3/4] Update tests Signed-off-by: Dimitar Dimitrov --- pkg/storegateway/bucket_stores_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/storegateway/bucket_stores_test.go b/pkg/storegateway/bucket_stores_test.go index 00b4d8bfe63..6723bdfe2c9 100644 --- a/pkg/storegateway/bucket_stores_test.go +++ b/pkg/storegateway/bucket_stores_test.go @@ -120,6 +120,7 @@ func TestBucketStores_InitialSync(t *testing.T) { # HELP cortex_bucket_stores_gate_queries_in_flight Number of queries that are currently in flight. # TYPE cortex_bucket_stores_gate_queries_in_flight gauge cortex_bucket_stores_gate_queries_in_flight{gate="query"} 0 + cortex_bucket_stores_gate_queries_in_flight{gate="index_header"} 0 `), "cortex_bucket_store_blocks_loaded", "cortex_bucket_store_block_loads_total", @@ -253,6 +254,7 @@ func TestBucketStores_SyncBlocks(t *testing.T) { # HELP cortex_bucket_stores_gate_queries_in_flight Number of queries that are currently in flight. # TYPE cortex_bucket_stores_gate_queries_in_flight gauge cortex_bucket_stores_gate_queries_in_flight{gate="query"} 0 + cortex_bucket_stores_gate_queries_in_flight{gate="index_header"} 0 `), "cortex_bucket_store_blocks_loaded", "cortex_bucket_store_block_loads_total", From 970ca55bab31da49341724051ff51e30b59ebe90 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Tue, 12 Sep 2023 15:18:28 +0200 Subject: [PATCH 4/4] Update tests Signed-off-by: Dimitar Dimitrov --- pkg/storegateway/bucket_stores_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/storegateway/bucket_stores_test.go b/pkg/storegateway/bucket_stores_test.go index 6723bdfe2c9..7a9085ac9ff 100644 --- a/pkg/storegateway/bucket_stores_test.go +++ b/pkg/storegateway/bucket_stores_test.go @@ -116,6 +116,7 @@ func TestBucketStores_InitialSync(t *testing.T) { # HELP cortex_bucket_stores_gate_queries_concurrent_max Number of maximum concurrent queries allowed. # TYPE cortex_bucket_stores_gate_queries_concurrent_max gauge cortex_bucket_stores_gate_queries_concurrent_max{gate="query"} 100 + cortex_bucket_stores_gate_queries_concurrent_max{gate="index_header"} 4 # HELP cortex_bucket_stores_gate_queries_in_flight Number of queries that are currently in flight. # TYPE cortex_bucket_stores_gate_queries_in_flight gauge @@ -250,6 +251,7 @@ func TestBucketStores_SyncBlocks(t *testing.T) { # HELP cortex_bucket_stores_gate_queries_concurrent_max Number of maximum concurrent queries allowed. # TYPE cortex_bucket_stores_gate_queries_concurrent_max gauge cortex_bucket_stores_gate_queries_concurrent_max{gate="query"} 100 + cortex_bucket_stores_gate_queries_concurrent_max{gate="index_header"} 4 # HELP cortex_bucket_stores_gate_queries_in_flight Number of queries that are currently in flight. # TYPE cortex_bucket_stores_gate_queries_in_flight gauge