Remove mmap-based index header reader

CHANGELOG.md

@@ -24,6 +24,10 @@ Querying with using `{__mimir_storage__="ephemeral"}` selector no longer works.
   * `cortex_ingester_queries_ephemeral_total`
   * `cortex_ingester_queried_ephemeral_samples`
   * `cortex_ingester_queried_ephemeral_series`
+* [CHANGE] Store-gateway: use mmap-less index-header reader by default and remove mmap-based index header reader. The following flags have changed: #4280
+   * `-blocks-storage.bucket-store.index-header.map-populate-enabled` has been removed
+   * `-blocks-storage.bucket-store.index-header.stream-reader-enabled` has been removed
+   * `-blocks-storage.bucket-store.index-header.stream-reader-max-idle-file-handles` has been renamed to `-blocks-storage.bucket-store.index-header.max-idle-file-handles`, and the corresponding configuration file option has been renamed from `stream_reader_max_idle_file_handles` to `max_idle_file_handles`
 * [FEATURE] Ruler: added `keep_firing_for` support to alerting rules. #4099
 * [FEATURE] Distributor, ingester: ingestion of native histograms. The new per-tenant limit `-ingester.native-histograms-ingestion-enabled` controls whether native histograms are stored or ignored. #4159
 * [FEATURE] Query-frontend: Introduce experimental `-query-frontend.query-sharding-target-series-per-shard` to allow query sharding to take into account cardinality of similar requests executed previously. This feature uses the same cache that's used for results caching. #4121 #4177 #4188 #4254

cmd/mimir/config-descriptor.json

@@ -5779,36 +5779,14 @@
               "blockEntries": [
                 {
                   "kind": "field",
-                  "name": "map_populate_enabled",
+                  "name": "max_idle_file_handles",
                   "required": false,
-                  "desc": "If enabled, the store-gateway will attempt to pre-populate the file system cache when memory-mapping index-header files.",
-                  "fieldValue": null,
-                  "fieldDefaultValue": false,
-                  "fieldFlag": "blocks-storage.bucket-store.index-header.map-populate-enabled",
-                  "fieldType": "boolean",
-                  "fieldCategory": "experimental"
-                },
-                {
-                  "kind": "field",
-                  "name": "stream_reader_enabled",
-                  "required": false,
-                  "desc": "If enabled, the store-gateway will use an experimental streaming reader to load and parse index-header files.",
-                  "fieldValue": null,
-                  "fieldDefaultValue": false,
-                  "fieldFlag": "blocks-storage.bucket-store.index-header.stream-reader-enabled",
-                  "fieldType": "boolean",
-                  "fieldCategory": "experimental"
-                },
-                {
-                  "kind": "field",
-                  "name": "stream_reader_max_idle_file_handles",
-                  "required": false,
-                  "desc": "Maximum number of idle file handles the store-gateway keeps open for each index-header file when using the streaming reader. This option is used only when the index-header streaming reader is enabled.",
+                  "desc": "Maximum number of idle file handles the store-gateway keeps open for each index-header file.",
                   "fieldValue": null,
                   "fieldDefaultValue": 1,
-                  "fieldFlag": "blocks-storage.bucket-store.index-header.stream-reader-max-idle-file-handles",
+                  "fieldFlag": "blocks-storage.bucket-store.index-header.max-idle-file-handles",
                   "fieldType": "int",
-                  "fieldCategory": "experimental"
+                  "fieldCategory": "advanced"
                 }
               ],
               "fieldValue": null,

cmd/mimir/help-all.txt.tmpl

@@ -343,12 +343,8 @@ Usage of ./cmd/mimir/mimir:
     	If enabled, store-gateway will lazy load an index-header only once required by a query. (default true)
   -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout duration
     	If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity. (default 1h0m0s)
-  -blocks-storage.bucket-store.index-header.map-populate-enabled
-    	[experimental] If enabled, the store-gateway will attempt to pre-populate the file system cache when memory-mapping index-header files.
-  -blocks-storage.bucket-store.index-header.stream-reader-enabled
-    	[experimental] If enabled, the store-gateway will use an experimental streaming reader to load and parse index-header files.
-  -blocks-storage.bucket-store.index-header.stream-reader-max-idle-file-handles uint
-    	[experimental] Maximum number of idle file handles the store-gateway keeps open for each index-header file when using the streaming reader. This option is used only when the index-header streaming reader is enabled. (default 1)
+  -blocks-storage.bucket-store.index-header.max-idle-file-handles uint
+    	Maximum number of idle file handles the store-gateway keeps open for each index-header file. (default 1)
   -blocks-storage.bucket-store.max-chunk-pool-bytes uint
     	Max size - in bytes - of a chunks pool, used to reduce memory allocations. The pool is shared across all tenants. 0 to disable the limit. (default 2147483648)
   -blocks-storage.bucket-store.max-concurrent int

docs/sources/mimir/operators-guide/configure/about-versioning.md

@@ -91,9 +91,6 @@ The following features are currently experimental:
   - `-query-scheduler.querier-forget-delay`
   - Max number of used instances (`-query-scheduler.max-used-instances`)
 - Store-gateway
-  - `-blocks-storage.bucket-store.index-header.map-populate-enabled`
-  - `-blocks-storage.bucket-store.index-header.stream-reader-enabled`
-  - `-blocks-storage.bucket-store.index-header.stream-reader-max-idle-file-handles`
   - `-blocks-storage.bucket-store.batch-series-size`
   - `-blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled`
 - Blocks Storage, Alertmanager, and Ruler support for partitioning access to the same storage bucket

docs/sources/mimir/references/configuration-parameters/index.md

@@ -3143,21 +3143,10 @@ bucket_store:
   [postings_offsets_in_mem_sampling: <int> | default = 32]
 
   index_header:
-    # (experimental) If enabled, the store-gateway will attempt to pre-populate
-    # the file system cache when memory-mapping index-header files.
-    # CLI flag: -blocks-storage.bucket-store.index-header.map-populate-enabled
-    [map_populate_enabled: <boolean> | default = false]
-
-    # (experimental) If enabled, the store-gateway will use an experimental
-    # streaming reader to load and parse index-header files.
-    # CLI flag: -blocks-storage.bucket-store.index-header.stream-reader-enabled
-    [stream_reader_enabled: <boolean> | default = false]
-
-    # (experimental) Maximum number of idle file handles the store-gateway keeps
-    # open for each index-header file when using the streaming reader. This
-    # option is used only when the index-header streaming reader is enabled.
-    # CLI flag: -blocks-storage.bucket-store.index-header.stream-reader-max-idle-file-handles
-    [stream_reader_max_idle_file_handles: <int> | default = 1]
+    # (advanced) Maximum number of idle file handles the store-gateway keeps
+    # open for each index-header file.
+    # CLI flag: -blocks-storage.bucket-store.index-header.max-idle-file-handles
+    [max_idle_file_handles: <int> | default = 1]
 
   # (experimental) If larger than 0, this option enables store-gateway series
   # streaming. The store-gateway will load series from the bucket in batches

go.mod

@@ -67,7 +67,6 @@ require (
 	go.opentelemetry.io/otel/trace v1.11.2
 	go.uber.org/multierr v1.9.0
 	golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
-	golang.org/x/sys v0.4.0
 	google.golang.org/api v0.108.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	sigs.k8s.io/kustomize/kyaml v0.13.7
@@ -213,6 +212,7 @@ require (
 	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/mod v0.7.0 // indirect
 	golang.org/x/oauth2 v0.4.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
 	golang.org/x/text v0.6.0 // indirect
 	golang.org/x/tools v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect

pkg/storegateway/bucket_test.go

@@ -846,7 +846,7 @@ func prepareTestBucket(tb test.TB, dataSetup ...func(tb testing.TB, appender sto
 func prepareTestBlockWithBinaryReader(tb test.TB, dataSetup ...func(tb testing.TB, appender storage.Appender)) func() *bucketBlock {
 	bkt, tmpDir, id, minT, maxT := prepareTestBucket(tb, dataSetup...)
 
-	r, err := indexheader.NewBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, id, mimir_tsdb.DefaultPostingOffsetInMemorySampling, indexheader.Config{})
+	r, err := indexheader.NewStreamBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, id, mimir_tsdb.DefaultPostingOffsetInMemorySampling, indexheader.NewStreamBinaryReaderMetrics(nil), indexheader.Config{})
 	require.NoError(tb, err)
 
 	return newBucketBlockFactory(bkt, r, id, minT, maxT)
@@ -1504,7 +1504,7 @@ func TestBucketStore_Series_OneBlock_InMemIndexCacheSegfault(t *testing.T) {
 			chunkObjs:    []string{filepath.Join(id.String(), "chunks", "000001")},
 			chunkPool:    chunkPool,
 		}
-		b1.indexHeaderReader, err = indexheader.NewBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, b1.meta.ULID, mimir_tsdb.DefaultPostingOffsetInMemorySampling, indexheader.Config{})
+		b1.indexHeaderReader, err = indexheader.NewStreamBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, b1.meta.ULID, mimir_tsdb.DefaultPostingOffsetInMemorySampling, indexheader.NewStreamBinaryReaderMetrics(nil), indexheader.Config{})
 		assert.NoError(t, err)
 	}
 
@@ -1543,7 +1543,7 @@ func TestBucketStore_Series_OneBlock_InMemIndexCacheSegfault(t *testing.T) {
 			chunkObjs:    []string{filepath.Join(id.String(), "chunks", "000001")},
 			chunkPool:    chunkPool,
 		}
-		b2.indexHeaderReader, err = indexheader.NewBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, b2.meta.ULID, mimir_tsdb.DefaultPostingOffsetInMemorySampling, indexheader.Config{})
+		b2.indexHeaderReader, err = indexheader.NewStreamBinaryReader(context.Background(), log.NewNopLogger(), bkt, tmpDir, b2.meta.ULID, mimir_tsdb.DefaultPostingOffsetInMemorySampling, indexheader.NewStreamBinaryReaderMetrics(nil), indexheader.Config{})
 		assert.NoError(t, err)
 	}
 
@@ -2567,7 +2567,7 @@ func prepareBucket(b *testing.B) (*bucketBlock, *metadata.Meta) {
 	partitioner := newGapBasedPartitioners(mimir_tsdb.DefaultPartitionerMaxGapSize, nil)
 
 	// Create an index header reader.
-	indexHeaderReader, err := indexheader.NewBinaryReader(ctx, logger, bkt, tmpDir, blockMeta.ULID, mimir_tsdb.DefaultPostingOffsetInMemorySampling, indexheader.Config{})
+	indexHeaderReader, err := indexheader.NewStreamBinaryReader(ctx, logger, bkt, tmpDir, blockMeta.ULID, mimir_tsdb.DefaultPostingOffsetInMemorySampling, indexheader.NewStreamBinaryReaderMetrics(nil), indexheader.Config{})
 	assert.NoError(b, err)
 	indexCache, err := indexcache.NewInMemoryIndexCacheWithConfig(logger, nil, indexcache.DefaultInMemoryIndexCacheConfig)
 	assert.NoError(b, err)