Add S3 part-size flag to configure the S3 minimum file size in bytes used for multipart uploads

CHANGELOG.md

@@ -23,6 +23,7 @@
 * [CHANGE] Upgrade Node.js to v20. #6540
 * [FEATURE] Query-frontend: add experimental support for query blocking. Queries are blocked on a per-tenant basis and is configured via the limit `blocked_queries`. #5609
 * [FEATURE] Vault: Added support for new Vault authentication methods: `AppRole`, `Kubernetes`, `UserPass` and `Token`. #6143
+* [FEATURE] Added `-<prefix>.s3.part-size` flag to configure the S3 minimum file size in bytes used for multipart uploads. #6592
 * [ENHANCEMENT] Ingester: exported summary `cortex_ingester_inflight_push_requests_summary` tracking total number of inflight requests in percentile buckets. #5845
 * [ENHANCEMENT] Query-scheduler: add `cortex_query_scheduler_enqueue_duration_seconds` metric that records the time taken to enqueue or reject a query request. #5879
 * [ENHANCEMENT] Query-frontend: add `cortex_query_frontend_enqueue_duration_seconds` metric that records the time taken to enqueue or reject a query request. When query-scheduler is in use, the metric has the `scheduler_address` label to differentiate the enqueue duration by query-scheduler backend. #5879 #6087 #6120

cmd/mimir/config-descriptor.json

@@ -5640,6 +5640,17 @@
               "fieldType": "boolean",
               "fieldCategory": "experimental"
             },
+            {
+              "kind": "field",
+              "name": "part_size",
+              "required": false,
+              "desc": "The minimum file size in bytes used for multipart uploads. If 0, a built-in default value is used.",
+              "fieldValue": null,
+              "fieldDefaultValue": 67108864,
+              "fieldFlag": "blocks-storage.s3.part-size",
+              "fieldType": "int",
+              "fieldCategory": "experimental"
+            },
             {
               "kind": "block",
               "name": "sse",
@@ -11504,6 +11515,17 @@
               "fieldType": "boolean",
               "fieldCategory": "experimental"
             },
+            {
+              "kind": "field",
+              "name": "part_size",
+              "required": false,
+              "desc": "The minimum file size in bytes used for multipart uploads. If 0, a built-in default value is used.",
+              "fieldValue": null,
+              "fieldDefaultValue": 67108864,
+              "fieldFlag": "ruler-storage.s3.part-size",
+              "fieldType": "int",
+              "fieldCategory": "experimental"
+            },
             {
               "kind": "block",
               "name": "sse",
@@ -13524,6 +13546,17 @@
               "fieldType": "boolean",
               "fieldCategory": "experimental"
             },
+            {
+              "kind": "field",
+              "name": "part_size",
+              "required": false,
+              "desc": "The minimum file size in bytes used for multipart uploads. If 0, a built-in default value is used.",
+              "fieldValue": null,
+              "fieldDefaultValue": 67108864,
+              "fieldFlag": "alertmanager-storage.s3.part-size",
+              "fieldType": "int",
+              "fieldCategory": "experimental"
+            },
             {
               "kind": "block",
               "name": "sse",
@@ -15788,6 +15821,17 @@
                   "fieldType": "boolean",
                   "fieldCategory": "experimental"
                 },
+                {
+                  "kind": "field",
+                  "name": "part_size",
+                  "required": false,
+                  "desc": "The minimum file size in bytes used for multipart uploads. If 0, a built-in default value is used.",
+                  "fieldValue": null,
+                  "fieldDefaultValue": 67108864,
+                  "fieldFlag": "common.storage.s3.part-size",
+                  "fieldType": "int",
+                  "fieldCategory": "experimental"
+                },
                 {
                   "kind": "block",
                   "name": "sse",

cmd/mimir/help-all.txt.tmpl

@@ -53,6 +53,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100)
   -alertmanager-storage.s3.native-aws-auth-enabled
     	[experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.
+  -alertmanager-storage.s3.part-size uint
+    	[experimental] The minimum file size in bytes used for multipart uploads. If 0, a built-in default value is used. (default 67108864)
   -alertmanager-storage.s3.region string
     	S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it.
   -alertmanager-storage.s3.secret-access-key string
@@ -695,6 +697,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100)
   -blocks-storage.s3.native-aws-auth-enabled
     	[experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.
+  -blocks-storage.s3.part-size uint
+    	[experimental] The minimum file size in bytes used for multipart uploads. If 0, a built-in default value is used. (default 67108864)
   -blocks-storage.s3.region string
     	S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it.
   -blocks-storage.s3.secret-access-key string
@@ -855,6 +859,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100)
   -common.storage.s3.native-aws-auth-enabled
     	[experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.
+  -common.storage.s3.part-size uint
+    	[experimental] The minimum file size in bytes used for multipart uploads. If 0, a built-in default value is used. (default 67108864)
   -common.storage.s3.region string
     	S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it.
   -common.storage.s3.secret-access-key string
@@ -2235,6 +2241,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100)
   -ruler-storage.s3.native-aws-auth-enabled
     	[experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.
+  -ruler-storage.s3.part-size uint
+    	[experimental] The minimum file size in bytes used for multipart uploads. If 0, a built-in default value is used. (default 67108864)
   -ruler-storage.s3.region string
     	S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it.
   -ruler-storage.s3.secret-access-key string

docs/sources/mimir/references/configuration-parameters/index.md

@@ -4513,6 +4513,11 @@ The s3_backend block configures the connection to Amazon S3 object storage backe
 # CLI flag: -<prefix>.s3.native-aws-auth-enabled
 [native_aws_auth_enabled: <boolean> | default = false]
 
+# (experimental) The minimum file size in bytes used for multipart uploads. If
+# 0, a built-in default value is used.
+# CLI flag: -<prefix>.s3.part-size
+[part_size: <int> | default = 67108864]
+
 sse:
   # Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3.
   # CLI flag: -<prefix>.s3.sse.type

pkg/storage/bucket/s3/bucket_client.go

@@ -60,6 +60,7 @@ func newS3Config(cfg Config) (s3.Config, error) {
 		SSEConfig:          sseCfg,
 		ListObjectsVersion: cfg.ListObjectsVersion,
 		AWSSDKAuth:         cfg.NativeAWSAuthEnabled,
+		PartSize:           cfg.PartSize,
 		HTTPConfig: s3.HTTPConfig{
 			IdleConnTimeout:       model.Duration(cfg.HTTP.IdleConnTimeout),
 			ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout),

pkg/storage/bucket/s3/config.go

@@ -85,6 +85,7 @@ type Config struct {
 	ListObjectsVersion   string         `yaml:"list_objects_version" category:"advanced"`
 	StorageClass         string         `yaml:"storage_class" category:"experimental"`
 	NativeAWSAuthEnabled bool           `yaml:"native_aws_auth_enabled" category:"experimental"`
+	PartSize             uint64         `yaml:"part_size" category:"experimental"`
 
 	SSE  SSEConfig  `yaml:"sse"`
 	HTTP HTTPConfig `yaml:"http"`
@@ -107,6 +108,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.ListObjectsVersion, prefix+"s3.list-objects-version", "", "Use a specific version of the S3 list object API. Supported values are v1 or v2. Default is unset.")
 	f.StringVar(&cfg.StorageClass, prefix+"s3.storage-class", "", "The S3 storage class to use, not set by default. Details can be found at https://aws.amazon.com/s3/storage-classes/. Supported values are: "+strings.Join(supportedStorageClasses, ", "))
 	f.BoolVar(&cfg.NativeAWSAuthEnabled, prefix+"s3.native-aws-auth-enabled", false, "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.")
+	f.Uint64Var(&cfg.PartSize, prefix+"s3.part-size", s3.DefaultConfig.PartSize, "The minimum file size in bytes used for multipart uploads. If 0, a built-in default value is used.")
 	cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f)
 	cfg.HTTP.RegisterFlagsWithPrefix(prefix, f)
 }