diff --git a/CHANGELOG.md b/CHANGELOG.md index 254bcd7290a..61e0f7515dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ * [CHANGE] Memcached: Change default read timeout for chunks and index caches to `750ms` from `450ms`. #7778 * [ENHANCEMENT] Compactor: add `$._config.cortex_compactor_concurrent_rollout_enabled` option (disabled by default) that makes use of rollout-operator to speed up the rollout of compactors. #7783 +* [ENHANCEMENT] Shuffle-sharding: add `$._config.shuffle_sharding.ingest_storage_partitions_enabled` and `$._config.shuffle_sharding.ingester_partitions_shard_size` options, that allow configuring partitions shard size in ingest-storage mode. #7804 * [BUGFIX] Guard against missing samples in KEDA queries. #7691 ### Mimirtool diff --git a/operations/mimir-tests/test-shuffle-sharding-partitions-enabled-generated.yaml b/operations/mimir-tests/test-shuffle-sharding-partitions-enabled-generated.yaml new file mode 100644 index 00000000000..8d901f711cb --- /dev/null +++ b/operations/mimir-tests/test-shuffle-sharding-partitions-enabled-generated.yaml @@ -0,0 +1,1598 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: default +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: alertmanager + name: alertmanager + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: alertmanager +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: compactor +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: distributor + name: distributor + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: distributor +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: ingester + name: ingester + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: ingester +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: memcached + name: memcached + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: memcached +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: memcached-frontend + name: memcached-frontend + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: memcached-frontend +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: memcached-index-queries + name: memcached-index-queries + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: memcached-index-queries +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: memcached-metadata + name: memcached-metadata + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: memcached-metadata +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: querier + name: querier + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: querier +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: query-frontend + name: query-frontend + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: query-frontend +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: query-scheduler + name: query-scheduler + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: query-scheduler +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: ruler + name: ruler + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: ruler +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: store-gateway + name: store-gateway + namespace: default +spec: + maxUnavailable: 2 + selector: + matchLabels: + name: store-gateway +--- +apiVersion: v1 +data: + overrides.yaml: | + overrides: {} +kind: ConfigMap +metadata: + name: overrides + namespace: default +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: alertmanager + name: alertmanager + namespace: default +spec: + clusterIP: None + ports: + - name: alertmanager-http-metrics + port: 8080 + targetPort: 8080 + - name: alertmanager-grpc + port: 9095 + targetPort: 9095 + - name: alertmanager-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: alertmanager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + - name: compactor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: distributor + name: distributor + namespace: default +spec: + clusterIP: None + ports: + - name: distributor-http-metrics + port: 8080 + targetPort: 8080 + - name: distributor-grpc + port: 9095 + targetPort: 9095 + - name: distributor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: distributor +--- +apiVersion: v1 +kind: Service +metadata: + name: gossip-ring + namespace: default +spec: + clusterIP: None + ports: + - appProtocol: tcp + name: gossip-ring + port: 7946 + protocol: TCP + targetPort: 7946 + selector: + gossip_ring_member: "true" +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: ingester + name: ingester + namespace: default +spec: + ports: + - name: ingester-http-metrics + port: 8080 + targetPort: 8080 + - name: ingester-grpc + port: 9095 + targetPort: 9095 + - name: ingester-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: ingester +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached + name: memcached + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached-frontend + name: memcached-frontend + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached-frontend +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached-index-queries + name: memcached-index-queries + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached-index-queries +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached-metadata + name: memcached-metadata + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached-metadata +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: querier + name: querier + namespace: default +spec: + ports: + - name: querier-http-metrics + port: 8080 + targetPort: 8080 + - name: querier-grpc + port: 9095 + targetPort: 9095 + - name: querier-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: querier +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-frontend + name: query-frontend + namespace: default +spec: + ports: + - name: query-frontend-http-metrics + port: 8080 + targetPort: 8080 + - name: query-frontend-grpc + port: 9095 + targetPort: 9095 + selector: + name: query-frontend +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-scheduler + name: query-scheduler + namespace: default +spec: + ports: + - name: query-scheduler-http-metrics + port: 8080 + targetPort: 8080 + - name: query-scheduler-grpc + port: 9095 + targetPort: 9095 + selector: + name: query-scheduler +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-scheduler + name: query-scheduler-discovery + namespace: default +spec: + clusterIP: None + ports: + - name: query-scheduler-http-metrics + port: 8080 + targetPort: 8080 + - name: query-scheduler-grpc + port: 9095 + targetPort: 9095 + publishNotReadyAddresses: true + selector: + name: query-scheduler +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: ruler + name: ruler + namespace: default +spec: + ports: + - name: ruler-http-metrics + port: 8080 + targetPort: 8080 + - name: ruler-grpc + port: 9095 + targetPort: 9095 + selector: + name: ruler +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: store-gateway + name: store-gateway + namespace: default +spec: + ports: + - name: store-gateway-http-metrics + port: 8080 + targetPort: 8080 + - name: store-gateway-grpc + port: 9095 + targetPort: 9095 + - name: store-gateway-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: store-gateway +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: distributor + namespace: default +spec: + minReadySeconds: 10 + replicas: 3 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: distributor + strategy: + rollingUpdate: + maxSurge: 15% + maxUnavailable: 0 + template: + metadata: + labels: + gossip_ring_member: "true" + name: distributor + spec: + containers: + - args: + - -distributor.ha-tracker.enable=true + - -distributor.ha-tracker.enable-for-all-users=true + - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 + - -distributor.ha-tracker.prefix=prom_ha/ + - -distributor.ha-tracker.store=etcd + - -distributor.health-check-ingesters=true + - -distributor.ingestion-burst-size=200000 + - -distributor.ingestion-rate-limit=10000 + - -distributor.ingestion-tenant-shard-size=3 + - -distributor.ring.heartbeat-period=1m + - -distributor.ring.heartbeat-timeout=4m + - -distributor.ring.prefix= + - -distributor.ring.store=memberlist + - -ingest-storage.ingestion-partition-tenant-shard-size=1 + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -mem-ballast-size-bytes=1073741824 + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.max-connection-age=60s + - -server.grpc.keepalive.max-connection-age-grace=5m + - -server.grpc.keepalive.max-connection-idle=1m + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -shutdown-delay=90s + - -target=distributor + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "8" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.12.0 + imagePullPolicy: IfNotPresent + name: distributor + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 4Gi + requests: + cpu: "2" + memory: 2Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + terminationGracePeriodSeconds: 100 + topologySpreadConstraints: + - labelSelector: + matchLabels: + name: distributor + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: querier + namespace: default +spec: + minReadySeconds: 10 + replicas: 6 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: querier + strategy: + rollingUpdate: + maxSurge: 15% + maxUnavailable: 0 + template: + metadata: + labels: + gossip_ring_member: "true" + name: querier + spec: + containers: + - args: + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -distributor.health-check-ingesters=true + - -distributor.ingestion-tenant-shard-size=3 + - -ingest-storage.ingestion-partition-tenant-shard-size=1 + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -mem-ballast-size-bytes=268435456 + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -querier.frontend-client.grpc-max-send-msg-size=104857600 + - -querier.max-concurrent=8 + - -querier.max-partial-query-length=768h + - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local.:9095 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.heartbeat-timeout=4m + - -store-gateway.sharding-ring.prefix= + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=memberlist + - -store-gateway.tenant-shard-size=3 + - -target=querier + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "5" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "5000" + image: grafana/mimir:2.12.0 + imagePullPolicy: IfNotPresent + name: querier + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 24Gi + requests: + cpu: "1" + memory: 12Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + topologySpreadConstraints: + - labelSelector: + matchLabels: + name: querier + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: query-frontend + namespace: default +spec: + minReadySeconds: 10 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: query-frontend + strategy: + rollingUpdate: + maxSurge: 15% + maxUnavailable: 0 + template: + metadata: + labels: + name: query-frontend + spec: + containers: + - args: + - -query-frontend.cache-results=true + - -query-frontend.max-cache-freshness=10m + - -query-frontend.max-queriers-per-tenant=10 + - -query-frontend.max-total-query-length=12000h + - -query-frontend.query-sharding-target-series-per-shard=2500 + - -query-frontend.results-cache.backend=memcached + - -query-frontend.results-cache.memcached.addresses=dnssrvnoa+memcached-frontend.default.svc.cluster.local.:11211 + - -query-frontend.results-cache.memcached.max-item-size=5242880 + - -query-frontend.results-cache.memcached.timeout=500ms + - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local.:9095 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.max-connection-age=30s + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -shutdown-delay=90s + - -target=query-frontend + - -usage-stats.installation-mode=jsonnet + env: + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "5000" + image: grafana/mimir:2.12.0 + imagePullPolicy: IfNotPresent + name: query-frontend + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 1200Mi + requests: + cpu: "2" + memory: 600Mi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + terminationGracePeriodSeconds: 390 + topologySpreadConstraints: + - labelSelector: + matchLabels: + name: query-frontend + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: query-scheduler + namespace: default +spec: + minReadySeconds: 10 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: query-scheduler + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + name: query-scheduler + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: query-scheduler + topologyKey: kubernetes.io/hostname + containers: + - args: + - -query-frontend.max-queriers-per-tenant=10 + - -query-scheduler.max-outstanding-requests-per-tenant=100 + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=query-scheduler + - -usage-stats.installation-mode=jsonnet + image: grafana/mimir:2.12.0 + imagePullPolicy: IfNotPresent + name: query-scheduler + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 2Gi + requests: + cpu: "2" + memory: 1Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ruler + namespace: default +spec: + minReadySeconds: 10 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ruler + strategy: + rollingUpdate: + maxSurge: 50% + maxUnavailable: 0 + template: + metadata: + labels: + gossip_ring_member: "true" + name: ruler + spec: + containers: + - args: + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -distributor.health-check-ingesters=true + - -distributor.ingestion-tenant-shard-size=3 + - -distributor.remote-timeout=10s + - -ingest-storage.ingestion-partition-tenant-shard-size=1 + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -querier.max-partial-query-length=768h + - -ruler-storage.cache.backend=memcached + - -ruler-storage.cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -ruler-storage.cache.memcached.max-async-concurrency=50 + - -ruler-storage.cache.memcached.max-item-size=1048576 + - -ruler-storage.gcs.bucket-name=rules-bucket + - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local./alertmanager + - -ruler.max-rule-groups-per-tenant=70 + - -ruler.max-rules-per-rule-group=20 + - -ruler.ring.store=memberlist + - -ruler.rule-path=/rules + - -ruler.tenant-shard-size=2 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.heartbeat-timeout=4m + - -store-gateway.sharding-ring.prefix= + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=memberlist + - -store-gateway.tenant-shard-size=3 + - -target=ruler + - -usage-stats.installation-mode=jsonnet + env: + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.12.0 + imagePullPolicy: IfNotPresent + name: ruler + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + cpu: "16" + memory: 16Gi + requests: + cpu: "1" + memory: 6Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + terminationGracePeriodSeconds: 600 + topologySpreadConstraints: + - labelSelector: + matchLabels: + name: ruler + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: alertmanager + name: alertmanager + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: alertmanager + serviceName: alertmanager + template: + metadata: + labels: + gossip_ring_member: "true" + name: alertmanager + spec: + containers: + - args: + - -alertmanager-storage.gcs.bucket-name=alerts-bucket + - -alertmanager.sharding-ring.replication-factor=3 + - -alertmanager.sharding-ring.store=memberlist + - -alertmanager.storage.path=/data + - -alertmanager.web.external-url=http://test/alertmanager + - -common.storage.backend=gcs + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=alertmanager + - -usage-stats.installation-mode=jsonnet + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: grafana/mimir:2.12.0 + imagePullPolicy: IfNotPresent + name: alertmanager + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 15Gi + requests: + cpu: "2" + memory: 10Gi + volumeMounts: + - mountPath: /data + name: alertmanager-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 900 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: alertmanager-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + name: compactor + serviceName: compactor + template: + metadata: + labels: + gossip_ring_member: "true" + name: compactor + spec: + containers: + - args: + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -compactor.block-ranges=2h,12h,24h + - -compactor.blocks-retention-period=0 + - -compactor.cleanup-interval=15m + - -compactor.compaction-concurrency=1 + - -compactor.compaction-interval=30m + - -compactor.compactor-tenant-shard-size=1 + - -compactor.data-dir=/data + - -compactor.deletion-delay=2h + - -compactor.first-level-compaction-wait-period=25m + - -compactor.max-closing-blocks-concurrency=2 + - -compactor.max-opening-blocks-concurrency=4 + - -compactor.ring.heartbeat-period=1m + - -compactor.ring.heartbeat-timeout=4m + - -compactor.ring.prefix= + - -compactor.ring.store=memberlist + - -compactor.ring.wait-stability-min-duration=1m + - -compactor.split-and-merge-shards=0 + - -compactor.split-groups=1 + - -compactor.symbols-flushers-concurrency=4 + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=compactor + - -usage-stats.installation-mode=jsonnet + image: grafana/mimir:2.12.0 + imagePullPolicy: IfNotPresent + name: compactor + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 6Gi + requests: + cpu: 1 + memory: 6Gi + volumeMounts: + - mountPath: /data + name: compactor-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 900 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: compactor-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 250Gi + storageClassName: standard +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: ingester + name: ingester + namespace: default +spec: + podManagementPolicy: Parallel + replicas: 3 + selector: + matchLabels: + name: ingester + serviceName: ingester + template: + metadata: + labels: + gossip_ring_member: "true" + name: ingester + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: ingester + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -blocks-storage.tsdb.block-ranges-period=2h + - -blocks-storage.tsdb.dir=/data/tsdb + - -blocks-storage.tsdb.head-compaction-interval=15m + - -blocks-storage.tsdb.ship-interval=1m + - -blocks-storage.tsdb.wal-replay-concurrency=3 + - -common.storage.backend=gcs + - -distributor.health-check-ingesters=true + - -distributor.ingestion-tenant-shard-size=3 + - -ingest-storage.ingestion-partition-tenant-shard-size=1 + - -ingester.max-global-metadata-per-metric=10 + - -ingester.max-global-metadata-per-user=30000 + - -ingester.max-global-series-per-user=150000 + - -ingester.ring.heartbeat-period=2m + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.num-tokens=512 + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -ingester.ring.tokens-file-path=/data/tokens + - -ingester.ring.unregister-on-shutdown=true + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc-max-concurrent-streams=500 + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=ingester + - -usage-stats.installation-mode=jsonnet + env: + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.12.0 + imagePullPolicy: IfNotPresent + name: ingester + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 25Gi + requests: + cpu: "4" + memory: 15Gi + volumeMounts: + - mountPath: /data + name: ingester-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 1200 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ingester-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi + storageClassName: fast +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: memcached + serviceName: memcached + template: + metadata: + labels: + name: memcached + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 6144 + - -I 1m + - -c 16384 + - -v + - --extended=track_sizes + image: memcached:1.6.22-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 9Gi + requests: + cpu: 500m + memory: 6552Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.14.3 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached-frontend + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: memcached-frontend + serviceName: memcached-frontend + template: + metadata: + labels: + name: memcached-frontend + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached-frontend + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 1024 + - -I 5m + - -c 16384 + - -v + - --extended=track_sizes + image: memcached:1.6.22-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 1536Mi + requests: + cpu: 500m + memory: 1176Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.14.3 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached-index-queries + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: memcached-index-queries + serviceName: memcached-index-queries + template: + metadata: + labels: + name: memcached-index-queries + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached-index-queries + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 1024 + - -I 5m + - -c 16384 + - -v + - --extended=track_sizes + image: memcached:1.6.22-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 1536Mi + requests: + cpu: 500m + memory: 1176Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.14.3 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached-metadata + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + name: memcached-metadata + serviceName: memcached-metadata + template: + metadata: + labels: + name: memcached-metadata + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached-metadata + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 512 + - -I 1m + - -c 16384 + - -v + - --extended=track_sizes + image: memcached:1.6.22-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 768Mi + requests: + cpu: 500m + memory: 638Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.14.3 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: store-gateway + name: store-gateway + namespace: default +spec: + podManagementPolicy: Parallel + replicas: 3 + selector: + matchLabels: + name: store-gateway + serviceName: store-gateway + template: + metadata: + labels: + gossip_ring_member: "true" + name: store-gateway + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: store-gateway + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.bucket-store.chunks-cache.backend=memcached + - -blocks-storage.bucket-store.chunks-cache.memcached.addresses=dnssrvnoa+memcached.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=750ms + - -blocks-storage.bucket-store.index-cache.backend=memcached + - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.index-cache.memcached.max-item-size=5242880 + - -blocks-storage.bucket-store.index-cache.memcached.timeout=750ms + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.heartbeat-period=1m + - -store-gateway.sharding-ring.heartbeat-timeout=4m + - -store-gateway.sharding-ring.prefix= + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=memberlist + - -store-gateway.sharding-ring.tokens-file-path=/data/tokens + - -store-gateway.sharding-ring.unregister-on-shutdown=false + - -store-gateway.sharding-ring.wait-stability-min-duration=1m + - -store-gateway.tenant-shard-size=3 + - -target=store-gateway + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "5" + - name: GOMEMLIMIT + value: "12884901888" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.12.0 + imagePullPolicy: IfNotPresent + name: store-gateway + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 18Gi + requests: + cpu: "1" + memory: 12Gi + volumeMounts: + - mountPath: /data + name: store-gateway-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: store-gateway-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: standard +--- +apiVersion: etcd.database.coreos.com/v1beta2 +kind: EtcdCluster +metadata: + annotations: + etcd.database.coreos.com/scope: clusterwide + name: etcd + namespace: default +spec: + pod: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + etcd_cluster: etcd + topologyKey: kubernetes.io/hostname + annotations: + prometheus.io/port: "2379" + prometheus.io/scrape: "true" + etcdEnv: + - name: ETCD_AUTO_COMPACTION_RETENTION + value: 1h + labels: + name: etcd + resources: + limits: + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + size: 3 + version: 3.3.13 diff --git a/operations/mimir-tests/test-shuffle-sharding-partitions-enabled.jsonnet b/operations/mimir-tests/test-shuffle-sharding-partitions-enabled.jsonnet new file mode 100644 index 00000000000..b73f51545f3 --- /dev/null +++ b/operations/mimir-tests/test-shuffle-sharding-partitions-enabled.jsonnet @@ -0,0 +1,26 @@ +local mimir = import 'mimir/mimir.libsonnet'; + +mimir { + _config+:: { + namespace: 'default', + external_url: 'http://test', + + storage_backend: 'gcs', + blocks_storage_bucket_name: 'blocks-bucket', + + ruler_enabled: true, + ruler_storage_bucket_name: 'rules-bucket', + + alertmanager_enabled: true, + alertmanager_storage_bucket_name: 'alerts-bucket', + + shuffle_sharding+:: { + ingester_write_path_enabled: true, + ingester_read_path_enabled: true, + querier_enabled: true, + ruler_enabled: true, + store_gateway_enabled: true, + ingest_storage_partitions_enabled: true, + }, + }, +} diff --git a/operations/mimir/shuffle-sharding.libsonnet b/operations/mimir/shuffle-sharding.libsonnet index 9e4ce2f3cc7..9f62ae7e3f0 100644 --- a/operations/mimir/shuffle-sharding.libsonnet +++ b/operations/mimir/shuffle-sharding.libsonnet @@ -8,9 +8,15 @@ ruler_enabled: false, store_gateway_enabled: false, + // Enables shuffle sharding for partitions, for (experimental) ingest storage. + ingest_storage_partitions_enabled: false, + // Default shard sizes. We want the shard size to be divisible by the number of zones. // We typically run 3 zones ingester_shard_size: 3, + // Default partitions shard size. With ingest storage we shard series across Kafka topic's partitions, + // not across ingesters. + ingester_partitions_shard_size: 1, querier_shard_size: 10, store_gateway_shard_size: 3, ruler_shard_size: 2, @@ -32,6 +38,10 @@ $._config.shuffle_sharding.store_gateway_enabled || $._config.shuffle_sharding.ruler_enabled, + // Check if shuffle-sharding is enabled for partitions. + local partitions_shuffle_sharding_enabled = + shuffle_sharding_enabled && + $._config.shuffle_sharding.ingest_storage_partitions_enabled, local roundUpToMultipleOfThree(n) = std.ceil(n / 3) * 3, @@ -49,62 +59,98 @@ roundUpToMultipleOfThree(series * 3 * $._config.shuffle_sharding.target_utilization_percentage / 100 / $._config.shuffle_sharding.target_series_per_ingester) ), + // The partitions shard size is computed with the notion, that, in the ingest storage mode, Kafka's partition + // is the unit of sharding, and one partition is assigned to RF number of ingesters. + // We expect ingesters tenants shard size to be always multi-zone ready, so here we divide its value by 3, + // to get the shard size for the partitions. + // Note, to gracefully migrate the running Mimir cluster, we guard the value with the extra "partitions_shuffle_sharding_enabled" flag. + local ingesterPartitionsTenantShardSize(series) = + if !partitions_shuffle_sharding_enabled then null else + std.max( + $._config.shuffle_sharding.ingester_partitions_shard_size, + ingesterTenantShardSize(series) / 3, + ), + overrides+:: if !shuffle_sharding_enabled then {} else { // Use defaults for this user class. extra_small_user+:: {}, // Target 300K active series. medium_small_user+:: { - ingestion_tenant_shard_size: ingesterTenantShardSize(300e3), + local series = 300e3, + + ingestion_tenant_shard_size: ingesterTenantShardSize(series), + ingestion_partitions_tenant_shard_size: ingesterPartitionsTenantShardSize(series), store_gateway_tenant_shard_size: std.max(3, $._config.shuffle_sharding.store_gateway_shard_size), ruler_tenant_shard_size: std.max(2, $._config.shuffle_sharding.ruler_shard_size), }, // Target 1M active series. small_user+:: { - ingestion_tenant_shard_size: ingesterTenantShardSize(1e6), + local series = 1e6, + + ingestion_tenant_shard_size: ingesterTenantShardSize(series), + ingestion_partitions_tenant_shard_size: ingesterPartitionsTenantShardSize(series), store_gateway_tenant_shard_size: std.max(6, $._config.shuffle_sharding.store_gateway_shard_size), ruler_tenant_shard_size: std.max(2, $._config.shuffle_sharding.ruler_shard_size), }, // Target 3M active series. medium_user+:: { - ingestion_tenant_shard_size: ingesterTenantShardSize(3e6), + local series = 3e6, + + ingestion_tenant_shard_size: ingesterTenantShardSize(series), + ingestion_partitions_tenant_shard_size: ingesterPartitionsTenantShardSize(series), store_gateway_tenant_shard_size: std.max(9, $._config.shuffle_sharding.store_gateway_shard_size), ruler_tenant_shard_size: std.max(2, $._config.shuffle_sharding.ruler_shard_size), }, // Target 6M active series. big_user+:: { - ingestion_tenant_shard_size: ingesterTenantShardSize(6e6), + local series = 6e6, + + ingestion_tenant_shard_size: ingesterTenantShardSize(series), + ingestion_partitions_tenant_shard_size: ingesterPartitionsTenantShardSize(series), store_gateway_tenant_shard_size: std.max(12, $._config.shuffle_sharding.store_gateway_shard_size), ruler_tenant_shard_size: std.max(3, $._config.shuffle_sharding.ruler_shard_size), }, // Target 12M active series. super_user+:: { - ingestion_tenant_shard_size: ingesterTenantShardSize(12e6), + local series = 12e6, + + ingestion_tenant_shard_size: ingesterTenantShardSize(series), + ingestion_partitions_tenant_shard_size: ingesterPartitionsTenantShardSize(series), store_gateway_tenant_shard_size: std.max(18, $._config.shuffle_sharding.store_gateway_shard_size), ruler_tenant_shard_size: std.max(6, $._config.shuffle_sharding.ruler_shard_size), }, // Target 16M active series. mega_user+:: { - ingestion_tenant_shard_size: ingesterTenantShardSize(16e6), + local series = 16e6, + + ingestion_tenant_shard_size: ingesterTenantShardSize(series), + ingestion_partitions_tenant_shard_size: ingesterPartitionsTenantShardSize(series), store_gateway_tenant_shard_size: std.max(24, $._config.shuffle_sharding.store_gateway_shard_size), ruler_tenant_shard_size: std.max(8, $._config.shuffle_sharding.ruler_shard_size), }, // Target 24M active series. user_24M+:: { - ingestion_tenant_shard_size: ingesterTenantShardSize(24e6), + local series = 24e6, + + ingestion_tenant_shard_size: ingesterTenantShardSize(series), + ingestion_partitions_tenant_shard_size: ingesterPartitionsTenantShardSize(series), store_gateway_tenant_shard_size: std.max(30, $._config.shuffle_sharding.store_gateway_shard_size), ruler_tenant_shard_size: std.max(8, $._config.shuffle_sharding.ruler_shard_size), }, // Target 32M active series. user_32M+:: { - ingestion_tenant_shard_size: ingesterTenantShardSize(32e6), + local series = 32e6, + + ingestion_tenant_shard_size: ingesterTenantShardSize(series), + ingestion_partitions_tenant_shard_size: ingesterPartitionsTenantShardSize(series), store_gateway_tenant_shard_size: std.max(42, $._config.shuffle_sharding.store_gateway_shard_size), ruler_tenant_shard_size: std.max(12, $._config.shuffle_sharding.ruler_shard_size), }, @@ -113,11 +159,15 @@ distributor_args+:: if !$._config.shuffle_sharding.ingester_write_path_enabled then {} else { 'distributor.ingestion-tenant-shard-size': $._config.shuffle_sharding.ingester_shard_size, + [if $._config.shuffle_sharding.ingest_storage_partitions_enabled + then 'ingest-storage.ingestion-partition-tenant-shard-size']: $._config.shuffle_sharding.ingester_partitions_shard_size, }, ingester_args+:: if !$._config.shuffle_sharding.ingester_write_path_enabled then {} else { // The shuffle sharding configuration is required on ingesters too because of global limits. 'distributor.ingestion-tenant-shard-size': $._config.shuffle_sharding.ingester_shard_size, + [if $._config.shuffle_sharding.ingest_storage_partitions_enabled + then 'ingest-storage.ingestion-partition-tenant-shard-size']: $._config.shuffle_sharding.ingester_partitions_shard_size, }, query_frontend_args+:: if !$._config.shuffle_sharding.querier_enabled then {} else { @@ -140,6 +190,8 @@ ) + ( if !$._config.shuffle_sharding.ingester_read_path_enabled then {} else { 'distributor.ingestion-tenant-shard-size': $._config.shuffle_sharding.ingester_shard_size, + [if $._config.shuffle_sharding.ingest_storage_partitions_enabled + then 'ingest-storage.ingestion-partition-tenant-shard-size']: $._config.shuffle_sharding.ingester_partitions_shard_size, } ), @@ -155,10 +207,14 @@ if !$._config.shuffle_sharding.ingester_write_path_enabled then {} else { // Required because the ruler directly writes to ingesters. 'distributor.ingestion-tenant-shard-size': $._config.shuffle_sharding.ingester_shard_size, + [if $._config.shuffle_sharding.ingest_storage_partitions_enabled + then 'ingest-storage.ingestion-partition-tenant-shard-size']: $._config.shuffle_sharding.ingester_partitions_shard_size, } ) + ( if !$._config.shuffle_sharding.ingester_read_path_enabled then {} else { 'distributor.ingestion-tenant-shard-size': $._config.shuffle_sharding.ingester_shard_size, + [if $._config.shuffle_sharding.ingest_storage_partitions_enabled + then 'ingest-storage.ingestion-partition-tenant-shard-size']: $._config.shuffle_sharding.ingester_partitions_shard_size, } ) + ( if !($._config.shuffle_sharding.ingester_write_path_enabled && !$._config.shuffle_sharding.ingester_read_path_enabled) then {} else {