Increase default Jaeger queue size for store-gateways and queriers

CHANGELOG.md

@@ -45,6 +45,7 @@
 
 * [CHANGE] Querier: Increase `JAEGER_REPORTER_MAX_QUEUE_SIZE` from 1000 to 5000, to avoid dropping tracing spans. #6764
 * [CHANGE] rollout-operator: remove default CPU limit. #7066
+* [CHANGE] Store-gateway: Increase `JAEGER_REPORTER_MAX_QUEUE_SIZE` from the default (100) to 1000, to avoid dropping tracing spans. #7068
 * [FEATURE] Added support for the following root-level settings to configure the list of matchers to apply to node affinity: #6782 #6829
   * `alertmanager_node_affinity_matchers`
   * `compactor_node_affinity_matchers`

operations/compare-helm-with-jsonnet/jsonnet/07-pods/kustomization.yaml

@@ -5,14 +5,6 @@ components:
   - ../../components/pods
 
 patches:
-  # TODO(logiraptor): Jsonnet sets JAEGER_REPORTER_MAX_QUEUE_SIZE on the querier
-  - target:
-      name: mimir-querier
-      kind: Deployment
-    patch: |-
-      - op: remove
-        path: /spec/template/spec/containers/0/env/1
-
   # TODO(logiraptor): Jsonnet sets POD_IP on the alertmanager
   - target:
       name: mimir-alertmanager

operations/helm/charts/mimir-distributed/CHANGELOG.md

@@ -34,6 +34,8 @@ Entries should include a reference to the Pull Request that introduced the chang
 * [CHANGE] Reduce `-server.grpc-max-concurrent-streams` from 1000 to 500 for ingester and to 100 for all components. #5666
 * [CHANGE] Changed default `clusterDomain` from `cluster.local` to `cluster.local.` to reduce the number of DNS lookups made by Mimir. #6389
 * [CHANGE] Change the default timeout used for index-queries caches from `200ms` to `450ms`. #6786
+* [CHANGE] Querier: Increase `JAEGER_REPORTER_MAX_QUEUE_SIZE` from the default (100) to 5000, to avoid dropping tracing spans. #7068
+* [CHANGE] Store-gateway: Increase `JAEGER_REPORTER_MAX_QUEUE_SIZE` from the default (100) to 1000, to avoid dropping tracing spans. #7068
 * [FEATURE] Added option to enable StatefulSetAutoDeletePVC for StatefulSets for compactor, ingester, store-gateway, and alertmanager via `*.persistance.enableRetentionPolicy`, `*.persistance.whenDeleted`, and `*.persistance.whenScaled`. #6106
 * [FEATURE] Add pure Ingress option instead of the gateway service. #6932
 * [ENHANCEMENT] Update the `rollout-operator` subchart to `0.10.0`. #6022 #6110 #6558 #6681

operations/helm/charts/mimir-distributed/templates/querier/querier-dep.yaml

@@ -109,6 +109,11 @@ spec:
             - name: "GOMAXPROCS"
               value: {{ max $cpu_request_doubled $cpu_request_plus_four | toString | toYaml }}
             {{- end }}
+            {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.querier }}
+            {{- if $jaeger_queue_size }}
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: {{$jaeger_queue_size | toString | toYaml }}
+            {{- end }}
           envFrom:
             {{- with .Values.global.extraEnvFrom }}
               {{- toYaml . | nindent 12 }}

operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -184,6 +184,11 @@ spec:
             - name: "GOMEMLIMIT"
               value: {{include "mimir.siToBytes" (dict "value" $mem_request) | toString | toYaml }}
             {{- end }}
+            {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.store_gateway }}
+            {{- if $jaeger_queue_size }}
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: {{$jaeger_queue_size | toString | toYaml }}
+            {{- end }}
           envFrom:
             {{- with .Values.global.extraEnvFrom }}
               {{- toYaml . | nindent 12 }}

operations/helm/charts/mimir-distributed/values.yaml

@@ -1213,6 +1213,7 @@ querier:
   extraVolumeMounts: []
   env: []
   extraEnvFrom: []
+  jaegerReporterMaxQueueSize: 5000
 
 query_frontend:
   # Setting it to null will produce a deployment without replicas set, allowing you to use autoscaling with the deployment
@@ -1491,6 +1492,7 @@ store_gateway:
   extraVolumeMounts: []
   env: []
   extraEnvFrom: []
+  jaegerReporterMaxQueueSize: 1000
 
   # -- Options to configure zone-aware replication for store-gateway
   # Example configuration with full geographical redundancy:

operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -100,6 +100,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "5"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -153,6 +153,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -309,6 +311,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -465,4 +469,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:

operations/helm/tests/gateway-enterprise-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -95,6 +95,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "5"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/gateway-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -144,6 +144,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -291,6 +293,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -438,4 +442,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:

operations/helm/tests/gateway-nginx-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -93,6 +93,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "5"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/gateway-nginx-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -139,6 +139,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -281,6 +283,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -423,4 +427,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:

operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -95,6 +95,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "5"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -135,4 +135,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:

operations/helm/tests/large-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -95,6 +95,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "6"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/large-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -153,6 +153,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "6442450944"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -309,6 +311,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "6442450944"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -465,4 +469,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "6442450944"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:

operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -93,6 +93,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "5"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -130,4 +130,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:

operations/helm/tests/openshift-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -92,6 +92,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "5"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/openshift-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -141,6 +141,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -285,6 +287,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -429,4 +433,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:

operations/helm/tests/scheduler-name-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -93,6 +93,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "5"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/scheduler-name-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -139,6 +139,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -281,6 +283,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -423,4 +427,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:

operations/helm/tests/small-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -95,6 +95,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "6"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/small-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -153,6 +153,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "1073741824"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -309,6 +311,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "1073741824"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -465,4 +469,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "1073741824"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:

operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -92,6 +92,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
             - secretRef:
                 name: mimir-minio-secret

operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -123,6 +123,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
             - secretRef:
                 name: mimir-minio-secret

operations/helm/tests/test-enterprise-k8s-1.25-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -95,6 +95,8 @@ spec:
           env:
             - name: "GOMAXPROCS"
               value: "5"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "5000"
           envFrom:
       nodeSelector:
         {}

operations/helm/tests/test-enterprise-k8s-1.25-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -144,6 +144,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -291,6 +293,8 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom:
 ---
 # Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml
@@ -438,4 +442,6 @@ spec:
               value: "5"
             - name: "GOMEMLIMIT"
               value: "536870912"
+            - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
+              value: "1000"
           envFrom: