Increase default Jaeger queue size for store-gateways and queriers

CHANGELOG.md

@@ -42,6 +42,7 @@
 ### Jsonnet
 
 * [CHANGE] Querier: Increase `JAEGER_REPORTER_MAX_QUEUE_SIZE` from 1000 to 5000, to avoid dropping tracing spans. #6764
+* [CHANGE] Store-gateway: Increase `JAEGER_REPORTER_MAX_QUEUE_SIZE` from the default (100) to 1000, to avoid dropping tracing spans. #7068
 * [FEATURE] Added support for the following root-level settings to configure the list of matchers to apply to node affinity: #6782 #6829
   * `alertmanager_node_affinity_matchers`
   * `compactor_node_affinity_matchers`

operations/compare-helm-with-jsonnet/jsonnet/07-pods/kustomization.yaml

@@ -5,14 +5,6 @@ components:
   - ../../components/pods
 
 patches:
-  # TODO(logiraptor): Jsonnet sets JAEGER_REPORTER_MAX_QUEUE_SIZE on the querier
-  - target:
-      name: mimir-querier
-      kind: Deployment
-    patch: |-
-      - op: remove
-        path: /spec/template/spec/containers/0/env/1
-
   # TODO(logiraptor): Jsonnet sets POD_IP on the alertmanager
   - target:
       name: mimir-alertmanager

operations/helm/charts/mimir-distributed/CHANGELOG.md

@@ -34,6 +34,8 @@ Entries should include a reference to the Pull Request that introduced the chang
 * [CHANGE] Reduce `-server.grpc-max-concurrent-streams` from 1000 to 500 for ingester and to 100 for all components. #5666
 * [CHANGE] Changed default `clusterDomain` from `cluster.local` to `cluster.local.` to reduce the number of DNS lookups made by Mimir. #6389
 * [CHANGE] Change the default timeout used for index-queries caches from `200ms` to `450ms`. #6786
+* [CHANGE] Querier: Increase `JAEGER_REPORTER_MAX_QUEUE_SIZE` from the default (100) to 5000, to avoid dropping tracing spans. #7068
+* [CHANGE] Store-gateway: Increase `JAEGER_REPORTER_MAX_QUEUE_SIZE` from the default (100) to 1000, to avoid dropping tracing spans. #7068
 * [ENHANCEMENT] Update the `rollout-operator` subchart to `0.10.0`. #6022 #6110 #6558 #6681
 * [ENHANCEMENT] Add support for not setting replicas for distributor, querier, and query-frontend. #6373
 * [ENHANCEMENT] Make Memcached connection limit configurable. #6715

operations/helm/charts/mimir-distributed/values.yaml

@@ -1211,7 +1211,9 @@ querier:
   extraContainers: []
   extraVolumes: []
   extraVolumeMounts: []
-  env: []
+  env:
+    - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+      value: "5000"
   extraEnvFrom: []
 
 query_frontend:
@@ -1489,7 +1491,9 @@ store_gateway:
   extraContainers: []
   extraVolumes: []
   extraVolumeMounts: []
-  env: []
+  env:
+    - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+      value: "1000"
   extraEnvFrom: []
 
   # -- Options to configure zone-aware replication for store-gateway

operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -98,6 +98,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "5"
           envFrom:

operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -149,6 +149,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -305,6 +307,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -461,6 +465,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"

operations/helm/tests/gateway-enterprise-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -93,6 +93,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "5"
           envFrom:

operations/helm/tests/gateway-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -140,6 +140,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -287,6 +289,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -434,6 +438,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"

operations/helm/tests/gateway-nginx-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -91,6 +91,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "5"
           envFrom:

operations/helm/tests/gateway-nginx-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -135,6 +135,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -277,6 +279,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -419,6 +423,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"

operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -93,6 +93,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "5"
           envFrom:

operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -131,6 +131,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"

operations/helm/tests/large-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -93,6 +93,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "6"
           envFrom:

operations/helm/tests/large-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -149,6 +149,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -305,6 +307,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -461,6 +465,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"

operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -91,6 +91,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "5"
           envFrom:

operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -126,6 +126,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"

operations/helm/tests/openshift-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -90,6 +90,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "5"
           envFrom:

operations/helm/tests/openshift-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -137,6 +137,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -281,6 +283,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -425,6 +429,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"

operations/helm/tests/scheduler-name-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -91,6 +91,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "5"
           envFrom:

operations/helm/tests/scheduler-name-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -135,6 +135,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -277,6 +279,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -419,6 +423,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"

operations/helm/tests/small-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -93,6 +93,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "6"
           envFrom:

operations/helm/tests/small-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -149,6 +149,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -305,6 +307,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -461,6 +465,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"

operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -92,6 +92,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
           envFrom:
             - secretRef:
                 name: mimir-minio-secret

operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -123,6 +123,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
           envFrom:
             - secretRef:
                 name: mimir-minio-secret

operations/helm/tests/test-enterprise-k8s-1.25-values-generated/mimir-distributed/templates/querier/querier-dep.yaml

@@ -93,6 +93,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "5000"
             - name: "GOMAXPROCS"
               value: "5"
           envFrom:

operations/helm/tests/test-enterprise-k8s-1.25-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml

@@ -140,6 +140,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -287,6 +289,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"
@@ -434,6 +438,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+            - name: JAEGER_REPORTER_MAX_QUEUE_SIZE
+              value: "1000"
             - name: "GOMAXPROCS"
               value: "5"
             - name: "GOMEMLIMIT"