Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Split querier GRPCClientConfig into separate frontend and scheduler client configs #6445

Merged
merged 10 commits into from
Oct 23, 2023
Merged

Split querier GRPCClientConfig into separate frontend and scheduler client configs #6445

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
880c5ca
Split querier GRPC client config into two
fayzal-g Oct 20, 2023
6c5afef
Update Helm chart and jsonnet with new config value
fayzal-g Oct 20, 2023
21adce5
Update jsonnet values
fayzal-g Oct 20, 2023
9b3b8ba
Update helm tests
fayzal-g Oct 20, 2023
7512241
Ensure backwards compatibility
fayzal-g Oct 23, 2023
fcd87a7
Update helm-tests
fayzal-g Oct 23, 2023
48c8289
Remove helm/jsonnet changes and remove config
fayzal-g Oct 23, 2023
3e1270c
Update CHANGELOG.md
fayzal-g Oct 23, 2023
949dbac
Update CHANGELOG
fayzal-g Oct 23, 2023
770c417
Improve CHANGELOG msg
fayzal-g Oct 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
262 changes: 262 additions & 0 deletions cmd/mimir/config-descriptor.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4136,6 +4136,268 @@
],
"fieldValue": null,
"fieldDefaultValue": null
},
{
"kind": "block",
"name": "query_scheduler_grpc_client_config",
"required": false,
"desc": "",
"blockEntries": [
{
"kind": "field",
"name": "max_recv_msg_size",
"required": false,
"desc": "gRPC client max receive message size (bytes).",
"fieldValue": null,
"fieldDefaultValue": 104857600,
"fieldFlag": "querier.scheduler-client.grpc-max-recv-msg-size",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "max_send_msg_size",
"required": false,
"desc": "gRPC client max send message size (bytes).",
"fieldValue": null,
"fieldDefaultValue": 104857600,
"fieldFlag": "querier.scheduler-client.grpc-max-send-msg-size",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "grpc_compression",
"required": false,
"desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "querier.scheduler-client.grpc-compression",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "rate_limit",
"required": false,
"desc": "Rate limit for gRPC client; 0 means disabled.",
"fieldValue": null,
"fieldDefaultValue": 0,
"fieldFlag": "querier.scheduler-client.grpc-client-rate-limit",
"fieldType": "float",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "rate_limit_burst",
"required": false,
"desc": "Rate limit burst for gRPC client.",
"fieldValue": null,
"fieldDefaultValue": 0,
"fieldFlag": "querier.scheduler-client.grpc-client-rate-limit-burst",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "backoff_on_ratelimits",
"required": false,
"desc": "Enable backoff and retry when we hit rate limits.",
"fieldValue": null,
"fieldDefaultValue": false,
"fieldFlag": "querier.scheduler-client.backoff-on-ratelimits",
"fieldType": "boolean",
"fieldCategory": "advanced"
},
{
"kind": "block",
"name": "backoff_config",
"required": false,
"desc": "",
"blockEntries": [
{
"kind": "field",
"name": "min_period",
"required": false,
"desc": "Minimum delay when backing off.",
"fieldValue": null,
"fieldDefaultValue": 100000000,
"fieldFlag": "querier.scheduler-client.backoff-min-period",
"fieldType": "duration",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "max_period",
"required": false,
"desc": "Maximum delay when backing off.",
"fieldValue": null,
"fieldDefaultValue": 10000000000,
"fieldFlag": "querier.scheduler-client.backoff-max-period",
"fieldType": "duration",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "max_retries",
"required": false,
"desc": "Number of times to backoff and retry before failing.",
"fieldValue": null,
"fieldDefaultValue": 10,
"fieldFlag": "querier.scheduler-client.backoff-retries",
"fieldType": "int",
"fieldCategory": "advanced"
}
],
"fieldValue": null,
"fieldDefaultValue": null
},
{
"kind": "field",
"name": "initial_stream_window_size",
"required": false,
"desc": "Initial stream window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator.",
"fieldValue": null,
"fieldDefaultValue": null,
"fieldFlag": "querier.scheduler-client.initial-stream-window-size",
"fieldType": "int",
"fieldCategory": "experimental"
},
{
"kind": "field",
"name": "initial_connection_window_size",
"required": false,
"desc": "Initial connection window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator.",
"fieldValue": null,
"fieldDefaultValue": null,
"fieldFlag": "querier.scheduler-client.initial-connection-window-size",
"fieldType": "int",
"fieldCategory": "experimental"
},
{
"kind": "field",
"name": "tls_enabled",
"required": false,
"desc": "Enable TLS in the gRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.",
"fieldValue": null,
"fieldDefaultValue": false,
"fieldFlag": "querier.scheduler-client.tls-enabled",
"fieldType": "boolean",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_cert_path",
"required": false,
"desc": "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "querier.scheduler-client.tls-cert-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_key_path",
"required": false,
"desc": "Path to the key for the client certificate. Also requires the client certificate to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "querier.scheduler-client.tls-key-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_ca_path",
"required": false,
"desc": "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "querier.scheduler-client.tls-ca-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_server_name",
"required": false,
"desc": "Override the expected name on the server certificate.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "querier.scheduler-client.tls-server-name",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_insecure_skip_verify",
"required": false,
"desc": "Skip validating server certificate.",
"fieldValue": null,
"fieldDefaultValue": false,
"fieldFlag": "querier.scheduler-client.tls-insecure-skip-verify",
"fieldType": "boolean",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_cipher_suites",
"required": false,
"desc": "Override the default cipher suite list (separated by commas). Allowed values:\n\nSecure Ciphers:\n- TLS_RSA_WITH_AES_128_CBC_SHA\n- TLS_RSA_WITH_AES_256_CBC_SHA\n- TLS_RSA_WITH_AES_128_GCM_SHA256\n- TLS_RSA_WITH_AES_256_GCM_SHA384\n- TLS_AES_128_GCM_SHA256\n- TLS_AES_256_GCM_SHA384\n- TLS_CHACHA20_POLY1305_SHA256\n- TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA\n- TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA\n- TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA\n- TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\n- TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n- TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n- TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n- TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n- TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n- TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n\nInsecure Ciphers:\n- TLS_RSA_WITH_RC4_128_SHA\n- TLS_RSA_WITH_3DES_EDE_CBC_SHA\n- TLS_RSA_WITH_AES_128_CBC_SHA256\n- TLS_ECDHE_ECDSA_WITH_RC4_128_SHA\n- TLS_ECDHE_RSA_WITH_RC4_128_SHA\n- TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA\n- TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256\n- TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256\n",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "querier.scheduler-client.tls-cipher-suites",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_min_version",
"required": false,
"desc": "Override the default minimum TLS version. Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "querier.scheduler-client.tls-min-version",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "connect_timeout",
"required": false,
"desc": "The maximum amount of time to establish a connection. A value of 0 means default gRPC client connect timeout and backoff.",
"fieldValue": null,
"fieldDefaultValue": 5000000000,
"fieldFlag": "querier.scheduler-client.connect-timeout",
"fieldType": "duration",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "connect_backoff_base_delay",
"required": false,
"desc": "Initial backoff delay after first connection failure. Only relevant if ConnectTimeout \u003e 0.",
"fieldValue": null,
"fieldDefaultValue": 1000000000,
"fieldFlag": "querier.scheduler-client.connect-backoff-base-delay",
"fieldType": "duration",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "connect_backoff_max_delay",
"required": false,
"desc": "Maximum backoff delay when establishing a connection. Only relevant if ConnectTimeout \u003e 0.",
"fieldValue": null,
"fieldDefaultValue": 5000000000,
"fieldFlag": "querier.scheduler-client.connect-backoff-max-delay",
"fieldType": "duration",
"fieldCategory": "advanced"
}
],
"fieldValue": null,
"fieldDefaultValue": null
}
],
"fieldValue": null,
Expand Down
44 changes: 44 additions & 0 deletions cmd/mimir/help-all.txt.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1673,6 +1673,50 @@ Usage of ./cmd/mimir/mimir:
The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. If this option is enabled, the time range of the query sent to the store-gateway will be manipulated to ensure the query end is not more recent than 'now - query-store-after'. (default 12h0m0s)
-querier.scheduler-address string
Address of the query-scheduler component, in host:port format. The host should resolve to all query-scheduler instances. This option should be set only when query-scheduler component is in use and -query-scheduler.service-discovery-mode is set to 'dns'.
-querier.scheduler-client.backoff-max-period duration
Maximum delay when backing off. (default 10s)
-querier.scheduler-client.backoff-min-period duration
Minimum delay when backing off. (default 100ms)
-querier.scheduler-client.backoff-on-ratelimits
Enable backoff and retry when we hit rate limits.
-querier.scheduler-client.backoff-retries int
Number of times to backoff and retry before failing. (default 10)
-querier.scheduler-client.connect-backoff-base-delay duration
Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
-querier.scheduler-client.connect-backoff-max-delay duration
Maximum backoff delay when establishing a connection. Only relevant if ConnectTimeout > 0. (default 5s)
-querier.scheduler-client.connect-timeout duration
The maximum amount of time to establish a connection. A value of 0 means default gRPC client connect timeout and backoff. (default 5s)
-querier.scheduler-client.grpc-client-rate-limit float
Rate limit for gRPC client; 0 means disabled.
-querier.scheduler-client.grpc-client-rate-limit-burst int
Rate limit burst for gRPC client.
-querier.scheduler-client.grpc-compression string
Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
-querier.scheduler-client.grpc-max-recv-msg-size int
gRPC client max receive message size (bytes). (default 104857600)
-querier.scheduler-client.grpc-max-send-msg-size int
gRPC client max send message size (bytes). (default 104857600)
-querier.scheduler-client.initial-connection-window-size value
[experimental] Initial connection window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B)
-querier.scheduler-client.initial-stream-window-size value
[experimental] Initial stream window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B)
-querier.scheduler-client.tls-ca-path string
Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.
-querier.scheduler-client.tls-cert-path string
Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.
-querier.scheduler-client.tls-cipher-suites string
Override the default cipher suite list (separated by commas).
-querier.scheduler-client.tls-enabled
Enable TLS in the gRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.
-querier.scheduler-client.tls-insecure-skip-verify
Skip validating server certificate.
-querier.scheduler-client.tls-key-path string
Path to the key for the client certificate. Also requires the client certificate to be configured.
-querier.scheduler-client.tls-min-version string
Override the default minimum TLS version. Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13
-querier.scheduler-client.tls-server-name string
Override the expected name on the server certificate.
-querier.shuffle-sharding-ingesters-enabled
Fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since -querier.query-ingesters-within. If this setting is false or -querier.query-ingesters-within is '0', queriers always query all ingesters (ingesters shuffle sharding on read path is disabled). (default true)
-querier.store-gateway-client.tls-ca-path string
Expand Down
10 changes: 8 additions & 2 deletions docs/sources/mimir/references/configuration-parameters/index.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2254,6 +2254,7 @@ The `grpc_client` block configures the gRPC client used to communicate between t

- `ingester.client`
- `querier.frontend-client`
- `querier.scheduler-client`
- `query-frontend.grpc-client-config`
- `query-scheduler.grpc-client-config`
- `ruler.client`
Expand Down Expand Up @@ -2425,10 +2426,15 @@ The `frontend_worker` block configures the worker running within the querier, pi
# CLI flag: -querier.id
[id: <string> | default = ""]

# Configures the gRPC client used to communicate between the queriers and the
# query-frontends / query-schedulers.
# Configures the gRPC client used to communicate between the querier and the
# query-frontend.
# The CLI flags prefix for this block configuration is: querier.frontend-client
[grpc_client_config: <grpc_client>]

# Configures the gRPC client used to communicate between the querier and the
# query-scheduler.
# The CLI flags prefix for this block configuration is: querier.scheduler-client
[query_scheduler_grpc_client_config: <grpc_client>]
```

### etcd
Expand Down
2 changes: 2 additions & 0 deletions operations/helm/charts/mimir-distributed/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,8 @@ mimir:
frontend_worker:
grpc_client_config:
max_send_msg_size: 419430400 # 400MiB
query_scheduler_grpc_client_config:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd check with @krajorama how to introduce new flag into Helm like this.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As suggested, will address Helm/Jsonnet changes in follow-up PR.

max_send_msg_size: 419430400 # 400MiB
{{- if .Values.query_scheduler.enabled }}
scheduler_address: {{ template "mimir.fullname" . }}-query-scheduler-headless.{{ .Release.Namespace }}.svc:{{ include "mimir.serverGrpcListenPort" . }}
{{- else }}
Expand Down
2 changes: 2 additions & 0 deletions ...elm/tests/enterprise-https-values-generated/mimir-distributed/templates/mimir-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,8 @@ data:
tls_insecure_skip_verify: false
tls_key_path: /certs/tls.key
tls_server_name: gem.grafana.com
query_scheduler_grpc_client_config:
max_send_msg_size: 419430400
scheduler_address: enterprise-https-values-mimir-query-scheduler-headless.citestns.svc:9095
gateway:
proxy:
Expand Down
2 changes: 2 additions & 0 deletions ...m/tests/gateway-enterprise-values-generated/mimir-distributed/templates/mimir-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ data:
frontend_worker:
grpc_client_config:
max_send_msg_size: 419430400
query_scheduler_grpc_client_config:
max_send_msg_size: 419430400
scheduler_address: gateway-enterprise-values-mimir-query-scheduler-headless.citestns.svc:9095
gateway:
proxy:
Expand Down
2 changes: 2 additions & 0 deletions ...s/helm/tests/gateway-nginx-values-generated/mimir-distributed/templates/mimir-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ data:
frontend_worker:
grpc_client_config:
max_send_msg_size: 419430400
query_scheduler_grpc_client_config:
max_send_msg_size: 419430400
scheduler_address: gateway-nginx-values-mimir-query-scheduler-headless.citestns.svc:9095
ingester:
ring:
Expand Down
2 changes: 2 additions & 0 deletions ...elm/tests/graphite-enabled-values-generated/mimir-distributed/templates/mimir-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ data:
frontend_worker:
grpc_client_config:
max_send_msg_size: 419430400
query_scheduler_grpc_client_config:
max_send_msg_size: 419430400
scheduler_address: graphite-enabled-values-mimir-query-scheduler-headless.citestns.svc:9095
gateway:
proxy:
Expand Down
2 changes: 2 additions & 0 deletions operations/helm/tests/large-values-generated/mimir-distributed/templates/mimir-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ data:
frontend_worker:
grpc_client_config:
max_send_msg_size: 419430400
query_scheduler_grpc_client_config:
max_send_msg_size: 419430400
scheduler_address: large-values-mimir-query-scheduler-headless.citestns.svc:9095
ingester:
ring:
Expand Down
Loading
Loading