make ingesters use heartbeat timeout instead of period to fix the bug…

… where they sometimes appear as unhealthy
grafana · May 26, 2022 · a78ec87 · a78ec87
1 parent 46eddd3
commit a78ec87
Show file tree

Hide file tree

Showing 16 changed files with 25 additions and 12 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,6 +18,7 @@
 * [ENHANCEMENT] Memberlist KV: incoming messages are now processed on per-key goroutine. This may reduce loss of "maintanance" packets in busy memberlist installations, but use more CPU. New `memberlist_client_received_broadcasts_dropped_total` counter tracks number of dropped per-key messages. #1912
 * [BUGFIX] Fix regexp parsing panic for regexp label matchers with start/end quantifiers. #1883
 * [BUGFIX] Ingester: fixed deceiving error log "failed to update cached shipped blocks after shipper initialisation", occurring for each new tenant in the ingester. #1893
+* [BUGFIX] Ring: fix bug where ingesters may appear unhealthy even though they are not #1933
 
 ### Mixin
 

diff --git a/go.mod b/go.mod
@@ -18,7 +18,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/google/gopacket v1.1.19
 	github.com/gorilla/mux v1.8.0
-	github.com/grafana/dskit v0.0.0-20220523143435-f5a17a2c14c8
+	github.com/grafana/dskit v0.0.0-20220526081034-789ec0ca4a3b
 	github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/json-iterator/go v1.1.12

diff --git a/go.sum b/go.sum
@@ -1047,8 +1047,8 @@ github.com/grafana-tools/sdk v0.0.0-20211220201350-966b3088eec9 h1:LQAhgcUPnzdjU
 github.com/grafana-tools/sdk v0.0.0-20211220201350-966b3088eec9/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4=
 github.com/grafana/dskit v0.0.0-20211021180445-3bd016e9d7f1/go.mod h1:uPG2nyK4CtgNDmWv7qyzYcdI+S90kHHRWvHnBtEMBXM=
 github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d/go.mod h1:M0/dlftwBvH7+hdNNpjMa/CUXD7gsew67mbkCuDlFXE=
-github.com/grafana/dskit v0.0.0-20220523143435-f5a17a2c14c8 h1:u7rtjSEjrX/WOGTgRq8ZsNXOv62tRqhkb1gJtNTEzR8=
-github.com/grafana/dskit v0.0.0-20220523143435-f5a17a2c14c8/go.mod h1:9It/K30QPyj/FuTqBb/SYnaS4/BJCP5YL4SRfXB7dG0=
+github.com/grafana/dskit v0.0.0-20220526081034-789ec0ca4a3b h1:9h79WowXGj6wErMzoi35pBECxnh7ucKJlQTs4Gs0yOI=
+github.com/grafana/dskit v0.0.0-20220526081034-789ec0ca4a3b/go.mod h1:9It/K30QPyj/FuTqBb/SYnaS4/BJCP5YL4SRfXB7dG0=
 github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe h1:mxrRWDjKtob43xF9nEhJthdtCzX35/800Sk7nE//YHQ=
 github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe/go.mod h1:+26VJWpczg2OU3D0537acnHSHzhJORpxOs6F+M27tZo=
 github.com/grafana/memberlist v0.3.1-0.20220425183535-6b97a09b7167 h1:PgEQkGHR4YimSCEGT5IoswN9gJKZDVskf+he6UClCLw=

diff --git a/pkg/alertmanager/alertmanager_ring.go b/pkg/alertmanager/alertmanager_ring.go
@@ -114,6 +114,7 @@ func (cfg *RingConfig) ToLifecyclerConfig(logger log.Logger) (ring.BasicLifecycl
 		ID:                  cfg.InstanceID,
 		Addr:                fmt.Sprintf("%s:%d", instanceAddr, instancePort),
 		HeartbeatPeriod:     cfg.HeartbeatPeriod,
+		HeartbeatTimeout:    cfg.HeartbeatTimeout,
 		TokensObservePeriod: 0,
 		Zone:                cfg.InstanceZone,
 		NumTokens:           RingNumTokens,

diff --git a/pkg/compactor/compactor_ring.go b/pkg/compactor/compactor_ring.go
@@ -102,6 +102,7 @@ func (cfg *RingConfig) ToLifecyclerConfig() ring.LifecyclerConfig {
 	lc.InfNames = cfg.InstanceInterfaceNames
 	lc.UnregisterOnShutdown = true
 	lc.HeartbeatPeriod = cfg.HeartbeatPeriod
+	lc.HeartbeatTimeout = cfg.HeartbeatTimeout
 	lc.ObservePeriod = cfg.ObservePeriod
 	lc.JoinAfter = 0
 	lc.MinReadyDuration = 0

diff --git a/pkg/compactor/compactor_ring_test.go b/pkg/compactor/compactor_ring_test.go
@@ -51,6 +51,7 @@ func TestRingConfig_CustomConfigToLifecyclerConfig(t *testing.T) {
 	// The lifecycler config should be generated based upon the compactor
 	// ring config
 	expected.HeartbeatPeriod = cfg.HeartbeatPeriod
+	expected.HeartbeatTimeout = cfg.HeartbeatTimeout
 	expected.RingConfig.HeartbeatTimeout = cfg.HeartbeatTimeout
 	expected.RingConfig.SubringCacheDisabled = true
 	expected.RingConfig.KVStore.Store = "memberlist"

diff --git a/pkg/distributor/distributor_ring.go b/pkg/distributor/distributor_ring.go
@@ -86,6 +86,7 @@ func (cfg *RingConfig) ToLifecyclerConfig() ring.LifecyclerConfig {
 	lc.InfNames = cfg.InstanceInterfaceNames
 	lc.UnregisterOnShutdown = true
 	lc.HeartbeatPeriod = cfg.HeartbeatPeriod
+	lc.HeartbeatTimeout = cfg.HeartbeatTimeout
 	lc.ObservePeriod = 0
 	lc.NumTokens = 1
 	lc.JoinAfter = 0

diff --git a/pkg/distributor/distributor_ring_test.go b/pkg/distributor/distributor_ring_test.go
@@ -54,6 +54,7 @@ func TestRingConfig_CustomConfigToLifecyclerConfig(t *testing.T) {
 	// The lifecycler config should be generated based upon the distributor
 	// ring config
 	expected.HeartbeatPeriod = cfg.HeartbeatPeriod
+	expected.HeartbeatTimeout = cfg.HeartbeatTimeout
 	expected.RingConfig.HeartbeatTimeout = cfg.HeartbeatTimeout
 	expected.RingConfig.KVStore.Store = "memberlist"
 	expected.ID = cfg.InstanceID

diff --git a/pkg/ingester/ingester_ring.go b/pkg/ingester/ingester_ring.go
@@ -121,6 +121,7 @@ func (cfg *RingConfig) ToLifecyclerConfig() ring.LifecyclerConfig {
 	lc.RingConfig = cfg.ToRingConfig()
 	lc.NumTokens = cfg.NumTokens
 	lc.HeartbeatPeriod = cfg.HeartbeatPeriod
+	lc.HeartbeatTimeout = cfg.HeartbeatTimeout
 	lc.ObservePeriod = cfg.ObservePeriod
 	lc.JoinAfter = cfg.JoinAfter
 	lc.MinReadyDuration = cfg.MinReadyDuration

diff --git a/pkg/ingester/ingester_ring_test.go b/pkg/ingester/ingester_ring_test.go
@@ -65,6 +65,7 @@ func TestRingConfig_CustomConfigToLifecyclerConfig(t *testing.T) {
 
 	expected.NumTokens = cfg.NumTokens
 	expected.HeartbeatPeriod = cfg.HeartbeatPeriod
+	expected.HeartbeatTimeout = cfg.HeartbeatTimeout
 	expected.ObservePeriod = cfg.ObservePeriod
 	expected.JoinAfter = cfg.JoinAfter
 	expected.MinReadyDuration = cfg.MinReadyDuration

diff --git a/pkg/ruler/ruler_ring.go b/pkg/ruler/ruler_ring.go
@@ -90,6 +90,7 @@ func (cfg *RingConfig) ToLifecyclerConfig(logger log.Logger) (ring.BasicLifecycl
 		ID:                  cfg.InstanceID,
 		Addr:                fmt.Sprintf("%s:%d", instanceAddr, instancePort),
 		HeartbeatPeriod:     cfg.HeartbeatPeriod,
+		HeartbeatTimeout:    cfg.HeartbeatTimeout,
 		TokensObservePeriod: 0,
 		NumTokens:           cfg.NumTokens,
 	}, nil

diff --git a/pkg/storegateway/gateway_ring.go b/pkg/storegateway/gateway_ring.go
@@ -154,6 +154,7 @@ func (cfg *RingConfig) ToLifecyclerConfig(logger log.Logger) (ring.BasicLifecycl
 		Addr:                            fmt.Sprintf("%s:%d", instanceAddr, instancePort),
 		Zone:                            cfg.InstanceZone,
 		HeartbeatPeriod:                 cfg.HeartbeatPeriod,
+		HeartbeatTimeout:                cfg.HeartbeatTimeout,
 		TokensObservePeriod:             0,
 		NumTokens:                       RingNumTokens,
 		KeepInstanceInTheRingOnShutdown: !cfg.UnregisterOnShutdown,

diff --git a/vendor/github.com/grafana/dskit/ring/basic_lifecycler.go b/vendor/github.com/grafana/dskit/ring/basic_lifecycler.go
diff --git a/vendor/github.com/grafana/dskit/ring/http.go b/vendor/github.com/grafana/dskit/ring/http.go
diff --git a/vendor/github.com/grafana/dskit/ring/lifecycler.go b/vendor/github.com/grafana/dskit/ring/lifecycler.go
diff --git a/vendor/modules.txt b/vendor/modules.txt