From 1a28bf0e8331afd723d250c06cefcedbe56a3e10 Mon Sep 17 00:00:00 2001 From: Mustafain Ali Khan Date: Wed, 20 Mar 2024 13:47:34 -0700 Subject: [PATCH 1/3] Support keep_firing_for field for alert rules Signed-off-by: Mustafain Ali Khan --- pkg/ruler/rulespb/compat.go | 22 +++++++------- pkg/ruler/rulespb/compat_test.go | 49 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 pkg/ruler/rulespb/compat_test.go diff --git a/pkg/ruler/rulespb/compat.go b/pkg/ruler/rulespb/compat.go index e385051d7e..f735eb1734 100644 --- a/pkg/ruler/rulespb/compat.go +++ b/pkg/ruler/rulespb/compat.go @@ -28,12 +28,13 @@ func formattedRuleToProto(rls []rulefmt.RuleNode) []*RuleDesc { rules := make([]*RuleDesc, len(rls)) for i := range rls { rules[i] = &RuleDesc{ - Expr: rls[i].Expr.Value, - Record: rls[i].Record.Value, - Alert: rls[i].Alert.Value, - For: time.Duration(rls[i].For), - Labels: cortexpb.FromLabelsToLabelAdapters(labels.FromMap(rls[i].Labels)), - Annotations: cortexpb.FromLabelsToLabelAdapters(labels.FromMap(rls[i].Annotations)), + Expr: rls[i].Expr.Value, + Record: rls[i].Record.Value, + Alert: rls[i].Alert.Value, + For: time.Duration(rls[i].For), + KeepFiringFor: time.Duration(rls[i].KeepFiringFor), + Labels: cortexpb.FromLabelsToLabelAdapters(labels.FromMap(rls[i].Labels)), + Annotations: cortexpb.FromLabelsToLabelAdapters(labels.FromMap(rls[i].Annotations)), } } @@ -54,10 +55,11 @@ func FromProto(rg *RuleGroupDesc) rulefmt.RuleGroup { exprNode.SetString(rl.GetExpr()) newRule := rulefmt.RuleNode{ - Expr: exprNode, - Labels: cortexpb.FromLabelAdaptersToLabels(rl.Labels).Map(), - Annotations: cortexpb.FromLabelAdaptersToLabels(rl.Annotations).Map(), - For: model.Duration(rl.GetFor()), + Expr: exprNode, + Labels: cortexpb.FromLabelAdaptersToLabels(rl.Labels).Map(), + Annotations: cortexpb.FromLabelAdaptersToLabels(rl.Annotations).Map(), + For: model.Duration(rl.GetFor()), + KeepFiringFor: model.Duration(rl.GetKeepFiringFor()), } if rl.GetRecord() != "" { diff --git a/pkg/ruler/rulespb/compat_test.go b/pkg/ruler/rulespb/compat_test.go new file mode 100644 index 0000000000..de93792bc2 --- /dev/null +++ b/pkg/ruler/rulespb/compat_test.go @@ -0,0 +1,49 @@ +package rulespb + +import ( + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/rulefmt" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + "testing" + "time" +) + +func TestProto(t *testing.T) { + rules := make([]rulefmt.RuleNode, 0) + + alertNode := yaml.Node{} + alertNode.SetString("test_rule") + exprNode := yaml.Node{} + exprNode.SetString("test_expr") + + testRule := rulefmt.RuleNode{ + Alert: alertNode, + Expr: exprNode, + Labels: map[string]string{"label1": "value1"}, + Annotations: map[string]string{"key1": "value1"}, + For: model.Duration(time.Minute * 2), + KeepFiringFor: model.Duration(time.Hour), + } + + rules = append(rules, testRule) + + rg := rulefmt.RuleGroup{ + Name: "group1", + Rules: rules, + Interval: model.Duration(time.Minute), + } + desc := ToProto("test", "namespace", rg) + + assert.Equal(t, len(rules), len(desc.Rules)) + + ruleDesc := desc.Rules[0] + + assert.Equal(t, "test_rule", ruleDesc.Alert) + assert.Equal(t, "test_expr", ruleDesc.Expr) + assert.Equal(t, time.Minute*2, ruleDesc.For) + assert.Equal(t, time.Hour, ruleDesc.KeepFiringFor) + + formatted := FromProto(desc) + assert.Equal(t, rg, formatted) +} From 7140b26b407e31ab0af6e6b23964406ee7aef143 Mon Sep 17 00:00:00 2001 From: Mustafain Ali Khan Date: Thu, 21 Mar 2024 00:27:31 -0700 Subject: [PATCH 2/3] Include keepFiringFor and keepFiringSince in API response Signed-off-by: Mustafain Ali Khan --- pkg/ruler/api.go | 29 +++-- pkg/ruler/ruler.go | 19 ++-- pkg/ruler/ruler.pb.go | 188 ++++++++++++++++++++----------- pkg/ruler/ruler.proto | 4 +- pkg/ruler/rulespb/compat_test.go | 5 +- 5 files changed, 159 insertions(+), 86 deletions(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index e76bc0830b..bcb3df70dd 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -46,11 +46,12 @@ type AlertDiscovery struct { // Alert has info for an alert. type Alert struct { - Labels labels.Labels `json:"labels"` - Annotations labels.Labels `json:"annotations"` - State string `json:"state"` - ActiveAt *time.Time `json:"activeAt"` - Value string `json:"value"` + Labels labels.Labels `json:"labels"` + Annotations labels.Labels `json:"annotations"` + State string `json:"state"` + ActiveAt *time.Time `json:"activeAt"` + KeepFiringSince *time.Time `json:"keepFiringSince,omitempty"` + Value string `json:"value"` } // RuleDiscovery has info for all rules @@ -80,6 +81,7 @@ type alertingRule struct { Name string `json:"name"` Query string `json:"query"` Duration float64 `json:"duration"` + KeepFiringFor float64 `json:"keepFiringFor"` Labels labels.Labels `json:"labels"` Annotations labels.Labels `json:"annotations"` Alerts []*Alert `json:"alerts"` @@ -211,13 +213,17 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) { if g.ActiveRules[i].Rule.Alert != "" { alerts := make([]*Alert, 0, len(rl.Alerts)) for _, a := range rl.Alerts { - alerts = append(alerts, &Alert{ + alert := &Alert{ Labels: cortexpb.FromLabelAdaptersToLabels(a.Labels), Annotations: cortexpb.FromLabelAdaptersToLabels(a.Annotations), State: a.GetState(), ActiveAt: &a.ActiveAt, Value: strconv.FormatFloat(a.Value, 'e', -1, 64), - }) + } + if !a.KeepFiringSince.IsZero() { + alert.KeepFiringSince = &a.KeepFiringSince + } + alerts = append(alerts, alert) } grp.Rules[i] = alertingRule{ State: rl.GetState(), @@ -232,6 +238,7 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) { LastEvaluation: rl.GetEvaluationTimestamp(), EvaluationTime: rl.GetEvaluationDuration().Seconds(), Type: v1.RuleTypeAlerting, + KeepFiringFor: rl.Rule.KeepFiringFor.Seconds(), } } else { grp.Rules[i] = recordingRule{ @@ -296,13 +303,17 @@ func (a *API) PrometheusAlerts(w http.ResponseWriter, req *http.Request) { for _, rl := range g.ActiveRules { if rl.Rule.Alert != "" { for _, a := range rl.Alerts { - alerts = append(alerts, &Alert{ + alert := &Alert{ Labels: cortexpb.FromLabelAdaptersToLabels(a.Labels), Annotations: cortexpb.FromLabelAdaptersToLabels(a.Annotations), State: a.GetState(), ActiveAt: &a.ActiveAt, Value: strconv.FormatFloat(a.Value, 'e', -1, 64), - }) + } + if !a.KeepFiringSince.IsZero() { + alert.KeepFiringSince = &a.KeepFiringSince + } + alerts = append(alerts, alert) } } } diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index b846515526..b0c46cb694 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -829,15 +829,16 @@ func (r *Ruler) getLocalRules(userID string, rulesRequest RulesRequest) ([]*Grou alerts := []*AlertStateDesc{} for _, a := range rule.ActiveAlerts() { alerts = append(alerts, &AlertStateDesc{ - State: a.State.String(), - Labels: cortexpb.FromLabelsToLabelAdapters(a.Labels), - Annotations: cortexpb.FromLabelsToLabelAdapters(a.Annotations), - Value: a.Value, - ActiveAt: a.ActiveAt, - FiredAt: a.FiredAt, - ResolvedAt: a.ResolvedAt, - LastSentAt: a.LastSentAt, - ValidUntil: a.ValidUntil, + State: a.State.String(), + Labels: cortexpb.FromLabelsToLabelAdapters(a.Labels), + Annotations: cortexpb.FromLabelsToLabelAdapters(a.Annotations), + Value: a.Value, + ActiveAt: a.ActiveAt, + FiredAt: a.FiredAt, + ResolvedAt: a.ResolvedAt, + LastSentAt: a.LastSentAt, + ValidUntil: a.ValidUntil, + KeepFiringSince: a.KeepFiringSince, }) } ruleDesc = &RuleStateDesc{ diff --git a/pkg/ruler/ruler.pb.go b/pkg/ruler/ruler.pb.go index 93dfd18123..412fe757f2 100644 --- a/pkg/ruler/ruler.pb.go +++ b/pkg/ruler/ruler.pb.go @@ -309,15 +309,16 @@ func (m *RuleStateDesc) GetEvaluationDuration() time.Duration { } type AlertStateDesc struct { - State string `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` - Labels []github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter `protobuf:"bytes,2,rep,name=labels,proto3,customtype=github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter" json:"labels"` - Annotations []github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter `protobuf:"bytes,3,rep,name=annotations,proto3,customtype=github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter" json:"annotations"` - Value float64 `protobuf:"fixed64,4,opt,name=value,proto3" json:"value,omitempty"` - ActiveAt time.Time `protobuf:"bytes,5,opt,name=active_at,json=activeAt,proto3,stdtime" json:"active_at"` - FiredAt time.Time `protobuf:"bytes,6,opt,name=fired_at,json=firedAt,proto3,stdtime" json:"fired_at"` - ResolvedAt time.Time `protobuf:"bytes,7,opt,name=resolved_at,json=resolvedAt,proto3,stdtime" json:"resolved_at"` - LastSentAt time.Time `protobuf:"bytes,8,opt,name=last_sent_at,json=lastSentAt,proto3,stdtime" json:"last_sent_at"` - ValidUntil time.Time `protobuf:"bytes,9,opt,name=valid_until,json=validUntil,proto3,stdtime" json:"valid_until"` + State string `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` + Labels []github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter `protobuf:"bytes,2,rep,name=labels,proto3,customtype=github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter" json:"labels"` + Annotations []github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter `protobuf:"bytes,3,rep,name=annotations,proto3,customtype=github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter" json:"annotations"` + Value float64 `protobuf:"fixed64,4,opt,name=value,proto3" json:"value,omitempty"` + ActiveAt time.Time `protobuf:"bytes,5,opt,name=active_at,json=activeAt,proto3,stdtime" json:"active_at"` + FiredAt time.Time `protobuf:"bytes,6,opt,name=fired_at,json=firedAt,proto3,stdtime" json:"fired_at"` + ResolvedAt time.Time `protobuf:"bytes,7,opt,name=resolved_at,json=resolvedAt,proto3,stdtime" json:"resolved_at"` + LastSentAt time.Time `protobuf:"bytes,8,opt,name=last_sent_at,json=lastSentAt,proto3,stdtime" json:"last_sent_at"` + ValidUntil time.Time `protobuf:"bytes,9,opt,name=valid_until,json=validUntil,proto3,stdtime" json:"valid_until"` + KeepFiringSince time.Time `protobuf:"bytes,10,opt,name=keep_firing_since,json=keepFiringSince,proto3,stdtime" json:"keep_firing_since"` } func (m *AlertStateDesc) Reset() { *m = AlertStateDesc{} } @@ -401,6 +402,13 @@ func (m *AlertStateDesc) GetValidUntil() time.Time { return time.Time{} } +func (m *AlertStateDesc) GetKeepFiringSince() time.Time { + if m != nil { + return m.KeepFiringSince + } + return time.Time{} +} + func init() { proto.RegisterType((*RulesRequest)(nil), "ruler.RulesRequest") proto.RegisterType((*RulesResponse)(nil), "ruler.RulesResponse") @@ -412,52 +420,54 @@ func init() { func init() { proto.RegisterFile("ruler.proto", fileDescriptor_9ecbec0a4cfddea6) } var fileDescriptor_9ecbec0a4cfddea6 = []byte{ - // 720 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0x4f, 0x6b, 0x13, 0x4d, - 0x1c, 0xde, 0x49, 0xbb, 0x69, 0x32, 0xdb, 0xb7, 0x2f, 0x4c, 0xf3, 0xbe, 0xac, 0x41, 0x26, 0x25, - 0x82, 0x14, 0xc1, 0x0d, 0xc4, 0x82, 0x78, 0xa8, 0xb2, 0xa5, 0xd5, 0x8b, 0x88, 0x6c, 0xd5, 0x6b, - 0x99, 0x24, 0xd3, 0xed, 0xea, 0x66, 0x67, 0x9d, 0x99, 0x0d, 0xf5, 0x22, 0xfd, 0x08, 0x3d, 0x7a, - 0xf6, 0xe4, 0x47, 0xe9, 0xb1, 0xc7, 0x22, 0x52, 0xed, 0xf6, 0xe2, 0xb1, 0x1f, 0x41, 0x66, 0x66, - 0xd7, 0x24, 0xb5, 0x82, 0x41, 0x7a, 0x49, 0xf6, 0xf7, 0xe7, 0x79, 0x7e, 0x7f, 0x9e, 0x99, 0x81, - 0x0e, 0xcf, 0x62, 0xca, 0xbd, 0x94, 0x33, 0xc9, 0x90, 0xad, 0x8d, 0x66, 0x23, 0x64, 0x21, 0xd3, - 0x9e, 0x8e, 0xfa, 0x32, 0xc1, 0x26, 0x0e, 0x19, 0x0b, 0x63, 0xda, 0xd1, 0x56, 0x2f, 0xdb, 0xed, - 0x0c, 0x32, 0x4e, 0x64, 0xc4, 0x92, 0x22, 0xde, 0xba, 0x1c, 0x97, 0xd1, 0x90, 0x0a, 0x49, 0x86, - 0x69, 0x91, 0xf0, 0x20, 0x8c, 0xe4, 0x5e, 0xd6, 0xf3, 0xfa, 0x6c, 0xd8, 0xe9, 0x33, 0x2e, 0xe9, - 0x7e, 0xca, 0xd9, 0x6b, 0xda, 0x97, 0x85, 0xd5, 0x49, 0xdf, 0x84, 0x65, 0xa0, 0x57, 0x7c, 0x14, - 0xd0, 0xf5, 0x3f, 0x81, 0xea, 0xe6, 0xf5, 0xaf, 0x48, 0x7b, 0xe6, 0xdf, 0xc0, 0xdb, 0xef, 0xe1, - 0x62, 0xa0, 0xcc, 0x80, 0xbe, 0xcd, 0xa8, 0x90, 0xe8, 0x26, 0xac, 0xab, 0xf0, 0x33, 0x32, 0xa4, - 0xc2, 0x05, 0x2b, 0x73, 0xab, 0xf5, 0x60, 0xec, 0x40, 0xb7, 0xe1, 0x92, 0x32, 0x9e, 0x70, 0x96, - 0xa5, 0x26, 0xa5, 0xa2, 0x53, 0x2e, 0x79, 0x51, 0x03, 0xda, 0xbb, 0x51, 0x4c, 0x85, 0x3b, 0xa7, - 0xc3, 0xc6, 0x40, 0x08, 0xce, 0xcb, 0x77, 0x29, 0x75, 0xe7, 0x57, 0xc0, 0x6a, 0x3d, 0xd0, 0xdf, - 0xed, 0x87, 0xf0, 0x9f, 0xa2, 0xbe, 0x48, 0x59, 0x22, 0x28, 0xba, 0x0b, 0xab, 0xa1, 0x22, 0x32, - 0xd5, 0x9d, 0xee, 0x7f, 0x9e, 0x91, 0x41, 0xb3, 0x6f, 0x4b, 0x22, 0xe9, 0x26, 0x15, 0xfd, 0xa0, - 0x48, 0x6a, 0x7f, 0xac, 0xc0, 0xa5, 0xe9, 0x10, 0xba, 0x03, 0x6d, 0x1d, 0x74, 0xc1, 0x0a, 0x58, - 0x75, 0xba, 0x0d, 0xcf, 0xcc, 0x1b, 0x94, 0x2d, 0x6a, 0xbc, 0x49, 0x41, 0xf7, 0xe1, 0x22, 0xe9, - 0xcb, 0x68, 0x44, 0x77, 0x74, 0x92, 0x1e, 0xa7, 0x84, 0x70, 0x0d, 0x19, 0x97, 0x74, 0x4c, 0xa6, - 0x6e, 0x17, 0xbd, 0x82, 0xcb, 0x74, 0x44, 0xe2, 0x4c, 0xcb, 0xfc, 0xa2, 0x94, 0xd3, 0x9d, 0xd3, - 0x25, 0x9b, 0x9e, 0x11, 0xdc, 0x2b, 0x05, 0xf7, 0x7e, 0x66, 0x6c, 0xd4, 0x8e, 0x4e, 0x5b, 0xd6, - 0xe1, 0xd7, 0x16, 0x08, 0xae, 0x22, 0x40, 0xdb, 0x10, 0x8d, 0xdd, 0x9b, 0xc5, 0x31, 0xd2, 0x1b, - 0x73, 0xba, 0x37, 0x7e, 0xa1, 0x2d, 0x13, 0x0c, 0xeb, 0x07, 0xc5, 0x7a, 0x05, 0xbc, 0xfd, 0xa5, - 0x62, 0xb6, 0x3c, 0xde, 0xd1, 0x2d, 0x38, 0xaf, 0x46, 0x2c, 0x56, 0xf4, 0xef, 0xc4, 0x8a, 0xf4, - 0xa8, 0x3a, 0xa8, 0x54, 0x14, 0x0a, 0xe1, 0x56, 0xb4, 0x60, 0xc6, 0x40, 0xff, 0xc3, 0xea, 0x1e, - 0x25, 0xb1, 0xdc, 0xd3, 0xc3, 0xd6, 0x83, 0xc2, 0x52, 0x27, 0x27, 0x26, 0x42, 0x6e, 0x71, 0xce, - 0x78, 0x21, 0xf1, 0xd8, 0xa1, 0x64, 0x25, 0x31, 0xe5, 0x52, 0xb8, 0xf6, 0x94, 0xac, 0xbe, 0x72, - 0x4e, 0xc8, 0x6a, 0x92, 0x7e, 0xb7, 0xde, 0xea, 0xf5, 0xac, 0x77, 0xe1, 0xef, 0xd6, 0x7b, 0x60, - 0xc3, 0xa5, 0xe9, 0x39, 0xc6, 0xab, 0x03, 0x93, 0xab, 0x4b, 0x60, 0x35, 0x26, 0x3d, 0x1a, 0x97, - 0xe7, 0x6c, 0xd9, 0x2b, 0xef, 0xb4, 0xf7, 0x54, 0xf9, 0x9f, 0x93, 0x88, 0x6f, 0xf8, 0xaa, 0xd6, - 0xe7, 0xd3, 0xd6, 0x4c, 0x6f, 0x82, 0xc1, 0xfb, 0x03, 0x92, 0x4a, 0xca, 0x83, 0xa2, 0x0a, 0xda, - 0x87, 0x0e, 0x49, 0x12, 0x26, 0x75, 0x9b, 0xe6, 0x32, 0x5e, 0x5f, 0xd1, 0xc9, 0x52, 0x6a, 0x7e, - 0xb5, 0x27, 0x73, 0xd7, 0x41, 0x60, 0x0c, 0xe4, 0xc3, 0x7a, 0x71, 0xdb, 0x88, 0x74, 0xed, 0x19, - 0xb4, 0xac, 0x19, 0x98, 0x2f, 0xd1, 0x23, 0x58, 0xdb, 0x8d, 0x38, 0x1d, 0x28, 0x86, 0x59, 0x4e, - 0xc3, 0x82, 0x46, 0xf9, 0x12, 0x6d, 0x41, 0x87, 0x53, 0xc1, 0xe2, 0x91, 0xe1, 0x58, 0x98, 0x81, - 0x03, 0x96, 0x40, 0x5f, 0xa2, 0xc7, 0x70, 0x51, 0x1d, 0xee, 0x1d, 0x41, 0x13, 0xa9, 0x78, 0x6a, - 0xb3, 0xf0, 0x28, 0xe4, 0x36, 0x4d, 0xa4, 0x69, 0x67, 0x44, 0xe2, 0x68, 0xb0, 0x93, 0x25, 0x32, - 0x8a, 0xdd, 0xfa, 0x2c, 0x34, 0x1a, 0xf8, 0x52, 0xe1, 0xba, 0xeb, 0xd0, 0x56, 0x97, 0x97, 0xa3, - 0x35, 0xf3, 0x21, 0xd0, 0xf2, 0xc4, 0x1b, 0x56, 0xbe, 0xee, 0xcd, 0xc6, 0xb4, 0xd3, 0x3c, 0xb9, - 0x6d, 0x6b, 0x63, 0xed, 0xf8, 0x0c, 0x5b, 0x27, 0x67, 0xd8, 0xba, 0x38, 0xc3, 0xe0, 0x20, 0xc7, - 0xe0, 0x53, 0x8e, 0xc1, 0x51, 0x8e, 0xc1, 0x71, 0x8e, 0xc1, 0xb7, 0x1c, 0x83, 0xef, 0x39, 0xb6, - 0x2e, 0x72, 0x0c, 0x0e, 0xcf, 0xb1, 0x75, 0x7c, 0x8e, 0xad, 0x93, 0x73, 0x6c, 0xf5, 0xaa, 0xba, - 0xbd, 0x7b, 0x3f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x7f, 0xb5, 0x75, 0x2a, 0x29, 0x07, 0x00, 0x00, + // 750 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x4f, 0x6b, 0x13, 0x4f, + 0x18, 0xde, 0x49, 0x9b, 0x34, 0x99, 0xf4, 0xd7, 0xf2, 0x9b, 0x46, 0x59, 0x83, 0x4c, 0x4a, 0x04, + 0x29, 0x82, 0x1b, 0x88, 0x05, 0xf1, 0x50, 0x25, 0xa5, 0xad, 0x17, 0x91, 0xb2, 0x51, 0xaf, 0x61, + 0x92, 0x4c, 0xb6, 0x6b, 0x37, 0x3b, 0xeb, 0xcc, 0x6c, 0xa8, 0x17, 0xf1, 0x23, 0xf4, 0xe8, 0xd9, + 0x93, 0x1f, 0xa5, 0xc7, 0xe2, 0xa9, 0x88, 0x54, 0x9b, 0x5e, 0x3c, 0xf6, 0x23, 0xc8, 0xcc, 0xec, + 0x9a, 0xa4, 0x56, 0x70, 0x91, 0x5e, 0x92, 0x79, 0xff, 0x3c, 0xcf, 0x3b, 0xef, 0xfb, 0xcc, 0xcc, + 0xc2, 0x32, 0x8f, 0x03, 0xca, 0x9d, 0x88, 0x33, 0xc9, 0x50, 0x5e, 0x1b, 0xd5, 0x8a, 0xc7, 0x3c, + 0xa6, 0x3d, 0x0d, 0xb5, 0x32, 0xc1, 0x2a, 0xf6, 0x18, 0xf3, 0x02, 0xda, 0xd0, 0x56, 0x37, 0x1e, + 0x34, 0xfa, 0x31, 0x27, 0xd2, 0x67, 0x61, 0x12, 0xaf, 0x5d, 0x8e, 0x4b, 0x7f, 0x48, 0x85, 0x24, + 0xc3, 0x28, 0x49, 0x78, 0xe4, 0xf9, 0x72, 0x2f, 0xee, 0x3a, 0x3d, 0x36, 0x6c, 0xf4, 0x18, 0x97, + 0xf4, 0x20, 0xe2, 0xec, 0x35, 0xed, 0xc9, 0xc4, 0x6a, 0x44, 0xfb, 0x5e, 0x1a, 0xe8, 0x26, 0x8b, + 0x04, 0xba, 0xf1, 0x37, 0x50, 0xbd, 0x79, 0xfd, 0x2b, 0xa2, 0xae, 0xf9, 0x37, 0xf0, 0xfa, 0x3b, + 0xb8, 0xe8, 0x2a, 0xd3, 0xa5, 0x6f, 0x62, 0x2a, 0x24, 0xba, 0x0d, 0x4b, 0x2a, 0xfc, 0x9c, 0x0c, + 0xa9, 0xb0, 0xc1, 0xea, 0xdc, 0x5a, 0xc9, 0x9d, 0x38, 0xd0, 0x5d, 0xb8, 0xa4, 0x8c, 0xa7, 0x9c, + 0xc5, 0x91, 0x49, 0xc9, 0xe9, 0x94, 0x4b, 0x5e, 0x54, 0x81, 0xf9, 0x81, 0x1f, 0x50, 0x61, 0xcf, + 0xe9, 0xb0, 0x31, 0x10, 0x82, 0xf3, 0xf2, 0x6d, 0x44, 0xed, 0xf9, 0x55, 0xb0, 0x56, 0x72, 0xf5, + 0xba, 0xfe, 0x18, 0xfe, 0x97, 0xd4, 0x17, 0x11, 0x0b, 0x05, 0x45, 0xf7, 0x61, 0xc1, 0x53, 0x44, + 0xa6, 0x7a, 0xb9, 0x79, 0xc3, 0x31, 0x32, 0x68, 0xf6, 0xb6, 0x24, 0x92, 0x6e, 0x51, 0xd1, 0x73, + 0x93, 0xa4, 0xfa, 0xc7, 0x1c, 0x5c, 0x9a, 0x0d, 0xa1, 0x7b, 0x30, 0xaf, 0x83, 0x36, 0x58, 0x05, + 0x6b, 0xe5, 0x66, 0xc5, 0x31, 0xfd, 0xba, 0xe9, 0x16, 0x35, 0xde, 0xa4, 0xa0, 0x87, 0x70, 0x91, + 0xf4, 0xa4, 0x3f, 0xa2, 0x1d, 0x9d, 0xa4, 0xdb, 0x49, 0x21, 0x5c, 0x43, 0x26, 0x25, 0xcb, 0x26, + 0x53, 0x6f, 0x17, 0xbd, 0x82, 0x2b, 0x74, 0x44, 0x82, 0x58, 0xcb, 0xfc, 0x22, 0x95, 0xd3, 0x9e, + 0xd3, 0x25, 0xab, 0x8e, 0x11, 0xdc, 0x49, 0x05, 0x77, 0x7e, 0x65, 0x6c, 0x16, 0x8f, 0x4e, 0x6b, + 0xd6, 0xe1, 0xb7, 0x1a, 0x70, 0xaf, 0x22, 0x40, 0x6d, 0x88, 0x26, 0xee, 0xad, 0xe4, 0x18, 0xe9, + 0x89, 0x95, 0x9b, 0xb7, 0x7e, 0xa3, 0x4d, 0x13, 0x0c, 0xeb, 0x07, 0xc5, 0x7a, 0x05, 0xbc, 0xfe, + 0x35, 0x67, 0xa6, 0x3c, 0x99, 0xd1, 0x1d, 0x38, 0xaf, 0x5a, 0x4c, 0x46, 0xb4, 0x3c, 0x35, 0x22, + 0xdd, 0xaa, 0x0e, 0x2a, 0x15, 0x85, 0x42, 0xd8, 0x39, 0x2d, 0x98, 0x31, 0xd0, 0x4d, 0x58, 0xd8, + 0xa3, 0x24, 0x90, 0x7b, 0xba, 0xd9, 0x92, 0x9b, 0x58, 0xea, 0xe4, 0x04, 0x44, 0xc8, 0x6d, 0xce, + 0x19, 0x4f, 0x24, 0x9e, 0x38, 0x94, 0xac, 0x24, 0xa0, 0x5c, 0x0a, 0x3b, 0x3f, 0x23, 0x6b, 0x4b, + 0x39, 0xa7, 0x64, 0x35, 0x49, 0x7f, 0x1a, 0x6f, 0xe1, 0x7a, 0xc6, 0xbb, 0xf0, 0x6f, 0xe3, 0xfd, + 0x9c, 0x87, 0x4b, 0xb3, 0x7d, 0x4c, 0x46, 0x07, 0xa6, 0x47, 0x17, 0xc2, 0x42, 0x40, 0xba, 0x34, + 0x48, 0xcf, 0xd9, 0x8a, 0x93, 0xde, 0x69, 0xe7, 0x99, 0xf2, 0xef, 0x12, 0x9f, 0x6f, 0xb6, 0x54, + 0xad, 0x2f, 0xa7, 0xb5, 0x4c, 0x6f, 0x82, 0xc1, 0xb7, 0xfa, 0x24, 0x92, 0x94, 0xbb, 0x49, 0x15, + 0x74, 0x00, 0xcb, 0x24, 0x0c, 0x99, 0xd4, 0xdb, 0x34, 0x97, 0xf1, 0xfa, 0x8a, 0x4e, 0x97, 0x52, + 0xfd, 0xab, 0x39, 0x99, 0xbb, 0x0e, 0x5c, 0x63, 0xa0, 0x16, 0x2c, 0x25, 0xb7, 0x8d, 0x48, 0x3b, + 0x9f, 0x41, 0xcb, 0xa2, 0x81, 0xb5, 0x24, 0x7a, 0x02, 0x8b, 0x03, 0x9f, 0xd3, 0xbe, 0x62, 0xc8, + 0x72, 0x1a, 0x16, 0x34, 0xaa, 0x25, 0xd1, 0x36, 0x2c, 0x73, 0x2a, 0x58, 0x30, 0x32, 0x1c, 0x0b, + 0x19, 0x38, 0x60, 0x0a, 0x6c, 0x49, 0xb4, 0x03, 0x17, 0xd5, 0xe1, 0xee, 0x08, 0x1a, 0x4a, 0xc5, + 0x53, 0xcc, 0xc2, 0xa3, 0x90, 0x6d, 0x1a, 0x4a, 0xb3, 0x9d, 0x11, 0x09, 0xfc, 0x7e, 0x27, 0x0e, + 0xa5, 0x1f, 0xd8, 0xa5, 0x2c, 0x34, 0x1a, 0xf8, 0x52, 0xe1, 0xd0, 0x2e, 0xfc, 0x7f, 0x9f, 0xd2, + 0xa8, 0x33, 0xf0, 0xb9, 0x1f, 0x7a, 0x1d, 0xe1, 0x87, 0x3d, 0x6a, 0xc3, 0x0c, 0x64, 0xcb, 0x0a, + 0xbe, 0xa3, 0xd1, 0x6d, 0x05, 0x6e, 0x6e, 0xc0, 0xbc, 0x7a, 0x0e, 0x38, 0x5a, 0x37, 0x0b, 0x81, + 0x56, 0xa6, 0x5e, 0xc5, 0xf4, 0x7b, 0x51, 0xad, 0xcc, 0x3a, 0xcd, 0x23, 0x5e, 0xb7, 0x36, 0xd7, + 0x8f, 0xcf, 0xb0, 0x75, 0x72, 0x86, 0xad, 0x8b, 0x33, 0x0c, 0xde, 0x8f, 0x31, 0xf8, 0x34, 0xc6, + 0xe0, 0x68, 0x8c, 0xc1, 0xf1, 0x18, 0x83, 0xef, 0x63, 0x0c, 0x7e, 0x8c, 0xb1, 0x75, 0x31, 0xc6, + 0xe0, 0xf0, 0x1c, 0x5b, 0xc7, 0xe7, 0xd8, 0x3a, 0x39, 0xc7, 0x56, 0xb7, 0xa0, 0xf7, 0xf8, 0xe0, + 0x67, 0x00, 0x00, 0x00, 0xff, 0xff, 0x77, 0xf3, 0x36, 0x0c, 0x7b, 0x07, 0x00, 0x00, } func (this *RulesRequest) Equal(that interface{}) bool { @@ -678,6 +688,9 @@ func (this *AlertStateDesc) Equal(that interface{}) bool { if !this.ValidUntil.Equal(that1.ValidUntil) { return false } + if !this.KeepFiringSince.Equal(that1.KeepFiringSince) { + return false + } return true } func (this *RulesRequest) GoString() string { @@ -746,7 +759,7 @@ func (this *AlertStateDesc) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 13) + s := make([]string, 0, 14) s = append(s, "&ruler.AlertStateDesc{") s = append(s, "State: "+fmt.Sprintf("%#v", this.State)+",\n") s = append(s, "Labels: "+fmt.Sprintf("%#v", this.Labels)+",\n") @@ -757,6 +770,7 @@ func (this *AlertStateDesc) GoString() string { s = append(s, "ResolvedAt: "+fmt.Sprintf("%#v", this.ResolvedAt)+",\n") s = append(s, "LastSentAt: "+fmt.Sprintf("%#v", this.LastSentAt)+",\n") s = append(s, "ValidUntil: "+fmt.Sprintf("%#v", this.ValidUntil)+",\n") + s = append(s, "KeepFiringSince: "+fmt.Sprintf("%#v", this.KeepFiringSince)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1114,45 +1128,53 @@ func (m *AlertStateDesc) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - n7, err7 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.ValidUntil, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.ValidUntil):]) + n7, err7 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.KeepFiringSince, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.KeepFiringSince):]) if err7 != nil { return 0, err7 } i -= n7 i = encodeVarintRuler(dAtA, i, uint64(n7)) i-- - dAtA[i] = 0x4a - n8, err8 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.LastSentAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.LastSentAt):]) + dAtA[i] = 0x52 + n8, err8 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.ValidUntil, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.ValidUntil):]) if err8 != nil { return 0, err8 } i -= n8 i = encodeVarintRuler(dAtA, i, uint64(n8)) i-- - dAtA[i] = 0x42 - n9, err9 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.ResolvedAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.ResolvedAt):]) + dAtA[i] = 0x4a + n9, err9 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.LastSentAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.LastSentAt):]) if err9 != nil { return 0, err9 } i -= n9 i = encodeVarintRuler(dAtA, i, uint64(n9)) i-- - dAtA[i] = 0x3a - n10, err10 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.FiredAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.FiredAt):]) + dAtA[i] = 0x42 + n10, err10 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.ResolvedAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.ResolvedAt):]) if err10 != nil { return 0, err10 } i -= n10 i = encodeVarintRuler(dAtA, i, uint64(n10)) i-- - dAtA[i] = 0x32 - n11, err11 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.ActiveAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.ActiveAt):]) + dAtA[i] = 0x3a + n11, err11 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.FiredAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.FiredAt):]) if err11 != nil { return 0, err11 } i -= n11 i = encodeVarintRuler(dAtA, i, uint64(n11)) i-- + dAtA[i] = 0x32 + n12, err12 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.ActiveAt, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.ActiveAt):]) + if err12 != nil { + return 0, err12 + } + i -= n12 + i = encodeVarintRuler(dAtA, i, uint64(n12)) + i-- dAtA[i] = 0x2a if m.Value != 0 { i -= 8 @@ -1348,6 +1370,8 @@ func (m *AlertStateDesc) Size() (n int) { n += 1 + l + sovRuler(uint64(l)) l = github_com_gogo_protobuf_types.SizeOfStdTime(m.ValidUntil) n += 1 + l + sovRuler(uint64(l)) + l = github_com_gogo_protobuf_types.SizeOfStdTime(m.KeepFiringSince) + n += 1 + l + sovRuler(uint64(l)) return n } @@ -1438,6 +1462,7 @@ func (this *AlertStateDesc) String() string { `ResolvedAt:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.ResolvedAt), "Timestamp", "timestamp.Timestamp", 1), `&`, ``, 1) + `,`, `LastSentAt:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.LastSentAt), "Timestamp", "timestamp.Timestamp", 1), `&`, ``, 1) + `,`, `ValidUntil:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.ValidUntil), "Timestamp", "timestamp.Timestamp", 1), `&`, ``, 1) + `,`, + `KeepFiringSince:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.KeepFiringSince), "Timestamp", "timestamp.Timestamp", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -2497,6 +2522,39 @@ func (m *AlertStateDesc) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field KeepFiringSince", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRuler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRuler + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRuler + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(&m.KeepFiringSince, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRuler(dAtA[iNdEx:]) diff --git a/pkg/ruler/ruler.proto b/pkg/ruler/ruler.proto index de74967425..da653b8651 100644 --- a/pkg/ruler/ruler.proto +++ b/pkg/ruler/ruler.proto @@ -70,4 +70,6 @@ message AlertStateDesc { [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; google.protobuf.Timestamp valid_until = 9 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; -} \ No newline at end of file + google.protobuf.Timestamp keep_firing_since = 10 + [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; +} diff --git a/pkg/ruler/rulespb/compat_test.go b/pkg/ruler/rulespb/compat_test.go index de93792bc2..c438a3fd07 100644 --- a/pkg/ruler/rulespb/compat_test.go +++ b/pkg/ruler/rulespb/compat_test.go @@ -1,12 +1,13 @@ package rulespb import ( + "testing" + "time" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/rulefmt" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" - "testing" - "time" ) func TestProto(t *testing.T) { From bb37d296bed3704f2ad817488769cecbdf02c589 Mon Sep 17 00:00:00 2001 From: Mustafain Ali Khan Date: Thu, 21 Mar 2024 14:01:10 -0700 Subject: [PATCH 3/3] Add integration test for keep_firing_for field Signed-off-by: Mustafain Ali Khan --- integration/ruler_test.go | 170 +++++++++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 4 deletions(-) diff --git a/integration/ruler_test.go b/integration/ruler_test.go index cd2b37a29a..8cc8da26ad 100644 --- a/integration/ruler_test.go +++ b/integration/ruler_test.go @@ -8,6 +8,7 @@ import ( "context" "crypto/x509" "crypto/x509/pkix" + "encoding/json" "fmt" "math" "math/rand" @@ -19,10 +20,7 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/ruler" - - "github.com/cortexproject/cortex/pkg/storage/tsdb" - + v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" @@ -37,6 +35,8 @@ import ( "github.com/cortexproject/cortex/integration/e2e" e2edb "github.com/cortexproject/cortex/integration/e2e/db" "github.com/cortexproject/cortex/integration/e2ecortex" + "github.com/cortexproject/cortex/pkg/ruler" + "github.com/cortexproject/cortex/pkg/storage/tsdb" ) func TestRulerAPI(t *testing.T) { @@ -1038,6 +1038,168 @@ func TestRulerDisablesRuleGroups(t *testing.T) { }) } +func TestRulerKeepFiring(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + // Start dependencies. + consul := e2edb.NewConsul() + minio := e2edb.NewMinio(9000, bucketName, rulestoreBucketName) + require.NoError(t, s.StartAndWaitReady(consul, minio)) + + // Configure the ruler. + flags := mergeFlags( + BlocksStorageFlags(), + RulerFlags(), + map[string]string{ + // Since we're not going to run any rule (our only rule is invalid), we don't need the + // store-gateway to be configured to a valid address. + "-querier.store-gateway-addresses": "localhost:12345", + // Enable the bucket index so we can skip the initial bucket scan. + "-blocks-storage.bucket-store.bucket-index.enabled": "true", + // Evaluate rules often, so that we don't need to wait for metrics to show up. + "-ruler.evaluation-interval": "2s", + "-ruler.poll-interval": "2s", + // No delay + "-ruler.evaluation-delay-duration": "0", + + "-blocks-storage.tsdb.block-ranges-period": "1h", + "-blocks-storage.bucket-store.sync-interval": "1s", + "-blocks-storage.tsdb.retention-period": "2h", + + // We run single ingester only, no replication. + "-distributor.replication-factor": "1", + + "-querier.max-fetched-chunks-per-query": "50", + }, + ) + + const namespace = "test" + const user = "user" + + distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), flags, "") + ingester := e2ecortex.NewIngester("ingester", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + require.NoError(t, s.StartAndWaitReady(distributor, ingester, ruler)) + + // Wait until both the distributor and ruler have updated the ring. The querier will also watch + // the store-gateway ring if blocks sharding is enabled. + require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) + require.NoError(t, ruler.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) + + c, err := e2ecortex.NewClient(distributor.HTTPEndpoint(), "", "", ruler.HTTPEndpoint(), user) + require.NoError(t, err) + + expression := "vector(1) > 0" // Alert will fire + groupName := "rule_group_1" + ruleName := "rule_keep_firing" + + require.NoError(t, c.SetRuleGroup(alertRuleWithKeepFiringFor(groupName, ruleName, expression, model.Duration(10*time.Second)), namespace)) + + m := ruleGroupMatcher(user, namespace, groupName) + + // Wait until ruler has loaded the group. + require.NoError(t, ruler.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_prometheus_rule_group_rules"}, e2e.WithLabelMatchers(m), e2e.WaitMissingMetrics)) + // Wait until rule group has tried to evaluate the rule. + require.NoError(t, ruler.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"cortex_prometheus_rule_evaluations_total"}, e2e.WithLabelMatchers(m), e2e.WaitMissingMetrics)) + + groups, err := c.GetPrometheusRules(e2ecortex.RuleFilter{ + RuleNames: []string{ruleName}, + }) + require.NoError(t, err) + require.NotEmpty(t, groups) + require.Equal(t, 1, len(groups[0].Rules)) + alert := parseAlertFromRule(t, groups[0].Rules[0]) + require.Equal(t, float64(10), alert.KeepFiringFor) + require.Equal(t, 1, len(alert.Alerts)) + require.Empty(t, alert.Alerts[0].KeepFiringSince) //Alert expression not resolved, keepFiringSince should be empty + + expression = "vector(1) > 1" // Resolve, should keep firing for set duration + ts := time.Now() + require.NoError(t, c.SetRuleGroup(alertRuleWithKeepFiringFor(groupName, ruleName, expression, model.Duration(10*time.Second)), namespace)) + // Wait until rule group has tried to evaluate the rule. + require.NoError(t, ruler.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(2), []string{"cortex_prometheus_rule_evaluations_total"}, e2e.WithLabelMatchers(m), e2e.WaitMissingMetrics)) + + updatedGroups, err := c.GetPrometheusRules(e2ecortex.RuleFilter{ + RuleNames: []string{ruleName}, + }) + require.NoError(t, err) + require.NotEmpty(t, updatedGroups) + require.Equal(t, 1, len(updatedGroups[0].Rules)) + + alert = parseAlertFromRule(t, updatedGroups[0].Rules[0]) + require.Equal(t, "firing", alert.State) + require.Equal(t, float64(10), alert.KeepFiringFor) + require.Equal(t, 1, len(alert.Alerts)) + require.NotEmpty(t, alert.Alerts[0].KeepFiringSince) + require.Greater(t, alert.Alerts[0].KeepFiringSince.UnixNano(), ts.UnixNano(), "KeepFiringSince value should be after expression is resolved") + + time.Sleep(10 * time.Second) // Sleep beyond keepFiringFor time + updatedGroups, err = c.GetPrometheusRules(e2ecortex.RuleFilter{ + RuleNames: []string{ruleName}, + }) + require.NoError(t, err) + require.NotEmpty(t, updatedGroups) + require.Equal(t, 1, len(updatedGroups[0].Rules)) + alert = parseAlertFromRule(t, updatedGroups[0].Rules[0]) + require.Equal(t, 0, len(alert.Alerts)) // alert should be resolved once keepFiringFor time expires +} + +func parseAlertFromRule(t *testing.T, rules interface{}) *alertingRule { + responseJson, err := json.Marshal(rules) + require.NoError(t, err) + + alertResp := &alertingRule{} + require.NoError(t, json.Unmarshal(responseJson, alertResp)) + return alertResp +} + +type alertingRule struct { + // State can be "pending", "firing", "inactive". + State string `json:"state"` + Name string `json:"name"` + Query string `json:"query"` + Duration float64 `json:"duration"` + KeepFiringFor float64 `json:"keepFiringFor"` + Labels labels.Labels `json:"labels"` + Annotations labels.Labels `json:"annotations"` + Alerts []*Alert `json:"alerts"` + Health string `json:"health"` + LastError string `json:"lastError"` + Type v1.RuleType `json:"type"` + LastEvaluation time.Time `json:"lastEvaluation"` + EvaluationTime float64 `json:"evaluationTime"` +} + +// Alert has info for an alert. +type Alert struct { + Labels labels.Labels `json:"labels"` + Annotations labels.Labels `json:"annotations"` + State string `json:"state"` + ActiveAt *time.Time `json:"activeAt"` + KeepFiringSince *time.Time `json:"keepFiringSince,omitempty"` + Value string `json:"value"` +} + +func alertRuleWithKeepFiringFor(groupName string, ruleName string, expression string, keepFiring model.Duration) rulefmt.RuleGroup { + var recordNode = yaml.Node{} + var exprNode = yaml.Node{} + + recordNode.SetString(ruleName) + exprNode.SetString(expression) + + return rulefmt.RuleGroup{ + Name: groupName, + Interval: 10, + Rules: []rulefmt.RuleNode{{ + Alert: recordNode, + Expr: exprNode, + KeepFiringFor: keepFiring, + }}, + } +} + func ruleGroupMatcher(user, namespace, groupName string) *labels.Matcher { return labels.MustNewMatcher(labels.MatchEqual, "rule_group", fmt.Sprintf("/rules/%s/%s;%s", user, namespace, groupName)) }