From 45d6bb8d83c927b1e490172a2cb66307b0db2546 Mon Sep 17 00:00:00 2001
From: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>
Date: Mon, 9 Mar 2020 20:23:57 +0000
Subject: [PATCH] ruler: Fix #2204 bug where alert queue is unpoppable causing
 full queue and dropped alerts (#2238)

* Add test for alert queue Pop after multiple Push

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>

* Fix alert queue bug by resignal after Pop (#2204)

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>

* Fix alert queue test and simplify

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>

* Update CHANGELOG.md

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>

* Link to thanos-io/thanos PR in CHANGELOG.md

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>
---
 CHANGELOG.md            |  4 ++++
 pkg/alert/alert.go      |  6 ++++++
 pkg/alert/alert_test.go | 25 +++++++++++++++++++++++++
 3 files changed, 35 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c33357a391..87b84f14d0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,10 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 ## Unreleased
 
+### Fixed
+
+- [#2238](https://github.com/thanos-io/thanos/pull/2238) Ruler: Fixed Issue #2204 bug in alert queue signalling filled up queue and alerts were dropped
+
 ## [v0.11.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0-rc.1) - 2020.03.02
 
 ### Fixed
diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go
index 0bb500e77a..0a7b4b30e1 100644
--- a/pkg/alert/alert.go
+++ b/pkg/alert/alert.go
@@ -193,6 +193,12 @@ func (q *Queue) Pop(termc <-chan struct{}) []*Alert {
 
 	q.popped.Add(float64(n))
 
+	if len(q.queue) > 0 {
+		select {
+		case q.morec <- struct{}{}:
+		default:
+		}
+	}
 	return as[:n]
 }
 
diff --git a/pkg/alert/alert_test.go b/pkg/alert/alert_test.go
index 7139dfc652..67f280280d 100644
--- a/pkg/alert/alert_test.go
+++ b/pkg/alert/alert_test.go
@@ -19,6 +19,31 @@ import (
 	"github.com/thanos-io/thanos/pkg/testutil"
 )
 
+func TestQueue_Pop_all_Pushed(t *testing.T) {
+	qcapacity := 10
+	batchsize := 1
+	pushes := 3
+
+	q := NewQueue(
+		nil, nil, qcapacity, batchsize, nil, nil,
+	)
+	for i := 0; i < pushes; i++ {
+		q.Push([]*Alert{
+			{},
+			{},
+		})
+	}
+
+	timeoutc := make(chan struct{}, 1)
+	time.AfterFunc(time.Second, func() { timeoutc <- struct{}{} })
+	popped := 0
+	for p := q.Pop(timeoutc); p != nil; p = q.Pop(timeoutc) {
+		popped += len(p)
+	}
+
+	testutil.Equals(t, pushes*2, popped)
+}
+
 func TestQueue_Push_Relabelled(t *testing.T) {
 	q := NewQueue(
 		nil, nil, 10, 10,