diff --git a/runtime/events/recorder.go b/runtime/events/recorder.go index 3f703711..1c0cb3ae 100644 --- a/runtime/events/recorder.go +++ b/runtime/events/recorder.go @@ -17,8 +17,10 @@ limitations under the License. package events import ( + "bytes" "encoding/json" "fmt" + "net/http" "net/url" "os" "time" @@ -49,6 +51,7 @@ func NewRecorder(webhook, reportingController string) (*Recorder, error) { httpClient := retryablehttp.NewClient() httpClient.HTTPClient.Timeout = 5 * time.Second + httpClient.CheckRetry = retryablehttp.ErrorPropagatedRetryPolicy httpClient.Logger = nil return &Recorder{ @@ -120,6 +123,12 @@ func (r *Recorder) Eventf( return fmt.Errorf("failed to marshal object into json, error: %w", err) } + // avoid retrying rate limited requests + if res, _ := r.Client.HTTPClient.Post(r.Webhook, "application/json", bytes.NewReader(body)); res != nil && + (res.StatusCode == http.StatusTooManyRequests || res.StatusCode == http.StatusAccepted) { + return nil + } + if _, err := r.Client.Post(r.Webhook, "application/json", body); err != nil { return err } diff --git a/runtime/events/recorder_test.go b/runtime/events/recorder_test.go index 73aea76b..01cbcaa7 100644 --- a/runtime/events/recorder_test.go +++ b/runtime/events/recorder_test.go @@ -18,7 +18,6 @@ package events import ( "encoding/json" - "fmt" "io/ioutil" "net/http" "net/http/httptest" @@ -86,5 +85,32 @@ func TestEventRecorder_Eventf_Retry(t *testing.T) { } err = eventRecorder.EventErrorf(obj, nil, "sync", "sync %s", obj.Name) - require.EqualError(t, err, fmt.Sprintf("POST %s giving up after 3 attempt(s)", ts.URL)) + require.Error(t, err) +} + +func TestEventRecorder_Eventf_RateLimited(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + + var payload Event + err = json.Unmarshal(b, &payload) + require.NoError(t, err) + + w.WriteHeader(http.StatusTooManyRequests) + })) + defer ts.Close() + + eventRecorder, err := NewRecorder(ts.URL, "test-controller") + require.NoError(t, err) + eventRecorder.Client.RetryMax = 2 + + obj := corev1.ObjectReference{ + Kind: "GitRepository", + Namespace: "gitops-system", + Name: "webapp", + } + + err = eventRecorder.EventInfof(obj, nil, "sync", "sync %s", obj.Name) + require.NoError(t, err) }