From 8927742d859eb3afcc1c4449e70d30a4a327c23e Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Wed, 16 Jan 2019 14:32:53 -0500 Subject: [PATCH] Allow disabling of failure thresholds --- controller/controller.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/controller/controller.go b/controller/controller.go index 0b5b866..f848343 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -249,7 +249,7 @@ func CreateProvisionedPVInterval(createProvisionedPVInterval time.Duration) func } // FailedProvisionThreshold is the threshold for max number of retries on -// failures of Provision. Defaults to 15. +// failures of Provision. Set to 0 to retry indefinitely. Defaults to 15. func FailedProvisionThreshold(failedProvisionThreshold int) func(*ProvisionController) error { return func(c *ProvisionController) error { if c.HasRun() { @@ -261,7 +261,7 @@ func FailedProvisionThreshold(failedProvisionThreshold int) func(*ProvisionContr } // FailedDeleteThreshold is the threshold for max number of retries on failures -// of Delete. Defaults to 15. +// of Delete. Set to 0 to retry indefinitely. Defaults to 15. func FailedDeleteThreshold(failedDeleteThreshold int) func(*ProvisionController) error { return func(c *ProvisionController) error { if c.HasRun() { @@ -684,7 +684,10 @@ func (ctrl *ProvisionController) processNextClaimWorkItem() bool { } if err := ctrl.syncClaimHandler(key); err != nil { - if ctrl.claimQueue.NumRequeues(obj) < ctrl.failedProvisionThreshold { + if ctrl.failedProvisionThreshold == 0 { + glog.Warningf("Retrying syncing claim %q, failure %v", key, ctrl.claimQueue.NumRequeues(obj)) + ctrl.claimQueue.AddRateLimited(obj) + } else if ctrl.claimQueue.NumRequeues(obj) < ctrl.failedProvisionThreshold { glog.Warningf("Retrying syncing claim %q because failures %v < threshold %v", key, ctrl.claimQueue.NumRequeues(obj), ctrl.failedProvisionThreshold) ctrl.claimQueue.AddRateLimited(obj) } else { @@ -725,11 +728,14 @@ func (ctrl *ProvisionController) processNextVolumeWorkItem() bool { } if err := ctrl.syncVolumeHandler(key); err != nil { - if ctrl.volumeQueue.NumRequeues(obj) < ctrl.failedDeleteThreshold { - glog.Warningf("Retrying syncing volume %q because failures %v < threshold %v", key, ctrl.volumeQueue.NumRequeues(obj), ctrl.failedProvisionThreshold) + if ctrl.failedDeleteThreshold == 0 { + glog.Warningf("Retrying syncing volume %q, failure %v", key, ctrl.volumeQueue.NumRequeues(obj)) + ctrl.volumeQueue.AddRateLimited(obj) + } else if ctrl.volumeQueue.NumRequeues(obj) < ctrl.failedDeleteThreshold { + glog.Warningf("Retrying syncing volume %q because failures %v < threshold %v", key, ctrl.volumeQueue.NumRequeues(obj), ctrl.failedDeleteThreshold) ctrl.volumeQueue.AddRateLimited(obj) } else { - glog.Errorf("Giving up syncing volume %q because failures %v >= threshold %v", key, ctrl.volumeQueue.NumRequeues(obj), ctrl.failedProvisionThreshold) + glog.Errorf("Giving up syncing volume %q because failures %v >= threshold %v", key, ctrl.volumeQueue.NumRequeues(obj), ctrl.failedDeleteThreshold) // Done but do not Forget: it will not be in the queue but NumRequeues // will be saved until the obj is deleted from kubernetes }