diff --git a/main.go b/main.go
index 2f8349744..9d87b6edf 100644
--- a/main.go
+++ b/main.go
@@ -228,8 +228,10 @@ func main() {
 		},
 	}
 
-	mgrConfig.Cache.DefaultNamespaces = map[string]ctrlcache.Config{
-		watchNamespace: {},
+	if watchNamespace != "" {
+		mgrConfig.Cache.DefaultNamespaces = map[string]ctrlcache.Config{
+			watchNamespace: ctrlcache.Config{},
+		}
 	}
 
 	mgr, err := ctrl.NewManager(restConfig, mgrConfig)