Add a new config Precedence

Add a new config Precedence to allow user to configure which Node IP should be used as tunnel endpoint IP. Signed-off-by: Lan Luo <luola@vmware.com>
antrea-io · Apr 7, 2022 · 5dd5619 · 5dd5619
1 parent 67749de
commit 5dd5619
Show file tree

Hide file tree

Showing 15 changed files with 335 additions and 64 deletions.
diff --git a/multicluster/apis/multicluster/v1alpha1/multiclusterconfig_types.go b/multicluster/apis/multicluster/v1alpha1/multiclusterconfig_types.go
@@ -21,6 +21,14 @@ import (
 	config "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
 )
 
+// Precedence defines the precedence of Node IP type.
+type Precedence string
+
+const (
+	PrecedencePrivate = "private"
+	PrecedencePublic  = "public"
+)
+
 //+kubebuilder:object:root=true
 
 // MultiClusterConfig is the Schema for the multiclusterconfigs API
@@ -30,6 +38,9 @@ type MultiClusterConfig struct {
 	config.ControllerManagerConfigurationSpec `json:",inline"`
 	// ServiceCIDR allows user to set the Cluster IP range of the cluster manually.
 	ServiceCIDR string `json:"serviceCIDR,omitempty"`
+	// The precedence about which IP (private or public one) of Node is preferred to
+	// be used as tunnel endpoint. If it's empty, private IP will be chose.
+	Precedence Precedence `json:"precedence,omitempty"`
 }
 
 func init() {

diff --git a/multicluster/apis/multicluster/v1alpha1/resourceexport_types.go b/multicluster/apis/multicluster/v1alpha1/resourceexport_types.go
@@ -69,7 +69,7 @@ type ResourceExportSpec struct {
 	// If exported resource is AntreaClusterNetworkPolicy.
 	ClusterNetworkPolicy *v1alpha1.ClusterNetworkPolicySpec `json:"clusternetworkpolicy,omitempty"`
 	// If exported resource Kind is unknown.
-	Raw RawResourceExport `json:"raw,omitempty"`
+	Raw *RawResourceExport `json:"raw,omitempty"`
 }
 
 type ResourceExportConditionType string

diff --git a/multicluster/apis/multicluster/v1alpha1/zz_generated.deepcopy.go b/multicluster/apis/multicluster/v1alpha1/zz_generated.deepcopy.go
diff --git a/multicluster/build/yamls/antrea-multicluster-member.yml b/multicluster/build/yamls/antrea-multicluster-member.yml
@@ -853,6 +853,14 @@ rules:
   verbs:
   - get
   - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -1027,6 +1035,72 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - resourceimports/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - resourceimports/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpointimports
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpointimports/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpointimports/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpoints
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpoints/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpoints/status
+  verbs:
+  - get
+  - patch
+  - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/multicluster/cmd/multicluster-controller/member.go b/multicluster/cmd/multicluster-controller/member.go
@@ -85,7 +85,8 @@ func runMember(o *Options) error {
 		mgr.GetScheme(),
 		env.GetPodNamespace(),
 		&clusterSetReconciler.RemoteCommonAreaManager,
-		opts.ServiceCIDR)
+		opts.ServiceCIDR,
+		opts.Precedence)
 	if err = gwNodeReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating Gateway Node controller: %v", err)
 	}

diff --git a/multicluster/cmd/multicluster-controller/options.go b/multicluster/cmd/multicluster-controller/options.go
@@ -32,6 +32,9 @@ type Options struct {
 	options        ctrl.Options
 	// The Service ClusterIP range used in the member cluster.
 	ServiceCIDR string
+	// The precedence about which IP (private or public one) of Node is preferred to
+	// be used as tunnel endpoint. If it's empty, private IP will be chose.
+	Precedence mcsv1alpha1.Precedence
 }
 
 func newOptions() *Options {
@@ -54,6 +57,7 @@ func (o *Options) complete(args []string) error {
 		}
 		o.options = options
 		o.ServiceCIDR = ctrlConfig.ServiceCIDR
+		o.Precedence = ctrlConfig.Precedence
 		klog.InfoS("Using config from file", "config", o.options)
 	} else {
 		klog.InfoS("Using default config", "config", o.options)

diff --git a/multicluster/config/overlays/member/role.yaml b/multicluster/config/overlays/member/role.yaml
@@ -6,6 +6,14 @@ metadata:
   creationTimestamp: null
   name: controller-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -180,3 +188,69 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - resourceimports/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - resourceimports/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpointimports
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpointimports/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpointimports/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpoints
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpoints/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - multicluster.crd.antrea.io
+  resources:
+  - tunnelendpoints/status
+  verbs:
+  - get
+  - patch
+  - update
diff --git a/multicluster/controllers/multicluster/commonarea/leader_elector.go b/multicluster/controllers/multicluster/commonarea/leader_elector.go
@@ -122,6 +122,8 @@ func (r *remoteCommonAreaManager) setElectedLeader(cluster RemoteCommonArea) {
 	}
 	r.electedLeaderCluster = cluster
 	if cluster != nil {
-		cluster.StartWatching()
+		if err := cluster.StartWatching(); err != nil {
+			klog.ErrorS(err, "Failed to start watching events")
+		}
 	}
 }
diff --git a/multicluster/controllers/multicluster/commonarea/remote_common_area.go b/multicluster/controllers/multicluster/commonarea/remote_common_area.go
@@ -147,7 +147,6 @@ func NewRemoteCommonArea(clusterID common.ClusterID, clusterSetID common.Cluster
 	if e != nil {
 		return nil, e
 	}
-
 	remote := &remoteCommonArea{
 		Client:                  remoteClient,
 		ClusterManager:          mgr,
@@ -410,6 +409,17 @@ func (r *remoteCommonArea) StopWatching() {
 	}
 	r.managerStopFunc()
 	r.managerStopFunc = nil
+
+	// Reset ClusterManager so this common area can be started again when it's reconnected.
+	mgr, err := ctrl.NewManager(r.config, ctrl.Options{
+		Scheme:             r.scheme,
+		MetricsBindAddress: "0",
+		Namespace:          r.Namespace,
+	})
+	if err != nil {
+		klog.ErrorS(err, "Error to reset manager for RemoteCommonArea", "Cluster", r.ClusterID)
+	}
+	r.ClusterManager = mgr
 }
 
 func (r *remoteCommonArea) GetStatus() []multiclusterv1alpha1.ClusterCondition {

diff --git a/multicluster/controllers/multicluster/commonarea/resourceimport_controller.go b/multicluster/controllers/multicluster/commonarea/resourceimport_controller.go
@@ -218,7 +218,6 @@ func (r *ResourceImportReconciler) handleResImpUpdateForService(ctx context.Cont
 			klog.ErrorS(err, "Failed to update imported Service", "service", svcName.String())
 			return ctrl.Result{}, err
 		}
-		r.installedResImports.Update(*resImp)
 	}
 
 	if !apiequality.Semantic.DeepEqual(svcImp.Spec, svcImpObj.Spec) {
@@ -229,8 +228,8 @@ func (r *ResourceImportReconciler) handleResImpUpdateForService(ctx context.Cont
 			klog.ErrorS(err, "Failed to update ServiceImport", "serviceimport", svcImpName.String())
 			return ctrl.Result{}, err
 		}
-		r.installedResImports.Update(*resImp)
 	}
+	r.installedResImports.Update(*resImp)
 	return ctrl.Result{}, nil
 }
 
@@ -332,8 +331,8 @@ func (r *ResourceImportReconciler) handleResImpUpdateForEndpoints(ctx context.Co
 			klog.ErrorS(err, "Failed to update MCS Endpoints", "endpoints", epNamespaced.String())
 			return ctrl.Result{}, err
 		}
-		r.installedResImports.Update(*resImp)
 	}
+	r.installedResImports.Update(*resImp)
 	return ctrl.Result{}, nil
 }
 

diff --git a/multicluster/controllers/multicluster/commonarea/tunnelendpoint_importer.go b/multicluster/controllers/multicluster/commonarea/tunnelendpoint_importer.go
@@ -62,6 +62,7 @@ func (r *ResourceImportReconciler) handleResImpForTunnelEndpoint(ctx context.Con
 	if reflect.DeepEqual(teImport.Spec, teSpec) {
 		klog.InfoS("No change on TunnelEndpointImport spec, skip reconciling", "tunnelendpointimport", teImportNamespaced.String(),
 			"resourceimport", req.NamespacedName.String())
+		r.installedResImports.Update(*resImp)
 		return ctrl.Result{}, nil
 	}
 	teImport.Spec = teSpec