Add Gateway, ClusterInfo exchange controllers

* Add a Node controller to watch Node changes in member cluster. It will do following things:
    * When the Node is annotated with `multicluster.antrea.io/gateway=true`, it will
      create a new Gateway CR like below:
    ```yaml
    apiVersion: multicluster.crd.antrea.io/v1alpha1
    kind: Gateway
    metadata:
      name: k8s-node-1
      namespace: kube-system
    gatewayIP: 172.16.27.224
    internalIP: 172.16.27.224
    ```
    * When the Node is annotated with both `multicluster.antrea.io/gateway=true` and `multicluster.antrea.io/gateway-ip="10.10.10.10"` ('10.10.10.10' is an IP example), the existing Gateway CR's gatewayIP will be updated like below:
    ```yaml
    apiVersion: multicluster.crd.antrea.io/v1alpha1
    kind: Gateway
    metadata:
      name: k8s-node-1
      namespace: kube-system
    gatewayIP: 10.10.10.10
    internalIP: 172.16.27.224
    ```
* When the Node's annotation `multicluster.antrea.io/gateway=true` is removed, the corresponding Gateway will be deleted.
* Add a Gateway controller to watch Gateway events in member cluster. It will do following things:
    * When a Gateway is created, a new ClusterInfo type of ResourceExport will be created in leader cluster, a sample yaml is like below:
    ```yaml
    apiVersion: multicluster.crd.antrea.io/v1alpha1
    kind: ResourceExport
    metadata:
      name: test-cluster-west-kube-system-clusterinfo
      namespace: antrea-mcs-ns
    spec:
      clusterID: test-cluster-west
      clusterinfo:
        clusterID: test-cluster-west
        gatewayInfos:
        - gatewayIP: 10.10.10.10
        serviceCIDR: 10.19.0.0/18
      kind: ClusterInfo
      name: test-cluster-west
      namespace: kube-system
    ```
    * When a Gateway is updated or one of many Gateways are deleted, the corresponding ResourceExport will be updated in leader cluster.
    Please note, we support one Gateway only for now, so the last created Gateway will be wrapped into ResourceExport if there are multiple Gateways.
    * When the last Gateway is deleted, the corresponding ResourceExport will be deleted in leader cluster.
* There are two new fields `serviceCIDR` and `gatewayIPPrecedence` are added in `MultiClusterConfig`.
  - By default, MC controller will detect ClusterIP range automatically. If admin sets the `serviceCIDR` config manually, it will use the value of `serviceCIDR` in the config.
  - By default, MC controller will choose InternalIP of a Node as the GatewayIP, if admin sets `gatewayIPPrecedence` config as `public`, it will use ExternalIP as GatewayIP.
* Add a new ClusterInfo kind handler to convert any new or updated ClusterInfo kind of ResourceExports into ResourceImports.
* Add a new ClusterInfo importer to watch any new or updated ClusterInfo kind of ResourceImport events. It will create or update a ClusterInfoImport locally.
  A sample ClusterInfoImport is like below:
  ```yaml
  apiVersion: multicluster.crd.antrea.io/v1alpha1
  kind: ClusterInfoImport
  metadata:
    name: test-cluster-west-kube-system-clusterinfo
    namespace: kube-system
  spec:
    clusterID: test-cluster-west
    gatewayInfos:
    - gatewayIP: 10.10.10.10
    serviceCIDR: 10.19.0.0/18
  ```

Signed-off-by: Lan Luo <luola@vmware.com>

multicluster/cmd/multicluster-controller/member.go

@@ -51,28 +51,48 @@ func runMember(o *Options) error {
 		return err
 	}
 
-	clusterSetReconciler := &multiclustercontrollers.MemberClusterSetReconciler{
-		Client:    mgr.GetClient(),
-		Scheme:    mgr.GetScheme(),
-		Namespace: env.GetPodNamespace(),
-	}
+	clusterSetReconciler := multiclustercontrollers.NewMemberClusterSetReconciler(mgr.GetClient(),
+		mgr.GetScheme(),
+		env.GetPodNamespace(),
+	)
 	if err = clusterSetReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating ClusterSet controller: %v", err)
 	}
 
+	commonAreaGetter := clusterSetReconciler
 	svcExportReconciler := multiclustercontrollers.NewServiceExportReconciler(
 		mgr.GetClient(),
 		mgr.GetScheme(),
-		&clusterSetReconciler.RemoteCommonAreaManager)
+		commonAreaGetter)
 	if err = svcExportReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating ServiceExport controller: %v", err)
 	}
 
+	gwReconciler := multiclustercontrollers.NewGatewayReconciler(
+		mgr.GetClient(),
+		mgr.GetScheme(),
+		env.GetPodNamespace(),
+		opts.ServiceCIDR,
+		commonAreaGetter)
+	if err = gwReconciler.SetupWithManager(mgr); err != nil {
+		return fmt.Errorf("error creating Gateway controller: %v", err)
+	}
+
+	nodeReconciler := multiclustercontrollers.NewNodeReconciler(
+		mgr.GetClient(),
+		mgr.GetScheme(),
+		env.GetPodNamespace(),
+		opts.GatewayIPPrecedence)
+	if err = nodeReconciler.SetupWithManager(mgr); err != nil {
+		return fmt.Errorf("error creating Node controller: %v", err)
+	}
+
 	stopCh := signals.RegisterSignalHandlers()
-	staleController := multiclustercontrollers.NewStaleController(
+	staleController := multiclustercontrollers.NewStaleResCleanupController(
 		mgr.GetClient(),
 		mgr.GetScheme(),
-		&clusterSetReconciler.RemoteCommonAreaManager)
+		env.GetPodNamespace(),
+		commonAreaGetter)
 
 	go staleController.Run(stopCh)
 	// Member runs ResourceImportReconciler from RemoteCommonArea only

multicluster/cmd/multicluster-controller/options.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"fmt"
+	"net"
 	"time"
 
 	"github.com/spf13/pflag"
@@ -30,6 +31,11 @@ type Options struct {
 	configFile     string
 	SelfSignedCert bool
 	options        ctrl.Options
+	// The Service ClusterIP range used in the member cluster.
+	ServiceCIDR string
+	// The precedence about which IP (private or public one) of Node is preferred to
+	// be used as tunnel endpoint. If not specified, private IP will be chosen.
+	GatewayIPPrecedence mcsv1alpha1.Precedence
 }
 
 func newOptions() *Options {
@@ -51,6 +57,13 @@ func (o *Options) complete(args []string) error {
 			return fmt.Errorf("failed to load options from configuration file %s", o.configFile)
 		}
 		o.options = options
+		if ctrlConfig.ServiceCIDR != "" {
+			if _, _, err := net.ParseCIDR(ctrlConfig.ServiceCIDR); err != nil {
+				return fmt.Errorf("failed to parse serviceCIDR, invalid CIDR string %s", ctrlConfig.ServiceCIDR)
+			}
+		}
+		o.ServiceCIDR = ctrlConfig.ServiceCIDR
+		o.GatewayIPPrecedence = ctrlConfig.GatewayIPPrecedence
 		klog.InfoS("Using config from file", "config", o.options)
 	} else {
 		klog.InfoS("Using default config", "config", o.options)

multicluster/config/rbac/gatewaynode_editor_role.yaml → multicluster/config/rbac/gateway_editor_role.yaml

@@ -2,7 +2,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: gatewaynode-editor-role
+  name: gateway-editor-role
 rules:
 - apiGroups:
   - multicluster.crd.antrea.io

multicluster/config/rbac/gatewaynode_viewer_role.yaml → multicluster/config/rbac/gateway_viewer_role.yaml

@@ -2,7 +2,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: gatewaynode-viewer-role
+  name: gateway-viewer-role
 rules:
 - apiGroups:
   - multicluster.crd.antrea.io

multicluster/controllers/multicluster/clusterinfoexport_handler.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2022 Antrea Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package multicluster
+
+import (
+	"context"
+	"reflect"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	mcsv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1"
+	"antrea.io/antrea/multicluster/controllers/multicluster/common"
+)
+
+func (r *ResourceExportReconciler) handleClusterInfo(ctx context.Context, req ctrl.Request, resExport mcsv1alpha1.ResourceExport) (ctrl.Result, error) {
+	resImport := &mcsv1alpha1.ResourceImport{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      req.Name,
+			Namespace: req.Namespace,
+		},
+	}
+
+	if !resExport.DeletionTimestamp.IsZero() {
+		if common.StringExistsInSlice(resExport.Finalizers, common.ResourceExportFinalizer) {
+			err := r.Client.Delete(ctx, resImport, &client.DeleteOptions{})
+			if err == nil || apierrors.IsNotFound(err) {
+				return r.deleteResourceExport(&resExport)
+			}
+			return ctrl.Result{}, err
+		}
+		return ctrl.Result{}, nil
+	}
+
+	resImport.Spec = mcsv1alpha1.ResourceImportSpec{
+		Kind:      common.ClusterInfoKind,
+		Name:      resExport.Name,
+		Namespace: resExport.Namespace,
+	}
+	resImportName := types.NamespacedName{
+		Name:      req.Name,
+		Namespace: req.Namespace,
+	}
+
+	var err error
+	if err = r.Client.Get(ctx, resImportName, resImport); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return ctrl.Result{}, err
+		}
+		// Create a new ClusterInfo of ResourceImport
+		resImport.Spec.ClusterInfo = resExport.Spec.ClusterInfo
+		if err = r.Client.Create(ctx, resImport, &client.CreateOptions{}); err != nil {
+			return ctrl.Result{}, err
+		}
+		return ctrl.Result{}, nil
+	}
+	if reflect.DeepEqual(resImport.Spec.ClusterInfo, resExport.Spec.ClusterInfo) {
+		klog.V(2).InfoS("No data change from ResourceExport, skip reconciling", "resourceexport", klog.KObj(&resExport))
+		return ctrl.Result{}, nil
+	}
+	// Update an existing ClusterInfo of ResourceImport
+	resImport.Spec.ClusterInfo = resExport.Spec.ClusterInfo
+	klog.InfoS("Updating ResourceImport", "resourceimport", klog.KObj(&resExport))
+	if err = r.Client.Update(ctx, resImport, &client.UpdateOptions{}); err != nil {
+		return ctrl.Result{}, err
+	}
+	return ctrl.Result{}, nil
+}

multicluster/controllers/multicluster/common/helper.go

@@ -19,12 +19,15 @@ const (
 	AntreaMCServiceAnnotation   = "multicluster.antrea.io/imported-service"
 	AntreaMCACNPAnnotation      = "multicluster.antrea.io/imported-acnp"
 	AntreaMCClusterIDAnnotation = "multicluster.antrea.io/local-cluster-id"
+	GatewayAnnotation           = "multicluster.antrea.io/gateway"
+	GatewayIPAnnotation         = "multicluster.antrea.io/gateway-ip"
 
 	AntreaMCSPrefix                = "antrea-mc-"
 	ServiceKind                    = "Service"
 	EndpointsKind                  = "Endpoints"
 	AntreaClusterNetworkPolicyKind = "AntreaClusterNetworkPolicy"
 	ServiceImportKind              = "ServiceImport"
+	ClusterInfoKind                = "ClusterInfo"
 
 	SourceName      = "sourceName"
 	SourceNamespace = "sourceNamespace"

multicluster/controllers/multicluster/commonarea/acnp_resourceimport_controller_test.go

@@ -107,13 +107,13 @@ var (
 )
 
 func TestResourceImportReconciler_handleCopySpanACNPCreateEvent(t *testing.T) {
-	remoteMgr := NewRemoteCommonAreaManager("test-clusterset", common.ClusterID(localClusterID))
+	remoteMgr := NewRemoteCommonAreaManager("test-clusterset", common.ClusterID(localClusterID), "kube-system")
 	go remoteMgr.Start()
 	defer remoteMgr.Stop()
 
 	fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(securityOpsTier).Build()
 	fakeRemoteClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(acnpResImport, acnpResImportNoMatchingTier).Build()
-	remoteCluster := NewFakeRemoteCommonArea(scheme, &remoteMgr, fakeRemoteClient, "leader-cluster", "default")
+	remoteCluster := NewFakeRemoteCommonArea(scheme, remoteMgr, fakeRemoteClient, "leader-cluster", "default")
 
 	tests := []struct {
 		name            string
@@ -134,7 +134,7 @@ func TestResourceImportReconciler_handleCopySpanACNPCreateEvent(t *testing.T) {
 			expectedSuccess: false,
 		},
 	}
-	r := NewResourceImportReconciler(fakeClient, scheme, fakeClient, localClusterID, remoteCluster)
+	r := NewResourceImportReconciler(fakeClient, scheme, fakeClient, localClusterID, "default", remoteCluster)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			if _, err := r.Reconcile(ctx, tt.req); err != nil {
@@ -164,7 +164,7 @@ func TestResourceImportReconciler_handleCopySpanACNPCreateEvent(t *testing.T) {
 }
 
 func TestResourceImportReconciler_handleCopySpanACNPDeleteEvent(t *testing.T) {
-	remoteMgr := NewRemoteCommonAreaManager("test-clusterset", common.ClusterID(localClusterID))
+	remoteMgr := NewRemoteCommonAreaManager("test-clusterset", common.ClusterID(localClusterID), "kube-system")
 	go remoteMgr.Start()
 	defer remoteMgr.Stop()
 
@@ -176,9 +176,9 @@ func TestResourceImportReconciler_handleCopySpanACNPDeleteEvent(t *testing.T) {
 
 	fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(existingACNP).Build()
 	fakeRemoteClient := fake.NewClientBuilder().WithScheme(scheme).Build()
-	remoteCluster := NewFakeRemoteCommonArea(scheme, &remoteMgr, fakeRemoteClient, "leader-cluster", "default")
+	remoteCluster := NewFakeRemoteCommonArea(scheme, remoteMgr, fakeRemoteClient, "leader-cluster", "default")
 
-	r := NewResourceImportReconciler(fakeClient, scheme, fakeClient, localClusterID, remoteCluster)
+	r := NewResourceImportReconciler(fakeClient, scheme, fakeClient, localClusterID, "default", remoteCluster)
 	r.installedResImports.Add(*acnpResImport)
 
 	if _, err := r.Reconcile(ctx, acnpImpReq); err != nil {
@@ -191,7 +191,7 @@ func TestResourceImportReconciler_handleCopySpanACNPDeleteEvent(t *testing.T) {
 }
 
 func TestResourceImportReconciler_handleCopySpanACNPUpdateEvent(t *testing.T) {
-	remoteMgr := NewRemoteCommonAreaManager("test-clusterset", common.ClusterID(localClusterID))
+	remoteMgr := NewRemoteCommonAreaManager("test-clusterset", common.ClusterID(localClusterID), "kube-system")
 	go remoteMgr.Start()
 	defer remoteMgr.Stop()
 
@@ -290,9 +290,9 @@ func TestResourceImportReconciler_handleCopySpanACNPUpdateEvent(t *testing.T) {
 
 	fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(existingACNP1, existingACNP3, existingACNP4, securityOpsTier).Build()
 	fakeRemoteClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(acnpResImport, updatedResImport2, updatedResImport3).Build()
-	remoteCluster := NewFakeRemoteCommonArea(scheme, &remoteMgr, fakeRemoteClient, "leader-cluster", "default")
+	remoteCluster := NewFakeRemoteCommonArea(scheme, remoteMgr, fakeRemoteClient, "leader-cluster", "default")
 
-	r := NewResourceImportReconciler(fakeClient, scheme, fakeClient, localClusterID, remoteCluster)
+	r := NewResourceImportReconciler(fakeClient, scheme, fakeClient, localClusterID, "default", remoteCluster)
 	r.installedResImports.Add(*acnpResImport)
 	r.installedResImports.Add(*acnpResImportNoMatchingTier)
 	r.installedResImports.Add(*updatedResImport3)