Auto update ClusterSet in leader cluster

Signed-off-by: hujiajing <hjiajing@vmware.com>

multicluster/build/yamls/antrea-multicluster-leader-namespaced.yml

@@ -23,6 +23,12 @@ metadata:
   name: antrea-mc-controller-role
   namespace: antrea-multicluster
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
 - apiGroups:
   - ""
   resources:

multicluster/cmd/multicluster-controller/leader.go

@@ -21,6 +21,7 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	multiclusterv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1"
@@ -63,10 +64,15 @@ func runLeader(o *Options) error {
 	if err = memberClusterStatusManager.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating MemberClusterAnnounce controller: %v", err)
 	}
+
+	noCachedClient, err := client.New(mgr.GetConfig(), client.Options{Scheme: mgr.GetScheme(), Mapper: mgr.GetRESTMapper()})
+	if err != nil {
+		return err
+	}
 	hookServer := mgr.GetWebhookServer()
 	hookServer.Register("/validate-multicluster-crd-antrea-io-v1alpha1-memberclusterannounce",
 		&webhook.Admission{Handler: &memberClusterAnnounceValidator{
-			Client:    mgr.GetClient(),
+			Client:    noCachedClient,
 			namespace: env.GetPodNamespace()}})
 
 	clusterSetReconciler := &multiclustercontrollers.LeaderClusterSetReconciler{

multicluster/cmd/multicluster-controller/memberclusterannounce_webhook.go

@@ -18,9 +18,11 @@ package main
 
 import (
 	"context"
-	"fmt"
+	"encoding/json"
 	"net/http"
 
+	admissionv1 "k8s.io/api/admission/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -54,33 +56,53 @@ func (v *memberClusterAnnounceValidator) Handle(ctx context.Context, req admissi
 		return admission.Errored(http.StatusBadRequest, err)
 	}
 
-	// read the ClusterSet info
-	clusterSetList := &multiclusterv1alpha1.ClusterSetList{}
-	if err := v.Client.List(context.TODO(), clusterSetList, client.InNamespace(v.namespace)); err != nil {
-		klog.ErrorS(err, "Error reading ClusterSet", "Namespace", v.namespace)
+	serviceAccount := &v1.ServiceAccount{}
+	if err := v.Client.Get(ctx, client.ObjectKey{Namespace: v.namespace, Name: saName}, serviceAccount); err != nil {
+		klog.ErrorS(err, "Error getting ServiceAccount", "ServiceAccount", saName, "Namespace", v.namespace, "MemberClusterAnnounce", klog.KObj(memberClusterAnnounce))
 		return admission.Errored(http.StatusPreconditionFailed, err)
 	}
 
-	if len(clusterSetList.Items) != 1 {
-		return admission.Errored(http.StatusPreconditionFailed,
-			fmt.Errorf("invalid ClusterSet config in the leader cluster, please contact your administrator"))
+	var newObj *multiclusterv1alpha1.MemberClusterAnnounce
+	var oldObj *multiclusterv1alpha1.MemberClusterAnnounce
+	if req.Object.Raw != nil {
+		if err := json.Unmarshal(req.Object.Raw, &newObj); err != nil {
+			klog.ErrorS(err, "Error while decoding new MemberClusterAnnounce", "MemberClusterAnnounce", klog.KObj(memberClusterAnnounce))
+			return admission.Errored(http.StatusBadRequest, err)
+		}
 	}
-
-	clusterSet := clusterSetList.Items[0]
-	if clusterSet.Name == memberClusterAnnounce.ClusterSetID {
-		for _, member := range clusterSet.Spec.Members {
-			if member.ClusterID == memberClusterAnnounce.ClusterID {
-				// validate the ServiceAccount used is correct
-				if member.ServiceAccount == saName {
-					return admission.Allowed("")
-				} else {
-					return admission.Denied("Member does not have permissions")
-				}
-			}
+	if req.OldObject.Raw != nil {
+		if err := json.Unmarshal(req.OldObject.Raw, &oldObj); err != nil {
+			klog.ErrorS(err, "Error while decoding old MemberClusterAnnounce", "MemberClusterAnnounce", klog.KObj(memberClusterAnnounce))
+			return admission.Errored(http.StatusBadRequest, err)
 		}
 	}
 
-	return admission.Denied("Unknown member")
+	switch req.Operation {
+	case admissionv1.Create:
+		// Read the ClusterSet info
+		clusterSetList := &multiclusterv1alpha1.ClusterSetList{}
+		if err := v.Client.List(context.TODO(), clusterSetList, client.InNamespace(v.namespace)); err != nil {
+			klog.ErrorS(err, "Error reading ClusterSet", "Namespace", v.namespace)
+			return admission.Errored(http.StatusPreconditionFailed, err)
+		}
+
+		clusterSet := clusterSetList.Items[0]
+		if clusterSet.Name != memberClusterAnnounce.ClusterSetID {
+			return admission.Denied("Unknown ClusterSet ID")
+		}
+		if clusterSet.Spec.Leaders[0].ClusterID != memberClusterAnnounce.LeaderClusterID {
+			return admission.Denied("Leader cluster ID in the MemberClusterAnnounce does not match that in the ClusterSet")
+		}
+		return admission.Allowed("")
+	case admissionv1.Update:
+		// Member cluster will never change ClusterSet ID in MemberClusterAnnounce
+		if newObj.ClusterSetID != oldObj.ClusterSetID || newObj.LeaderClusterID != oldObj.LeaderClusterID {
+			return admission.Denied("ClusterSetID or LeaderClusterID cannot be changed")
+		}
+		return admission.Allowed("")
+	default:
+		return admission.Allowed("")
+	}
 }
 
 func (v *memberClusterAnnounceValidator) InjectDecoder(d *admission.Decoder) error {

multicluster/cmd/multicluster-controller/memberclusterannounce_webhook_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/admission/v1"
 	authenticationv1 "k8s.io/api/authentication/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -63,11 +64,28 @@ func setup() {
 		},
 	}
 
+	existingServiceAccounts := &corev1.ServiceAccountList{
+		Items: []corev1.ServiceAccount{
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "mcs1",
+					Name:      "east-access-sa",
+				},
+			},
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "mcs1",
+					Name:      "west-access-sa",
+				},
+			},
+		},
+	}
+
 	newScheme := runtime.NewScheme()
 	utilruntime.Must(clientgoscheme.AddToScheme(newScheme))
 	utilruntime.Must(k8smcsv1alpha1.AddToScheme(newScheme))
 	utilruntime.Must(mcsv1alpha1.AddToScheme(newScheme))
-	fakeClient := fake.NewClientBuilder().WithScheme(newScheme).WithObjects(existingClusterSet).Build()
+	fakeClient := fake.NewClientBuilder().WithScheme(newScheme).WithObjects(existingClusterSet).WithLists(existingServiceAccounts).Build()
 
 	mcaWebhookUnderTest = &memberClusterAnnounceValidator{
 		Client:    fakeClient,
@@ -130,7 +148,56 @@ func TestWebhookAllow(t *testing.T) {
 	assert.Equal(t, true, response.Allowed)
 }
 
-func TestWebhookDeniedUnknownMember(t *testing.T) {
+func TestWebhookJoinAllow(t *testing.T) {
+	setup()
+
+	mca := &mcsv1alpha1.MemberClusterAnnounce{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "member-announce-from-south",
+			Namespace: "mcs1",
+		},
+		ClusterID:       "south",
+		ClusterSetID:    "clusterset1",
+		LeaderClusterID: "leader1",
+	}
+	b, _ := j.Marshal(mca)
+
+	req := admission.Request{
+		AdmissionRequest: v1.AdmissionRequest{
+			UID: "07e52e8d-4513-11e9-a716-42010a800270",
+			Kind: metav1.GroupVersionKind{
+				Group:   "multicluster.crd.antrea.io",
+				Version: "v1alpha1",
+				Kind:    "MemberClusterAnnounce",
+			},
+			Resource: metav1.GroupVersionResource{
+				Group:    "multicluster.crd.antrea.io",
+				Version:  "v1alpha1",
+				Resource: "memberclusterannounces",
+			},
+			Name:      "member-announce-from-south",
+			Namespace: "mcs1",
+			Operation: v1.Create,
+			Object: runtime.RawExtension{
+				Raw: b,
+			},
+			UserInfo: authenticationv1.UserInfo{
+				Username: "system:serviceaccount:mcs1:east-access-sa",
+				UID:      "4842eb60-68e3-4e38-adad-3abfd6117241",
+				Groups: []string{
+					"system:serviceaccounts",
+					"system:serviceaccounts:mcs1",
+					"system:authenticated",
+				},
+			},
+		},
+	}
+
+	response := mcaWebhookUnderTest.Handle(context.Background(), req)
+	assert.Equal(t, true, response.Allowed)
+}
+
+func TestWebhookDeniedDifferentClusterSet(t *testing.T) {
 	setup()
 
 	mca := &mcsv1alpha1.MemberClusterAnnounce{
@@ -139,7 +206,7 @@ func TestWebhookDeniedUnknownMember(t *testing.T) {
 			Namespace: "mcs1",
 		},
 		ClusterID:       "north",
-		ClusterSetID:    "clusterset1",
+		ClusterSetID:    "another-clusterset",
 		LeaderClusterID: "leader1",
 	}
 	b, _ := j.Marshal(mca)
@@ -177,18 +244,66 @@ func TestWebhookDeniedUnknownMember(t *testing.T) {
 
 	response := mcaWebhookUnderTest.Handle(context.Background(), req)
 	assert.Equal(t, false, response.Allowed)
-	assert.Equal(t, metav1.StatusReason("Unknown member"), response.Result.Reason)
 }
 
-func TestWebhookDeniedNoPermission(t *testing.T) {
+func TestWebhookDeniedDifferentLeaderCluster(t *testing.T) {
 	setup()
 
 	mca := &mcsv1alpha1.MemberClusterAnnounce{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "member-announce-from-east",
+			Name:      "member-announce-from-north",
 			Namespace: "mcs1",
 		},
-		ClusterID:       "east",
+		ClusterID:       "north",
+		ClusterSetID:    "clusterset1",
+		LeaderClusterID: "different-leader",
+	}
+	b, _ := j.Marshal(mca)
+
+	req := admission.Request{
+		AdmissionRequest: v1.AdmissionRequest{
+			UID: "07e52e8d-4513-11e9-a716-42010a800270",
+			Kind: metav1.GroupVersionKind{
+				Group:   "multicluster.crd.antrea.io",
+				Version: "v1alpha1",
+				Kind:    "MemberClusterAnnounce",
+			},
+			Resource: metav1.GroupVersionResource{
+				Group:    "multicluster.crd.antrea.io",
+				Version:  "v1alpha1",
+				Resource: "memberclusterannounces",
+			},
+			Name:      "member-announce-from-north",
+			Namespace: "mcs1",
+			Operation: v1.Create,
+			Object: runtime.RawExtension{
+				Raw: b,
+			},
+			UserInfo: authenticationv1.UserInfo{
+				Username: "system:serviceaccount:mcs1:east-access-sa",
+				UID:      "4842eb60-68e3-4e38-adad-3abfd6117241",
+				Groups: []string{
+					"system:serviceaccounts",
+					"system:serviceaccounts:mcs1",
+					"system:authenticated",
+				},
+			},
+		},
+	}
+
+	response := mcaWebhookUnderTest.Handle(context.Background(), req)
+	assert.Equal(t, false, response.Allowed)
+}
+
+func TestWebhookDeniedUnknownServiceAccount(t *testing.T) {
+	setup()
+
+	mca := &mcsv1alpha1.MemberClusterAnnounce{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "member-announce-from-south",
+			Namespace: "mcs1",
+		},
+		ClusterID:       "south",
 		ClusterSetID:    "clusterset1",
 		LeaderClusterID: "leader1",
 	}
@@ -207,14 +322,76 @@ func TestWebhookDeniedNoPermission(t *testing.T) {
 				Version:  "v1alpha1",
 				Resource: "memberclusterannounces",
 			},
-			Name:      "member-announce-from-east",
+			Name:      "member-announce-from-south",
 			Namespace: "mcs1",
 			Operation: v1.Create,
 			Object: runtime.RawExtension{
 				Raw: b,
 			},
 			UserInfo: authenticationv1.UserInfo{
-				Username: "system:serviceaccount:mcs1:north-access-sa",
+				Username: "system:serviceaccount:mcs1:unknown-access-sa",
+				UID:      "4842eb60-68e3-4e38-adad-3abfd6117241",
+				Groups: []string{
+					"system:serviceaccounts",
+					"system:serviceaccounts:mcs1",
+					"system:authenticated",
+				},
+			},
+		},
+	}
+
+	response := mcaWebhookUnderTest.Handle(context.Background(), req)
+	assert.Equal(t, false, response.Allowed)
+}
+
+func TestUpdateClusterSetID(t *testing.T) {
+	setup()
+
+	mca := &mcsv1alpha1.MemberClusterAnnounce{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "member-announce-from-south",
+			Namespace: "mcs1",
+		},
+		ClusterID:       "south",
+		ClusterSetID:    "clusterset-changed",
+		LeaderClusterID: "leader1",
+	}
+	oldMca := &mcsv1alpha1.MemberClusterAnnounce{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "member-announce-from-south",
+			Namespace: "mcs1",
+		},
+		ClusterID:       "south",
+		ClusterSetID:    "clusterset",
+		LeaderClusterID: "leader1",
+	}
+	b, _ := j.Marshal(mca)
+	old, _ := j.Marshal(oldMca)
+
+	req := admission.Request{
+		AdmissionRequest: v1.AdmissionRequest{
+			UID: "07e52e8d-4513-11e9-a716-42010a800270",
+			Kind: metav1.GroupVersionKind{
+				Group:   "multicluster.crd.antrea.io",
+				Version: "v1alpha1",
+				Kind:    "MemberClusterAnnounce",
+			},
+			Resource: metav1.GroupVersionResource{
+				Group:    "multicluster.crd.antrea.io",
+				Version:  "v1alpha1",
+				Resource: "memberclusterannounces",
+			},
+			Name:      "member-announce-from-south",
+			Namespace: "mcs1",
+			Operation: v1.Update,
+			Object: runtime.RawExtension{
+				Raw: b,
+			},
+			OldObject: runtime.RawExtension{
+				Raw: old,
+			},
+			UserInfo: authenticationv1.UserInfo{
+				Username: "system:serviceaccount:mcs1:east-access-sa",
 				UID:      "4842eb60-68e3-4e38-adad-3abfd6117241",
 				Groups: []string{
 					"system:serviceaccounts",
@@ -227,5 +404,4 @@ func TestWebhookDeniedNoPermission(t *testing.T) {
 
 	response := mcaWebhookUnderTest.Handle(context.Background(), req)
 	assert.Equal(t, false, response.Allowed)
-	assert.Equal(t, metav1.StatusReason("Member does not have permissions"), response.Result.Reason)
 }

multicluster/controllers/multicluster/commonarea/remote_common_area.go

@@ -247,6 +247,7 @@ func (r *remoteCommonArea) SendMemberAnnounce() error {
 		localClusterMemberAnnounce.Name = "member-announce-from-" + r.GetLocalClusterID()
 		localClusterMemberAnnounce.Namespace = r.Namespace
 		localClusterMemberAnnounce.ClusterSetID = string(r.ClusterSetID)
+		localClusterMemberAnnounce.LeaderClusterID = string(r.GetClusterID())
 		if err := r.Create(context.TODO(), &localClusterMemberAnnounce, &client.CreateOptions{}); err != nil {
 			klog.ErrorS(err, "Error creating MemberClusterAnnounce", "cluster", r.GetClusterID())
 			return err