Auto update ClusterSet in leader cluster

Signed-off-by: hujiajing <hjiajing@vmware.com>

multicluster/cmd/multicluster-controller/memberclusterannounce_webhook.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"net/http"
 
-	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -47,13 +46,6 @@ func (v *memberClusterAnnounceValidator) Handle(ctx context.Context, req admissi
 		return admission.Errored(http.StatusBadRequest, e)
 	}
 
-	ui := req.UserInfo
-	_, saName, err := serviceaccount.SplitUsername(ui.Username)
-	if err != nil {
-		klog.ErrorS(err, "Error getting ServiceAccount name", "request", req)
-		return admission.Errored(http.StatusBadRequest, err)
-	}
-
 	// read the ClusterSet info
 	clusterSetList := &multiclusterv1alpha1.ClusterSetList{}
 	if err := v.Client.List(context.TODO(), clusterSetList, client.InNamespace(v.namespace)); err != nil {
@@ -67,20 +59,20 @@ func (v *memberClusterAnnounceValidator) Handle(ctx context.Context, req admissi
 	}
 
 	clusterSet := clusterSetList.Items[0]
-	if clusterSet.Name == memberClusterAnnounce.ClusterSetID {
-		for _, member := range clusterSet.Spec.Members {
-			if member.ClusterID == memberClusterAnnounce.ClusterID {
-				// validate the ServiceAccount used is correct
-				if member.ServiceAccount == saName {
-					return admission.Allowed("")
-				} else {
-					return admission.Denied("Member does not have permissions")
-				}
-			}
-		}
+	if len(clusterSet.Spec.Leaders) != 1 {
+		return admission.Errored(http.StatusPreconditionFailed,
+			fmt.Errorf("invalid ClusterSet config in the leader cluster, there must be one leader cluster in the ClusterSet"))
 	}
 
-	return admission.Denied("Unknown member")
+	if clusterSet.Name == memberClusterAnnounce.ClusterSetID &&
+		clusterSet.Spec.Leaders[0].ClusterID == memberClusterAnnounce.LeaderClusterID {
+		return admission.Allowed("")
+	}
+
+	if clusterSet.Name != memberClusterAnnounce.ClusterSetID {
+		return admission.Denied("Unknow ClusterSet ID")
+	}
+	return admission.Denied("The leader cluster ID in the MemberClusterAnnounce is not same with the leader cluster config")
 }
 
 func (v *memberClusterAnnounceValidator) InjectDecoder(d *admission.Decoder) error {

multicluster/cmd/multicluster-controller/memberclusterannounce_webhook_test.go

@@ -132,7 +132,56 @@ func TestWebhookAllow(t *testing.T) {
 	assert.Equal(t, true, response.Allowed)
 }
 
-func TestWebhookDeniedUnknownMember(t *testing.T) {
+func TestWebhookJoinAllow(t *testing.T) {
+	setup()
+
+	mca := &mcsv1alpha1.MemberClusterAnnounce{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "member-announce-from-south",
+			Namespace: "mcs1",
+		},
+		ClusterID:       "south",
+		ClusterSetID:    "clusterset1",
+		LeaderClusterID: "leader1",
+	}
+	b, _ := j.Marshal(mca)
+
+	req := admission.Request{
+		AdmissionRequest: v1.AdmissionRequest{
+			UID: "07e52e8d-4513-11e9-a716-42010a800270",
+			Kind: metav1.GroupVersionKind{
+				Group:   "multicluster.crd.antrea.io",
+				Version: "v1alpha1",
+				Kind:    "MemberClusterAnnounce",
+			},
+			Resource: metav1.GroupVersionResource{
+				Group:    "multicluster.crd.antrea.io",
+				Version:  "v1alpha1",
+				Resource: "memberclusterannounces",
+			},
+			Name:      "member-announce-from-south",
+			Namespace: "mcs1",
+			Operation: v1.Create,
+			Object: runtime.RawExtension{
+				Raw: b,
+			},
+			UserInfo: authenticationv1.UserInfo{
+				Username: "system:serviceaccount:mcs1:east-access-sa",
+				UID:      "4842eb60-68e3-4e38-adad-3abfd6117241",
+				Groups: []string{
+					"system:serviceaccounts",
+					"system:serviceaccounts:mcs1",
+					"system:authenticated",
+				},
+			},
+		},
+	}
+
+	response := mcaWebhookUnderTest.Handle(context.Background(), req)
+	assert.Equal(t, true, response.Allowed)
+}
+
+func TestWebhookDeniedDifferentClusterSet(t *testing.T) {
 	setup()
 
 	mca := &mcsv1alpha1.MemberClusterAnnounce{
@@ -141,7 +190,7 @@ func TestWebhookDeniedUnknownMember(t *testing.T) {
 			Namespace: "mcs1",
 		},
 		ClusterID:       "north",
-		ClusterSetID:    "clusterset1",
+		ClusterSetID:    "another-clusterset",
 		LeaderClusterID: "leader1",
 	}
 	b, _ := j.Marshal(mca)
@@ -179,20 +228,19 @@ func TestWebhookDeniedUnknownMember(t *testing.T) {
 
 	response := mcaWebhookUnderTest.Handle(context.Background(), req)
 	assert.Equal(t, false, response.Allowed)
-	assert.Equal(t, metav1.StatusReason("Unknown member"), response.Result.Reason)
 }
 
-func TestWebhookDeniedNoPermission(t *testing.T) {
+func TestWebhookDeniedDifferentLeaderCluster(t *testing.T) {
 	setup()
 
 	mca := &mcsv1alpha1.MemberClusterAnnounce{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "member-announce-from-east",
+			Name:      "member-announce-from-north",
 			Namespace: "mcs1",
 		},
-		ClusterID:       "east",
+		ClusterID:       "north",
 		ClusterSetID:    "clusterset1",
-		LeaderClusterID: "leader1",
+		LeaderClusterID: "different-leader",
 	}
 	b, _ := j.Marshal(mca)
 
@@ -209,14 +257,14 @@ func TestWebhookDeniedNoPermission(t *testing.T) {
 				Version:  "v1alpha1",
 				Resource: "memberclusterannounces",
 			},
-			Name:      "member-announce-from-east",
+			Name:      "member-announce-from-north",
 			Namespace: "mcs1",
 			Operation: v1.Create,
 			Object: runtime.RawExtension{
 				Raw: b,
 			},
 			UserInfo: authenticationv1.UserInfo{
-				Username: "system:serviceaccount:mcs1:north-access-sa",
+				Username: "system:serviceaccount:mcs1:east-access-sa",
 				UID:      "4842eb60-68e3-4e38-adad-3abfd6117241",
 				Groups: []string{
 					"system:serviceaccounts",
@@ -229,5 +277,4 @@ func TestWebhookDeniedNoPermission(t *testing.T) {
 
 	response := mcaWebhookUnderTest.Handle(context.Background(), req)
 	assert.Equal(t, false, response.Allowed)
-	assert.Equal(t, metav1.StatusReason("Member does not have permissions"), response.Result.Reason)
 }

multicluster/controllers/multicluster/member_clusterset_controller.go

@@ -84,6 +84,9 @@ func (r *MemberClusterSetReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			return ctrl.Result{}, err
 		}
 		klog.InfoS("Received ClusterSet delete", "clusterset", req.NamespacedName)
+		if err := r.deleteMemberAnnounce(); err != nil {
+			return ctrl.Result{}, err
+		}
 		stopErr := r.remoteCommonAreaManager.Stop()
 		r.remoteCommonAreaManager = nil
 		r.clusterSetConfig = nil
@@ -345,3 +348,27 @@ func (r *MemberClusterSetReconciler) GetRemoteCommonAreaAndLocalID() (commonarea
 	}
 	return nil, "", errors.New("no connected remote common area")
 }
+
+func (r *MemberClusterSetReconciler) deleteMemberAnnounce() error {
+	memberClusterAnnounce := &multiclusterv1alpha1.MemberClusterAnnounce{}
+
+	commonArea, ok := r.remoteCommonAreaManager.GetRemoteCommonAreas()[r.remoteCommonAreaManager.GetElectedLeaderClusterID()]
+	if !ok {
+		return fmt.Errorf("no common area for ClusetSet %s", r.clusterSetID)
+	}
+
+	if err := commonArea.Get(context.TODO(), types.NamespacedName{
+		Namespace: commonArea.GetNamespace(),
+		Name:      "member-announce-from-" + string(r.clusterID),
+	}, memberClusterAnnounce); err != nil {
+		klog.ErrorS(err, "Failed to get MemberClusterAnnounce in leader cluster", "MemberClusterAnnounce", "member-announce-from-"+string(r.clusterID))
+		return err
+	}
+	memberClusterAnnounce.Annotations[isDeletedAnnotation] = "true"
+	if err := commonArea.Update(context.TODO(), memberClusterAnnounce); err != nil {
+		klog.ErrorS(err, "Failed to update MemberClusterAnnounce", "member-announce-from-"+string(r.clusterID))
+		return err
+	}
+
+	return nil
+}

multicluster/controllers/multicluster/memberclusterannounce_controller.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -44,6 +45,8 @@ var (
 
 	TimerInterval     = 5 * time.Second
 	ConnectionTimeout = 3 * TimerInterval
+
+	isDeletedAnnotation = "multicluster.antrea.io/is-member-deleted"
 )
 
 type leaderStatus struct {
@@ -137,10 +140,57 @@ func (r *MemberClusterAnnounceReconciler) Reconcile(ctx context.Context, req ctr
 				}
 			}
 		}
-		// If err != nil, probably ClusterClaims were deleted during the processing of MemberClusterAnnounce.
-		// Nothing to handle in this case and MemberClusterAnnounce will also be deleted soon.
-		// TODO: Add ClusterClaim webhook to make sure it cannot be deleted while ClusterSet is present.
 	}
+	// If err != nil, probably ClusterClaims were deleted during the processing of MemberClusterAnnounce.
+	// Nothing to handle in this case and MemberClusterAnnounce will also be deleted soon.
+	// TODO: Add ClusterClaim webhook to make sure it cannot be deleted while ClusterSet is present.
+
+	clusterSetID := memberAnnounce.ClusterSetID
+	clusterSet := &multiclusterv1alpha1.ClusterSet{}
+	if err := r.Get(context.TODO(), types.NamespacedName{Namespace: memberAnnounce.Namespace, Name: clusterSetID}, clusterSet); err != nil {
+		if errors.IsNotFound(err) {
+			klog.ErrorS(err, "ClusterSet not found in leader cluster", "ClusterSet", clusterSetID)
+			return ctrl.Result{}, err
+		}
+		klog.ErrorS(err, "Failed to get ClusterSet in leader cluster", "ClusterSet", clusterSetID)
+		return ctrl.Result{}, err
+	}
+	if memberAnnounce.Annotations[isDeletedAnnotation] == "true" {
+		reservedMembers := []multiclusterv1alpha1.MemberCluster{}
+
+		for _, member := range clusterSet.Spec.Members {
+			if member.ClusterID != memberAnnounce.ClusterID {
+				reservedMembers = append(reservedMembers, member)
+			}
+		}
+		clusterSet.Spec.Members = reservedMembers
+		if err := r.Update(context.TODO(), clusterSet); err != nil {
+			klog.ErrorS(err, "Failed to delete member cluster in ClusterSet", "memberCluster", memberAnnounce.ClusterID, "ClusterSet", clusterSet.Name)
+			return ctrl.Result{}, err
+		}
+
+		if err := r.Delete(context.TODO(), memberAnnounce); err != nil {
+			klog.ErrorS(err, "Failed to delete MemberClusterAnnounce", "MemberClusterAnnounce", memberAnnounce.Name)
+			return ctrl.Result{}, err
+		}
+	} else {
+		isExist := false
+		for _, member := range clusterSet.Spec.Members {
+			if member.ClusterID == memberAnnounce.ClusterID {
+				isExist = true
+				break
+			}
+		}
+
+		if !isExist {
+			clusterSet.Spec.Members = append(clusterSet.Spec.Members, multiclusterv1alpha1.MemberCluster{ClusterID: memberAnnounce.ClusterID})
+			if err := r.Update(context.TODO(), clusterSet); err != nil {
+				klog.ErrorS(err, "Failed to add member cluster in ClusterSet", "memberCluster", memberAnnounce.ClusterID, "ClusterSet", clusterSet.Name)
+				return ctrl.Result{}, err
+			}
+		}
+	}
+
 	// Member not found. If this happens, the MemberClusterAnnounce should soon be deleted.
 	// Nothing to do here.