From 04b9dc6bbb283c5f49251532e5370f5b7f68eb4f Mon Sep 17 00:00:00 2001 From: Jiajing Hu Date: Thu, 11 Aug 2022 05:28:07 +0800 Subject: [PATCH] Add antctl commands for Multi-cluster ClusterSet setup Add 4 new commands for the users to set up and delete ClusterSets. And delete a few old commands. antctl mc init: to set up a ClusterSet in a leader cluster. antctl mc join: to join a ClusterSet in a member cluster. antctl mc destroy: to delete a ClusterSet in a leader cluster. antctl mc leave: to leave a ClusterSet in a member cluster. Signed-off-by: hujiajing --- docs/multicluster/antctl.md | 99 ++-- go.mod | 4 +- .../commonarea/remote_common_area.go | 6 +- pkg/antctl/antctl.go | 20 +- .../raw/multicluster/add/member_cluster.go | 115 ----- pkg/antctl/raw/multicluster/commands.go | 23 +- pkg/antctl/raw/multicluster/common/cleanup.go | 54 +++ pkg/antctl/raw/multicluster/common/common.go | 425 ++++++++++++++++++ .../raw/multicluster/common/rollback.go | 37 ++ .../raw/multicluster/create/access_token.go | 179 ++------ .../raw/multicluster/create/clusterclaim.go | 150 ------- .../raw/multicluster/create/clusterset.go | 142 ------ .../raw/multicluster/delete/clusterclaim.go | 124 ----- .../raw/multicluster/delete/clusterset.go | 101 ----- .../raw/multicluster/delete/member_cluster.go | 114 ----- pkg/antctl/raw/multicluster/destroy.go | 51 +++ pkg/antctl/raw/multicluster/init.go | 179 ++++++++ pkg/antctl/raw/multicluster/join.go | 315 +++++++++++++ pkg/antctl/raw/multicluster/leave.go | 51 +++ 19 files changed, 1248 insertions(+), 941 deletions(-) delete mode 100644 pkg/antctl/raw/multicluster/add/member_cluster.go create mode 100644 pkg/antctl/raw/multicluster/common/cleanup.go create mode 100644 pkg/antctl/raw/multicluster/common/common.go create mode 100644 pkg/antctl/raw/multicluster/common/rollback.go delete mode 100644 pkg/antctl/raw/multicluster/create/clusterclaim.go delete mode 100644 pkg/antctl/raw/multicluster/create/clusterset.go delete mode 100644 pkg/antctl/raw/multicluster/delete/clusterclaim.go delete mode 100644 pkg/antctl/raw/multicluster/delete/clusterset.go delete mode 100644 pkg/antctl/raw/multicluster/delete/member_cluster.go create mode 100644 pkg/antctl/raw/multicluster/destroy.go create mode 100644 pkg/antctl/raw/multicluster/init.go create mode 100644 pkg/antctl/raw/multicluster/join.go create mode 100644 pkg/antctl/raw/multicluster/leave.go diff --git a/docs/multicluster/antctl.md b/docs/multicluster/antctl.md index d03881086af..ccc4e83d162 100644 --- a/docs/multicluster/antctl.md +++ b/docs/multicluster/antctl.md @@ -36,57 +36,96 @@ To see the usage examples of these commands, you may also run `antctl mc get [su ## antctl mc create -`antctl mc create` command can create access-token and other resources like ClusterSet, ClusterClaims for -Antrea Multi-cluster setup - -+ `antctl mc create accesstoken` command can create accesstoken for member clusters. -+ `antctl mc create clusterclaims` command can create two ClusterClaims in a leader or member cluster. One for the leader or member cluster, and another for the ClusterSet. -+ `antctl mc create clusterset` command can create a ClusterSet in a leader or member cluster. +`antctl mc create` command can create tokens for member clusters to join a ClusterSet. The command will +also create a Secret to store the token, as well as a ServiceAccount and a RoleBinding. The `--output-file` +option saves the member token Secret manifest to a file. ```bash -anctcl mc create accesstoken [NAME] [-n NAMESPACE] [--serviceaccount SERVICE_ACCOUNT] [--role-binding ROLE_BINDING] -antctl mc create clusterclaims [-n NAMESPACE] [--clusterset-id CLUSTERSET_ID] [--cluster-id CLUSTER_ID] -antctl mc create clusterset [NAME] [-n NAMESPACE] [--leader-server LEADER_SERVER] [--service-account SERVICE_ACCOUNT] [--secret SECRET] [--leader-cluster LEADER_CLUSTER_ID] +anctcl mc create membertoken NAME -n NAMESPACE [-o OUTPUT_FILE] ``` To see the usage examples of these commands, you may also run `antctl mc create [subcommand] --help`. -## antctl mc add +## antctl mc deploy + +`antctl mc deploy` command deploys Antrea Multi-cluster Controller to a leader or member cluster. -`antctl mc add` command can add a new member cluster to a ClusterSet. ++ `antctl mc deploy leadercluster` command deploys Antrea Multi-cluster Controller to a leader cluster and imports + all the Antrea Multi-cluster CRDs. ++ `antctl mc deploy membercluster` command deploys Antrea Multi-cluster Controller to a member cluster and imports + all the Antrea Multi-cluster CRDs. ```bash -antctl mc add membercluster [CLUSTER_ID] [-n NAMESPACE] [--clusterset CLUSTERSET] [--service-account SERVICE_ACCOUNT] +antctl mc deploy leadercluster -n NAMESPACE [--antrea-version ANTREA_VERSION] [-f PATH_TO_MANIFEST] +antctl mc deploy membercluster -n NAMESPACE [--antrea-version ANTREA_VERSION] [-f PATH_TO_MANIFEST] ``` -To see the usage examples of these commands, you may also run `antctl mc add [subcommand] --help`. - -## antctl mc delete +To see the usage examples of these commands, you may also run `antctl mc deploy [subcommand] --help`. -`antctl mc delete` command can delete resources in an Antrea Multi-cluster ClusterSet. +## antctl mc init -+ `antctl mc delete clusterclaims` command can delete the two ClusterClaims in a specified Namespace. One for the leader or member cluster, and another for the ClusterSet. -+ `antctl mc delete clusterset` command can delete a ClusterSet in a leader or member cluster. -+ `antctl mc delete member-cluster` command can delete a member cluster in a specified Antrea Multi-cluster ClusterSet. +`antctl mc init` command initializes an Antrea Multi-cluster ClusterSet in a leader cluster. It will create a +ClusterSet and ClusterClaims for the leader cluster. If the `--output-file` option is specified, the config arguments +for member clusters to join the ClusterSet will be saved to the specified file. ```bash -antctl mc delete clusterclaims [-n NAMESPACE] -antctl mc delete clusterset [NAME] [-n NAMESPACE] -antctl mc delete membercluster [MEMBER_CLUSTER_ID] [-n NAMESPACE] [--clusterset CLUSTERSET] +antctl mc init -n NAMESPACE --clusterset CLUSTERSET_ID --clusterid CLUSTERID [--create-token] [-o OUTPUT_FILE] ``` -To see the usage examples of these commands, you may also run `antctl mc delete [subcommand] --help`. +To see the usage examples of this command, you may also run `antctl mc init --help`. -## antctl mc deploy +## antctl mc join -`antctl mc deploy` command can deploy Antrea Multi-cluster Controller to a leader or member cluster. +`antctl mc join` command lets a member cluster join an existing Antrea Multi-cluster ClusterSet. It will create a +ClusterSet and ClusterClaims for the member cluster. Users can use command line options or a config file (which can +be the output file of the `anctl mc init` command) to specify the ClusterSet join arguments. -+ `antctl mc deploy leadercluster` command can deploy Antrea Multi-cluster Controller to a leader cluster, and define all the CRDs the leader cluster needed. -+ `antctl mc deploy membercluster` command can deploy Antrea Multi-cluster Controller to a member cluster, and define all the CRDs the member cluster needed. +When the config file is provided, the command line options may be overwritten by the file. A token is needed for a +member cluster to access the leader cluster API server. Users can either specify a pre-created token Secret with the +`--token-secret-name` option, or pass a Secret manifest to create the Secret with either the `--token-secret-file` +option or the config file. ```bash -antctl mc deploy leadercluster [--antrea-version ANTREA_VERSION] [-n NAMESPACE] [-f PATH_TO_MANIFEST] -antctl mc deploy membercluster [--antrea-version ANTREA_VERSION] [-n NAMESPACE] [-f PATH_TO_MANIFEST] +antctl mc join --clusterset=CLUSTERSET_ID \ + --clusterid=CLUSTER_ID \ + --namespace=[MEMBER_NAMESPACE] \ + --leader-clusterid=LEADER_CLUSTER_ID \ + --leader-namespace=LEADER_NAMESPACE \ + --leader-apiserver=LEADER_APISERVER \ + --token-secret-name=[TOKEN_SECRET_NAME] \ + --token-secret-file=[TOKEN_SECRET_FILE] + +antctl mc join --config-file PATH_TO_CONFIG_FILE [--clusterid=CLUSTER_ID] [--token-secret-name=TOKEN_SECRET_NAME] [--token-secret-file=TOKEN_SECRET_FILE] ``` -To see the usage examples of these commands, you may also run `antctl mc deploy [subcommand] --help`. +Below is a config file example: + +```yaml +apiVersion: multicluster.antrea.io/v1alpha1 +kind: ClusterSetJoinConfig +clusterSetID: clusterset1 +clusterID: cluster-east +namespace: kube-system +leaderClusterID: cluster-north +leaderNamespace: antrea-multicluster +leaderAPIServer: https://172.18.0.3:6443 +tokenSecretName: cluster-east-token +``` + +## antctl mc leave + +`antctl mc leave` command lets a member cluster leave a ClusterSet. It will delete the ClusterSet and ClusterClaims +and other resources created by antctl for the member cluster. + +```bash +antctl mc leave --clusterset CLUSTERSET_ID --namespace [NAMESPACE] +``` + +## antctl mc destroy + +`antctl mc destroy` command can destroy an Antrea Multi-cluster ClusterSet in a leader cluster. It will delete the +ClusterSet and ClusterClaims and other resources created by antctl for the leader cluster. + +```bash +antctl mc destroy --clusterset=CLUSTERSET_ID --namespace NAMESPACE +``` diff --git a/go.mod b/go.mod index cafc7a3b16f..2e23e763442 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( google.golang.org/protobuf v1.27.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b k8s.io/api v0.24.0 k8s.io/apiextensions-apiserver v0.24.0 k8s.io/apimachinery v0.24.0 @@ -75,6 +76,7 @@ require ( k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 sigs.k8s.io/controller-runtime v0.12.1 sigs.k8s.io/mcs-api v0.1.0 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -183,11 +185,9 @@ require ( google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) // Newer version of github.com/googleapis/gnostic make use of newer gopkg.in/yaml(v3), which conflicts with diff --git a/multicluster/controllers/multicluster/commonarea/remote_common_area.go b/multicluster/controllers/multicluster/commonarea/remote_common_area.go index 353dd05b6de..8e19a71e44e 100644 --- a/multicluster/controllers/multicluster/commonarea/remote_common_area.go +++ b/multicluster/controllers/multicluster/commonarea/remote_common_area.go @@ -136,7 +136,7 @@ func NewRemoteCommonArea(clusterID common.ClusterID, clusterSetID common.Cluster scheme *runtime.Scheme, localClusterClient client.Client, clusterSetNamespace string, localNamespace string) (RemoteCommonArea, error) { klog.InfoS("Create a RemoteCommonArea", "cluster", clusterID) - crtData, token, err := getSecretCACrtAndToken(secret) + crtData, token, err := GetSecretCACrtAndToken(secret) if err != nil { return nil, err } @@ -189,9 +189,9 @@ func NewRemoteCommonArea(clusterID common.ClusterID, clusterSetID common.Cluster } /** - * getSecretCACrtAndToken returns the access credentials from Secret. + * GetSecretCACrtAndToken returns the access credentials from Secret. */ -func getSecretCACrtAndToken(secretObj *v1.Secret) ([]byte, []byte, error) { +func GetSecretCACrtAndToken(secretObj *v1.Secret) ([]byte, []byte, error) { caData, found := secretObj.Data[v1.ServiceAccountRootCAKey] if !found { return nil, nil, fmt.Errorf("ca.crt data not found in Secret %v", secretObj.GetName()) diff --git a/pkg/antctl/antctl.go b/pkg/antctl/antctl.go index 88038d12eb3..1bece42a8d9 100644 --- a/pkg/antctl/antctl.go +++ b/pkg/antctl/antctl.go @@ -598,25 +598,37 @@ $ antctl get podmulticaststats pod -n namespace`, commandGroup: mc, }, { - cobraCommand: multicluster.AddCmd, + cobraCommand: multicluster.CreateCmd, supportAgent: false, supportController: false, commandGroup: mc, }, { - cobraCommand: multicluster.CreateCmd, + cobraCommand: multicluster.DeployCmd, supportAgent: false, supportController: false, commandGroup: mc, }, { - cobraCommand: multicluster.DeleteCmd, + cobraCommand: multicluster.JoinCmd, supportAgent: false, supportController: false, commandGroup: mc, }, { - cobraCommand: multicluster.DeployCmd, + cobraCommand: multicluster.LeaveCmd, + supportAgent: false, + supportController: false, + commandGroup: mc, + }, + { + cobraCommand: multicluster.InitCmd, + supportAgent: false, + supportController: false, + commandGroup: mc, + }, + { + cobraCommand: multicluster.DestroyCmd, supportAgent: false, supportController: false, commandGroup: mc, diff --git a/pkg/antctl/raw/multicluster/add/member_cluster.go b/pkg/antctl/raw/multicluster/add/member_cluster.go deleted file mode 100644 index 666237dcdc2..00000000000 --- a/pkg/antctl/raw/multicluster/add/member_cluster.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2022 Antrea Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package add - -import ( - "context" - "fmt" - "strings" - - "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - - multiclusterv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" - "antrea.io/antrea/pkg/antctl/raw" - multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme" -) - -type memberClusterOptions struct { - namespace string - clusterSet string - serviceAccount string -} - -var memberClusterOpt *memberClusterOptions - -var memberClusterExamples = strings.Trim(` -# Add a new member cluster to a ClusterSet - $ antctl mc add membercluster -n --clusterset --service-account -`, "\n") - -func (o *memberClusterOptions) validateAndComplete() error { - if o.namespace == "" { - return fmt.Errorf("the Namespace cannot be empty") - } - if o.clusterSet == "" { - return fmt.Errorf("the ClusterSet cannot be empty") - } - if o.serviceAccount == "" { - return fmt.Errorf("the ServiceAccount cannot be empty") - } - - return nil -} - -func NewMemberClusterCmd() *cobra.Command { - command := &cobra.Command{ - Use: "membercluster", - Args: cobra.MaximumNArgs(1), - Short: "Add a new member cluster to a ClusterSet", - Long: "Add a new member cluster to a ClusterSet", - Example: memberClusterExamples, - RunE: memberClusterRunE, - } - - o := &memberClusterOptions{} - memberClusterOpt = o - command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of member cluster") - command.Flags().StringVarP(&o.clusterSet, "clusterset", "", "", "The name of target ClusterSet to add a new member cluster") - command.Flags().StringVarP(&o.serviceAccount, "service-account", "", "", "ServiceAccount of the member cluster") - - return command -} - -func memberClusterRunE(cmd *cobra.Command, args []string) error { - if err := memberClusterOpt.validateAndComplete(); err != nil { - return err - } - if len(args) != 1 { - return fmt.Errorf("exactly one NAME is required, got %d", len(args)) - } - - kubeconfig, err := raw.ResolveKubeconfig(cmd) - if err != nil { - return err - } - restconfigTmpl := rest.CopyConfig(kubeconfig) - raw.SetupKubeconfig(restconfigTmpl) - - k8sClient, err := client.New(kubeconfig, client.Options{Scheme: multiclusterscheme.Scheme}) - if err != nil { - return err - } - - memberClusterID := args[0] - clusterSet := &multiclusterv1alpha1.ClusterSet{} - if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: memberClusterOpt.clusterSet, Namespace: memberClusterOpt.namespace}, clusterSet); err != nil { - return err - } - for _, member := range clusterSet.Spec.Members { - if member.ClusterID == memberClusterID { - return fmt.Errorf(`the member cluster "%s" was already added to the ClusterSet "%s"`, memberClusterID, memberClusterOpt.clusterSet) - } - } - clusterSet.Spec.Members = append(clusterSet.Spec.Members, multiclusterv1alpha1.MemberCluster{ClusterID: memberClusterID, ServiceAccount: memberClusterOpt.serviceAccount}) - if err := k8sClient.Update(context.TODO(), clusterSet); err != nil { - return err - } - - fmt.Fprintf(cmd.OutOrStdout(), "The member cluster \"%s\" is added to the ClusterSet \"%s\" successfully\n", memberClusterID, memberClusterOpt.clusterSet) - return nil -} diff --git a/pkg/antctl/raw/multicluster/commands.go b/pkg/antctl/raw/multicluster/commands.go index 7367d3cb03a..2d997238599 100644 --- a/pkg/antctl/raw/multicluster/commands.go +++ b/pkg/antctl/raw/multicluster/commands.go @@ -17,9 +17,7 @@ package multicluster import ( "github.com/spf13/cobra" - "antrea.io/antrea/pkg/antctl/raw/multicluster/add" "antrea.io/antrea/pkg/antctl/raw/multicluster/create" - deleteCmd "antrea.io/antrea/pkg/antctl/raw/multicluster/delete" "antrea.io/antrea/pkg/antctl/raw/multicluster/deploy" "antrea.io/antrea/pkg/antctl/raw/multicluster/get" ) @@ -34,32 +32,21 @@ var CreateCmd = &cobra.Command{ Short: "Create multi-cluster resources", } -var AddCmd = &cobra.Command{ - Use: "add", - Short: "Add a new member cluster to a ClusterSet", -} - -var DeleteCmd = &cobra.Command{ - Use: "delete", - Short: "Delete multi-cluster resources", -} - var DeployCmd = &cobra.Command{ Use: "deploy", Short: "Deploy Antrea Multi-cluster Controller to a leader or member cluster", } +var JoinCmd = NewJoinCommand() +var LeaveCmd = NewLeaveCommand() +var InitCmd = NewInitCommand() +var DestroyCmd = NewDestroyCommand() + func init() { GetCmd.AddCommand(get.NewClusterSetCommand()) GetCmd.AddCommand(get.NewResourceImportCommand()) GetCmd.AddCommand(get.NewResourceExportCommand()) - CreateCmd.AddCommand(create.NewClusterClaimCmd()) CreateCmd.AddCommand(create.NewAccessTokenCmd()) - CreateCmd.AddCommand(create.NewClusterSetCmd()) - DeleteCmd.AddCommand(deleteCmd.NewMemberClusterCmd()) - DeleteCmd.AddCommand(deleteCmd.NewClusterSetCmd()) - DeleteCmd.AddCommand(deleteCmd.NewClusterClaimCmd()) - AddCmd.AddCommand(add.NewMemberClusterCmd()) DeployCmd.AddCommand(deploy.NewLeaderClusterCmd()) DeployCmd.AddCommand(deploy.NewMemberClusterCmd()) } diff --git a/pkg/antctl/raw/multicluster/common/cleanup.go b/pkg/antctl/raw/multicluster/common/cleanup.go new file mode 100644 index 00000000000..82d3eadff96 --- /dev/null +++ b/pkg/antctl/raw/multicluster/common/cleanup.go @@ -0,0 +1,54 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Common codes for both leader and member cluster to clean up the Antrea +// Multi-cluster resources in the given Namespace. + +package common + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +type CleanOptions struct { + Namespace string + ClusterSet string +} + +func (o *CleanOptions) validate() error { + if o.ClusterSet == "" { + return fmt.Errorf("ClusterSet is required") + } + + return nil +} + +func Cleanup(cmd *cobra.Command, cleanOpts *CleanOptions) error { + if err := cleanOpts.validate(); err != nil { + return err + } + k8sClient, err := NewClient(cmd) + if err != nil { + return err + } + deleteClusterSet(cmd, k8sClient, cleanOpts.Namespace, cleanOpts.ClusterSet) + deleteClusterClaims(cmd, k8sClient, cleanOpts.Namespace) + deleteSecrets(cmd, k8sClient, cleanOpts.Namespace) + deleteServiceAccounts(cmd, k8sClient, cleanOpts.Namespace) + deleteRoleBindings(cmd, k8sClient, cleanOpts.Namespace) + + return nil +} diff --git a/pkg/antctl/raw/multicluster/common/common.go b/pkg/antctl/raw/multicluster/common/common.go new file mode 100644 index 00000000000..f962af32563 --- /dev/null +++ b/pkg/antctl/raw/multicluster/common/common.go @@ -0,0 +1,425 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + multiclusterv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" + multiclusterv1alpha2 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha2" + "antrea.io/antrea/pkg/antctl/raw" + multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme" +) + +const ( + ClusterSetJoinConfigAPIVersion = "multicluster.antrea.io/v1alpha1" + ClusterSetJoinConfigKind = "ClusterSetJoinConfig" + + CreateByAntctlAnnotation = "multicluster.antrea.io/created-by-antctl" +) + +func NewClient(cmd *cobra.Command) (client.Client, error) { + kubeconfig, err := raw.ResolveKubeconfig(cmd) + if err != nil { + return nil, err + } + restConfigTmpl := rest.CopyConfig(kubeconfig) + raw.SetupKubeconfig(restConfigTmpl) + + k8sClient, err := client.New(restConfigTmpl, client.Options{Scheme: multiclusterscheme.Scheme}) + if err != nil { + return nil, err + } + + return k8sClient, nil +} + +func CreateClusterClaim(cmd *cobra.Command, k8sClient client.Client, namespace string, clusterset string, clusterID string, createdRes *[]map[string]interface{}) error { + var createErr error + var unstructuredClusterClaim map[string]interface{} + clusterClaim := newClusterClaim(clusterID, namespace, false) + unstructuredClusterClaim, _ = runtime.DefaultUnstructuredConverter.ToUnstructured(clusterClaim) + + if createErr = k8sClient.Create(context.TODO(), clusterClaim); createErr != nil { + if !apierrors.IsAlreadyExists(createErr) { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to create ClusterClaim \"%s\": %v\n", multiclusterv1alpha2.WellKnownClusterClaimID, createErr) + return createErr + + } + fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" already exists in Namespace %s\n", multiclusterv1alpha2.WellKnownClusterClaimID, namespace) + createErr = nil + } else { + fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" created in Namespace %s\n", multiclusterv1alpha2.WellKnownClusterClaimID, namespace) + *createdRes = append(*createdRes, unstructuredClusterClaim) + } + + clusterClaim = newClusterClaim(clusterset, namespace, true) + unstructuredClusterClaim, _ = runtime.DefaultUnstructuredConverter.ToUnstructured(clusterClaim) + + if createErr = k8sClient.Create(context.TODO(), clusterClaim); createErr != nil { + if !apierrors.IsAlreadyExists(createErr) { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to create ClusterClaim \"%s\": %v\n", multiclusterv1alpha2.WellKnownClusterClaimClusterSet, createErr) + return createErr + } + fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" already exists in Namespace %s\n", multiclusterv1alpha2.WellKnownClusterClaimClusterSet, namespace) + createErr = nil + } else { + fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" created in Namespace %s\n", multiclusterv1alpha2.WellKnownClusterClaimClusterSet, namespace) + *createdRes = append(*createdRes, unstructuredClusterClaim) + } + + return nil +} + +func CreateClusterSet(cmd *cobra.Command, k8sClient client.Client, namespace string, clusterset string, + leaderServer string, secret string, memberClusterID string, leaderClusterID string, leaderClusterNamespace string, createdRes *[]map[string]interface{}) error { + var unstructuredClusterSet map[string]interface{} + clusterSet := newClusterSet(clusterset, namespace, leaderServer, secret, memberClusterID, leaderClusterID, leaderClusterNamespace) + unstructuredClusterSet, _ = runtime.DefaultUnstructuredConverter.ToUnstructured(clusterSet) + + if err := k8sClient.Create(context.TODO(), clusterSet); err != nil { + if !apierrors.IsAlreadyExists(err) { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to create ClusterSet \"%s\": %v\n", clusterSet.Name, err) + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "ClusterSet \"%s\" already exists in Namespace %s\n", clusterSet.Name, clusterSet.Namespace) + } else { + fmt.Fprintf(cmd.OutOrStdout(), "ClusterSet \"%s\" created in Namespace %s\n", clusterSet.Name, clusterSet.Namespace) + *createdRes = append(*createdRes, unstructuredClusterSet) + } + + return nil +} + +func deleteClusterClaims(cmd *cobra.Command, k8sClient client.Client, namespace string) error { + var e error + clusterClaimNames := []string{multiclusterv1alpha2.WellKnownClusterClaimID, multiclusterv1alpha2.WellKnownClusterClaimClusterSet} + for _, name := range clusterClaimNames { + if err := k8sClient.Delete(context.TODO(), newClusterClaim(name, namespace, name == multiclusterv1alpha2.WellKnownClusterClaimClusterSet)); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete ClusterClaim \"%s\": %v\n", name, err) + e = err + } + fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" deleted in Namespace %s\n", name, namespace) + } + return e +} + +func deleteClusterSet(cmd *cobra.Command, k8sClient client.Client, namespace string, clusterSet string) error { + var err error + if err = k8sClient.Delete(context.TODO(), newClusterSet(clusterSet, namespace, "", "", "", "", "")); err != nil && !apierrors.IsNotFound(err) { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete ClusterSet \"%s\": %v\n", clusterSet, err) + return err + } + if err == nil { + fmt.Fprintf(cmd.OutOrStdout(), "ClusterSet \"%s\" deleted in Namespace %s\n", clusterSet, namespace) + } + return nil +} + +func deleteSecrets(cmd *cobra.Command, k8sClient client.Client, namespace string) error { + secretList := &corev1.SecretList{} + if err := k8sClient.List(context.TODO(), secretList, client.InNamespace(namespace)); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to list Secrets in Namespace %s: %v\n", namespace, err) + } + + for _, s := range secretList.Items { + secret := s + if secret.Annotations[CreateByAntctlAnnotation] != "true" { + continue + } + + if err := k8sClient.Delete(context.TODO(), &secret); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete Secret \"%s\": %v\n", secret.Name, err) + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Secret \"%s\" deleted in Namespace %s\n", secret.Name, namespace) + } + + return nil +} + +func deleteRoleBindings(cmd *cobra.Command, k8sClient client.Client, namespace string) error { + roleBindingList := &rbacv1.RoleBindingList{} + if err := k8sClient.List(context.TODO(), roleBindingList, client.InNamespace(namespace)); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to list RoleBindings in Namespace %s: %v\n", namespace, err) + } + + for _, r := range roleBindingList.Items { + roleBinding := r + if roleBinding.Annotations[CreateByAntctlAnnotation] != "true" { + continue + } + + if err := k8sClient.Delete(context.TODO(), &roleBinding); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete RoleBinding \"%s\": %v\n", roleBinding.Name, err) + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "RoleBinding \"%s\" deleted in Namespace %s\n", roleBinding.Name, namespace) + } + + return nil +} + +func deleteServiceAccounts(cmd *cobra.Command, k8sClient client.Client, namespace string) error { + serviceAccountList := &corev1.ServiceAccountList{} + if err := k8sClient.List(context.TODO(), serviceAccountList, client.InNamespace(namespace)); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to list ServiceAccounts in Namespace %s: %v\n", namespace, err) + } + + for _, sa := range serviceAccountList.Items { + serviceAccount := sa + if serviceAccount.Annotations[CreateByAntctlAnnotation] != "true" { + continue + } + + if err := k8sClient.Delete(context.TODO(), &serviceAccount); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete ServiceAccount \"%s\": %v\n", serviceAccount.Name, err) + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "ServiceAccount \"%s\" deleted in Namespace %s\n", serviceAccount.Name, namespace) + } + + return nil +} + +func CreateMemberToken(cmd *cobra.Command, k8sClient client.Client, name string, namespace string, file *os.File, createdRes *[]map[string]interface{}) error { + var createErr error + serviceAccount := newServiceAccount(name, namespace) + unstructuredSA, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(serviceAccount) + + createErr = k8sClient.Create(context.TODO(), serviceAccount) + if createErr != nil { + if !apierrors.IsAlreadyExists(createErr) { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to create ServiceAccount \"%s\", error: %s\n", name, createErr.Error()) + return createErr + } + fmt.Fprintf(cmd.OutOrStderr(), "ServiceAccount \"%s\" already exists\n", name) + createErr = nil + } else { + fmt.Fprintf(cmd.OutOrStdout(), "ServiceAccount \"%s\" created\n", serviceAccount.Name) + *createdRes = append(*createdRes, unstructuredSA) + } + + roleBinding := newRoleBinding(name, name, namespace) + unstructuredRoleBinding, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(roleBinding) + + createErr = k8sClient.Create(context.TODO(), roleBinding) + if createErr != nil { + if !apierrors.IsAlreadyExists(createErr) { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to create RoleBinding \"%s\", error: %s\n", name, createErr.Error()) + return createErr + } + fmt.Fprintf(cmd.OutOrStderr(), "RoleBinding \"%s\" already exists\n", name) + createErr = nil + } else { + fmt.Fprintf(cmd.OutOrStdout(), "RoleBinding \"%s\" created\n", roleBinding.Name) + *createdRes = append(*createdRes, unstructuredRoleBinding) + } + + secret := newSecret(name, name, namespace) + createErr = k8sClient.Create(context.TODO(), secret) + if createErr != nil { + if !apierrors.IsAlreadyExists(createErr) { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to create Secret \"%s\", start rollback\n", name) + return createErr + } + fmt.Fprintf(cmd.OutOrStderr(), "Secret \"%s\" already exists\n", name) + } + // It will take one or two seconds to wait for the Data.token to be created. + if err := waitForSecretReady(k8sClient, name, namespace); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Secret \"%s\" created\n", secret.Name) + + if file == nil { + return nil + } + + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, secret); err != nil { + return err + } + s := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Data: secret.Data, + Type: corev1.SecretTypeOpaque, + } + + b, err := yaml.Marshal(s) + if err != nil { + return err + } + if _, err := file.Write([]byte("# Manifest to create a Secret for a member cluster token.\n")); err != nil { + return err + } + if _, err := file.Write([]byte("---\n")); err != nil { + return err + } + if _, err := file.Write(b); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Member token saved to %s\n", file.Name()) + + return nil +} + +func waitForSecretReady(client client.Client, secretName string, namespace string) error { + return wait.PollImmediate( + 1*time.Second, + 5*time.Second, + func() (bool, error) { + secret := &corev1.Secret{} + if err := client.Get(context.TODO(), types.NamespacedName{Name: secretName, Namespace: namespace}, secret); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return string(secret.Data["token"]) != "", nil + }) +} + +func newClusterClaim(name string, namespace string, clusterSet bool) *multiclusterv1alpha2.ClusterClaim { + clusterClaim := &multiclusterv1alpha2.ClusterClaim{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "multicluster.crd.antrea.io/v1alpha2", + Kind: "ClusterClaim", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: multiclusterv1alpha2.WellKnownClusterClaimID, + }, + Value: name, + } + + if clusterSet { + clusterClaim.Name = multiclusterv1alpha2.WellKnownClusterClaimClusterSet + } + + return clusterClaim +} + +func newClusterSet(name, namespace, leaderServer, secret, memberClusterID, leaderClusterID, leaderNamespace string) *multiclusterv1alpha1.ClusterSet { + clusterSet := &multiclusterv1alpha1.ClusterSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "multicluster.crd.antrea.io/v1alpha1", + Kind: "ClusterSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: multiclusterv1alpha1.ClusterSetSpec{ + Leaders: []multiclusterv1alpha1.MemberCluster{ + { + ClusterID: leaderClusterID, + }, + }, + Namespace: namespace, + }, + } + if leaderServer != "" { + clusterSet.Spec.Namespace = leaderNamespace + clusterSet.Spec.Leaders[0].Secret = secret + clusterSet.Spec.Leaders[0].Server = leaderServer + clusterSet.Spec.Members = append(clusterSet.Spec.Members, multiclusterv1alpha1.MemberCluster{ + ClusterID: memberClusterID, + }) + } + + return clusterSet +} + +func newRoleBinding(name string, saName string, namespace string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + CreateByAntctlAnnotation: "true", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "antrea-mc-member-cluster-role", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: saName, + Namespace: namespace, + }, + }, + } +} + +func newSecret(name string, saName string, namespace string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "kubernetes.io/service-account.name": saName, + CreateByAntctlAnnotation: "true", + }, + }, + Type: "kubernetes.io/service-account-token", + } +} + +func newServiceAccount(name string, namespace string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + CreateByAntctlAnnotation: "true", + }, + }, + } +} diff --git a/pkg/antctl/raw/multicluster/common/rollback.go b/pkg/antctl/raw/multicluster/common/rollback.go new file mode 100644 index 00000000000..e32d32823ac --- /dev/null +++ b/pkg/antctl/raw/multicluster/common/rollback.go @@ -0,0 +1,37 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func Rollback(cmd *cobra.Command, k8sClient client.Client, res []map[string]interface{}) error { + for _, obj := range res { + u := &unstructured.Unstructured{Object: obj} + if err := k8sClient.Delete(context.TODO(), u); err != nil && !apierrors.IsNotFound(err) { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete %s %s/%s: %v\n", u.GetKind(), u.GetNamespace(), u.GetName(), err) + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s \"%s/%s\" deleted\n", u.GetKind(), u.GetNamespace(), u.GetName()) + } + return nil +} diff --git a/pkg/antctl/raw/multicluster/create/access_token.go b/pkg/antctl/raw/multicluster/create/access_token.go index c38c540f3c0..8a909072df3 100644 --- a/pkg/antctl/raw/multicluster/create/access_token.go +++ b/pkg/antctl/raw/multicluster/create/access_token.go @@ -15,186 +15,89 @@ package create import ( - "context" "fmt" + "os" "strings" "github.com/spf13/cobra" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - - "antrea.io/antrea/pkg/antctl/raw" - multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme" + + "antrea.io/antrea/pkg/antctl/raw/multicluster/common" ) -type accessTokenOptions struct { - namespace string - serviceAccount string - roleBinding string +type memberTokenOptions struct { + namespace string + output string } -var accessTokenOpts *accessTokenOptions +var memberTokenOpts *memberTokenOptions -var accessTokenExamples = strings.Trim(` -# Create an access token in a leader cluster for one or more member clusters. If the ServiceAccount or RoleBinding does not exist, the command will create them too - $ antctl mc create accesstoken -n --service-account --role-binding +var memberTokenExamples = strings.Trim(` +# Create a member token. + $ antctl mc create membertoken cluster-east-token -n antrea-multicluster +# Create a member token and save the Secret manifest to a file. + $ antctl mc create membertoken cluster-east-token -n antrea-multicluster -o token-secret.yml `, "\n") -func (o *accessTokenOptions) validateAndComplete() error { +func (o *memberTokenOptions) validateAndComplete() error { if o.namespace == "" { - return fmt.Errorf("the Namespace cannot be empty") - } - if o.serviceAccount == "" { - return fmt.Errorf("the ServiceAccount cannot be empty") - } - if o.roleBinding == "" { - return fmt.Errorf("the RoleBinding cannot be empty") + return fmt.Errorf("the Namespace is required") } return nil } func NewAccessTokenCmd() *cobra.Command { command := &cobra.Command{ - Use: "accesstoken", + Use: "membertoken", Args: cobra.MaximumNArgs(1), - Short: "Create an accesstoken in a leader cluster", - Long: "Create an accesstoken in a leader cluster", - Example: accessTokenExamples, - RunE: accessTokenRunE, + Short: "Create a member token in a leader cluster", + Long: "Create a member token in a leader cluster, which will be saved in a Secret. A ServiceAccount and a RoleBinding will be created too if they do not exist.", + Example: memberTokenExamples, + RunE: memberTokenRunE, } - o := &accessTokenOptions{} - accessTokenOpts = o - command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of the ClusterClaim") - command.Flags().StringVarP(&o.serviceAccount, "service-account", "", "", "ServiceAccount of the access token") - command.Flags().StringVarP(&o.roleBinding, "role-binding", "", "", "RoleBinding of the ServiceAccount") + o := &memberTokenOptions{} + memberTokenOpts = o + command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of the ClusterSet") + command.Flags().StringVarP(&o.output, "output-file", "o", "", "Output file to save the token Secret manifest") return command } -func accessTokenRunE(cmd *cobra.Command, args []string) error { - if err := accessTokenOpts.validateAndComplete(); err != nil { +func memberTokenRunE(cmd *cobra.Command, args []string) error { + if err := memberTokenOpts.validateAndComplete(); err != nil { return err } if len(args) != 1 { return fmt.Errorf("exactly one NAME is required, got %d", len(args)) } - kubeconfig, err := raw.ResolveKubeconfig(cmd) - if err != nil { - return err - } - restconfigTmpl := rest.CopyConfig(kubeconfig) - raw.SetupKubeconfig(restconfigTmpl) - k8sClient, err := client.New(kubeconfig, client.Options{Scheme: multiclusterscheme.Scheme}) + k8sClient, err := common.NewClient(cmd) if err != nil { return err } - serviceAccount := corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: accessTokenOpts.serviceAccount, - Namespace: accessTokenOpts.namespace, - }, - } - var createErr error - fmt.Fprintf(cmd.OutOrStdout(), "Creating ServiceAccount \"%s\"\n", accessTokenOpts.serviceAccount) - createErr = k8sClient.Create(context.TODO(), &serviceAccount) - if createErr != nil { - if errors.IsAlreadyExists(createErr) { - fmt.Fprintf(cmd.OutOrStderr(), "ServiceAccount \"%s\" already exists\n", accessTokenOpts.serviceAccount) - createErr = nil - } else { - fmt.Fprintf(cmd.OutOrStdout(), "Failed to create ServiceAccount \"%s\", error: %s\n", accessTokenOpts.serviceAccount, createErr.Error()) - return createErr - } - } else { - fmt.Fprintf(cmd.OutOrStdout(), "ServiceAccount \"%s\" created\n", serviceAccount.Name) - defer func() { - if createErr != nil { - err := k8sClient.Delete(context.TODO(), &serviceAccount) - if err != nil { - fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete ServiceAccount \"%s\", error: %s\n", serviceAccount.Name, err.Error()) - } else { - fmt.Fprintf(cmd.OutOrStdout(), "ServiceAccount \"%s\" deleted\n", serviceAccount.Name) - } + createdRes := []map[string]interface{}{} + defer func() { + if createErr != nil { + if err := common.Rollback(cmd, k8sClient, createdRes); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to rollback: %v\n", err) } - }() - } - - fmt.Fprintf(cmd.OutOrStdout(), "Creating RoleBinding \"%s\"\n", accessTokenOpts.roleBinding) - roleBinding := rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: accessTokenOpts.roleBinding, - Namespace: accessTokenOpts.namespace, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: "antrea-mc-member-cluster-role", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: accessTokenOpts.serviceAccount, - Namespace: accessTokenOpts.namespace, - }, - }, - } + } + }() - createErr = k8sClient.Create(context.TODO(), &roleBinding) - if createErr != nil { - if errors.IsAlreadyExists(createErr) { - fmt.Fprintf(cmd.OutOrStderr(), "RoleBinding \"%s\" already exists\n", accessTokenOpts.roleBinding) - createErr = nil - } else { - fmt.Fprintf(cmd.OutOrStdout(), "Failed to create RoleBingding \"%s\", error: %s\n", accessTokenOpts.roleBinding, createErr.Error()) - return createErr + var file *os.File + if memberTokenOpts.output != "" { + if file, err = os.OpenFile(memberTokenOpts.output, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600); err != nil { + return err } - } else { - fmt.Fprintf(cmd.OutOrStdout(), "RoleBinding \"%s\" created\n", roleBinding.Name) - defer func() { - if createErr != nil { - err := k8sClient.Delete(context.TODO(), &roleBinding) - if err != nil { - fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete RoleBinding \"%s\", error: %s\n", roleBinding.Name, err.Error()) - } else { - fmt.Fprintf(cmd.OutOrStdout(), "RoleBinding \"%s\" deleted\n", roleBinding.Name) - } - } - }() } + defer file.Close() - secretName := args[0] - - fmt.Fprintf(cmd.OutOrStdout(), "Creating Secret \"%s\"\n", secretName) - secret := corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName + "A", - Namespace: accessTokenOpts.namespace, - Annotations: map[string]string{ - "kubernetes.io/service-account.name": accessTokenOpts.serviceAccount, - }, - }, - Type: "kubernetes.io/service-account-token", + if createErr = common.CreateMemberToken(cmd, k8sClient, args[0], memberTokenOpts.namespace, file, &createdRes); createErr != nil { + return createErr } - createErr = k8sClient.Create(context.TODO(), &secret) - if createErr != nil { - if errors.IsAlreadyExists(createErr) { - fmt.Fprintf(cmd.OutOrStderr(), "Secret \"%s\" already exists\n", secretName) - createErr = nil - } else { - fmt.Fprintf(cmd.OutOrStdout(), "Failed to create Secret \"%s\", start rollback\n", secretName) - return createErr - } - } else { - fmt.Fprintf(cmd.OutOrStdout(), "Secret \"%s\" with access token created\n", secretName) - } + fmt.Fprintf(cmd.OutOrStdout(), "You can now run the \"antctl mc join\" command with the token to have the cluster join the ClusterSet\n") return nil } diff --git a/pkg/antctl/raw/multicluster/create/clusterclaim.go b/pkg/antctl/raw/multicluster/create/clusterclaim.go deleted file mode 100644 index 8af72305114..00000000000 --- a/pkg/antctl/raw/multicluster/create/clusterclaim.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2022 Antrea Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package create - -import ( - "context" - "fmt" - "strings" - - "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - - multiclusterv1alpha2 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha2" - "antrea.io/antrea/pkg/antctl/raw" - multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme" -) - -type clusterClaimOptions struct { - namespace string - clusterSetID string - clusterID string -} - -var clusterClaimOpt *clusterClaimOptions - -var clusterClaimExamples = strings.Trim(` -# Create two ClusterClaims, one for the leader or member cluster and another for the ClusterSet - $ antctl mc create clusterclaims --cluster-id --clusterset -n -`, "\n") - -func (o *clusterClaimOptions) validateAndComplete() error { - if o.namespace == "" { - return fmt.Errorf("the Namespace cannot be empty") - } - if o.clusterSetID == "" { - return fmt.Errorf("the ClusterSet ID cannot be empty") - } - if o.clusterID == "" { - return fmt.Errorf("the cluster ID cannot be empty") - } - - return nil -} - -func NewClusterClaimCmd() *cobra.Command { - command := &cobra.Command{ - Use: "clusterclaims", - Args: cobra.MaximumNArgs(0), - Short: "Create two ClusterClaims in a leader or member cluster", - Long: "Create two ClusterClaims in a leader or member cluster. One for ClusterSet and another for the leader or member cluster", - Example: clusterClaimExamples, - RunE: clusterClaimRunE, - } - - o := &clusterClaimOptions{} - clusterClaimOpt = o - command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of the ClusterClaim") - command.Flags().StringVarP(&o.clusterSetID, "clusterset-id", "", "", "ClusterSet ID of the ClusterClaim for the ClusterSet") - command.Flags().StringVarP(&o.clusterID, "cluster-id", "", "", "cluster ID of the ClusterClaim for the leader or member cluster") - - return command -} - -func clusterClaimRunE(cmd *cobra.Command, args []string) error { - if err := clusterClaimOpt.validateAndComplete(); err != nil { - return err - } - - kubeconfig, err := raw.ResolveKubeconfig(cmd) - if err != nil { - return err - } - restconfigTmpl := rest.CopyConfig(kubeconfig) - raw.SetupKubeconfig(restconfigTmpl) - - k8sClient, err := client.New(kubeconfig, client.Options{Scheme: multiclusterscheme.Scheme}) - if err != nil { - return err - } - - clusterClaim := &multiclusterv1alpha2.ClusterClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: multiclusterv1alpha2.WellKnownClusterClaimID, - Namespace: clusterClaimOpt.namespace, - }, - Value: clusterClaimOpt.clusterID, - } - - var createErr error - createErr = k8sClient.Create(context.TODO(), clusterClaim) - if createErr != nil { - if errors.IsAlreadyExists(createErr) { - fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" already exists\n", multiclusterv1alpha2.WellKnownClusterClaimID) - createErr = nil - } else { - fmt.Fprintf(cmd.OutOrStdout(), "Failed to create ClusterClaim \"%s\", error: %s\n", multiclusterv1alpha2.WellKnownClusterClaimID, createErr.Error()) - return createErr - } - } else { - fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" with Value \"%s\" created\n", multiclusterv1alpha2.WellKnownClusterClaimID, clusterClaimOpt.clusterID) - defer func() { - if createErr != nil { - err := k8sClient.Delete(context.TODO(), clusterClaim) - if err != nil { - fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete ClusterClaim \"%s\", error: \n", err.Error()) - } else { - fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" deleted\n", multiclusterv1alpha2.WellKnownClusterClaimID) - } - } - }() - } - - clustersetClaim := &multiclusterv1alpha2.ClusterClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: multiclusterv1alpha2.WellKnownClusterClaimClusterSet, - Namespace: clusterClaimOpt.namespace, - }, - Value: clusterClaimOpt.clusterSetID, - } - - createErr = k8sClient.Create(context.TODO(), clustersetClaim) - if createErr != nil { - if errors.IsAlreadyExists(createErr) { - fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" already exists\n", multiclusterv1alpha2.WellKnownClusterClaimClusterSet) - createErr = nil - } else { - fmt.Fprintf(cmd.OutOrStdout(), "Failed to create ClusterClaim \"%s\", start rollback\n", multiclusterv1alpha2.WellKnownClusterClaimClusterSet) - return createErr - } - } else { - fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" with Value \"%s\" created\n", multiclusterv1alpha2.WellKnownClusterClaimClusterSet, clusterClaimOpt.clusterSetID) - } - - return nil -} diff --git a/pkg/antctl/raw/multicluster/create/clusterset.go b/pkg/antctl/raw/multicluster/create/clusterset.go deleted file mode 100644 index a50969a0632..00000000000 --- a/pkg/antctl/raw/multicluster/create/clusterset.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2022 Antrea Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package create - -import ( - "context" - "fmt" - "strings" - - "github.com/spf13/cobra" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - - multiclusterv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" - "antrea.io/antrea/pkg/antctl/raw" - multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme" -) - -type clusterSetOptions struct { - leaderCluster string - leaderClusterServer string - leaderClusterNamespace string - memberClusters map[string]string - namespace string - secret string -} - -var clusterSetOpt *clusterSetOptions - -var clusterSetExamples = strings.Trim(` -# Create a ClusterSet in a leader cluster - $ antctl mc create clusterset -n --service-account --leader-cluster -# Create a ClusterSet in a member cluster - $ antctl mc create clusterset -n --leader-apiserver --secret --leader-cluster -`, "\n") - -func (o *clusterSetOptions) validateAndComplete() error { - if o.namespace == "" { - return fmt.Errorf("the Namespace cannot be empty") - } - if o.leaderClusterNamespace == "" { - o.leaderClusterNamespace = metav1.NamespaceDefault - } - if o.leaderCluster == "" { - return fmt.Errorf("the leader-cluster-id cannot be empty") - } - if o.secret == "" && o.memberClusters == nil { - return fmt.Errorf("the ServiceAccounts list is required in leader cluster, the secret is required in member cluster") - } - if o.secret != "" && o.leaderClusterServer == "" { - return fmt.Errorf("the leader cluster apiserver is required in member cluster") - } - - return nil -} - -func NewClusterSetCmd() *cobra.Command { - command := &cobra.Command{ - Use: "clusterset", - Args: cobra.MaximumNArgs(1), - Short: "Create a ClusterSet in a leader or member cluster", - Long: "Create a ClusterSet in a leader or member cluster", - Example: clusterSetExamples, - RunE: clusterSetRunE, - } - - o := &clusterSetOptions{} - clusterSetOpt = o - command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of the ClusterSet") - command.Flags().StringVarP(&o.leaderCluster, "leader-cluster", "", "", "leader cluster ID of the ClusterSet") - command.Flags().StringVarP(&o.leaderClusterServer, "leader-apiserver", "", "", "leader cluster apiserver address of the ClusterSet. It is required only for a member cluster") - command.Flags().StringVarP(&o.leaderClusterNamespace, "leader-namespace", "", "", "the Namespace where Antrea Multi-cluster Controller is running in leader cluster") - command.Flags().StringToStringVarP(&o.memberClusters, "member-clusters", "", nil, "a map from cluster ID to ServiceAccount of the member clusters(e.g. --member-clusters member1=sa1,member2=sa2). It is required only for a leader cluster") - command.Flags().StringVarP(&o.secret, "secret", "", "", "Secret to access the leader cluster. It is required only when creating ClusterSet in a member cluster") - - return command -} - -func clusterSetRunE(cmd *cobra.Command, args []string) error { - if err := clusterSetOpt.validateAndComplete(); err != nil { - return err - } - if len(args) != 1 { - return fmt.Errorf("exactly one NAME is required, got %d", len(args)) - } - - kubeconfig, err := raw.ResolveKubeconfig(cmd) - if err != nil { - return err - } - restconfigTmpl := rest.CopyConfig(kubeconfig) - raw.SetupKubeconfig(restconfigTmpl) - - k8sClient, err := client.New(kubeconfig, client.Options{Scheme: multiclusterscheme.Scheme}) - if err != nil { - return err - } - clusterSetName := args[0] - clusterSet := &multiclusterv1alpha1.ClusterSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: clusterSetName, - Namespace: clusterSetOpt.namespace, - }, - Spec: multiclusterv1alpha1.ClusterSetSpec{ - Leaders: []multiclusterv1alpha1.MemberCluster{ - { - ClusterID: clusterSetOpt.leaderCluster, - Secret: clusterSetOpt.secret, - Server: fmt.Sprintf("https://%s", strings.Replace(clusterSetOpt.leaderClusterServer, "https://", "", 1)), - }, - }, - Namespace: clusterSetOpt.leaderClusterNamespace, - }, - } - - for memberCluster, serviceAccount := range clusterSetOpt.memberClusters { - clusterSet.Spec.Members = append(clusterSet.Spec.Members, multiclusterv1alpha1.MemberCluster{ - ClusterID: memberCluster, - ServiceAccount: serviceAccount, - }) - } - - if err := k8sClient.Create(context.TODO(), clusterSet); err != nil { - return err - } - fmt.Fprintf(cmd.OutOrStdout(), "ClusterSet \"%s\" created\n", clusterSetName) - - return nil -} diff --git a/pkg/antctl/raw/multicluster/delete/clusterclaim.go b/pkg/antctl/raw/multicluster/delete/clusterclaim.go deleted file mode 100644 index 174575a4be1..00000000000 --- a/pkg/antctl/raw/multicluster/delete/clusterclaim.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2022 Antrea Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package delete - -import ( - "context" - "fmt" - "strings" - - "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - - multiclusterv1alpha2 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha2" - "antrea.io/antrea/pkg/antctl/raw" - multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme" -) - -type clusterClaimOptions struct { - namespace string -} - -var clusterClaimOpt *clusterClaimOptions - -var clusterClaimExamples = strings.Trim(` -# Delete the two ClusterClaims in a specified Namespace. One for the leader or member cluster, and another for the ClusterSet - $ antctl mc delete clusterclaims -n -`, "\n") - -func (o *clusterClaimOptions) validateAndComplete() error { - if o.namespace == "" { - return fmt.Errorf("the Namespace cannot be empty") - } - - return nil -} - -func NewClusterClaimCmd() *cobra.Command { - command := &cobra.Command{ - Use: "clusterclaims", - Args: cobra.MaximumNArgs(0), - Short: "Delete the ClusterClaims in a specified Namespace", - Long: "Delete the ClusterClaims in a specified Namespace. One for the leader or member cluster, and another for the ClusterSet", - Example: clusterClaimExamples, - RunE: clusterClaimRunE, - } - - o := &clusterClaimOptions{} - clusterClaimOpt = o - command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of ClusterClaim") - - return command -} - -func clusterClaimRunE(cmd *cobra.Command, args []string) error { - if err := clusterClaimOpt.validateAndComplete(); err != nil { - return err - } - - kubeconfig, err := raw.ResolveKubeconfig(cmd) - if err != nil { - return err - } - restconfigTmpl := rest.CopyConfig(kubeconfig) - raw.SetupKubeconfig(restconfigTmpl) - - k8sClient, err := client.New(kubeconfig, client.Options{Scheme: multiclusterscheme.Scheme}) - if err != nil { - return err - } - - clusterClaims := []*multiclusterv1alpha2.ClusterClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: multiclusterv1alpha2.WellKnownClusterClaimID, - Namespace: clusterClaimOpt.namespace, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: multiclusterv1alpha2.WellKnownClusterClaimClusterSet, - Namespace: clusterClaimOpt.namespace, - }, - }, - } - - for _, clusterClaim := range clusterClaims { - if err := deleteClusterClaim(cmd, k8sClient, clusterClaim); err != nil { - return err - } - } - - return nil -} - -func deleteClusterClaim(cmd *cobra.Command, k8sClient client.Client, clusterClaim *multiclusterv1alpha2.ClusterClaim) error { - err := k8sClient.Delete(context.TODO(), clusterClaim) - if err != nil { - if !errors.IsNotFound(err) { - fmt.Fprintf(cmd.OutOrStdout(), "Failed to delete ClusterClaim \"%s\", error: %s\n", clusterClaim.ObjectMeta.Name, err.Error()) - return err - } else { - fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" not found\n", clusterClaim.ObjectMeta.Name) - } - } else { - fmt.Fprintf(cmd.OutOrStdout(), "ClusterClaim \"%s\" deleted\n", clusterClaim.ObjectMeta.Name) - } - - return nil -} diff --git a/pkg/antctl/raw/multicluster/delete/clusterset.go b/pkg/antctl/raw/multicluster/delete/clusterset.go deleted file mode 100644 index 35830033b59..00000000000 --- a/pkg/antctl/raw/multicluster/delete/clusterset.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2022 Antrea Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package delete - -import ( - "context" - "fmt" - "strings" - - "github.com/spf13/cobra" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - - multiclusterv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" - "antrea.io/antrea/pkg/antctl/raw" - multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme" -) - -type clusterSetOptions struct { - namespace string -} - -var clusterSetOpt *clusterSetOptions - -var clusterSetExamples = strings.Trim(` -# Delete a ClusterSet in a specified Namespace in a leader or member cluster - $ antctl mc delete clusterset -n -`, "\n") - -func (o *clusterSetOptions) validateAndComplete() error { - if o.namespace == "" { - return fmt.Errorf("the Namespace cannot be empty") - } - - return nil -} - -func NewClusterSetCmd() *cobra.Command { - command := &cobra.Command{ - Use: "clusterset", - Args: cobra.MaximumNArgs(1), - Short: "Delete a ClusterSet in a leader or member cluster", - Long: "Delete a ClusterSet in a leader or member cluster", - Example: clusterSetExamples, - RunE: clusterSetRunE, - } - - o := &clusterSetOptions{} - clusterSetOpt = o - command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of ClusterSet") - - return command -} - -func clusterSetRunE(cmd *cobra.Command, args []string) error { - if err := clusterSetOpt.validateAndComplete(); err != nil { - return err - } - if len(args) != 1 { - return fmt.Errorf("exactly one NAME is required, got %d", len(args)) - } - - kubeconfig, err := raw.ResolveKubeconfig(cmd) - if err != nil { - return err - } - restconfigTmpl := rest.CopyConfig(kubeconfig) - raw.SetupKubeconfig(restconfigTmpl) - - k8sClient, err := client.New(kubeconfig, client.Options{Scheme: multiclusterscheme.Scheme}) - if err != nil { - return err - } - - clusterSetName := args[0] - clusterSet := &multiclusterv1alpha1.ClusterSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: clusterSetName, - Namespace: clusterSetOpt.namespace, - }, - } - if err := k8sClient.Delete(context.TODO(), clusterSet); err != nil { - return err - } - - fmt.Fprintf(cmd.OutOrStdout(), "ClusterSet \"%s\" deleted\n", clusterSetName) - return nil -} diff --git a/pkg/antctl/raw/multicluster/delete/member_cluster.go b/pkg/antctl/raw/multicluster/delete/member_cluster.go deleted file mode 100644 index 565d490e52f..00000000000 --- a/pkg/antctl/raw/multicluster/delete/member_cluster.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2022 Antrea Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package delete - -import ( - "context" - "fmt" - "strings" - - "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - - multiclusterv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" - "antrea.io/antrea/pkg/antctl/raw" - multiclusterscheme "antrea.io/antrea/pkg/antctl/raw/multicluster/scheme" -) - -type memberClusterOptions struct { - namespace string - clusterSet string -} - -var memberClusterOpts *memberClusterOptions - -var memberClusterExamples = strings.Trim(` -# Delete a member cluster in a ClusterSet - $ antctl mc delete membercluster -n --clusterset -`, "\n") - -func (o *memberClusterOptions) validateAndComplete() error { - if o.namespace == "" { - return fmt.Errorf("the Namespace cannot be empty") - } - if o.clusterSet == "" { - return fmt.Errorf("the ClusterSet cannot be empty") - } - - return nil -} - -func NewMemberClusterCmd() *cobra.Command { - command := &cobra.Command{ - Use: "membercluster", - Args: cobra.MaximumNArgs(1), - Short: "Delete a member cluster in a ClusterSet", - Long: "Delete a member cluster in a ClusterSet", - Example: memberClusterExamples, - RunE: memberClusterRunE, - } - - o := &memberClusterOptions{} - memberClusterOpts = o - command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of member Cluster") - command.Flags().StringVarP(&o.clusterSet, "clusterset", "", "", "ClusterSet ID of the member Cluster") - - return command -} - -func memberClusterRunE(cmd *cobra.Command, args []string) error { - if err := memberClusterOpts.validateAndComplete(); err != nil { - return err - } - if len(args) != 1 { - return fmt.Errorf("exactly one ClusterID is required, got %d", len(args)) - } - - kubeconfig, err := raw.ResolveKubeconfig(cmd) - if err != nil { - return err - } - restconfigTmpl := rest.CopyConfig(kubeconfig) - raw.SetupKubeconfig(restconfigTmpl) - - k8sClient, err := client.New(kubeconfig, client.Options{Scheme: multiclusterscheme.Scheme}) - if err != nil { - return err - } - memberClusterID := args[0] - clusterSet := &multiclusterv1alpha1.ClusterSet{} - if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: memberClusterOpts.clusterSet, Namespace: memberClusterOpts.namespace}, clusterSet); err != nil { - return err - } - - var memberClusters []multiclusterv1alpha1.MemberCluster - for _, m := range clusterSet.Spec.Members { - if m.ClusterID != memberClusterID { - memberClusters = append(memberClusters, m) - } - } - if len(memberClusters) == len(clusterSet.Spec.Members) { - return fmt.Errorf(`member cluster "%s" not found in ClusterSet "%s"`, memberClusterID, memberClusterOpts.clusterSet) - } - clusterSet.Spec.Members = memberClusters - if err := k8sClient.Update(context.TODO(), clusterSet); err != nil { - return err - } - - fmt.Fprintf(cmd.OutOrStdout(), "Member cluster \"%s\" deleted\n", memberClusterID) - return nil -} diff --git a/pkg/antctl/raw/multicluster/destroy.go b/pkg/antctl/raw/multicluster/destroy.go new file mode 100644 index 00000000000..b308da680ab --- /dev/null +++ b/pkg/antctl/raw/multicluster/destroy.go @@ -0,0 +1,51 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multicluster + +import ( + "strings" + + "github.com/spf13/cobra" + + "antrea.io/antrea/pkg/antctl/raw/multicluster/common" +) + +var destroyOpts *common.CleanOptions + +var destroyExamples = strings.Trim(` +# Destroy ClusterSet in the leader cluster. + antctl mc destroy --namespace antrea-multicluster --clusterset clusterset1 +`, "\n") + +func NewDestroyCommand() *cobra.Command { + command := &cobra.Command{ + Use: "destroy", + Short: "Destroy ClusterSet in the given Namespace of the leader cluster", + Args: cobra.MaximumNArgs(0), + Example: destroyExamples, + RunE: destroyRunE, + } + + o := common.CleanOptions{} + destroyOpts = &o + command.Flags().StringVarP(&o.Namespace, "namespace", "n", "", "Namespace of the ClusterSet") + command.Flags().StringVarP(&o.ClusterSet, "clusterset", "", "", "ClusterSet ID") + + return command +} + +func destroyRunE(cmd *cobra.Command, args []string) error { + return common.Cleanup(cmd, destroyOpts) +} diff --git a/pkg/antctl/raw/multicluster/init.go b/pkg/antctl/raw/multicluster/init.go new file mode 100644 index 00000000000..9c15a59f687 --- /dev/null +++ b/pkg/antctl/raw/multicluster/init.go @@ -0,0 +1,179 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multicluster + +import ( + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + + "antrea.io/antrea/pkg/antctl/raw" + "antrea.io/antrea/pkg/antctl/raw/multicluster/common" +) + +const ( + defaultToken = "default-member-token" + + optionalFields = `#clusterID: "" +#namespace: "" +# Use the pre-created token Secret. +#tokenSecretName: "" +# Create a token Secret with the manifest file. +#toeknSecretFile: "" +` +) + +type initOptions struct { + namespace string + clusterSet string + clusterID string + createToken bool + output string +} + +var initOpts *initOptions + +func (o *initOptions) validate() error { + if o.namespace == "" { + return fmt.Errorf("Namespace is required") + } + if o.clusterSet == "" { + return fmt.Errorf("ClusterSet is required") + } + if o.clusterID == "" { + return fmt.Errorf("ClusterID is required") + } + return nil +} + +var initExample = strings.Trim(` +# Initialize ClusterSet in the given Namespace of the leader cluster. + $ antctl mc init --namespace antrea-multicluster --clusterset clusterset1 --clusterid cluster-north +# Initialize ClusterSet of the leader cluster and save the member cluster join config to a file. + $ antctl mc init --namespace antrea-multicluster --clusterset clusterset1 --clusterid cluster-north -o join-config.yml +# Initialize ClusterSet with a default member token, and save the join config as well as the token Secret to a file. + $ antctl mc init --namespace antrea-multicluster --clusterset clusterset1 --clusterid cluster-north --create-token -o join-config.yml +`, "\n") + +func NewInitCommand() *cobra.Command { + command := &cobra.Command{ + Use: "init", + Short: "Initialize ClusterSet in the given Namespace of the leader cluster", + Args: cobra.MaximumNArgs(0), + Example: initExample, + RunE: initRunE, + } + + o := initOptions{} + initOpts = &o + command.Flags().StringVarP(&o.namespace, "namespace", "n", "", "Namespace of the ClusterSet") + command.Flags().StringVarP(&o.clusterSet, "clusterset", "", "", "ClusterSet ID of the leader cluster") + command.Flags().StringVarP(&o.clusterID, "clusterid", "", "", "ClusterID of the leader cluster") + command.Flags().BoolVarP(&o.createToken, "create-token", "", false, "If specified, a default member token will be created") + command.Flags().StringVarP(&o.output, "output-file", "o", "", "Output file to save the member cluster join config") + + return command +} + +func initRunE(cmd *cobra.Command, args []string) error { + if err := initOpts.validate(); err != nil { + return err + } + k8sClient, err := common.NewClient(cmd) + if err != nil { + return err + } + createdRes := []map[string]interface{}{} + var createErr error + defer func() { + if createErr != nil { + fmt.Fprintf(cmd.OutOrStderr(), "Failed to init the Antrea Multi-cluster. Deleting the created resources\n") + if err := common.Rollback(cmd, k8sClient, createdRes); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to rollback: %v\n", err) + } + } + }() + createErr = common.CreateClusterClaim(cmd, k8sClient, initOpts.namespace, initOpts.clusterSet, initOpts.clusterID, &createdRes) + if createErr != nil { + return createErr + } + createErr = common.CreateClusterSet(cmd, k8sClient, initOpts.namespace, initOpts.clusterSet, "", "", "", initOpts.clusterID, initOpts.namespace, &createdRes) + if createErr != nil { + return createErr + } + + var file *os.File + if initOpts.output != "" { + if file, err = os.OpenFile(initOpts.output, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0644); err != nil { + fmt.Fprintf(cmd.OutOrStderr(), "Failed to open file %s: %v\n", initOpts.output, err) + } + } + defer file.Close() + + if err := outputConfig(cmd, file); err != nil { + return err + } + if initOpts.createToken { + if createErr = common.CreateMemberToken(cmd, k8sClient, defaultToken, initOpts.namespace, file, &createdRes); createErr != nil { + fmt.Fprintf(cmd.OutOrStderr(), "Failed to create Secret: %v\n", createErr) + return createErr + } + } + + fmt.Fprintf(cmd.OutOrStdout(), "Successfully initialized ClusterSet %s\n", initOpts.clusterSet) + + return nil +} + +func outputConfig(cmd *cobra.Command, file *os.File) error { + if file == nil { + return nil + } + kubeconfig, err := raw.ResolveKubeconfig(cmd) + if err != nil { + return err + } + + config := &ClusterSetJoinConfig{ + APIVersion: common.ClusterSetJoinConfigAPIVersion, + Kind: common.ClusterSetJoinConfigKind, + Namespace: "", + ClusterID: "", + LeaderClusterID: initOpts.clusterID, + LeaderAPIServer: kubeconfig.Host, + LeaderNamespace: initOpts.namespace, + ClusterSetID: initOpts.clusterSet, + } + + b, err := yaml.Marshal(config) + if err != nil { + return err + } + if _, err := file.Write([]byte("---\n")); err != nil { + return err + } + if _, err := file.Write(b); err != nil { + return err + } + if _, err := file.Write([]byte(optionalFields)); err != nil { + return err + } + + fmt.Fprintf(cmd.OutOrStdout(), "Successfully output the join config to %s\n", initOpts.output) + return nil +} diff --git a/pkg/antctl/raw/multicluster/join.go b/pkg/antctl/raw/multicluster/join.go new file mode 100644 index 00000000000..870347101b2 --- /dev/null +++ b/pkg/antctl/raw/multicluster/join.go @@ -0,0 +1,315 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multicluster + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "strings" + "time" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + + multiclusterv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" + "antrea.io/antrea/pkg/antctl/raw/multicluster/common" +) + +const ( + defaultMemberNamespace = "kube-system" +) + +type ClusterSetJoinConfig struct { + Kind string `yaml:"kind"` + APIVersion string `yaml:"apiVersion"` + ClusterSetID string `yaml:"clusterSetID"` + ClusterID string `yaml:"clusterID,omitempty"` + Namespace string `yaml:"namespace,omitempty"` + LeaderClusterID string `yaml:"leaderClusterID"` + LeaderNamespace string `yaml:"leaderNamespace"` + LeaderAPIServer string `yaml:"leaderAPIServer"` + TokenSecretName string `yaml:"tokenSecretName,omitempty"` + TokenSecretFile string `yaml:"tokenSecretFile,omitempty"` + ConfigFile string `yaml:"-"` + Secret *v1.Secret `yaml:"-"` +} + +var joinOpts *ClusterSetJoinConfig + +func (o *ClusterSetJoinConfig) validateAndComplete() error { + if o.ConfigFile != "" { + raw, err := ioutil.ReadFile(o.ConfigFile) + if err != nil { + return err + } + if err := yamlUnmarshall(raw, o); err != nil { + return err + } + if o.Kind != common.ClusterSetJoinConfigKind || o.APIVersion != common.ClusterSetJoinConfigAPIVersion { + return fmt.Errorf("unknown apiVersion or kind in config: %s", o.ConfigFile) + } + + if o.TokenSecretName == "" && o.TokenSecretFile == "" { + o.Secret, err = unmarshallSecret(raw) + if err != nil { + return fmt.Errorf("failed to unmarshall Secret from config file: %v", err) + } + } + } + + // The precedence order is that TokenSecretName > TokenSecretFile > JoinConfigFile. + if o.TokenSecretName == "" && o.TokenSecretFile != "" { + raw, err := ioutil.ReadFile(o.TokenSecretFile) + if err != nil { + return err + } + o.Secret, err = unmarshallSecret(raw) + if err != nil { + return fmt.Errorf("failed to unmarshall Secret from token Secret file: %v", err) + } + } + + if o.LeaderClusterID == "" { + return fmt.Errorf("the ClusterID of leader cluster is required") + } + if o.LeaderAPIServer == "" { + return fmt.Errorf("the API server of the leader cluster is required") + } + if o.TokenSecretName == "" && o.Secret == nil { + return fmt.Errorf("a member token Secret must be provided through the Secret name, or Secret file, or Secret manifest in the config file") + } + if o.LeaderNamespace == "" { + return fmt.Errorf("the leader cluster Namespace is required") + } + if o.ClusterSetID == "" { + return fmt.Errorf("the ClusterSet ID is required") + } + if o.ClusterID == "" { + return fmt.Errorf("the member ClusterID is required") + } + if o.Namespace == "" { + fmt.Printf("Antrea Multi-cluster Namespace is not specified. Use %s\n.", defaultMemberNamespace) + o.Namespace = defaultMemberNamespace + } + + // Always set the Secret Namespace with the member cluster Multi-cluster Namespace + if o.Secret != nil { + o.Secret.Namespace = o.Namespace + } + return nil +} + +func yamlUnmarshall(raw []byte, v interface{}) error { + d := yaml.NewDecoder(bytes.NewReader(raw)) + d.KnownFields(false) + return d.Decode(v) +} + +func unmarshallSecret(raw []byte) (*v1.Secret, error) { + decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(raw), 100) + secret := &v1.Secret{} + // We need to skip the first object, as the Secret object is always the second object when unmarshalling the + // config file or the token Secret file. + u := unstructured.Unstructured{} + if err := decoder.Decode(&u); err != nil { + return nil, err + } + if err := decoder.Decode(secret); err != nil { + return nil, err + } + + return secret, nil +} + +var joinExamples = strings.Trim(` +# Join the ClusterSet with a pre-created token Secret. + $ antctl mc join --clusterset=clusterset1 \ + --clusterid=cluster-east \ + --namespace=kube-system \ + --leader-clusterid=cluster-north \ + --leader-namespace=antrea-multicluster \ + --leader-apiserver=https://172.18.0.3:6443 \ + --token-secret-name=cluster-east-token + +# Join the ClusterSet with a token Secret manifest. + $ antctl mc join --clusterset=clusterset1 \ + --clusterid=cluster-east \ + --namespace=kube-system \ + --leader-clusterid=cluster-north \ + --leader-namespace=antrea-multicluster \ + --leader-apiserver=https://172.18.0.3:6443 \ + --token-secret-file=cluster-east-token.yml + +# Join the ClusterSet with a config manifest. + $ antctl mc join --config-file join-config.yml + +# Config file example: +--- +apiVersion: multicluster.antrea.io/v1alpha1 +kind: ClusterSetJoinConfig +clusterSetID: clusterset1 +clusterID: cluster-east +namespace: kube-system +leaderClusterID: cluster-north +leaderNamespace: antrea-multicluster +leaderAPIServer: https://172.18.0.3:6443 +# Use the pre-created token Secret. +#tokenSecretName: "" +# Create a token Secret with the manifest file. +#tokenSecretFile: "" +# Manifest to create a Secret for a member cluster token. +--- +apiVersion: v1 +kind: Secret +metadata: + name: token-secret +data: +# Generated by "init" or "create membertoken" command + ca.crt: ... + namespace: ... + token: ... +type: Opaque +`, "\n") + +func NewJoinCommand() *cobra.Command { + command := &cobra.Command{ + Use: "join", + Short: "Join the ClusterSet from a member cluster", + Args: cobra.MaximumNArgs(0), + Example: joinExamples, + RunE: joinRunE, + } + + o := ClusterSetJoinConfig{} + joinOpts = &o + command.Flags().StringVarP(&joinOpts.LeaderNamespace, "leader-namespace", "", "", "Namespace of the leader cluster") + command.Flags().StringVarP(&joinOpts.LeaderClusterID, "leader-clusterid", "", "", "Cluster ID of the leader cluster") + command.Flags().StringVarP(&joinOpts.TokenSecretName, "token-secret-name", "", "", "Name of the Secret resource that contains the member token. "+ + "Token Secret name takes precedence over token Secret file and the Secret manifest in the join config file") + command.Flags().StringVarP(&joinOpts.LeaderAPIServer, "leader-apiserver", "", "", "API Server endpoint of the leader cluster") + command.Flags().StringVarP(&joinOpts.Namespace, "namespace", "n", defaultMemberNamespace, "Antrea Multi-cluster Namespace. Defaults to "+defaultMemberNamespace) + command.Flags().StringVarP(&joinOpts.ClusterID, "clusterid", "", "", "Cluster ID of the member cluster") + command.Flags().StringVarP(&joinOpts.ClusterSetID, "clusterset", "", "", "ClusterSet ID") + command.Flags().StringVarP(&joinOpts.TokenSecretFile, "token-secret-file", "", "", "Secret manifest for the member token. If specified, a Secret will be created with the manifest. "+ + "Token Secret file takes precedence over the Secret manifest in the join config file, if both are specified.") + command.Flags().StringVarP(&joinOpts.ConfigFile, "config-file", "f", "", "Config file that defines all config options. If both command line options and config file are specified, "+ + "the arguments in config file will be used.") + + return command +} + +func joinRunE(cmd *cobra.Command, args []string) error { + var err error + if err = joinOpts.validateAndComplete(); err != nil { + return err + } + k8sClient, err := common.NewClient(cmd) + + memberClusterNamespace := joinOpts.Namespace + memberClusterID := joinOpts.ClusterID + memberClusterSet := joinOpts.ClusterSetID + createdRes := []map[string]interface{}{} + defer func() { + if err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to join the ClusterSet. Deleting the created resources\n") + if err := common.Rollback(cmd, k8sClient, createdRes); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to rollback: %v\n", err) + } + } + }() + + if joinOpts.Secret != nil { + joinOpts.Secret.Annotations = map[string]string{ + common.CreateByAntctlAnnotation: "true", + } + if err := k8sClient.Create(context.TODO(), joinOpts.Secret); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to create the Secret from the config file: %v\n", err) + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Created the Secret from the config file\n") + unstructuredSecret, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(joinOpts.Secret) + unstructuredSecret["apiVersion"] = "v1" + unstructuredSecret["kind"] = "Secret" + createdRes = append(createdRes, unstructuredSecret) + joinOpts.TokenSecretName = joinOpts.Secret.Name + } + + err = common.CreateClusterClaim(cmd, k8sClient, memberClusterNamespace, memberClusterSet, memberClusterID, &createdRes) + if err != nil { + return err + } + err = common.CreateClusterSet(cmd, k8sClient, memberClusterNamespace, memberClusterSet, joinOpts.LeaderAPIServer, joinOpts.TokenSecretName, + joinOpts.ClusterID, joinOpts.LeaderClusterID, joinOpts.LeaderNamespace, &createdRes) + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Waiting for member cluster ready\n") + if err = waitForMemberClusterReady(cmd, k8sClient); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to wait for member cluster ready: %v\n", err) + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Member cluster joined successfully\n") + + return nil +} + +func waitForMemberClusterReady(cmd *cobra.Command, k8sClient client.Client) error { + fmt.Fprintf(cmd.OutOrStdout(), "Waiting for ClusterSet ready\n") + + if err := waitForClusterSetReady(k8sClient, joinOpts.ClusterSetID, joinOpts.Namespace, joinOpts.LeaderClusterID); err != nil { + fmt.Fprintf(cmd.OutOrStdout(), "Failed to wait for ClusterSet \"%s\" in Namespace %s in member cluster: %v\n", joinOpts.ClusterSetID, joinOpts.Namespace, err) + return err + } + + return nil +} + +func waitForClusterSetReady(client client.Client, name string, namespace string, clusterID string) error { + return wait.PollImmediate( + 1*time.Second, + 3*time.Minute, + func() (bool, error) { + clusterSet := &multiclusterv1alpha1.ClusterSet{} + if err := client.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, clusterSet); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + for _, status := range clusterSet.Status.ClusterStatuses { + if status.ClusterID == clusterID { + for _, cond := range status.Conditions { + if cond.Type == multiclusterv1alpha1.ClusterReady { + return cond.Status == "True", nil + } + } + break + } + } + + return false, nil + }) +} diff --git a/pkg/antctl/raw/multicluster/leave.go b/pkg/antctl/raw/multicluster/leave.go new file mode 100644 index 00000000000..69a9612e7a3 --- /dev/null +++ b/pkg/antctl/raw/multicluster/leave.go @@ -0,0 +1,51 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multicluster + +import ( + "strings" + + "github.com/spf13/cobra" + + "antrea.io/antrea/pkg/antctl/raw/multicluster/common" +) + +var leaveOpts *common.CleanOptions + +var leaveExamples = strings.Trim(` +# Leave the ClusterSet from a member cluster. + antctl mc leave --clusterset clusterset1 --namespace kube-system +`, "\n") + +func NewLeaveCommand() *cobra.Command { + command := &cobra.Command{ + Use: "leave", + Short: "Leave the ClusterSet from a member cluster", + Args: cobra.MaximumNArgs(0), + Example: leaveExamples, + RunE: leaveRunE, + } + + o := common.CleanOptions{} + leaveOpts = &o + command.Flags().StringVarP(&o.Namespace, "namespace", "n", defaultMemberNamespace, "Antrea Multi-cluster Namespace. Defaults to "+defaultMemberNamespace) + command.Flags().StringVarP(&o.ClusterSet, "clusterset", "", "", "ClusterSet ID") + + return command +} + +func leaveRunE(cmd *cobra.Command, args []string) error { + return common.Cleanup(cmd, leaveOpts) +}