Pre-installation testing framework

Signed-off-by: Kanha gupta <kanhag4163@gmail.com>

.github/workflows/kind.yml

@@ -772,13 +772,16 @@ jobs:
       - name: Create Kind Cluster
         run: |
           kind create cluster --config ci/kind/config-3nodes.yml
+      - name: Build antctl binary
+        run: |
+          make antctl-linux
+      - name: Run Pre-installation checks
+        run: |
+          ./bin/antctl-linux check cluster
       - name: Load Docker images and deploy Antrea
         run: |
           kind load docker-image antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-agent-ubuntu-coverage:latest
           kubectl apply -f build/yamls/antrea.yml
-      - name: Build antctl binary
-        run: |
-          make antctl-linux
       - name: Run antctl command
         run: |
           ./bin/antctl-linux check installation

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/btree v1.1.2
 	github.com/google/uuid v1.6.0
+	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/memberlist v0.5.1
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0
 	github.com/k8snetworkplumbingwg/sriov-cni v2.1.0+incompatible

go.sum

@@ -413,6 +413,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=

pkg/antctl/antctl.go

@@ -19,6 +19,7 @@ import (
 
 	agentapis "antrea.io/antrea/pkg/agent/apis"
 	fallbackversion "antrea.io/antrea/pkg/antctl/fallback/version"
+	checkcluster "antrea.io/antrea/pkg/antctl/raw/check/cluster"
 	checkinstallation "antrea.io/antrea/pkg/antctl/raw/check/installation"
 	"antrea.io/antrea/pkg/antctl/raw/featuregates"
 	"antrea.io/antrea/pkg/antctl/raw/multicluster"
@@ -640,6 +641,12 @@ $ antctl get podmulticaststats pod -n namespace`,
 			supportController: false,
 			commandGroup:      check,
 		},
+		{
+			cobraCommand:      checkcluster.Command(),
+			supportAgent:      false,
+			supportController: false,
+			commandGroup:      check,
+		},
 		{
 			cobraCommand:      supportbundle.Command,
 			supportAgent:      true,

pkg/antctl/raw/check/cluster/command.go

@@ -0,0 +1,222 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+
+	"antrea.io/antrea/pkg/antctl/raw/check"
+)
+
+func Command() *cobra.Command {
+	o := newOptions()
+	command := &cobra.Command{
+		Use:   "cluster",
+		Short: "Runs pre installation checks",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return Run(o)
+		},
+	}
+	command.Flags().StringVarP(&o.antreaNamespace, "Namespace", "n", o.antreaNamespace, "Configure Namespace in which Antrea is running")
+	return command
+}
+
+type options struct {
+	antreaNamespace string
+}
+
+func newOptions() *options {
+	return &options{
+		antreaNamespace: "kube-system",
+	}
+}
+
+const (
+	antreaNamespace = "kube-system"
+	deploymentName  = "cluster-check"
+	podReadyTimeout = 1 * time.Minute
+)
+
+type Test interface {
+	Run(ctx context.Context, testContext *testContext) error
+}
+
+var testsRegistry = make(map[string]Test)
+
+func RegisterTest(name string, test Test) {
+	testsRegistry[name] = test
+}
+
+type testContext struct {
+	client          kubernetes.Interface
+	config          *rest.Config
+	clusterName     string
+	antreaNamespace string
+}
+
+func Run(o *options) error {
+	client, config, clusterName, err := check.NewClient()
+	if err != nil {
+		return fmt.Errorf("unable to create Kubernetes client: %s", err)
+	}
+	ctx := context.Background()
+	testContext := NewTestContext(client, config, clusterName, o)
+	if err := testContext.setup(ctx); err != nil {
+		return err
+	}
+	for name, test := range testsRegistry {
+		testContext.Header("Running test: %s", name)
+		if err := test.Run(ctx, testContext); err != nil {
+			testContext.Header("Test %s failed: %s", name, err)
+		} else {
+			testContext.Header("Test %s passed", name)
+		}
+	}
+	testContext.Log("Test finished")
+	testContext.teardown(ctx, deploymentName, antreaNamespace)
+	return nil
+}
+
+func (t *testContext) setup(ctx context.Context) error {
+	deployment := check.NewDeployment(check.DeploymentParameters{
+		Name:        deploymentName,
+		Image:       "alpine",
+		Replicas:    1,
+		Command:     []string{"sleep", "infinity"},
+		Labels:      map[string]string{"app": "cluster-check"},
+		HostNetwork: true,
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: "cni-conf", MountPath: "/etc/cni/net.d"},
+			{Name: "lib-modules", MountPath: "/lib/modules"},
+			{Name: "os-info", MountPath: "/etc/os-release"},
+		},
+		Tolerations: []corev1.Toleration{
+			{
+				Key:      "node-role.kubernetes.io/control-plane",
+				Operator: "Exists",
+				Effect:   "NoSchedule",
+			},
+			{
+				Key:      "node.kubernetes.io/not-ready",
+				Operator: "Exists",
+				Effect:   "NoSchedule",
+			},
+		},
+		Volumes: []corev1.Volume{
+			{
+				Name: "cni-conf",
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: "/etc/cni/net.d",
+					},
+				},
+			},
+			{
+				Name: "lib-modules",
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: "/lib/modules",
+						Type: hostPathTypePtr("Directory"),
+					},
+				},
+			},
+			{
+				Name: "os-info",
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: "/etc/os-release",
+						Type: hostPathTypePtr("File"),
+					},
+				},
+			},
+		},
+	})
+
+	t.Log("Creating Deployment")
+	_, err := t.client.AppsV1().Deployments(antreaNamespace).Create(ctx, deployment, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("unable to create Deployment: %w", err)
+	}
+
+	t.Log("Waiting for Deployment to become ready")
+	t.waitForDeploymentsReady(ctx, time.Second, podReadyTimeout, deploymentName)
+	if err != nil {
+		return fmt.Errorf("error while waiting for Deployment to become ready: %w", err)
+	}
+	return nil
+}
+
+func hostPathTypePtr(s string) *corev1.HostPathType {
+	v := corev1.HostPathType(s)
+	return &v
+}
+
+func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterName string, o *options) *testContext {
+	return &testContext{
+		client:          client,
+		config:          config,
+		clusterName:     clusterName,
+		antreaNamespace: o.antreaNamespace,
+	}
+}
+
+func (t *testContext) teardown(ctx context.Context, deploymentName, namespace string) error {
+	err := t.client.AppsV1().Deployments(namespace).Delete(ctx, deploymentName, metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
+	t.Log("Waiting for the deletion of Deployment %s in Namespace %s...", deploymentName, namespace)
+	err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) {
+		_, err := t.client.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{})
+		if errors.IsNotFound(err) {
+			return true, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		return false, nil
+	})
+	if err != nil {
+		return fmt.Errorf("error waiting for Deployment %s to be deleted in Namespace %s: %w", deploymentName, namespace, err)
+	}
+	t.Log("Deployment %s successfully deleted from Namespace %s", deploymentName, namespace)
+	return nil
+}
+
+func (t *testContext) waitForDeploymentsReady(ctx context.Context, interval, timeout time.Duration, deployments ...string) error {
+	for _, deployment := range deployments {
+		t.Log("Waiting for Deployment %s to become ready...", deployment)
+		err := wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) {
+			ready, err := check.DeploymentIsReady(ctx, t.client, t.antreaNamespace, deployment)
+			if err != nil {
+				return false, fmt.Errorf("error checking readiness of Deployment %s: %w", deployment, err)
+			}
+			return ready, nil
+		})
+		if err != nil {
+			return fmt.Errorf("waiting for Deployment %s to become ready has been interrupted: %w", deployment, err)
+		}
+		t.Log("Deployment %s is ready.", deployment)
+	}
+	return nil
+}
+
+func (t *testContext) Log(format string, a ...interface{}) {
+	fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+format+"\n", a...)
+}
+
+func (t *testContext) Header(format string, a ...interface{}) {
+	t.Log("-------------------------------------------------------------------------------------------")
+	t.Log(format, a...)
+	t.Log("-------------------------------------------------------------------------------------------")
+}

pkg/antctl/raw/check/cluster/test_checkCNIAvailability.go

@@ -0,0 +1,41 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"antrea.io/antrea/pkg/antctl/raw/check"
+)
+
+type checkCNIAvailability struct{}
+
+func init() {
+	RegisterTest("Check if another CNI is Present", &checkCNIAvailability{})
+}
+
+func (t *checkCNIAvailability) Run(ctx context.Context, testContext *testContext) error {
+	pods, err := testContext.client.CoreV1().Pods(antreaNamespace).List(ctx, metav1.ListOptions{LabelSelector: "app=cluster-check"})
+	if err != nil {
+		return fmt.Errorf("failed to list Pods: %v", err)
+	}
+	for _, pod := range pods.Items {
+		testContext.Log("Checking if CNI is present in Pod: %s", pod.Name)
+		command := []string{"ls", "/etc/cni/net.d"}
+		output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, antreaNamespace, pod.Name, "", command)
+		if err != nil {
+			testContext.Log("failed to execute command in pod: %s, error: %v", pod.Name, err)
+			continue
+		}
+		outputStr := strings.TrimSpace(output)
+		if outputStr == "" {
+			testContext.Log("No files present in /etc/cni/net.d in pod: %s", pod.Name)
+			return nil
+		} else {
+			return fmt.Errorf("error: files found in /host/etc/cni/net.d in pod: %s", outputStr)
+		}
+	}
+	return nil
+}

pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go

@@ -0,0 +1,41 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type checkControlPlaneAvailability struct{}
+
+func init() {
+	RegisterTest("Check Control Plane Availability", &checkControlPlaneAvailability{})
+}
+
+func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *testContext) error {
+	labelSelector := "component=kube-controller-manager,tier=control-plane"
+	namespaces, err := testContext.client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to list Namespaces: %w", err)
+	}
+	controlPlaneFound := false
+	for _, namespace := range namespaces.Items {
+		pods, err := testContext.client.CoreV1().Pods(namespace.Name).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
+		if err != nil {
+			return fmt.Errorf("failed to list Pods in Namespace %s: %w", namespace.Name, err)
+		}
+		if len(pods.Items) > 0 {
+			controlPlaneFound = true
+			for _, pod := range pods.Items {
+				testContext.Log("Control plane Pod %s found in Namespace %s\n", pod.Name, namespace.Name)
+			}
+		}
+	}
+	if !controlPlaneFound {
+		testContext.Log("Warning: No control plane Pods found in any Namespace.")
+	} else {
+		testContext.Log("Control plane Pods found in one or more Namespaces.")
+	}
+	return nil
+}

pkg/antctl/raw/check/cluster/test_checkk8sversion.go

@@ -0,0 +1,31 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/go-version"
+)
+
+type checkK8sVersion struct{}
+
+func init() {
+	RegisterTest("Check K8s Version", &checkK8sVersion{})
+}
+
+func (t *checkK8sVersion) Run(ctx context.Context, testContext *testContext) error {
+	discoveryClient := testContext.client.Discovery()
+	serverVersion, err := discoveryClient.ServerVersion()
+	if err != nil {
+		return fmt.Errorf("error getting server version: %v", err)
+	}
+	currentVersion, err := version.NewVersion(strings.TrimPrefix(serverVersion.GitVersion, "v"))
+	minVersion, _ := version.NewVersion("1.19")
+	if currentVersion.GreaterThanOrEqual(minVersion) {
+		testContext.Log("Kubernetes server version is compatible with Antrea. version : %s", serverVersion.GitVersion)
+	} else {
+		testContext.Log("Kubernetes min version required : 1.19")
+	}
+	return err
+}