e2e: topologyupdater: fix and stabilize tests

test/e2e/node_feature_discovery.go

@@ -38,9 +38,8 @@ import (
 	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
 	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
-	nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned"
-
 	nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1"
+	nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned"
 	"sigs.k8s.io/node-feature-discovery/source/custom"
 	testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils"
 )

test/e2e/topology_updater.go

@@ -28,14 +28,14 @@ import (
 	topologyclientset "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned"
 
 	corev1 "k8s.io/api/core/v1"
-	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/framework/kubelet"
 	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
+	admissionapi "k8s.io/pod-security-admission/api"
 
 	testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils"
 )
@@ -44,14 +44,13 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() {
 	var (
 		extClient           *extclient.Clientset
 		topologyClient      *topologyclientset.Clientset
-		crd                 *apiextensionsv1.CustomResourceDefinition
 		topologyUpdaterNode *corev1.Node
 		workerNodes         []corev1.Node
 		kubeletConfig       *kubeletconfig.KubeletConfiguration
 	)
 
 	f := framework.NewDefaultFramework("node-topology-updater")
-
+	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
 	BeforeEach(func() {
 		var err error
 
@@ -72,7 +71,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() {
 		By(fmt.Sprintf("Using config (%#v)", kcfg))
 
 		By("Creating the node resource topologies CRD")
-		crd, err = testutils.CreateNodeResourceTopologies(extClient)
+		_, err = testutils.CreateNodeResourceTopologies(extClient)
 		Expect(err).NotTo(HaveOccurred())
 
 		err = testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name)
@@ -209,32 +208,41 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() {
 			initialNodeTopo := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name)
 			By("creating a pod consuming exclusive CPUs")
 			sleeperPod := testutils.GuaranteedSleeperPod("1000m")
+			// in case there is more than a single node in the cluster
+			// we need to set the node name, so we'll have certainty about
+			// which node we need to examine
+			sleeperPod.Spec.NodeName = topologyUpdaterNode.Name
 
 			podMap := make(map[string]*corev1.Pod)
 			pod := f.PodClient().CreateSync(sleeperPod)
 			podMap[pod.Name] = pod
 			defer testutils.DeletePodsAsync(f, podMap)
 
-			By("getting the updated topology")
+			By("checking the changes in the updated topology")
 			var finalNodeTopo *v1alpha1.NodeResourceTopology
 			Eventually(func() bool {
 				finalNodeTopo, err = topologyClient.TopologyV1alpha1().NodeResourceTopologies().Get(context.TODO(), topologyUpdaterNode.Name, metav1.GetOptions{})
 				if err != nil {
 					framework.Logf("failed to get the node topology resource: %v", err)
 					return false
 				}
-				return finalNodeTopo.ObjectMeta.ResourceVersion != initialNodeTopo.ObjectMeta.ResourceVersion
-			}, time.Minute, 5*time.Second).Should(BeTrue(), "didn't get updated node topology info")
-			By("checking the changes in the updated topology")
+				if finalNodeTopo.ObjectMeta.ResourceVersion == initialNodeTopo.ObjectMeta.ResourceVersion {
+					framework.Logf("node topology resource %s was not updated", topologyUpdaterNode.Name)
+				}
 
-			initialAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(initialNodeTopo)
-			finalAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(finalNodeTopo)
-			if len(initialAllocRes) == 0 || len(finalAllocRes) == 0 {
-				Fail(fmt.Sprintf("failed to find allocatable resources from node topology initial=%v final=%v", initialAllocRes, finalAllocRes))
-			}
-			zoneName, resName, isLess := lessAllocatableResources(initialAllocRes, finalAllocRes)
-			framework.Logf("zone=%q resource=%q isLess=%v", zoneName, resName, isLess)
-			Expect(isLess).To(BeTrue(), fmt.Sprintf("final allocatable resources not decreased - initial=%v final=%v", initialAllocRes, finalAllocRes))
+				initialAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(initialNodeTopo)
+				finalAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(finalNodeTopo)
+				if len(initialAllocRes) == 0 || len(finalAllocRes) == 0 {
+					Fail(fmt.Sprintf("failed to find allocatable resources from node topology initial=%v final=%v", initialAllocRes, finalAllocRes))
+				}
+
+				zoneName, resName, isLess := lessAllocatableResources(initialAllocRes, finalAllocRes)
+				framework.Logf("zone=%q resource=%q isLess=%v", zoneName, resName, isLess)
+				if !isLess {
+					framework.Logf("final allocatable resources not decreased - initial=%v final=%v", initialAllocRes, finalAllocRes)
+				}
+				return true
+			}, time.Minute, 5*time.Second).Should(BeTrue(), "didn't get updated node topology info")
 		})
 
 	})
@@ -244,11 +252,6 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() {
 		if err != nil {
 			framework.Logf("failed to delete RBAC resources: %v", err)
 		}
-
-		err = extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{})
-		if err != nil {
-			framework.Logf("failed to delete node resources topologies CRD: %v", err)
-		}
 	})
 })
 

test/e2e/utils/noderesourcetopology.go

@@ -35,6 +35,7 @@ import (
 	extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/scheme"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -81,15 +82,26 @@ func CreateNodeResourceTopologies(extClient extclient.Interface) (*apiextensions
 		return nil, err
 	}
 
-	updatedCrd, err := extClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{})
+	// Delete existing CRD (if any) with this we also get rid of stale objects
+	err = extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{})
 	if err != nil && !errors.IsNotFound(err) {
-		return nil, err
+		return nil, fmt.Errorf("failed to delete NodeResourceTopology CRD: %w", err)
 	}
 
-	if err == nil {
-		return updatedCrd, nil
-	}
+	// It takes time for the delete operation, wait until the CRD completely gone
+	if err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
+		_, err = extClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{})
+		if err == nil {
+			return false, nil
+		}
 
+		if errors.IsNotFound(err) {
+			return true, nil
+		}
+		return false, err
+	}); err != nil {
+		return nil, fmt.Errorf("failed to get NodeResourceTopology CRD: %w", err)
+	}
 	return extClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
 }
 

test/e2e/utils/pod.go

@@ -203,6 +203,8 @@ func newDaemonSet(name string, podSpec *corev1.PodSpec) *appsv1.DaemonSet {
 }
 
 func nfdWorkerPodSpec(image string, extraArgs []string) *corev1.PodSpec {
+	yes := true
+	no := false
 	return &corev1.PodSpec{
 		Containers: []corev1.Container{
 			{
@@ -221,6 +223,15 @@ func nfdWorkerPodSpec(image string, extraArgs []string) *corev1.PodSpec {
 						},
 					},
 				},
+				SecurityContext: &corev1.SecurityContext{
+					Capabilities: &corev1.Capabilities{
+						Drop: []corev1.Capability{"ALL"},
+					},
+					Privileged:               &no,
+					RunAsNonRoot:             &yes,
+					ReadOnlyRootFilesystem:   &yes,
+					AllowPrivilegeEscalation: &no,
+				},
 				VolumeMounts: []corev1.VolumeMount{
 					{
 						Name:      "host-boot",