diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 141215445f..f166ca3e7b 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -38,9 +38,8 @@ import ( e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" - nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" + nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" "sigs.k8s.io/node-feature-discovery/source/custom" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" ) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 266625be2c..0d8a435012 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -28,7 +28,6 @@ import ( topologyclientset "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned" corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -36,6 +35,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/kubelet" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" + admissionapi "k8s.io/pod-security-admission/api" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" ) @@ -44,14 +44,13 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { var ( extClient *extclient.Clientset topologyClient *topologyclientset.Clientset - crd *apiextensionsv1.CustomResourceDefinition topologyUpdaterNode *corev1.Node workerNodes []corev1.Node kubeletConfig *kubeletconfig.KubeletConfiguration ) f := framework.NewDefaultFramework("node-topology-updater") - + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged BeforeEach(func() { var err error @@ -72,7 +71,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { By(fmt.Sprintf("Using config (%#v)", kcfg)) By("Creating the node resource topologies CRD") - crd, err = testutils.CreateNodeResourceTopologies(extClient) + _, err = testutils.CreateNodeResourceTopologies(extClient) Expect(err).NotTo(HaveOccurred()) err = testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name) @@ -209,13 +208,17 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { initialNodeTopo := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name) By("creating a pod consuming exclusive CPUs") sleeperPod := testutils.GuaranteedSleeperPod("1000m") + // in case there is more than a single node in the cluster + // we need to set the node name, so we'll have certainty about + // which node we need to examine + sleeperPod.Spec.NodeName = topologyUpdaterNode.Name podMap := make(map[string]*corev1.Pod) pod := f.PodClient().CreateSync(sleeperPod) podMap[pod.Name] = pod defer testutils.DeletePodsAsync(f, podMap) - By("getting the updated topology") + By("checking the changes in the updated topology") var finalNodeTopo *v1alpha1.NodeResourceTopology Eventually(func() bool { finalNodeTopo, err = topologyClient.TopologyV1alpha1().NodeResourceTopologies().Get(context.TODO(), topologyUpdaterNode.Name, metav1.GetOptions{}) @@ -223,18 +226,23 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { framework.Logf("failed to get the node topology resource: %v", err) return false } - return finalNodeTopo.ObjectMeta.ResourceVersion != initialNodeTopo.ObjectMeta.ResourceVersion - }, time.Minute, 5*time.Second).Should(BeTrue(), "didn't get updated node topology info") - By("checking the changes in the updated topology") + if finalNodeTopo.ObjectMeta.ResourceVersion == initialNodeTopo.ObjectMeta.ResourceVersion { + framework.Logf("node topology resource %s was not updated", topologyUpdaterNode.Name) + } - initialAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(initialNodeTopo) - finalAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(finalNodeTopo) - if len(initialAllocRes) == 0 || len(finalAllocRes) == 0 { - Fail(fmt.Sprintf("failed to find allocatable resources from node topology initial=%v final=%v", initialAllocRes, finalAllocRes)) - } - zoneName, resName, isLess := lessAllocatableResources(initialAllocRes, finalAllocRes) - framework.Logf("zone=%q resource=%q isLess=%v", zoneName, resName, isLess) - Expect(isLess).To(BeTrue(), fmt.Sprintf("final allocatable resources not decreased - initial=%v final=%v", initialAllocRes, finalAllocRes)) + initialAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(initialNodeTopo) + finalAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(finalNodeTopo) + if len(initialAllocRes) == 0 || len(finalAllocRes) == 0 { + Fail(fmt.Sprintf("failed to find allocatable resources from node topology initial=%v final=%v", initialAllocRes, finalAllocRes)) + } + + zoneName, resName, isLess := lessAllocatableResources(initialAllocRes, finalAllocRes) + framework.Logf("zone=%q resource=%q isLess=%v", zoneName, resName, isLess) + if !isLess { + framework.Logf("final allocatable resources not decreased - initial=%v final=%v", initialAllocRes, finalAllocRes) + } + return true + }, time.Minute, 5*time.Second).Should(BeTrue(), "didn't get updated node topology info") }) }) @@ -244,11 +252,6 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { if err != nil { framework.Logf("failed to delete RBAC resources: %v", err) } - - err = extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) - if err != nil { - framework.Logf("failed to delete node resources topologies CRD: %v", err) - } }) }) diff --git a/test/e2e/utils/noderesourcetopology.go b/test/e2e/utils/noderesourcetopology.go index 7ff6988e6d..a685680060 100644 --- a/test/e2e/utils/noderesourcetopology.go +++ b/test/e2e/utils/noderesourcetopology.go @@ -35,6 +35,7 @@ import ( extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/test/e2e/framework" @@ -81,15 +82,26 @@ func CreateNodeResourceTopologies(extClient extclient.Interface) (*apiextensions return nil, err } - updatedCrd, err := extClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{}) + // Delete existing CRD (if any) with this we also get rid of stale objects + err = extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { - return nil, err + return nil, fmt.Errorf("failed to delete NodeResourceTopology CRD: %w", err) } - if err == nil { - return updatedCrd, nil - } + // It takes time for the delete operation, wait until the CRD completely gone + if err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) { + _, err = extClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{}) + if err == nil { + return false, nil + } + if errors.IsNotFound(err) { + return true, nil + } + return false, err + }); err != nil { + return nil, fmt.Errorf("failed to get NodeResourceTopology CRD: %w", err) + } return extClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}) } diff --git a/test/e2e/utils/pod.go b/test/e2e/utils/pod.go index 8726f41df3..fd6c0c43e8 100644 --- a/test/e2e/utils/pod.go +++ b/test/e2e/utils/pod.go @@ -203,6 +203,8 @@ func newDaemonSet(name string, podSpec *corev1.PodSpec) *appsv1.DaemonSet { } func nfdWorkerPodSpec(image string, extraArgs []string) *corev1.PodSpec { + yes := true + no := false return &corev1.PodSpec{ Containers: []corev1.Container{ { @@ -221,6 +223,15 @@ func nfdWorkerPodSpec(image string, extraArgs []string) *corev1.PodSpec { }, }, }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + Privileged: &no, + RunAsNonRoot: &yes, + ReadOnlyRootFilesystem: &yes, + AllowPrivilegeEscalation: &no, + }, VolumeMounts: []corev1.VolumeMount{ { Name: "host-boot",