From e73b2b23a9f824d09dd378a15ae190c9578031e2 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Tue, 6 Jun 2023 15:00:13 +0200 Subject: [PATCH] test/e2e: also dump workload cluster kube-system pods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Büringer buringerst@vmware.com --- .../kubeadm/internal/controllers/status.go | 4 +- docs/book/src/developer/logging.md | 10 ++-- test/e2e/common.go | 9 ++++ test/framework/alltypes_helpers.go | 54 ++++++++++++++++++- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/controlplane/kubeadm/internal/controllers/status.go b/controlplane/kubeadm/internal/controllers/status.go index 3d742bdac7f4..426fec4b0977 100644 --- a/controlplane/kubeadm/internal/controllers/status.go +++ b/controlplane/kubeadm/internal/controllers/status.go @@ -66,8 +66,8 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, kcp *c return nil } - machinesWithHealthAPIServer := ownedMachines.Filter(collections.HealthyAPIServer()) - lowestVersion := machinesWithHealthAPIServer.LowestVersion() + machinesWithHealthyAPIServer := ownedMachines.Filter(collections.HealthyAPIServer()) + lowestVersion := machinesWithHealthyAPIServer.LowestVersion() if lowestVersion != nil { kcp.Status.Version = lowestVersion } diff --git a/docs/book/src/developer/logging.md b/docs/book/src/developer/logging.md index fc642c56528e..1da2b7e34daf 100644 --- a/docs/book/src/developer/logging.md +++ b/docs/book/src/developer/logging.md @@ -176,28 +176,28 @@ In the Log browser the following queries can be used to browse logs by controlle Will return logs from the `capi-controller-manager` which are parsed in json. Passing the query through the json parser allows filtering by key-value pairs that are part of nested json objects. For example `.cluster.name` becomes `cluster_name`. ``` -{app="capi-controller-manager"} | json | cluster_name="my-cluster" +{app="capi-controller-manager"} | json | Cluster_name="my-cluster" ``` Will return logs from the `capi-controller-manager` that are associated with the Cluster `my-cluster`. ``` -{app="capi-controller-manager"} | json | cluster_name="my-cluster" | v <= 2 +{app="capi-controller-manager"} | json | Cluster_name="my-cluster" | v <= 2 ``` Will return logs from the `capi-controller-manager` that are associated with the Cluster `my-cluster` with log level <= 2. ``` -{app="capi-controller-manager"} | json | cluster_name="my-cluster" reconcileID="6f6ad971-bdb6-4fa3-b803-xxxxxxxxxxxx" +{app="capi-controller-manager"} | json | Cluster_name="my-cluster" reconcileID="6f6ad971-bdb6-4fa3-b803-xxxxxxxxxxxx" ``` Will return logs from the `capi-controller-manager`, associated with the Cluster `my-cluster` and the Reconcile ID `6f6ad971-bdb6-4fa3-b803-xxxxxxxxxxxx`. Each reconcile loop will have a unique Reconcile ID. ``` -{app="capi-controller-manager"} | json | cluster_name="my-cluster" reconcileID="6f6ad971-bdb6-4fa3-b803-ef81c5c8f9d0" controller="cluster" | line_format "{{ .msg }}" +{app="capi-controller-manager"} | json | Cluster_name="my-cluster" reconcileID="6f6ad971-bdb6-4fa3-b803-ef81c5c8f9d0" controller="cluster" | line_format "{{ .msg }}" ``` Will return logs from the `capi-controller-manager`, associated with the Cluster `my-cluster` and the Reconcile ID `6f6ad971-bdb6-4fa3-b803-xxxxxxxxxxxx` it further selects only those logs which come from the Cluster controller. It will then format the logs so only the message is displayed. ``` -{app=~"capd-controller-manager|capi-kubeadm-bootstrap-controller-manager|capi-kubeadm-control-plane-controller-manager"} | json | cluster_name="my-cluster" machine_name="my-cluster-linux-worker-1" | line_format "{{.controller}} {{.msg}}" +{app=~"capd-controller-manager|capi-kubeadm-bootstrap-controller-manager|capi-kubeadm-control-plane-controller-manager"} | json | Cluster_name="my-cluster" Machine_name="my-cluster-linux-worker-1" | line_format "{{.controller}} {{.msg}}" ``` Will return the logs from four CAPI providers - the Core provider, Kubeadm Control Plane provider, Kubeadm Bootstrap provider and the Docker infrastructure provider. It filters by the cluster name and the machine name and then formats the log lines to show just the source controller and the message. This allows us to correlate logs and see actions taken by each of these four providers related to the machine `my-cluster-linux-worker-1`. diff --git a/test/e2e/common.go b/test/e2e/common.go index 19cd3fb38f55..8cf6c669a4cd 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -26,6 +26,7 @@ import ( "github.com/onsi/gomega/types" corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" @@ -78,6 +79,14 @@ func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterPr LogPath: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName(), "resources"), }) + // If the cluster still exists, dump kube-system pods in the workload cluster before deleting the cluster. + if err := clusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(cluster), &clusterv1.Cluster{}); err == nil { + framework.DumpKubeSystemPods(ctx, framework.DumpKubeSystemPodsInput{ + Lister: clusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name).GetClient(), + LogPath: filepath.Join(artifactFolder, "clusters", cluster.Name, "resources"), + }) + } + if !skipCleanup { Byf("Deleting cluster %s", klog.KObj(cluster)) // While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance diff --git a/test/framework/alltypes_helpers.go b/test/framework/alltypes_helpers.go index 6d1bf2dc7f9c..cd88ae94d799 100644 --- a/test/framework/alltypes_helpers.go +++ b/test/framework/alltypes_helpers.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -49,7 +50,7 @@ type GetCAPIResourcesInput struct { // This list includes all the types belonging to CAPI providers. func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstructured.Unstructured { Expect(ctx).NotTo(BeNil(), "ctx is required for GetCAPIResources") - Expect(input.Lister).NotTo(BeNil(), "input.Deleter is required for GetCAPIResources") + Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for GetCAPIResources") Expect(input.Namespace).NotTo(BeEmpty(), "input.Namespace is required for GetCAPIResources") types := getClusterAPITypes(ctx, input.Lister) @@ -76,6 +77,34 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr return objList } +// GetKubeSystemPodsInput is the input for GetKubeSystemPods. +type GetKubeSystemPodsInput struct { + Lister Lister +} + +// GetKubeSystemPods reads all pods in the kube-system namespace. +// Note: This function intentionally retrieves Pods as Unstructured, because we need the Pods +// as Unstructured eventually. +func GetKubeSystemPods(ctx context.Context, input GetKubeSystemPodsInput) []*unstructured.Unstructured { + Expect(ctx).NotTo(BeNil(), "ctx is required for GetKubeSystemPods") + Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for GetKubeSystemPods") + + podList := new(unstructured.UnstructuredList) + podList.SetAPIVersion(corev1.SchemeGroupVersion.String()) + podList.SetKind("Pod") + if err := input.Lister.List(ctx, podList, client.InNamespace(metav1.NamespaceSystem)); err != nil { + Fail(fmt.Sprintf("failed to list Pods in kube-system: %v", err)) + } + + objList := []*unstructured.Unstructured{} + for i := range podList.Items { + obj := podList.Items[i] + objList = append(objList, &obj) + } + + return objList +} + // getClusterAPITypes returns the list of TypeMeta to be considered for the move discovery phase. // This list includes all the types belonging to CAPI providers. func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { @@ -115,7 +144,7 @@ type DumpAllResourcesInput struct { // This dump includes all the types belonging to CAPI providers. func DumpAllResources(ctx context.Context, input DumpAllResourcesInput) { Expect(ctx).NotTo(BeNil(), "ctx is required for DumpAllResources") - Expect(input.Lister).NotTo(BeNil(), "input.Deleter is required for DumpAllResources") + Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for DumpAllResources") Expect(input.Namespace).NotTo(BeEmpty(), "input.Namespace is required for DumpAllResources") resources := GetCAPIResources(ctx, GetCAPIResourcesInput{ @@ -129,6 +158,27 @@ func DumpAllResources(ctx context.Context, input DumpAllResourcesInput) { } } +// DumpKubeSystemPodsInput is the input for DumpKubeSystemPods. +type DumpKubeSystemPodsInput struct { + Lister Lister + LogPath string +} + +// DumpKubeSystemPods dumps kube-system Pods to YAML. +func DumpKubeSystemPods(ctx context.Context, input DumpKubeSystemPodsInput) { + Expect(ctx).NotTo(BeNil(), "ctx is required for DumpAllResources") + Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for DumpAllResources") + + resources := GetKubeSystemPods(ctx, GetKubeSystemPodsInput{ + Lister: input.Lister, + }) + + for i := range resources { + r := resources[i] + dumpObject(r, input.LogPath) + } +} + func dumpObject(resource runtime.Object, logPath string) { resourceYAML, err := yaml.Marshal(resource) Expect(err).ToNot(HaveOccurred(), "Failed to marshal %s", resource.GetObjectKind().GroupVersionKind().String())