CAPD: add version aware patch for cgroupDriver to support v1.24 and l…

…ower versions at the same time

* Bumps kind to v0.13 for v1.24 support as kind management cluster
* Migrates CI upgrade tests to use cluster class and version aware patches for cgroupDriver
* Implement workaround for DockerMachinePools which are not supported by ClusterClass to use correct cgroupDriver on upgrade tests.

hack/ensure-kind.sh

@@ -21,7 +21,7 @@ set -o pipefail
 set -x
 
 GOPATH_BIN="$(go env GOPATH)/bin/"
-MINIMUM_KIND_VERSION=v0.12.0
+MINIMUM_KIND_VERSION=v0.13.0
 goarch="$(go env GOARCH)"
 goos="$(go env GOOS)"
 

hack/tools/go.sum

@@ -69,7 +69,7 @@ github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
@@ -1683,7 +1683,7 @@ sigs.k8s.io/controller-tools v0.8.0 h1:uUkfTGEwrguqYYfcI2RRGUnC8mYdCFDqfwPKUcNJh
 sigs.k8s.io/controller-tools v0.8.0/go.mod h1:qE2DXhVOiEq5ijmINcFbqi9GZrrUjzB1TuJU0xa6eoY=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
-sigs.k8s.io/kind v0.12.0/go.mod h1:EcgDSBVxz8Bvm19fx8xkioFrf9dC30fMJdOTXBSGNoM=
+sigs.k8s.io/kind v0.13.0/go.mod h1:UrFRPHG+2a5j0Q7qiR4gtJ4rEyn8TuMQwuOPf+m4oHg=
 sigs.k8s.io/kubebuilder/docs/book/utils v0.0.0-20211028165026-57688c578b5d h1:KLiQzLW3RZJR19+j4pw2h5iioyAyqCkDBEAFdnGa3N8=
 sigs.k8s.io/kubebuilder/docs/book/utils v0.0.0-20211028165026-57688c578b5d/go.mod h1:NRdZafr4zSCseLQggdvIMXa7umxf+Q+PJzrj3wFwiGE=
 sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8=

test/e2e/Makefile

@@ -94,6 +94,7 @@ cluster-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1beta
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-machine-pool --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-machine-pool.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-node-drain --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-node-drain.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-upgrades --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-upgrades.yaml
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-upgrades-cgroupfs --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-upgrades-cgroupfs.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-kcp-scale-in.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-ipv6 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-ipv6.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-topology.yaml

test/e2e/cluster_upgrade_test.go

@@ -20,14 +20,30 @@ limitations under the License.
 package e2e
 
 import (
+	"strings"
+
+	"github.com/blang/semver"
 	. "github.com/onsi/ginkgo"
 	"k8s.io/utils/pointer"
-
-	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 )
 
 var _ = Describe("When upgrading a workload cluster and testing K8S conformance [Conformance] [K8s-Upgrade]", func() {
 	ClusterUpgradeConformanceSpec(ctx, func() ClusterUpgradeConformanceSpecInput {
+
+		// For KubernetesVersionUpgradeFrom < v1.24 we have to use upgrades-cgroupfs flavor as source cluster.
+		// This is because kind and CAPD require to use cgroupDriver cgroupfs for Kubernetes <v1.24
+		// and systemd for >=v1.24.
+		if version, _ := e2eConfig.Variables[KubernetesVersionUpgradeFrom]; semver.MustParse(strings.TrimPrefix(version, "v")).LT(semver.MustParse("1.24.0")) {
+			return ClusterUpgradeConformanceSpecInput{
+				E2EConfig:             e2eConfig,
+				ClusterctlConfigPath:  clusterctlConfigPath,
+				BootstrapClusterProxy: bootstrapClusterProxy,
+				ArtifactFolder:        artifactFolder,
+				SkipCleanup:           skipCleanup,
+				Flavor:                pointer.String("upgrades-cgroupfs"),
+			}
+		}
+
 		return ClusterUpgradeConformanceSpecInput{
 			E2EConfig:             e2eConfig,
 			ClusterctlConfigPath:  clusterctlConfigPath,
@@ -67,7 +83,7 @@ var _ = Describe("When upgrading a workload cluster with a single control plane
 			SkipConformanceTests:     true,
 			ControlPlaneMachineCount: pointer.Int64(1),
 			WorkerMachineCount:       pointer.Int64(1),
-			Flavor:                   pointer.String(clusterctl.DefaultFlavor),
+			Flavor:                   pointer.String("topology"),
 		}
 	})
 })
@@ -85,7 +101,7 @@ var _ = Describe("When upgrading a workload cluster with a HA control plane", fu
 			SkipConformanceTests:     true,
 			ControlPlaneMachineCount: pointer.Int64(3),
 			WorkerMachineCount:       pointer.Int64(1),
-			Flavor:                   pointer.String(clusterctl.DefaultFlavor),
+			Flavor:                   pointer.String("topology"),
 		}
 	})
 })

test/e2e/config/docker.yaml

@@ -187,6 +187,7 @@ providers:
     - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-machine-pool.yaml"
     - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-node-drain.yaml"
     - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-upgrades.yaml"
+    - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-upgrades-cgroupfs.yaml"
     - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-kcp-scale-in.yaml"
     - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-ipv6.yaml"
     - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-topology.yaml"
@@ -199,10 +200,10 @@ variables:
   # allowing the same e2e config file to be re-used in different Prow jobs e.g. each one with a K8s version permutation.
   # The following Kubernetes versions should be the latest versions with already published kindest/node images.
   # This avoids building node images in the default case which improves the test duration significantly.
-  KUBERNETES_VERSION_MANAGEMENT: "v1.23.4"
-  KUBERNETES_VERSION: "v1.23.4"
-  KUBERNETES_VERSION_UPGRADE_FROM: "v1.22.7"
-  KUBERNETES_VERSION_UPGRADE_TO: "v1.23.4"
+  KUBERNETES_VERSION_MANAGEMENT: "v1.24.0"
+  KUBERNETES_VERSION: "v1.24.0"
+  KUBERNETES_VERSION_UPGRADE_FROM: "v1.23.6"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.24.0"
   ETCD_VERSION_UPGRADE_TO: "3.5.3-0"
   COREDNS_VERSION_UPGRADE_TO: "v1.8.6"
   DOCKER_SERVICE_DOMAIN: "cluster.local"
@@ -225,7 +226,7 @@ variables:
   # NOTE: We test the latest release with a previous contract.
   INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/clusterctl-{OS}-{ARCH}"
   INIT_WITH_PROVIDERS_CONTRACT: "v1alpha4"
-  INIT_WITH_KUBERNETES_VERSION: "v1.23.4"
+  INIT_WITH_KUBERNETES_VERSION: "v1.24.0"
 
 intervals:
   default/wait-controllers: ["3m", "10s"]

test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml

@@ -67,16 +67,10 @@ spec:
       nodeRegistration:
         criSocket: unix:///var/run/containerd/containerd.sock
         kubeletExtraArgs:
-          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-          cgroup-driver: cgroupfs
           eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
     joinConfiguration:
       nodeRegistration:
         criSocket: unix:///var/run/containerd/containerd.sock
         kubeletExtraArgs:
-          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-          cgroup-driver: cgroupfs
           eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
   version: "${KUBERNETES_VERSION}"

test/e2e/data/infrastructure-docker/v1alpha4/bases/cluster-with-kcp.yaml

@@ -68,16 +68,10 @@ spec:
       nodeRegistration:
         criSocket: unix:///var/run/containerd/containerd.sock
         kubeletExtraArgs:
-          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-          cgroup-driver: cgroupfs
           eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
     joinConfiguration:
       nodeRegistration:
         criSocket: unix:///var/run/containerd/containerd.sock
         kubeletExtraArgs:
-          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-          cgroup-driver: cgroupfs
           eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
   version: "${KUBERNETES_VERSION}"

test/e2e/data/infrastructure-docker/v1alpha4/bases/md.yaml

@@ -24,9 +24,6 @@ spec:
         nodeRegistration:
           criSocket: unix:///var/run/containerd/containerd.sock
           kubeletExtraArgs:
-            # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-            # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-            cgroup-driver: cgroupfs
             eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
 ---
 # MachineDeployment object

test/e2e/data/infrastructure-docker/v1alpha4/bases/mp.yaml

@@ -36,7 +36,4 @@ spec:
   joinConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
-        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-        cgroup-driver: cgroupfs
         eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%

test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-kcp-adoption/step1/cluster-with-cp0.yaml

@@ -48,17 +48,11 @@ spec:
     nodeRegistration:
       criSocket: unix:///var/run/containerd/containerd.sock
       kubeletExtraArgs:
-        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-        cgroup-driver: cgroupfs
         eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
   joinConfiguration:
     nodeRegistration:
       criSocket: unix:///var/run/containerd/containerd.sock
       kubeletExtraArgs:
-        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-        cgroup-driver: cgroupfs
         eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
 ---
 # cp0 Machine

test/e2e/data/infrastructure-docker/v1beta1/bases/cluster-with-kcp.yaml

@@ -86,16 +86,10 @@ spec:
       nodeRegistration:
         criSocket: unix:///var/run/containerd/containerd.sock
         kubeletExtraArgs:
-          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-          cgroup-driver: cgroupfs
           eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
     joinConfiguration:
       nodeRegistration:
         criSocket: unix:///var/run/containerd/containerd.sock
         kubeletExtraArgs:
-          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-          cgroup-driver: cgroupfs
           eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
   version: "${KUBERNETES_VERSION}"

test/e2e/data/infrastructure-docker/v1beta1/bases/md.yaml

@@ -24,9 +24,6 @@ spec:
         nodeRegistration:
           criSocket: unix:///var/run/containerd/containerd.sock
           kubeletExtraArgs:
-            # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-            # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-            cgroup-driver: cgroupfs
             eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
 ---
 # MachineDeployment object

test/e2e/data/infrastructure-docker/v1beta1/bases/mp.yaml

@@ -42,7 +42,4 @@ spec:
   joinConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
-        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-        cgroup-driver: cgroupfs
         eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%

test/e2e/data/infrastructure-docker/v1beta1/cluster-template-kcp-adoption/step1/cluster-with-cp0.yaml

@@ -48,17 +48,11 @@ spec:
     nodeRegistration:
       criSocket: unix:///var/run/containerd/containerd.sock
       kubeletExtraArgs:
-        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-        cgroup-driver: cgroupfs
         eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
   joinConfiguration:
     nodeRegistration:
       criSocket: unix:///var/run/containerd/containerd.sock
       kubeletExtraArgs:
-        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
-        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
-        cgroup-driver: cgroupfs
         eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
 ---
 # cp0 Machine

test/e2e/data/infrastructure-docker/v1beta1/cluster-template-kcp-scale-in/kcp-scale-in-variable.yaml

@@ -0,0 +1,5 @@
+- op: add
+  path: /spec/topology/variables/-
+  value:
+    name: kubeadmControlPlaneMaxSurge
+    value: "0"

test/e2e/data/infrastructure-docker/v1beta1/cluster-template-kcp-scale-in/kustomization.yaml

@@ -1,7 +1,10 @@
 bases:
 - ../bases/crs.yaml
-- ../bases/md.yaml
-- ../bases/cluster-with-kcp.yaml
+- ../bases/cluster-with-topology.yaml
 
-patchesStrategicMerge:
-- ./cluster-with-kcp.yaml
+patches:
+- path: ./kcp-scale-in-variable.yaml
+  target:
+    group: cluster.x-k8s.io
+    version: v1beta1
+    kind: Cluster

test/e2e/data/infrastructure-docker/v1beta1/cluster-template-upgrades-cgroupfs/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - ../cluster-template-upgrades
+  - ./mp-cgroupfs.yaml
+
+patches:
+- ./mp-default-cgroupfs.yaml

test/e2e/data/infrastructure-docker/v1beta1/cluster-template-upgrades-cgroupfs/mp-cgroupfs.yaml

@@ -0,0 +1,13 @@
+# KubeadmConfigTemplate referenced by the MachinePool for cgroupfs
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfig
+metadata:
+  name: "${CLUSTER_NAME}-mp-0-config-cgroupfs"
+spec:
+  joinConfiguration:
+    nodeRegistration:
+      kubeletExtraArgs:
+        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+        cgroup-driver: cgroupfs
+        eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%

test/e2e/data/infrastructure-docker/v1beta1/cluster-template-upgrades-cgroupfs/mp-default-cgroupfs.yaml

@@ -0,0 +1,10 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachinePool
+metadata:
+  name: "${CLUSTER_NAME}-mp-0"
+spec:
+  template:
+    spec:
+      bootstrap:
+        configRef:
+          name: "${CLUSTER_NAME}-mp-0-config-cgroupfs"

test/e2e/data/infrastructure-docker/v1beta1/cluster-template-upgrades/kustomization.yaml

@@ -1,5 +1,4 @@
 resources:
-  - ../bases/cluster-with-kcp.yaml
+  - ../bases/cluster-with-topology.yaml
   - ../bases/crs.yaml
-  - ../bases/md.yaml
   - ../bases/mp.yaml