Skip to content

Commit

Permalink
Add clusterctl upgrade test
Browse files Browse the repository at this point in the history
Signed-off-by: killianmuldoon <kmuldoon@vmware.com>
  • Loading branch information
killianmuldoon authored and chrischdi committed Dec 5, 2023
1 parent da206f1 commit 6e5d1f1
Show file tree
Hide file tree
Showing 47 changed files with 5,236 additions and 87 deletions.
6 changes: 4 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ _artifacts/

# E2E test templates
test/e2e/data/infrastructure-vsphere/**/cluster-template*.yaml
test/e2e/data/infrastructure-vsphere/main/clusterclass-quick-start.yaml
test/e2e/data/infrastructure-vsphere/main/**/clusterclass-quick-start.yaml
test/e2e/data/infrastructure-vsphere/*/clusterclass-quick-start.yaml

# env vars file used in getting-started.md and manifests generation
envvars.txt
Expand Down Expand Up @@ -75,6 +76,7 @@ _releasenotes
*~
*.tmp
.DS_Store
*.swp

# Ginkgo logs from test runs
*ginkgo-log.txt
*ginkgo-log.txt
48 changes: 29 additions & 19 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ GINKGO_SKIP ?=
GINKGO_TIMEOUT ?= 2h
E2E_CONF_FILE ?= "$(abspath test/e2e/config/vsphere-dev.yaml)"
INTEGRATION_CONF_FILE ?= "$(abspath test/integration/integration-dev.yaml)"
E2E_TEMPLATE_DIR := "$(abspath test/e2e/data/infrastructure-vsphere/)"
E2E_TEMPLATE_DIR := $(abspath test/e2e/data/infrastructure-vsphere/)
SKIP_RESOURCE_CLEANUP ?= false
USE_EXISTING_CLUSTER ?= false
GINKGO_NOCOLOR ?= false
Expand Down Expand Up @@ -286,32 +286,42 @@ generate-doctoc:
TRACE=$(TRACE) ./hack/generate-doctoc.sh

.PHONY: generate-e2e-templates
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, main) ## Generate test templates for all branches
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.7 v1.8 main) ## Generate test templates for all branches

.PHONY: generate-e2e-templates-main
generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the main branch
$(MAKE) e2e-flavors-main
cp $(RELEASE_DIR)/main/cluster-template.yaml $(E2E_TEMPLATE_DIR)/main/base/cluster-template.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/base > $(E2E_TEMPLATE_DIR)/main/cluster-template.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/hw-upgrade > $(E2E_TEMPLATE_DIR)/main/cluster-template-hw-upgrade.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/storage-policy > $(E2E_TEMPLATE_DIR)/main/cluster-template-storage-policy.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/remote-management > $(E2E_TEMPLATE_DIR)/main/cluster-template-remote-management.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/conformance > $(E2E_TEMPLATE_DIR)/main/cluster-template-conformance.yaml
cp "$(RELEASE_DIR)/main/cluster-template.yaml" "$(E2E_TEMPLATE_DIR)/main/base/cluster-template.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/base" > "$(E2E_TEMPLATE_DIR)/main/cluster-template.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/hw-upgrade" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-hw-upgrade.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/storage-policy" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-storage-policy.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/conformance" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-conformance.yaml"
# Since CAPI uses different flavor names for KCP and MD remediation using MHC
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/mhc-remediation/kcp > $(E2E_TEMPLATE_DIR)/main/cluster-template-kcp-remediation.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/mhc-remediation/md > $(E2E_TEMPLATE_DIR)/main/cluster-template-md-remediation.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/node-drain > $(E2E_TEMPLATE_DIR)/main/cluster-template-node-drain.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/ignition > $(E2E_TEMPLATE_DIR)/main/cluster-template-ignition.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/mhc-remediation/kcp" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-kcp-remediation.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/mhc-remediation/md" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-md-remediation.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/node-drain" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-node-drain.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/ignition" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-ignition.yaml"
# generate clusterclass and cluster topology
cp $(RELEASE_DIR)/main/cluster-template-topology.yaml $(E2E_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml
cp $(RELEASE_DIR)/main/clusterclass-template.yaml $(E2E_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/topology > $(E2E_TEMPLATE_DIR)/main/cluster-template-topology.yaml
cp "$(RELEASE_DIR)/main/clusterclass-template.yaml" "$(E2E_TEMPLATE_DIR)/main/clusterclass/clusterclass-quick-start.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml"
cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/topology" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-topology.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/remote-management" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-remote-management.yaml"
# for PCI passthrough template
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/pci > $(E2E_TEMPLATE_DIR)/main/cluster-template-pci.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/pci" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-pci.yaml"
# for DHCP overrides
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/dhcp-overrides > $(E2E_TEMPLATE_DIR)/main/cluster-template-dhcp-overrides.yaml
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/ownerreferences > $(E2E_TEMPLATE_DIR)/main/cluster-template-ownerreferences.yaml

"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/dhcp-overrides" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-dhcp-overrides.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/ownerreferences" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-ownerreferences.yaml"

.PHONY: generate-e2e-templates-v1.8
generate-e2e-templates-v1.8: $(KUSTOMIZE)
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/v1.8/clusterclass" > "$(E2E_TEMPLATE_DIR)/v1.8/clusterclass-quick-start.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/v1.8/workload" > "$(E2E_TEMPLATE_DIR)/v1.8/cluster-template-workload.yaml"

.PHONY: generate-e2e-templates-v1.7
generate-e2e-templates-v1.7: $(KUSTOMIZE)
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/v1.7/clusterclass" > "$(E2E_TEMPLATE_DIR)/v1.7/clusterclass-quick-start.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/v1.7/workload" > "$(E2E_TEMPLATE_DIR)/v1.7/cluster-template-workload.yaml"

## --------------------------------------
## Lint / Verify
Expand Down
9 changes: 2 additions & 7 deletions packaging/flavorgen/flavors/clusterclass_generators.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ func getWorkersClass() clusterv1.WorkersClass {

func getClusterClassPatches() []clusterv1.ClusterClassPatch {
return []clusterv1.ClusterClassPatch{
createFilesArrayPatch(),
enableSSHPatch(),
infraClusterPatch(),
kubeVipEnabledPatch(),
Expand Down Expand Up @@ -223,12 +224,6 @@ func newVSphereClusterTemplate() infrav1.VSphereClusterTemplate {
}

func newKubeadmControlPlaneTemplate(templateName string) controlplanev1.KubeadmControlPlaneTemplate {
files := []bootstrapv1.File{
{
Owner: "root:root",
Path: "/etc/kubernetes/manifests/kube-vip.yaml",
},
}
return controlplanev1.KubeadmControlPlaneTemplate{
TypeMeta: metav1.TypeMeta{
Kind: util.TypeToKind(&controlplanev1.KubeadmControlPlaneTemplate{}),
Expand All @@ -241,7 +236,7 @@ func newKubeadmControlPlaneTemplate(templateName string) controlplanev1.KubeadmC
Spec: controlplanev1.KubeadmControlPlaneTemplateSpec{
Template: controlplanev1.KubeadmControlPlaneTemplateResource{
Spec: controlplanev1.KubeadmControlPlaneTemplateResourceSpec{
KubeadmConfigSpec: defaultKubeadmInitSpec(files),
KubeadmConfigSpec: defaultKubeadmInitSpec([]bootstrapv1.File{}),
},
},
},
Expand Down
55 changes: 52 additions & 3 deletions packaging/flavorgen/flavors/patches.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package flavors
import (
"fmt"

apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/utils/pointer"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
Expand All @@ -29,6 +30,52 @@ import (
"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util"
)

func createFilesArrayPatch() clusterv1.ClusterClassPatch {
return clusterv1.ClusterClassPatch{
Name: "createFilesArray",
Definitions: []clusterv1.PatchDefinition{
{
Selector: clusterv1.PatchSelector{
APIVersion: controlplanev1.GroupVersion.String(),
Kind: util.TypeToKind(&controlplanev1.KubeadmControlPlaneTemplate{}),
MatchResources: clusterv1.PatchSelectorMatch{
ControlPlane: true,
},
},
JSONPatches: []clusterv1.JSONPatch{
{
Op: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/files",
Value: &apiextensionsv1.JSON{
Raw: []byte("[]"),
},
},
},
},
{
Selector: clusterv1.PatchSelector{
APIVersion: bootstrapv1.GroupVersion.String(),
Kind: util.TypeToKind(&bootstrapv1.KubeadmConfigTemplate{}),
MatchResources: clusterv1.PatchSelectorMatch{
MachineDeploymentClass: &clusterv1.PatchSelectorMatchMachineDeploymentClass{
Names: []string{fmt.Sprintf("%s-worker", env.ClusterClassNameVar)},
},
},
},
JSONPatches: []clusterv1.JSONPatch{
{
Op: "add",
Path: "/spec/template/spec/files",
Value: &apiextensionsv1.JSON{
Raw: []byte("[]"),
},
},
},
},
},
}
}

func enableSSHPatch() clusterv1.ClusterClassPatch {
return clusterv1.ClusterClassPatch{
Name: "enableSSHIntoNodes",
Expand Down Expand Up @@ -126,7 +173,7 @@ func infraClusterPatch() clusterv1.ClusterClassPatch {

func kubeVipEnabledPatch() clusterv1.ClusterClassPatch {
return clusterv1.ClusterClassPatch{
Name: "kubeVipEnabled",
Name: "kubeVipPodManifest",
Definitions: []clusterv1.PatchDefinition{
{
Selector: clusterv1.PatchSelector{
Expand All @@ -139,9 +186,11 @@ func kubeVipEnabledPatch() clusterv1.ClusterClassPatch {
JSONPatches: []clusterv1.JSONPatch{
{
Op: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/files/0/content",
Path: "/spec/template/spec/kubeadmConfigSpec/files/-",
ValueFrom: &clusterv1.JSONPatchValue{
Variable: pointer.String("kubeVipPodManifest"),
Template: pointer.String(`owner: root:root
path: "/etc/kubernetes/manifests/kube-vip.yaml"
content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest (printf "$1 %s" .controlPlaneIpAddr)) }}`),
},
},
},
Expand Down
34 changes: 28 additions & 6 deletions templates/clusterclass-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,28 @@ spec:
name: '${CLUSTER_CLASS_NAME}'
namespace: '${NAMESPACE}'
patches:
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files
value: []
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
- jsonPatches:
- op: add
path: /spec/template/spec/files
value: []
selector:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
matchResources:
machineDeploymentClass:
names:
- ${CLUSTER_CLASS_NAME}-worker
name: createFilesArray
- definitions:
- jsonPatches:
- op: add
Expand Down Expand Up @@ -96,15 +118,18 @@ spec:
- definitions:
- jsonPatches:
- op: add
path: /spec/template/spec/kubeadmConfigSpec/files/0/content
path: /spec/template/spec/kubeadmConfigSpec/files/-
valueFrom:
variable: kubeVipPodManifest
template: |-
owner: root:root
path: "/etc/kubernetes/manifests/kube-vip.yaml"
content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest (printf "$1 %s" .controlPlaneIpAddr)) }}
selector:
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
matchResources:
controlPlane: true
name: kubeVipEnabled
name: kubeVipPodManifest
variables:
- name: sshKey
required: false
Expand Down Expand Up @@ -228,9 +253,6 @@ spec:
controllerManager:
extraArgs:
cloud-provider: external
files:
- owner: root:root
path: /etc/kubernetes/manifests/kube-vip.yaml
initConfiguration:
nodeRegistration:
criSocket: /var/run/containerd/containerd.sock
Expand Down
70 changes: 70 additions & 0 deletions test/e2e/clusterctl_upgrade_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"

. "github.com/onsi/ginkgo/v2"
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
)

var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.8=>current, CAPI 1.5=>1.6) [ClusterClass]", func() {
capi_e2e.ClusterctlUpgradeSpec(context.TODO(), func() capi_e2e.ClusterctlUpgradeSpecInput {
return capi_e2e.ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithProvidersContract: "v1beta1",
MgmtFlavor: "remote-management",
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.3/clusterctl-{OS}-{ARCH}",
InitWithCoreProvider: "cluster-api:v1.5.3",
InitWithBootstrapProviders: []string{"kubeadm:v1.5.3"},
InitWithControlPlaneProviders: []string{"kubeadm:v1.5.3"},
InitWithInfrastructureProviders: []string{"vsphere:v1.8.4"},
InitWithRuntimeExtensionProviders: []string{},
InitWithKubernetesVersion: "v1.28.0",
WorkloadKubernetesVersion: "v1.28.0",
WorkloadFlavor: "workload",
}
})
})

var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.7=>current, CAPI 1.4=>1.6) [ClusterClass]", func() {
capi_e2e.ClusterctlUpgradeSpec(context.TODO(), func() capi_e2e.ClusterctlUpgradeSpecInput {
return capi_e2e.ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithProvidersContract: "v1beta1",
MgmtFlavor: "remote-management",
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.7/clusterctl-{OS}-{ARCH}",
InitWithCoreProvider: "cluster-api:v1.4.7",
InitWithBootstrapProviders: []string{"kubeadm:v1.4.7"},
InitWithControlPlaneProviders: []string{"kubeadm:v1.4.7"},
InitWithInfrastructureProviders: []string{"vsphere:v1.7.4"},
InitWithRuntimeExtensionProviders: []string{},
InitWithKubernetesVersion: "v1.27.3",
WorkloadKubernetesVersion: "v1.27.3",
WorkloadFlavor: "workload",
}
})
})
Loading

0 comments on commit 6e5d1f1

Please sign in to comment.