E2E: Run multiple BMO upgrade tests in sequence

Signed-off-by: Huy Mai <huy.mai@est.tech>
metal3-io · Mar 4, 2024 · a0cf589 · a0cf589
1 parent 466bafc
commit a0cf589
Show file tree

Hide file tree

Showing 8 changed files with 233 additions and 139 deletions.
diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh
@@ -24,6 +24,7 @@ case "${GINKGO_FOCUS:-}" in
     export DEPLOY_IRONIC="false"
     export DEPLOY_BMO="false"
     export DEPLOY_CERT_MANAGER="false"
+    export GINKGO_NODES=1
     ;;
   *)
     export GINKGO_SKIP="${GINKGO_SKIP:-upgrade}"
@@ -90,10 +91,6 @@ fi
 export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMO_E2E_EMULATOR}.yaml"
 "${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}" baremetal-e2e
 
-# Set the number of ginkgo processes to the number of BMCs
-n_vms=$(yq '. | length' "${E2E_BMCS_CONF_FILE}")
-export GINKGO_NODES="${n_vms}"
-
 # Image server variables
 CIRROS_VERSION="0.6.2"
 IMAGE_FILE="cirros-${CIRROS_VERSION}-x86_64-disk.img"
@@ -113,7 +110,7 @@ docker run --name image-server-e2e -d \
 ssh-keygen -t ed25519 -f "${IMAGE_DIR}/ssh_testkey" -q -N ""
 
 # Generate credentials
-BMO_OVERLAYS=("${REPO_ROOT}/config/overlays/e2e" "${REPO_ROOT}/config/overlays/e2e-release-0.4" "${REPO_ROOT}/config/overlays/e2e-release-0.5")
+BMO_OVERLAYS=("${REPO_ROOT}/config/overlays/e2e" "${REPO_ROOT}/config/overlays/e2e-release-0.3" "${REPO_ROOT}/config/overlays/e2e-release-0.4" "${REPO_ROOT}/config/overlays/e2e-release-0.5")
 IRONIC_OVERLAY="${REPO_ROOT}/ironic-deployment/overlays/e2e"
 
 IRONIC_USERNAME="$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 12 | head -n 1)"
@@ -142,6 +139,18 @@ envsubst < "${REPO_ROOT}/ironic-deployment/components/basic-auth/ironic-auth-con
 echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
   "${IRONIC_OVERLAY}/ironic-htpasswd"
 
+IRONIC_WITH_INSPECTOR_OVERLAY="${REPO_ROOT}/ironic-deployment/overlays/e2e-with-inspector"
+envsubst < "${REPO_ROOT}/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \
+  "${IRONIC_WITH_INSPECTOR_OVERLAY}/ironic-auth-config"
+
+echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
+  "${IRONIC_WITH_INSPECTOR_OVERLAY}/ironic-htpasswd"
+IRONIC_INSPECTOR_AUTH_CONFIG_TPL="/tmp/ironic-inspector-auth-config-tpl"
+curl -o "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" https://raw.githubusercontent.com/metal3-io/baremetal-operator/release-0.5/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl 
+envsubst < "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" > \
+  "${IRONIC_WITH_INSPECTOR_OVERLAY}/ironic-inspector-auth-config"
+echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
+  "${IRONIC_INSPECTOR_PASSWORD}")" > "${IRONIC_WITH_INSPECTOR_OVERLAY}/ironic-inspector-htpasswd"
 
 # We need to gather artifacts/logs before exiting also if there are errors
 set +e

diff --git a/ironic-deployment/overlays/e2e-with-inspector/ironic-patch.yaml b/ironic-deployment/overlays/e2e-with-inspector/ironic-patch.yaml
@@ -0,0 +1,10 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ironic
+spec:
+  template:
+    spec:
+      containers:
+        - name: ironic-dnsmasq
+          $patch: delete
diff --git a/ironic-deployment/overlays/e2e-with-inspector/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e-with-inspector/ironic_bmo_configmap.env
@@ -0,0 +1,7 @@
+HTTP_PORT=6180
+PROVISIONING_IP=192.168.222.199
+CACHEURL=http://192.168.222.199/images
+IRONIC_FAST_TRACK=true
+IRONIC_KERNEL_PARAMS=console=ttyS0
+IRONIC_INSPECTOR_VLAN_INTERFACES=all
+USE_IRONIC_INSPECTOR=true
diff --git a/ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml b/ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml
@@ -0,0 +1,51 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: baremetal-operator-system
+resources:
+- ../../../config/namespace
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5
+
+components:
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5
+
+configMapGenerator:
+- envs:
+  - ironic_bmo_configmap.env
+  name: ironic-bmo-configmap
+  behavior: create
+
+patches:
+  - path: ironic-patch.yaml
+
+# NOTE: These credentials are generated automatically in hack/ci-e2e.sh
+secretGenerator:
+- name: ironic-htpasswd
+  behavior: create
+  envs:
+  - ironic-htpasswd
+- name: ironic-inspector-htpasswd
+  behavior: create
+  envs:
+  - ironic-inspector-htpasswd
+- name: ironic-auth-config
+  files:
+  - auth-config=ironic-auth-config
+- name: ironic-inspector-auth-config
+  files:
+  - auth-config=ironic-inspector-auth-config
+
+replacements:
+  # Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap
+  - source:
+      kind: ConfigMap
+      name: ironic-bmo-configmap
+      fieldPath: .data.PROVISIONING_IP
+    targets:
+      - select:
+          version: v1
+          group: cert-manager.io
+          kind: Certificate
+          name:
+        fieldPaths:
+          - .spec.ipAddresses.0
diff --git a/test/e2e/config/fixture.yaml b/test/e2e/config/fixture.yaml
@@ -23,7 +23,7 @@ variables:
   UPGRADE_DEPLOY_IRONIC: "false"
   UPGRADE_DEPLOY_BMO: "true"
   UPGRADE_DEPLOY_CERT_MANAGER: "true"
-  UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.5"
+  UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.3 ../../config/overlays/fixture-release-0.4 ../../config/overlays/fixture-release-0.5"
 
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"

diff --git a/test/e2e/config/ironic.yaml b/test/e2e/config/ironic.yaml
@@ -22,12 +22,14 @@ variables:
   DEPLOY_CERT_MANAGER: "true"
   BMO_KUSTOMIZATION: "../../config/overlays/e2e"
   IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e"
+  # IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e-with-inspector"
 
   # These settings are for the separate cluster that are used for upgrade tests
   UPGRADE_DEPLOY_IRONIC: "true"
   UPGRADE_DEPLOY_BMO: "true"
   UPGRADE_DEPLOY_CERT_MANAGER: "true"
-  UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/e2e-release-0.5"
+  UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/e2e-release-0.3 ../../config/overlays/e2e-release-0.4 ../../config/overlays/e2e-release-0.5"
+  UPGRADE_BMO_IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e-with-inspector ../../ironic-deployment/overlays/e2e-with-inspector ../../ironic-deployment/overlays/e2e"
 
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -54,6 +54,12 @@ var (
 
 	// the BMC instance to use in a parallel test.
 	bmc BMC
+
+	// the list of BMO Kustomization to upgrade from in BMO upgrade tests.
+	bmoFromKustomizations []string
+
+	// the list of Ironic Kustomization to use in BMO upgrade tests.
+	bmoUpgradeIronicKustomizations []string
 )
 
 func init() {
@@ -72,13 +78,15 @@ func TestE2e(t *testing.T) {
 	g.Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder)
 
 	RegisterFailHandler(Fail)
+	Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.")
+	e2eConfig = LoadE2EConfig(configPath)
+	bmoFromKustomizations = strings.Split(e2eConfig.GetVariable("UPGRADE_BMO_KUSTOMIZATION_FROM"), " ")
+	bmoUpgradeIronicKustomizations = strings.Split(e2eConfig.GetVariable("UPGRADE_BMO_IRONIC_KUSTOMIZATION"), " ")
 	RunSpecs(t, "E2e Suite")
 }
 
 var _ = SynchronizedBeforeSuite(func() []byte {
 	var kubeconfigPath string
-	Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.")
-	e2eConfig = LoadE2EConfig(configPath)
 
 	if useExistingCluster {
 		kubeconfigPath = os.Getenv("KUBECONFIG")