E2E: Run multiple BMO upgrade tests in sequence

Signed-off-by: Huy Mai <huy.mai@est.tech>

hack/ci-e2e.sh

@@ -24,6 +24,7 @@ case "${GINKGO_FOCUS:-}" in
     export DEPLOY_IRONIC="false"
     export DEPLOY_BMO="false"
     export DEPLOY_CERT_MANAGER="false"
+    export GINKGO_NODES=1
     ;;
   *)
     export GINKGO_SKIP="${GINKGO_SKIP:-upgrade}"
@@ -94,10 +95,6 @@ fi
 export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMO_E2E_EMULATOR}.yaml"
 "${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}" baremetal-e2e
 
-# Set the number of ginkgo processes to the number of BMCs
-n_vms=$(yq '. | length' "${E2E_BMCS_CONF_FILE}")
-export GINKGO_NODES="${n_vms}"
-
 # Image server variables
 CIRROS_VERSION="0.6.2"
 IMAGE_FILE="cirros-${CIRROS_VERSION}-x86_64-disk.img"
@@ -117,7 +114,7 @@ docker run --name image-server-e2e -d \
 ssh-keygen -t ed25519 -f "${IMAGE_DIR}/ssh_testkey" -q -N ""
 
 # Generate credentials
-BMO_OVERLAYS=("${REPO_ROOT}/config/overlays/e2e" "${REPO_ROOT}/config/overlays/e2e-release-0.4" "${REPO_ROOT}/config/overlays/e2e-release-0.5")
+BMO_OVERLAYS=("${REPO_ROOT}/config/overlays/e2e" "${REPO_ROOT}/config/overlays/e2e-release-0.3" "${REPO_ROOT}/config/overlays/e2e-release-0.4" "${REPO_ROOT}/config/overlays/e2e-release-0.5")
 IRONIC_OVERLAY="${REPO_ROOT}/ironic-deployment/overlays/e2e"
 
 IRONIC_USERNAME="$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 12 | head -n 1)"
@@ -143,6 +140,18 @@ done
 echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
   "${IRONIC_OVERLAY}/ironic-htpasswd"
 
+IRONIC_WITH_INSPECTOR_OVERLAY="${REPO_ROOT}/ironic-deployment/overlays/e2e-with-inspector"
+envsubst < "${REPO_ROOT}/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \
+  "${IRONIC_WITH_INSPECTOR_OVERLAY}/ironic-auth-config"
+
+echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
+  "${IRONIC_WITH_INSPECTOR_OVERLAY}/ironic-htpasswd"
+IRONIC_INSPECTOR_AUTH_CONFIG_TPL="/tmp/ironic-inspector-auth-config-tpl"
+curl -o "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" https://raw.githubusercontent.com/metal3-io/baremetal-operator/release-0.5/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl 
+envsubst < "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" > \
+  "${IRONIC_WITH_INSPECTOR_OVERLAY}/ironic-inspector-auth-config"
+echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
+  "${IRONIC_INSPECTOR_PASSWORD}")" > "${IRONIC_WITH_INSPECTOR_OVERLAY}/ironic-inspector-htpasswd"
 
 # We need to gather artifacts/logs before exiting also if there are errors
 set +e

ironic-deployment/overlays/e2e-with-inspector/ironic-patch.yaml

@@ -0,0 +1,10 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ironic
+spec:
+  template:
+    spec:
+      containers:
+        - name: ironic-dnsmasq
+          $patch: delete

ironic-deployment/overlays/e2e-with-inspector/ironic_bmo_configmap.env

@@ -0,0 +1,7 @@
+HTTP_PORT=6180
+PROVISIONING_IP=192.168.222.199
+CACHEURL=http://192.168.222.199/images
+IRONIC_FAST_TRACK=true
+IRONIC_KERNEL_PARAMS=console=ttyS0
+IRONIC_INSPECTOR_VLAN_INTERFACES=all
+USE_IRONIC_INSPECTOR=true

ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml

@@ -0,0 +1,51 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: baremetal-operator-system
+resources:
+- ../../../config/namespace
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5
+
+components:
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5
+
+configMapGenerator:
+- envs:
+  - ironic_bmo_configmap.env
+  name: ironic-bmo-configmap
+  behavior: create
+
+patches:
+  - path: ironic-patch.yaml
+
+# NOTE: These credentials are generated automatically in hack/ci-e2e.sh
+secretGenerator:
+- name: ironic-htpasswd
+  behavior: create
+  envs:
+  - ironic-htpasswd
+- name: ironic-inspector-htpasswd
+  behavior: create
+  envs:
+  - ironic-inspector-htpasswd
+- name: ironic-auth-config
+  files:
+  - auth-config=ironic-auth-config
+- name: ironic-inspector-auth-config
+  files:
+  - auth-config=ironic-inspector-auth-config
+
+replacements:
+  # Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap
+  - source:
+      kind: ConfigMap
+      name: ironic-bmo-configmap
+      fieldPath: .data.PROVISIONING_IP
+    targets:
+      - select:
+          version: v1
+          group: cert-manager.io
+          kind: Certificate
+          name:
+        fieldPaths:
+          - .spec.ipAddresses.0

test/e2e/config/fixture.yaml

@@ -19,11 +19,8 @@ variables:
   DEPLOY_CERT_MANAGER: "true"
   BMO_KUSTOMIZATION: "../../config/overlays/fixture"
 
-  # These settings are for the separate cluster that are used for upgrade tests
-  UPGRADE_DEPLOY_IRONIC: "false"
-  UPGRADE_DEPLOY_BMO: "true"
+  # This setting is for the separate cluster that are used for upgrade tests
   UPGRADE_DEPLOY_CERT_MANAGER: "true"
-  UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.5"
 
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
@@ -49,3 +46,20 @@ intervals:
   default/wait-deleted: ["5s", "10ms"]
   default/wait-secret-deletion: ["5s", "10ms"]
   default/wait-power-state: ["5s", "10ms"]
+
+bmoIronicUpgradeSpecs:
+- deployIronic: false
+  deployBMO: true
+  initBMOKustomization: "../../config/overlays/fixture-release-0.3"
+  upgradeEntityKustomization: "../../config/overlays/fixture"
+  upgradeEntityName: "bmo"
+- deployIronic: false
+  deployBMO: true
+  initBMOKustomization: "../../config/overlays/fixture-release-0.4"
+  upgradeEntityKustomization: "../../config/overlays/fixture"
+  upgradeEntityName: "bmo"
+- deployIronic: false
+  deployBMO: true
+  initBMOKustomization: "../../config/overlays/fixture-release-0.5"
+  upgradeEntityKustomization: "../../config/overlays/fixture"
+  upgradeEntityName: "bmo"

test/e2e/config/ironic.yaml

@@ -23,11 +23,8 @@ variables:
   BMO_KUSTOMIZATION: "../../config/overlays/e2e"
   IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e"
 
-  # These settings are for the separate cluster that are used for upgrade tests
-  UPGRADE_DEPLOY_IRONIC: "true"
-  UPGRADE_DEPLOY_BMO: "true"
+  # This setting is for the separate cluster that are used for upgrade tests
   UPGRADE_DEPLOY_CERT_MANAGER: "true"
-  UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/e2e-release-0.5"
 
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
@@ -55,3 +52,23 @@ intervals:
   default/wait-deleted: ["20s", "10ms"]
   default/wait-secret-deletion: ["1m", "1s"]
   default/wait-connect-ssh: ["2m", "10s"]
+
+bmoIronicUpgradeSpecs:
+- deployIronic: true
+  deployBMO: true
+  initBMOKustomization: "../../config/overlays/e2e-release-0.3"
+  initIronicKustomization: "../../ironic-deployment/overlays/e2e-with-inspector"
+  upgradeEntityKustomization: "../../config/overlays/e2e"
+  upgradeEntityName: "bmo"
+- deployIronic: true
+  deployBMO: true
+  initBMOKustomization: "../../config/overlays/e2e-release-0.4"
+  initIronicKustomization: "../../ironic-deployment/overlays/e2e-with-inspector"
+  upgradeEntityKustomization: "../../config/overlays/e2e"
+  upgradeEntityName: "bmo"
+- deployIronic: true
+  deployBMO: true
+  initBMOKustomization: "../../config/overlays/e2e-release-0.5"
+  initIronicKustomization: "../../ironic-deployment/overlays/e2e"
+  upgradeEntityKustomization: "../../config/overlays/e2e"
+  upgradeEntityName: "bmo"

test/e2e/e2e_config.go

@@ -30,6 +30,26 @@ const (
 	TryLoadImage LoadImageBehavior = "tryLoad"
 )
 
+type BMOIronicUpgradeInput struct {
+	// DeployIronic determines if Ironic should be installed at the beginning of the test.
+	// This should be generally set to `true`, but can be `false` in case Ironic is either pre-installed
+	// or not required (for e.g. in `fixture` setup)
+	DeployIronic bool `yaml:"deployIronic,omitempty"`
+	// DeployBMO determines if BMO should be installed at the beginning of the test.
+	// This should be generally set to `true`, but can be `false` in case BMO is pre-installed
+	DeployBMO bool `yaml:"deployBMO,omitempty"`
+	// Path to the Ironic kustomization that should be installed at the beginning of the test.
+	// Not used if DeployIronic is false
+	InitIronicKustomization string `yaml:"initIronicKustomization,omitempty"`
+	// Path to the BMO kustomization that should be installed at the beginning of the test.
+	// Not used if DeployBMO is false
+	InitBMOKustomization string `yaml:"initBMOKustomization,omitempty"`
+	// Name of the entity that should be upgraded and tested. It should be either `bmo` or `ironic`.
+	UpgradeEntityName string `yaml:"upgradeEntityName,omitempty"`
+	// Path to the kustomization of the entity that should be used in upgrading.
+	UpgradeEntityKustomization string `yaml:"upgradeEntityKustomization,omitempty"`
+}
+
 // Config defines the configuration of an e2e test environment.
 type Config struct {
 	// Images is a list of container images to load into the Kind cluster.
@@ -41,6 +61,9 @@ type Config struct {
 
 	// Intervals to be used for long operations during tests.
 	Intervals map[string][]string `json:"intervals,omitempty"`
+
+	// BMOIronicUpgradeSpecs
+	BMOIronicUpgradeSpecs []BMOIronicUpgradeInput `yaml:"bmoIronicUpgradeSpecs,omitempty"`
 }
 
 // LoadE2EConfig loads the configuration for the e2e test environment.

test/e2e/e2e_suite_test.go

@@ -72,13 +72,13 @@ func TestE2e(t *testing.T) {
 	g.Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder)
 
 	RegisterFailHandler(Fail)
+	Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.")
+	e2eConfig = LoadE2EConfig(configPath)
 	RunSpecs(t, "E2e Suite")
 }
 
 var _ = SynchronizedBeforeSuite(func() []byte {
 	var kubeconfigPath string
-	Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.")
-	e2eConfig = LoadE2EConfig(configPath)
 
 	if useExistingCluster {
 		kubeconfigPath = os.Getenv("KUBECONFIG")