Update RP and Gateway vmss OS image to cbl-mariner-2-gen2-fips.

Restructure VMSS bootstrap bash scripts for increased reliability, and easier debugging

Move all shared code into a commonly shared file to be sourced by all
bootstrapping scripts. This allows for code reuse, minimal duplication.
Fix mdm mdsd certificate download script
Increase rpm retry time to 30 minutes total, every 30 seconds.

Install Azure Security Monitor via VMSS Extension

Remove RHUI and Microsoft repo configuration, add Mariner Extended repo config

Remove lvm disk resize
Mariner does not use lvm, the disk is automatically grown to the full size specified.

Firewalld configuration has been removed, as Mariner does not have the
requirements to support the nftables backend.
Firewall rules will be configured at the vnet level in Azure.

Remove semanage
Mariner Linux does not have selinux configured.

Add changes to remove CHECKACCESS
Merged in PR #3643

Remove gateway log rotation config
Log rotation for the podman level driver log was not the correct
approach. The podman log driver is now journald, so all logs will be
shipped to journald rather than a ctr.log file.

During mdm and mdsd setup, I've added wait steps for the download
scripts to complete getting certificates. Without this, the download
scripts run in a subshell and fixing up the certificates fails.

Add firewalld configuration, required for podman networking
Add podman aro network creation to isolate RP containers from possible
interaction on the default podman network.

pkg/deploy/assets/env-development.json

@@ -367,7 +367,7 @@
                                     "autoUpgradeMinorVersion": true,
                                     "settings": {},
                                     "protectedSettings": {
-                                        "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'PROXYIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImage')),''')\n','PROXYIMAGEAUTH=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImageAuth')),''')\n','PROXYCERT=''',parameters('proxyCert'),'''\n','PROXYCLIENTCERT=''',parameters('proxyClientCert'),'''\n','PROXYKEY=''',parameters('proxyKey'),'''\n','\n',base64ToString('I0FkZGluZyByZXRyeSBsb2dpYyB0byB5dW0gY29tbWFuZHMgaW4gb3JkZXIgdG8gYXZvaWQgc3RhbGxpbmcgb3V0IG9uIHJlc291cmNlIGxvY2tzCmVjaG8gImluc3RhbGxpbmcgbW9ieS1lbmdpbmUgKGRvY2tlcikiCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCgl0ZG5mIGluc3RhbGwgLXkgbW9ieS1lbmdpbmUgbW9ieS1jbGkgJiYgYnJlYWsKCWlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKc3lzdGVtY3RsIGVuYWJsZSBkb2NrZXIKc3lzdGVtY3RsIHN0YXJ0IGRvY2tlcgoKbWtkaXIgL3Jvb3QvLmRvY2tlcgpjYXQgPi9yb290Ly5kb2NrZXIvY29uZmlnLmpzb24gPDxFT0YKewoJImF1dGhzIjogewoJCSIke1BST1hZSU1BR0UlJS8qfSI6IHsKCQkJImF1dGgiOiAiJFBST1hZSU1BR0VBVVRIIgoJCX0KCX0KfQpFT0YKCmRvY2tlciBwdWxsICIkUFJPWFlJTUFHRSIKCm1rZGlyIC9ldGMvcHJveHkKYmFzZTY0IC1kIDw8PCIkUFJPWFlDRVJUIiA+L2V0Yy9wcm94eS9wcm94eS5jcnQKYmFzZTY0IC1kIDw8PCIkUFJPWFlLRVkiID4vZXRjL3Byb3h5L3Byb3h5LmtleQpiYXNlNjQgLWQgPDw8IiRQUk9YWUNMSUVOVENFUlQiID4vZXRjL3Byb3h5L3Byb3h5LWNsaWVudC5jcnQKY2hvd24gLVIgMTAwMDoxMDAwIC9ldGMvcHJveHkKY2htb2QgMDYwMCAvZXRjL3Byb3h5L3Byb3h5LmtleQoKY2F0ID4vZXRjL3N5c2NvbmZpZy9wcm94eSA8PEVPRgpQUk9YWV9JTUFHRT0nJFBST1hZSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3Byb3h5LnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9wcm94eQpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlbgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biAtLXJtIC0tbmFtZSAlbiAtcCA0NDM6ODQ0MyAtdiAvZXRjL3Byb3h5Oi9zZWNyZXRzICRQUk9YWV9JTUFHRQpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlbgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCnN5c3RlbWN0bCBlbmFibGUgcHJveHkuc2VydmljZQoKY2F0ID4vZXRjL2Nyb24ud2Vla2x5L3B1bGwtaW1hZ2UgPDwnRU9GJwojIS9iaW4vYmFzaAoKZG9ja2VyIHB1bGwgJFBST1hZSU1BR0UKc3lzdGVtY3RsIHJlc3RhcnQgcHJveHkuc2VydmljZQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLndlZWtseS9wdWxsLWltYWdlCgpjYXQgPi9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlIDw8J0VPRicKIyEvYmluL2Jhc2gKCnl1bSB1cGRhdGUgLXkKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlCgpjYXQgPi9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5IDw8J0VPRicKIyEvYmluL2Jhc2gKCnN5c3RlbWN0bCByZXN0YXJ0IHByb3h5LnNlcnZpY2UKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5CgooCglzbGVlcCAzMAoJcmVib290CikgJgo=')))]"
+                                        "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'PROXYIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImage')),''')\n','PROXYIMAGEAUTH=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImageAuth')),''')\n','PROXYCERT=''',parameters('proxyCert'),'''\n','PROXYCLIENTCERT=''',parameters('proxyClientCert'),'''\n','PROXYKEY=''',parameters('proxyKey'),'''\n','\n',base64ToString('IyEvYmluL2Jhc2gKI0FkZGluZyByZXRyeSBsb2dpYyB0byB5dW0gY29tbWFuZHMgaW4gb3JkZXIgdG8gYXZvaWQgc3RhbGxpbmcgb3V0IG9uIHJlc291cmNlIGxvY2tzCmVjaG8gImluc3RhbGxpbmcgbW9ieS1lbmdpbmUgKGRvY2tlcikiCmZvciBhdHRlbXB0IGluIHsxLi42MH07IGRvCgl0ZG5mIGluc3RhbGwgLXkgbW9ieS1lbmdpbmUgbW9ieS1jbGkgJiYgYnJlYWsKCWlmIFtbICR7YXR0ZW1wdH0gLWx0IDYwIF1dOyB0aGVuIHNsZWVwIDMwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKc3lzdGVtY3RsIGVuYWJsZSBkb2NrZXIKc3lzdGVtY3RsIHN0YXJ0IGRvY2tlcgoKbWtkaXIgL3Jvb3QvLmRvY2tlcgpjYXQgPi9yb290Ly5kb2NrZXIvY29uZmlnLmpzb24gPDxFT0YKewoJImF1dGhzIjogewoJCSIke1BST1hZSU1BR0UlJS8qfSI6IHsKCQkJImF1dGgiOiAiJFBST1hZSU1BR0VBVVRIIgoJCX0KCX0KfQpFT0YKCmRvY2tlciBwdWxsICIkUFJPWFlJTUFHRSIKCm1rZGlyIC9ldGMvcHJveHkKYmFzZTY0IC1kIDw8PCIkUFJPWFlDRVJUIiA+L2V0Yy9wcm94eS9wcm94eS5jcnQKYmFzZTY0IC1kIDw8PCIkUFJPWFlLRVkiID4vZXRjL3Byb3h5L3Byb3h5LmtleQpiYXNlNjQgLWQgPDw8IiRQUk9YWUNMSUVOVENFUlQiID4vZXRjL3Byb3h5L3Byb3h5LWNsaWVudC5jcnQKY2hvd24gLVIgMTAwMDoxMDAwIC9ldGMvcHJveHkKY2htb2QgMDYwMCAvZXRjL3Byb3h5L3Byb3h5LmtleQoKY2F0ID4vZXRjL3N5c2NvbmZpZy9wcm94eSA8PEVPRgpQUk9YWV9JTUFHRT0nJFBST1hZSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3Byb3h5LnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9wcm94eQpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlbgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biAtLXJtIC0tbmFtZSAlbiAtcCA0NDM6ODQ0MyAtdiAvZXRjL3Byb3h5Oi9zZWNyZXRzICRQUk9YWV9JTUFHRQpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlbgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCnN5c3RlbWN0bCBlbmFibGUgcHJveHkuc2VydmljZQoKY2F0ID4vZXRjL2Nyb24ud2Vla2x5L3B1bGwtaW1hZ2UgPDwnRU9GJwojIS9iaW4vYmFzaAoKZG9ja2VyIHB1bGwgJFBST1hZSU1BR0UKc3lzdGVtY3RsIHJlc3RhcnQgcHJveHkuc2VydmljZQpFT0YKY2htb2QgK3ggL2V0Yy9jcm9uLndlZWtseS9wdWxsLWltYWdlCgpjYXQgPi9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlIDw8J0VPRicKIyEvYmluL2Jhc2gKCnl1bSB1cGRhdGUgLXkKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi53ZWVrbHkveXVtdXBkYXRlCgpjYXQgPi9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5IDw8J0VPRicKIyEvYmluL2Jhc2gKCnN5c3RlbWN0bCByZXN0YXJ0IHByb3h5LnNlcnZpY2UKRU9GCmNobW9kICt4IC9ldGMvY3Jvbi5kYWlseS9yZXN0YXJ0LXByb3h5CgooCglzbGVlcCAzMAoJcmVib290CikgJgo=')))]"
                                     },
                                     "provisionAfterExtensions": [
                                         "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent",

pkg/deploy/generator/resources_dev.go

@@ -101,7 +101,11 @@ func (g *generator) devProxyVMSS() *arm.Resource {
 		)
 	}
 
-	trailer := base64.StdEncoding.EncodeToString(scriptDevProxyVMSS)
+	var sb strings.Builder
+
+	sb.WriteString(string(scriptDevProxyVMSS))
+
+	trailer := base64.StdEncoding.EncodeToString([]byte(sb.String()))
 
 	parts = append(parts, "'\n'", fmt.Sprintf("base64ToString('%s')", trailer))
 

pkg/deploy/generator/resources_gateway.go

@@ -241,7 +241,19 @@ func (g *generator) gatewayVMSS() *arm.Resource {
 		"''')\n'",
 	)
 
-	trailer := base64.StdEncoding.EncodeToString(scriptGatewayVMSS)
+	var sb strings.Builder
+
+	// VMSS extensions only support one custom script
+	// Because of this, the util-*.sh scripts are prefixed to the bootstrapping script
+	// main is called at the end of the bootstrapping script, so appending them will not work
+	sb.WriteString(string(scriptUtilCommon))
+	sb.WriteString(string(scriptUtilPackages))
+	sb.WriteString(string(scriptUtilServices))
+	sb.WriteString(string(scriptUtilSystem))
+	sb.WriteString("\n#Start of gatewayVMSS.sh\n")
+	sb.WriteString(string(scriptGatewayVMSS))
+
+	trailer := base64.StdEncoding.EncodeToString([]byte(sb.String()))
 
 	parts = append(parts, "'\n'", fmt.Sprintf("base64ToString('%s')", trailer))
 
@@ -340,6 +352,23 @@ func (g *generator) gatewayVMSS() *arm.Resource {
 									},
 								},
 							},
+							{
+								// az-secmonitor package no longer needs to be manually installed
+								// References:
+								// 		https://eng.ms/docs/products/azure-linux/gettingstarted/aks/monitoring
+								//		https://msazure.visualstudio.com/ASMDocs/_wiki/wikis/ASMDocs.wiki/179541/Linux-AzSecPack-AutoConfig-Onboarding-(manual-for-C-AI)?anchor=3.1.1-using-arm-template-resource-elements
+								Name: to.StringPtr("AzureMonitorLinuxAgent"),
+								VirtualMachineScaleSetExtensionProperties: &mgmtcompute.VirtualMachineScaleSetExtensionProperties{
+									Publisher:               to.StringPtr("Microsoft.Azure.Monitor"),
+									EnableAutomaticUpgrade:  to.BoolPtr(true),
+									AutoUpgradeMinorVersion: to.BoolPtr(true),
+									TypeHandlerVersion:      to.StringPtr("1.0"),
+									Type:                    to.StringPtr("AzureMonitorLinuxAgent"),
+									Settings: map[string]interface{}{
+										"GCS_AUTO_CONFIG": true,
+									},
+								},
+							},
 						},
 					},
 					DiagnosticsProfile: &mgmtcompute.DiagnosticsProfile{

pkg/deploy/generator/resources_rp.go

@@ -441,7 +441,19 @@ func (g *generator) rpVMSS() *arm.Resource {
 		"''')\n'",
 	)
 
-	trailer := base64.StdEncoding.EncodeToString(scriptRpVMSS)
+	var sb strings.Builder
+
+	// VMSS extensions only support one custom script
+	// Because of this, the util-*.sh scripts are prefixed to the bootstrapping script
+	// main is called at the end of the bootstrapping script, so appending them will not work
+	sb.WriteString(string(scriptUtilCommon))
+	sb.WriteString(string(scriptUtilPackages))
+	sb.WriteString(string(scriptUtilServices))
+	sb.WriteString(string(scriptUtilSystem))
+	sb.WriteString("\n#Start of rpVMSS.sh\n")
+	sb.WriteString(string(scriptRpVMSS))
+
+	trailer := base64.StdEncoding.EncodeToString([]byte(sb.String()))
 
 	parts = append(parts, "'\n'", fmt.Sprintf("base64ToString('%s')", trailer))
 
@@ -476,6 +488,7 @@ func (g *generator) rpVMSS() *arm.Resource {
 						},
 					},
 					StorageProfile: &mgmtcompute.VirtualMachineScaleSetStorageProfile{
+						// https://eng.ms/docs/products/azure-linux/gettingstarted/azurevm/azurevm
 						ImageReference: &mgmtcompute.ImageReference{
 							Publisher: to.StringPtr("RedHat"),
 							Offer:     to.StringPtr("RHEL"),
@@ -537,6 +550,23 @@ func (g *generator) rpVMSS() *arm.Resource {
 									},
 								},
 							},
+							{
+								// az-secmonitor package no longer needs to be manually installed
+								// References:
+								// 		https://eng.ms/docs/products/azure-linux/gettingstarted/aks/monitoring
+								//		https://msazure.visualstudio.com/ASMDocs/_wiki/wikis/ASMDocs.wiki/179541/Linux-AzSecPack-AutoConfig-Onboarding-(manual-for-C-AI)?anchor=3.1.1-using-arm-template-resource-elements
+								Name: to.StringPtr("AzureMonitorLinuxAgent"),
+								VirtualMachineScaleSetExtensionProperties: &mgmtcompute.VirtualMachineScaleSetExtensionProperties{
+									Publisher:               to.StringPtr("Microsoft.Azure.Monitor"),
+									EnableAutomaticUpgrade:  to.BoolPtr(true),
+									AutoUpgradeMinorVersion: to.BoolPtr(true),
+									TypeHandlerVersion:      to.StringPtr("1.0"),
+									Type:                    to.StringPtr("AzureMonitorLinuxAgent"),
+									Settings: map[string]interface{}{
+										"GCS_AUTO_CONFIG": true,
+									},
+								},
+							},
 						},
 					},
 					DiagnosticsProfile: &mgmtcompute.DiagnosticsProfile{

pkg/deploy/generator/scripts.go

@@ -15,3 +15,15 @@ var scriptGatewayVMSS []byte
 
 //go:embed scripts/rpVMSS.sh
 var scriptRpVMSS []byte
+
+//go:embed scripts/util-system.sh
+var scriptUtilSystem []byte
+
+//go:embed scripts/util-services.sh
+var scriptUtilServices []byte
+
+//go:embed scripts/util-packages.sh
+var scriptUtilPackages []byte
+
+//go:embed scripts/util-common.sh
+var scriptUtilCommon []byte

pkg/deploy/generator/scripts/devProxyVMSS.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 #Adding retry logic to yum commands in order to avoid stalling out on resource locks
 echo "installing moby-engine (docker)"
 for attempt in {1..60}; do