From 804be2825062e4ad8d3e592cd14a151fd0fde0ce Mon Sep 17 00:00:00 2001 From: Anand Kumar Date: Tue, 6 Sep 2022 17:08:00 +0530 Subject: [PATCH] Add antrea monitor service for cleanup. The antrea-monitor service would run at system start up and clean up OVS configuration. Also fix typo and style issues. Also update documentation. Fixes #4122 Signed-off-by: Anand Kumar --- docs/external-node.md | 8 +- hack/externalnode/install-vm.ps1 | 127 +++++++++++++++++++++++-------- hack/externalnode/install-vm.sh | 105 +++++++++++++++++++------ 3 files changed, 180 insertions(+), 60 deletions(-) diff --git a/docs/external-node.md b/docs/external-node.md index eb64f28e96d..77bec384415 100644 --- a/docs/external-node.md +++ b/docs/external-node.md @@ -195,7 +195,7 @@ spec: change `vm-ns` to the right Namespace. ```bash - kubectl apply -f https://raw.githubusercontent.com/antrea-io/antrea/feature/externalnode/build/yamls/externalnode/vm-agent-rbac.yml + kubectl apply -f https://raw.githubusercontent.com/antrea-io/antrea/main/build/yamls/externalnode/vm-agent-rbac.yml ``` 4. Create `antrea-agent.kubeconfig` file for `antrea-agent` to access the K8S @@ -204,8 +204,9 @@ spec: ```bash export CLUSTER_NAME="kubernetes" export SERVICE_ACCOUNT="vm-agent" + export NAMESPACE="vm-ns" APISERVER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$CLUSTER_NAME\")].cluster.server}") - TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) + TOKEN=$(kubectl -n $NAMESPACE get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) kubectl config --kubeconfig=antrea-agent.kubeconfig set-cluster $CLUSTER_NAME --server=$APISERVER --insecure-skip-tls-verify=true kubectl config --kubeconfig=antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN kubectl config --kubeconfig=antrea-agent.kubeconfig set-context antrea-agent@$CLUSTER_NAME --cluster=$CLUSTER_NAME --user=antrea-agent @@ -221,7 +222,8 @@ spec: # to be exposed via the Node IP or a public IP that is reachable from the VM export ANTREA_API_SERVER="https://172.18.0.1:443" export ANTREA_CLUSTER_NAME="antrea" - TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) + export NAMESPACE="vm-ns" + TOKEN=$(kubectl -n $NAMESPACE get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-cluster $ANTREA_CLUSTER_NAME --server=$ANTREA_API_SERVER --insecure-skip-tls-verify=true kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-context antrea-agent@$ANTREA_CLUSTER_NAME --cluster=$ANTREA_CLUSTER_NAME --user=antrea-agent diff --git a/hack/externalnode/install-vm.ps1 b/hack/externalnode/install-vm.ps1 index 00668a5fb44..db4cee63061 100644 --- a/hack/externalnode/install-vm.ps1 +++ b/hack/externalnode/install-vm.ps1 @@ -1,6 +1,6 @@ <# .SYNOPSIS - Installs Antrea-Agent service. + Installs Antrea-Agent and Antrea-Monitor service. .PARAMETER Namespace ExternalNode Namespace to be used. @@ -17,8 +17,17 @@ .PARAMETER AntreaKubeConfigPath Specifies the path of the kubeconfig to access Antrea API Server. + .PARAMETER NodeName + Specifies the ExternalNode name to be used by the antrea-agent. + + .PARAMETER OVSBridge + Specifies the OVS bridge name. + .PARAMETER InstallDir The target installation directory. The default path is "C:\antrea-agent". + + .PARAMETER ClearOVS + Clears OVS configuration. #> Param( [parameter(Mandatory = $true)] [string] $Namespace, @@ -27,28 +36,30 @@ Param( [parameter(Mandatory = $true)] [string] $KubeConfigPath, [parameter(Mandatory = $true)] [string] $AntreaKubeConfigPath, [parameter(Mandatory = $false)] [string] $NodeName = $(hostname), - [parameter(Mandatory = $false)] [string] $InstallDir = "C:\antrea-agent" + [parameter(Mandatory = $false)] [string] $OVSBridge = "br-int", + [parameter(Mandatory = $false)] [string] $InstallDir = "C:\antrea-agent", + [parameter(Mandatory = $false)] [string] $ClearOVS = "false" ) $ErrorActionPreference = "Stop" - -$WorkDir = [System.IO.Path]::GetDirectoryName($myInvocation.MyCommand.Definition) -$InstallLog = "$WorkDir\install_vm.log" - # Antrea paths $AntreaAgentPath = [io.path]::combine($InstallDir, "antrea-agent.exe") $AntreaAgentConfDir = [io.path]::combine($InstallDir, "conf") $AntreaAgentLogDir = [io.path]::combine($InstallDir, "logs") $AntreaAgentConfPath = [io.path]::combine($AntreaAgentConfDir, "antrea-agent.conf") $AntreaAgentLogFile = [io.path]::combine($AntreaAgentLogDir, "antrea-agent.log") +$InstallLog = [io.path]::combine($AntreaAgentLogDir, "install_vm.log") # Constants +$AntreaAgent = "antrea-agent" +$AntreaCleanup = "antrea-cleanup" +$AntreaSwitch = "antrea-switch" +$OVSServices = "ovsdb-server", "ovs-vswitchd" $K8sKubeconfig = "antrea-agent.kubeconfig" $AntreaKubeconfig = "antrea-agent.antrea.kubeconfig" -$OVSServices = "ovsdb-server", "ovs-vswitchd" -$AntreaAgent = "antrea-agent" -$Kubeconfig = "kubeconfig" +$Bridge = "ovsBridge" $ExternalNodeNamespace = "externalNodeNamespace" +$Kubeconfig = "kubeconfig" # List of supported OS versions, verified by antrea # Versions are named like Major.Minor.Build @@ -66,8 +77,20 @@ function ServiceExists($ServiceName) { return $false } +function StartService($name, $ignoreError=$false) { + try { + Start-Service $name + } catch { + if ( -Not $ignoreError) { + Log "Failed to start $name service, rc $_" + exit 1 + } + Log "Ignoring start error for $name service" + } +} + function CheckSupportedVersions() { - echo "Checking supported Windows OS versions" + Log "Checking supported Windows OS versions" $OSVersion = [System.Environment]::OSVersion.Version $Version = $OSVersion.Major.ToString() + "." + $OSVersion.Minor.ToString() + "." + $OSVersion.Build.ToString() foreach ($v in $SupportedVersions) { @@ -79,16 +102,14 @@ function CheckSupportedVersions() { exit 1 } -function PrintPrerequisites() -{ - echo "Please execute these commands to enable Hyper-V" - echo "Install-WindowsFeature Hyper-V-Powershell" - echo "Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All -NoRestart" +function PrintPrerequisites() { + Write-Host "Please execute these commands to enable Hyper-V" + Write-Host "Install-WindowsFeature Hyper-V-Powershell" + Write-Host "Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All -NoRestart" exit 1 } -function CheckPrerequisites() -{ +function CheckPrerequisites() { CheckSupportedVersions $valid = $true Log "Check Hyper-v feature is enabled" @@ -116,7 +137,6 @@ function CheckPrerequisites() } function SetupInstallDir() { - Log "Create install directories" if (-Not (Test-Path $AntreaAgentConfDir)) { New-Item $AntreaAgentConfDir -type directory -Force | Out-Null } @@ -167,6 +187,10 @@ function UpdateAgentConf() { Log "Updating $AntreaAgentConfPath with ${ExternalNodeNamespace}: ${Namespace}" [System.IO.File]::AppendAllText($AntreaAgentConfPath, " ${ExternalNodeNamespace}: ${Namespace}" + ([Environment]::NewLine)) + } elseif ($line -like "*$Bridge*") { + Log "Updating $AntreaAgentConfPath with ${Bridge}: ${OVSBridge}" + [System.IO.File]::AppendAllText($AntreaAgentConfPath, "${Bridge}: ${OVSBridge}" + + ([Environment]::NewLine)) } else { [System.IO.File]::AppendAllText($AntreaAgentConfPath, $line + ([Environment]::NewLine)) @@ -174,34 +198,71 @@ function UpdateAgentConf() { } } +function ClearOVSConfig() { + Log "Deleting OVS bridge $OVSBridge" + Stop-Service $AntreaAgent + try { + $adapterName = (Get-VMNetworkAdapter -SwitchName $AntreaSwitch -ManagementOS).Name + ovs-vsctl.exe del-br $OVSBridge + } catch { + Log "Failed to get VMSwitch $AntreaSwitch , rc $_" + exit 1 + } + + try { + Remove-VMSwitch -ComputerName $(hostname.exe) $AntreaSwitch -Force + } catch { + Log "Ignore error while removing VMSwitch, rc $_" + } + + try { + Rename-NetAdapter -Name "$adapterName~" -NewName "$adapterName" + } catch { + Log "Failed to rename network adapter $adapterName~ to $adapterName, rc $_" + exit 1 + } + Start-Service $AntreaAgent + Log "Done Deleting OVS bridge $OVSBridge" +} + +function ConfigureAntreaCleanupService() { + Log "Copying install script to $AntreaAgentConfDir" + Copy-Item $myInvocation.PSCommandPath "$AntreaAgentConfDir\install-vm.ps1" + $Binary = (Get-Command Powershell).Source + $Arguments = "-ExecutionPolicy Bypass -NoProfile -File $AntreaAgentConfDir\install-vm.ps1 -ClearOVS true -OVSBridge $OVSBridge -Namespace $Namespace -BinaryPath $BinaryPath -ConfigPath $ConfigPath -KubeConfigPath $KubeConfigPath -AntreaKubeConfigPath $AntreaKubeConfigPath" + & nssm install $AntreaCleanup $Binary $Arguments + # Configure service to avoid restarts upon exit + nssm set $AntreaCleanup AppExit Default Exit +} + function ConfigureAntreaAgentService() { # Set environment variables [Environment]::SetEnvironmentVariable("NODE_NAME", $NodeName, [System.EnvironmentVariableTarget]::Machine) # Assume nssm is installed and configure service $AntreaAgentArgs = "--config $AntreaAgentConfPath --log_file $AntreaAgentLogFile --logtostderr=false" - log "Creating service $AntreaAgent $AntreaAgentPath $AntreaAgentArgs" + Log "Creating service $AntreaAgent $AntreaAgentPath $AntreaAgentArgs" try { # Configured to auto-restart upon reboot & nssm install $AntreaAgent $AntreaAgentPath $AntreaAgentArgs } catch { - log "Failed to create service for $AntreaAgent, rc $_" + Log "Failed to create service for $AntreaAgent, rc $_" exit 1 } } -function StartAntreaAgentService() -{ - try { - & nssm start $AntreaAgent - } catch { - log "Failed to start service for $AntreaAgent, rc $_" - exit 1 - } +function StartAntreaServices() { + StartService $AntreaCleanup $true + StartService $AntreaAgent } -CheckPrerequisites SetupInstallDir -CopyAntreaAgentFiles -UpdateAgentConf -ConfigureAntreaAgentService -StartAntreaAgentService +if ( $ClearOVS -eq "true") { + ClearOVSConfig +} else { + CheckPrerequisites + CopyAntreaAgentFiles + UpdateAgentConf + ConfigureAntreaCleanupService + ConfigureAntreaAgentService + StartAntreaServices +} diff --git a/hack/externalnode/install-vm.sh b/hack/externalnode/install-vm.sh index 07c3964d2b8..45ea5213357 100644 --- a/hack/externalnode/install-vm.sh +++ b/hack/externalnode/install-vm.sh @@ -20,13 +20,15 @@ function echoerr { >&2 echo "$@" } -_usage="Usage: $0 [--ns ] [--bin ] [--config ] [--kubeconfig ] [--antrea-kubeconfig ] [--nodename ] [--help|-h] - --ns Namespace to be used by the antrea-agent. +_usage="Usage: $0 [--ns ] [--bin ] [--config ] [--kubeconfig ] [--antrea-kubeconfig ] [--nodename ] [--ovs-bridge ] [--clear-ovs] [--help|-h] + --ns Namespace to be used by the antrea-agent --bin Path of the antrea-agent binary --config Path of the antrea-agent configuration file --kubeconfig Path of the kubeconfig to access K8s API Server --antrea-kubeconfig Path of the kubeconfig to access Antrea API Server --nodename ExternalNode name to be used by the antrea-agent + --ovs-bridge Specify the OVS bridge name + --clear-ovs Clears OVS configuration --help, -h Print this message and exit Please run the script as sudo user" @@ -40,24 +42,26 @@ function print_help { } INSTALL_PATH="/usr/sbin" -AGENT_BIN_PATH="" -CONFIG_PATH="" -KUBECONFIG="" -ANTREAKUBECONFIG="" -AGENT_NAMESPACE="" -NODE_NAME="$(hostname)" +ANTREA_AGENT="antrea-agent" +ANTREA_CLEANUP="antrea-cleanup" AGENT_LOG_DIR="/var/log/antrea" AGENT_CONF_PATH="/etc/antrea" +OVS_BRIDGE="br-int" + +# Optional arguments +CLEAR_CONFIG=false +NODE_NAME="$(hostname)" + # List of supported OS versions, verified by antrea. -declare -a SUPPORTED_OS=("Ubuntu 18.04", "Ubuntu 20.04") +declare -a SUPPORTED_OS=("Ubuntu 18.04" "Ubuntu 20.04") check_supported_platform() { echo "Checking supported OS platform" dist_version="$(lsb_release -is) $(lsb_release -rs)" for ver in "${SUPPORTED_OS[@]}"; do - if [ "$ver" == "$dist_version" ]; then - return - fi + if [ "$ver" == "$dist_version" ]; then + return + fi done echoerr "Error ${SUPPORTED_OS[*]} are supported" exit 1 @@ -65,16 +69,16 @@ check_supported_platform() { copy_antrea_agent_files() { if [[ ! -f "$CONFIG_PATH" ]]; then - echoerr "Error $CONFIG_PATH file not found" - exit 1 + echoerr "Error $CONFIG_PATH file not found" + exit 1 fi mkdir -p $AGENT_CONF_PATH echo "Copying $CONFIG_PATH to $AGENT_CONF_PATH" cp $CONFIG_PATH $AGENT_CONF_PATH if [[ ! -f "$KUBECONFIG" ]]; then - echoerr "Error $KUBECONFIG file not found" - exit 1 + echoerr "Error $KUBECONFIG file not found" + exit 1 fi echo "Copying $KUBECONFIG to $AGENT_CONF_PATH" @@ -82,8 +86,8 @@ copy_antrea_agent_files() { chmod 600 "${AGENT_CONF_PATH}/antrea-agent.kubeconfig" if [[ ! -f "$ANTREA_KUBECONFIG" ]]; then - echoerr "Error $ANTREA_KUBECONFIG file not found" - exit 1 + echoerr "Error $ANTREA_KUBECONFIG file not found" + exit 1 fi echo "Copying $ANTREA_KUBECONFIG to $AGENT_CONF_PATH" cp "$ANTREA_KUBECONFIG" "${AGENT_CONF_PATH}/antrea-agent.antrea.kubeconfig" @@ -93,14 +97,19 @@ copy_antrea_agent_files() { update_antrea_agent_conf() { echo "Updating clientConnection and antreaClientConnection" sed -i "s|kubeconfig: |kubeconfig: $AGENT_CONF_PATH/|g" $AGENT_CONF_PATH/antrea-agent.conf + if [[ -z "$AGENT_NAMESPACE" ]]; then + AGENT_NAMESPACE="default" + fi echo "Updating externalNodeNamespace to $AGENT_NAMESPACE" sed -i "s|#externalNodeNamespace: default|externalNodeNamespace: $AGENT_NAMESPACE|g" $AGENT_CONF_PATH/antrea-agent.conf + echo "Updating ovsBridge to $OVS_BRIDGE" + sed -i "s|#ovsBridge: br-int|ovsBridge: $OVS_BRIDGE|g" $AGENT_CONF_PATH/antrea-agent.conf } start_antrea_agent_service() { if [[ ! -f "$AGENT_BIN_PATH" ]]; then - echoerr "Error $AGENT_BIN_PATH file not found" - exit 1 + echoerr "Error $AGENT_BIN_PATH file not found" + exit 1 fi mkdir -p $AGENT_LOG_DIR mkdir -p $INSTALL_PATH @@ -120,10 +129,36 @@ Restart=on-failure WantedBy=multi-user.target EOF systemctl daemon-reload - systemctl enable antrea-agent - echo "Starting antrea-agent service" - systemctl start antrea-agent - systemctl status antrea-agent + systemctl enable "$ANTREA_AGENT" + echo "Starting ${ANTREA_AGENT} service" + systemctl start "$ANTREA_AGENT" + systemctl status "$ANTREA_AGENT" +} + +start_antrea_cleanup_service() { + echo "Copying $BASH_SOURCE to ${AGENT_CONF_PATH}/install-vm.sh" + cp "$BASH_SOURCE" "${AGENT_CONF_PATH}/install-vm.sh" + chmod +x "${AGENT_CONF_PATH}/install-vm.sh" + cat >/etc/systemd/system/antrea-cleanup.service << EOF +[Unit] +Description="antrea cleanup as a systemd service" +After=network.target +[Service] +ExecStart="${AGENT_CONF_PATH}/install-vm.sh" --clear-ovs --ovs-bridge $OVS_BRIDGE +[Install] +WantedBy=multi-user.target +EOF + systemctl daemon-reload + systemctl enable "$ANTREA_CLEANUP" + echo "Starting ${ANTREA_CLEANUP} service" + systemctl start "$ANTREA_CLEANUP" +} + +clear_ovs_config() { + echo "Deleting OVS bridge $OVS_BRIDGE" + systemctl stop "$ANTREA_AGENT" + ovs-vsctl del-br "$OVS_BRIDGE" + systemctl start "$ANTREA_AGENT" } validate_argument() { @@ -168,6 +203,14 @@ case $key in validate_argument $1, $2 shift 2 ;; + --ovs-bridge) + OVS_BRIDGE="$2" + shift 2 + ;; + --clear-ovs) + CLEAR_CONFIG=true + shift 1 + ;; -h|--help) print_usage exit 0 @@ -179,7 +222,21 @@ case $key in esac done +# Check whether OVS configuration needs to be cleaned up. +if [ "$CLEAR_CONFIG" = true ] ; then + clear_ovs_config + exit 0 +fi + +# Check for mandatory arguments. +if [ -z "$AGENT_BIN_PATH" ] || [ -z "$CONFIG_PATH" ] || [ -z "$KUBECONFIG" ] || [ -z "$ANTREA_KUBECONFIG" ] ; then + echoerr "Missing argument(s)" + print_usage + exit 1 +fi + check_supported_platform copy_antrea_agent_files update_antrea_agent_conf +start_antrea_cleanup_service start_antrea_agent_service