Run cleanup before running antrea-agent service

Before running the antrea-agent, validate whether ovs configuration is in good state. If there is an error, cleanup ovs configuration and then start antrea-agent. Also fix typo and style issues. Also update documentation. Fixes #4122 Signed-off-by: Anand Kumar <kumaranand@vmware.com>
antrea-io · Oct 13, 2022 · f350ff6 · f350ff6
1 parent 0dceb75
commit f350ff6
Show file tree

Hide file tree

Showing 3 changed files with 202 additions and 109 deletions.
diff --git a/docs/external-node.md b/docs/external-node.md
@@ -25,7 +25,6 @@
   - [Non-IP packet](#non-ip-packet)
   - [IP packet](#ip-packet)
 - [Limitations](#limitations)
-- [Known issues](#known-issues)
 <!-- /toc -->
 
 ## What is ExternalNode?
@@ -195,21 +194,23 @@ spec:
    change `vm-ns` to the right Namespace.
 
    ```bash
-   kubectl apply -f https://raw.githubusercontent.com/antrea-io/antrea/feature/externalnode/build/yamls/externalnode/vm-agent-rbac.yml
+   kubectl apply -f https://raw.githubusercontent.com/antrea-io/antrea/main/build/yamls/externalnode/vm-agent-rbac.yml
    ```
 
 4. Create `antrea-agent.kubeconfig` file for `antrea-agent` to access the K8S
    API server.
 
    ```bash
-   export CLUSTER_NAME="kubernetes"
-   export SERVICE_ACCOUNT="vm-agent"
+   CLUSTER_NAME="kubernetes"
+   SERVICE_ACCOUNT="vm-agent"
+   NAMESPACE="vm-ns"
+   KUBECONFIG="antrea-agent.kubeconfig"
    APISERVER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$CLUSTER_NAME\")].cluster.server}")
-   TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode)
-   kubectl config --kubeconfig=antrea-agent.kubeconfig set-cluster $CLUSTER_NAME --server=$APISERVER --insecure-skip-tls-verify=true
-   kubectl config --kubeconfig=antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN
-   kubectl config --kubeconfig=antrea-agent.kubeconfig set-context antrea-agent@$CLUSTER_NAME --cluster=$CLUSTER_NAME --user=antrea-agent
-   kubectl config --kubeconfig=antrea-agent.kubeconfig use-context antrea-agent@$CLUSTER_NAME
+   TOKEN=$(kubectl -n $NAMESPACE get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode)
+   kubectl config --kubeconfig=$KUBECONFIG set-cluster $CLUSTER_NAME --server=$APISERVER --insecure-skip-tls-verify=true
+   kubectl config --kubeconfig=$KUBECONFIG set-credentials antrea-agent --token=$TOKEN
+   kubectl config --kubeconfig=$KUBECONFIG set-context antrea-agent@$CLUSTER_NAME --cluster=$CLUSTER_NAME --user=antrea-agent
+   kubectl config --kubeconfig=$KUBECONFIG use-context antrea-agent@$CLUSTER_NAME
    # Copy antrea-agent.kubeconfig to the VM
    ```
 
@@ -219,13 +220,15 @@ spec:
    ```bash
    # Specify the antrea-controller API server endpoint. Antrea-Controller needs
    # to be exposed via the Node IP or a public IP that is reachable from the VM
-   export ANTREA_API_SERVER="https://172.18.0.1:443"
-   export ANTREA_CLUSTER_NAME="antrea"
-   TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode)
-   kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-cluster $ANTREA_CLUSTER_NAME --server=$ANTREA_API_SERVER --insecure-skip-tls-verify=true
-   kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN
-   kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-context antrea-agent@$ANTREA_CLUSTER_NAME --cluster=$ANTREA_CLUSTER_NAME --user=antrea-agent
-   kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig use-context antrea-agent@$ANTREA_CLUSTER_NAME
+   ANTREA_API_SERVER="https://172.18.0.1:443"
+   ANTREA_CLUSTER_NAME="antrea"
+   NAMESPACE="vm-ns"
+   KUBECONFIG="antrea-agent.antrea.kubeconfig"
+   TOKEN=$(kubectl -n $NAMESPACE get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode)
+   kubectl config --kubeconfig=$KUBECONFIG set-cluster $ANTREA_CLUSTER_NAME --server=$ANTREA_API_SERVER --insecure-skip-tls-verify=true
+   kubectl config --kubeconfig=$KUBECONFIG set-credentials antrea-agent --token=$TOKEN
+   kubectl config --kubeconfig=$KUBECONFIG set-context antrea-agent@$ANTREA_CLUSTER_NAME --cluster=$ANTREA_CLUSTER_NAME --user=antrea-agent
+   kubectl config --kubeconfig=$KUBECONFIG use-context antrea-agent@$ANTREA_CLUSTER_NAME
    # Copy antrea-agent.antrea.kubeconfig to the VM
    ```
 
@@ -592,39 +595,3 @@ interfaces will be added in the future.
 
 `ExternalNode` name must be unique in the `cluster` scope even though it is
 itself a Namespaced resource.
-
-## Known issues
-
-`antrea-agent` will fail to re-attach the VM's network interface to the OVS
-bridge if the VM is rebooted. On a Windows VM, network connectivity will be
-lost after reboot.
-
-As a workaround for [this issue](https://github.com/antrea-io/antrea/issues/4122),
-you can manually remove OVS configurations and then restart `antrea-agent` after
-the VM is rebooted.
-
-To remove OVS configurations and restart `antrea-agent` on Linux VM,
-
-```shell
-sudo systemctl stop antrea-agent
-sudo ovs-vsctl del-br br-int
-sudo systemctl start antrea-agent
-```
-
-To remove OVS configurations and restart `antrea-agent` on Windows VM,
-
-```powershell
-$adapterName="Ethernet 0"
-Stop-Service antrea-agent
-ovs-vsctl.exe del-br br-int
-Remove-VMSwitch -ComputerName $(hostname.exe) antrea-switch -Force
-Rename-NetAdapter -Name "$adapterName~" -NewName "$adapterName"
-Start-Service antrea-agent
-```
-
-Note:
-
-- `$adapterName` should be set to the `ExternalNode` interface.
-- You may need a separate network interface to RDP into the Windows VM to run
-  these commands, since the network connectivity on the `ExternalNode` interface
-  is lost.
diff --git a/hack/externalnode/install-vm.ps1 b/hack/externalnode/install-vm.ps1
@@ -17,6 +17,12 @@
   .PARAMETER AntreaKubeConfigPath
   Specifies the path of the kubeconfig to access Antrea API Server.
 
+  .PARAMETER NodeName
+  Specifies the ExternalNode name to be used by the antrea-agent.
+
+  .PARAMETER OVSBridge
+  Specifies the OVS bridge name.
+
   .PARAMETER InstallDir
   The target installation directory. The default path is "C:\antrea-agent".
 #>
@@ -27,28 +33,32 @@ Param(
     [parameter(Mandatory = $true)] [string] $KubeConfigPath,
     [parameter(Mandatory = $true)] [string] $AntreaKubeConfigPath,
     [parameter(Mandatory = $false)] [string] $NodeName = $(hostname),
+    [parameter(Mandatory = $false)] [string] $OVSBridge = "br-int",
     [parameter(Mandatory = $false)] [string] $InstallDir = "C:\antrea-agent"
 )
 
 $ErrorActionPreference = "Stop"
 
-$WorkDir = [System.IO.Path]::GetDirectoryName($myInvocation.MyCommand.Definition)
-$InstallLog = "$WorkDir\install_vm.log"
+# System related
+$Powershell = (Get-Command powershell).Source
+$PowershellArgs = "-ExecutionPolicy Bypass -NoProfile -File"
 
 # Antrea paths
-$AntreaAgentPath = [io.path]::combine($InstallDir, "antrea-agent.exe")
 $AntreaAgentConfDir = [io.path]::combine($InstallDir, "conf")
 $AntreaAgentLogDir = [io.path]::combine($InstallDir, "logs")
 $AntreaAgentConfPath = [io.path]::combine($AntreaAgentConfDir, "antrea-agent.conf")
-$AntreaAgentLogFile = [io.path]::combine($AntreaAgentLogDir, "antrea-agent.log")
+$InstallLog = [io.path]::combine($AntreaAgentLogDir, "install_vm.log")
+$StartAntreaAgentScript = ""
 
 # Constants
+$AntreaAgent = "antrea-agent"
+$OVSServices = "ovsdb-server", "ovs-vswitchd"
+$OVSVswitchd = "ovs-vswitchd"
 $K8sKubeconfig = "antrea-agent.kubeconfig"
 $AntreaKubeconfig = "antrea-agent.antrea.kubeconfig"
-$OVSServices = "ovsdb-server", "ovs-vswitchd"
-$AntreaAgent = "antrea-agent"
-$Kubeconfig = "kubeconfig"
+$Bridge = "ovsBridge"
 $ExternalNodeNamespace = "externalNodeNamespace"
+$Kubeconfig = "kubeconfig"
 
 # List of supported OS versions, verified by antrea
 # Versions are named like Major.Minor.Build
@@ -67,7 +77,7 @@ function ServiceExists($ServiceName) {
 }
 
 function CheckSupportedVersions() {
-    echo "Checking supported Windows OS versions"
+    Log "Checking supported Windows OS versions"
     $OSVersion = [System.Environment]::OSVersion.Version
     $Version = $OSVersion.Major.ToString() + "." + $OSVersion.Minor.ToString() + "." + $OSVersion.Build.ToString()
     foreach ($v in $SupportedVersions) {
@@ -79,16 +89,14 @@ function CheckSupportedVersions() {
     exit 1
 }
 
-function PrintPrerequisites()
-{
-    echo "Please execute these commands to enable Hyper-V"
-    echo "Install-WindowsFeature Hyper-V-Powershell"
-    echo "Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All -NoRestart"
+function PrintPrerequisites() {
+    Write-Host "Please execute these commands to enable Hyper-V"
+    Write-Host "Install-WindowsFeature Hyper-V-Powershell"
+    Write-Host "Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All -NoRestart"
     exit 1
 }
 
-function CheckPrerequisites()
-{
+function CheckPrerequisites() {
     CheckSupportedVersions
     $valid = $true
     Log "Check Hyper-v feature is enabled"
@@ -116,7 +124,6 @@ function CheckPrerequisites()
 }
 
 function SetupInstallDir() {
-    Log "Create install directories"
     if (-Not (Test-Path $AntreaAgentConfDir)) {
         New-Item $AntreaAgentConfDir -type directory -Force | Out-Null
     }
@@ -167,41 +174,106 @@ function UpdateAgentConf() {
             Log "Updating $AntreaAgentConfPath with ${ExternalNodeNamespace}: ${Namespace}"
             [System.IO.File]::AppendAllText($AntreaAgentConfPath, "  ${ExternalNodeNamespace}: ${Namespace}" +
                     ([Environment]::NewLine))
+        } elseif ($line -like "*$Bridge*") {
+            Log "Updating $AntreaAgentConfPath with ${Bridge}: ${OVSBridge}"
+            [System.IO.File]::AppendAllText($AntreaAgentConfPath, "${Bridge}: ${OVSBridge}" +
+                    ([Environment]::NewLine))
         } else {
             [System.IO.File]::AppendAllText($AntreaAgentConfPath, $line +
                     ([Environment]::NewLine))
         }
     }
 }
 
-function ConfigureAntreaAgentService() {
-    # Set environment variables
-    [Environment]::SetEnvironmentVariable("NODE_NAME", $NodeName, [System.EnvironmentVariableTarget]::Machine)
-    # Assume nssm is installed and configure service
-    $AntreaAgentArgs = "--config $AntreaAgentConfPath --log_file $AntreaAgentLogFile --logtostderr=false"
-    log "Creating service $AntreaAgent $AntreaAgentPath $AntreaAgentArgs"
+function CreateAntreaAgentStartupScript() {
+    $Script:StartAntreaAgentScript = "$AntreaAgentConfDir\Start-AntreaAgent.ps1"
+    $StartAntreaAgentScriptContent = '
+Param(
+    [parameter(Mandatory = $true)] [string] $OVSBridge,
+    [parameter(Mandatory = $true)] [string] $InstallDir
+)
+
+$AntreaSwitch = "antrea-switch"
+$AntreaAgentConfDir = [io.path]::combine($InstallDir, "conf")
+$AntreaAgentLogDir = [io.path]::combine($InstallDir, "logs")
+$AntreaAgentConfPath = [io.path]::combine($AntreaAgentConfDir, "antrea-agent.conf")
+$AntreaAgentLogFile = [io.path]::combine($AntreaAgentLogDir, "antrea-agent.log")
+$AntreaAgentPath = [io.path]::combine($InstallDir, "antrea-agent.exe")
+$InstallLog = [io.path]::combine($AntreaAgentLogDir, "install_vm.log")
+
+function Log($Info) {
+    $time = $(get-date -Format g)
+    "$time $Info " | Tee-Object $InstallLog -Append | Write-Host
+}
+
+function ClearOVSConfig() {
+    Log "Deleting OVS bridge $OVSBridge"
     try {
-        # Configured to auto-restart upon reboot
-        & nssm install $AntreaAgent $AntreaAgentPath $AntreaAgentArgs
-    } catch {
-        log "Failed to create service for $AntreaAgent, rc $_"
+        $adapterName = (Get-VMNetworkAdapter -ComputerName $(hostname.exe) -SwitchName $AntreaSwitch -ManagementOS).Name
+        ovs-vsctl.exe del-br $OVSBridge
+    }  catch {
+        Log "Failed to get VMSwitch $AntreaSwitch, rc $_"
         exit 1
     }
-}
 
-function StartAntreaAgentService()
-{
     try {
-        & nssm start $AntreaAgent
+        Remove-VMSwitch -ComputerName $(hostname.exe) $AntreaSwitch  -Force
     } catch {
-        log "Failed to start service for $AntreaAgent, rc $_"
+        Log "Ignore error while removing VMSwitch, rc $_"
+    }
+
+    try {
+        Rename-NetAdapter -Name "$adapterName~" -NewName "$adapterName"
+    }  catch {
+        Log "Failed to rename network adapter $adapterName~ to $adapterName, rc $_"
         exit 1
     }
 }
 
-CheckPrerequisites
+function CheckOVSConfigAndCleanup() {
+    $bridges = ovs-vsctl list-br
+    foreach ($br in $bridges) {
+        if ($br -ne $OVSBridge) {
+            continue
+        }
+        $ports = ovs-vsctl list-ports $OVSBridge
+        foreach ($port in $ports) {
+            $output = ovs-vsctl --no-headings --columns=error list interface "$port"
+            if ($output -ne "[]") {
+                ClearOVSConfig
+                break
+            }
+        }
+    }
+}
+
+function StartAntreaAgent() {
+    $antreaAgentArgs = "--config $AntreaAgentConfPath --log_file $AntreaAgentLogFile --logtostderr=false"
+    $cmd = "$AntreaAgentPath $antreaAgentArgs"
+    Invoke-Expression $cmd
+}
+
+CheckOVSConfigAndCleanup
+StartAntreaAgent
+'
+    Set-Content -Path $StartAntreaAgentScript -Value $StartAntreaAgentScriptContent
+}
+
+function ConfigureAntreaAgentService() {
+    $AntreaAgentArgs = "$StartAntreaAgentScript -InstallDir $InstallDir -OVSBridge $OVSBridge"
+    nssm install $AntreaAgent $Powershell $PowershellArgs $AntreaAgentArgs
+    # Add OVS as a dependent service
+    nssm set $AntreaAgent DependOnService $OVSVswitchd
+}
+
+function StartAntreaAgentService() {
+    nssm start $AntreaAgent
+}
+
 SetupInstallDir
+CheckPrerequisites
 CopyAntreaAgentFiles
 UpdateAgentConf
+CreateAntreaAgentStartupScript
 ConfigureAntreaAgentService
 StartAntreaAgentService