Support configurable network V4 CIDR for Pod traffic across Node (#2473)

build/yamls/antrea-aks.yml

@@ -3911,9 +3911,20 @@ data:
     #tlsMinVersion:
 
     # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes.
-    # If there are multiple IP addresses configured on the interface, the first one is used.
-    # The interface configured with Node IP is used if this parameter is not set.
+    # If there are multiple IP addresses configured on the interface, the first one is used. The order for
+    # configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP.
     #transportInterface:
+
+    # The network V4 CIDR of the interface on Node which is used for tunneling or routing the traffic across
+    # Nodes. If there are multiple interfaces configured the same network V4 CIDR, the first one is used. The
+    # order for configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP
+    #transportV4CIDR:
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",

build/yamls/antrea-eks.yml

@@ -3911,9 +3911,20 @@ data:
     #tlsMinVersion:
 
     # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes.
-    # If there are multiple IP addresses configured on the interface, the first one is used.
-    # The interface configured with Node IP is used if this parameter is not set.
+    # If there are multiple IP addresses configured on the interface, the first one is used. The order for
+    # configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP.
     #transportInterface:
+
+    # The network V4 CIDR of the interface on Node which is used for tunneling or routing the traffic across
+    # Nodes. If there are multiple interfaces configured the same network V4 CIDR, the first one is used. The
+    # order for configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP
+    #transportV4CIDR:
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",

build/yamls/antrea-gke.yml

@@ -3911,9 +3911,20 @@ data:
     #tlsMinVersion:
 
     # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes.
-    # If there are multiple IP addresses configured on the interface, the first one is used.
-    # The interface configured with Node IP is used if this parameter is not set.
+    # If there are multiple IP addresses configured on the interface, the first one is used. The order for
+    # configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP.
     #transportInterface:
+
+    # The network V4 CIDR of the interface on Node which is used for tunneling or routing the traffic across
+    # Nodes. If there are multiple interfaces configured the same network V4 CIDR, the first one is used. The
+    # order for configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP
+    #transportV4CIDR:
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",

build/yamls/antrea-ipsec.yml

@@ -3916,9 +3916,20 @@ data:
     #tlsMinVersion:
 
     # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes.
-    # If there are multiple IP addresses configured on the interface, the first one is used.
-    # The interface configured with Node IP is used if this parameter is not set.
+    # If there are multiple IP addresses configured on the interface, the first one is used. The order for
+    # configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP.
     #transportInterface:
+
+    # The network V4 CIDR of the interface on Node which is used for tunneling or routing the traffic across
+    # Nodes. If there are multiple interfaces configured the same network V4 CIDR, the first one is used. The
+    # order for configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP
+    #transportV4CIDR:
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",

build/yamls/antrea-windows.yml

@@ -98,9 +98,20 @@ data:
     #trafficEncapMode: encap
 
     # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes.
-    # If there are multiple IP addresses configured on the interface, the first one is used.
-    # The interface configured with Node IP is used if this parameter is not set.
+    # If there are multiple IP addresses configured on the interface, the first one is used. The order for
+    # configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP.
     #transportInterface:
+
+    # The network V4 CIDR of the interface on Node which is used for tunneling or routing the traffic across
+    # Nodes. If there are multiple interfaces configured the same network V4 CIDR, the first one is used. The
+    # order for configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP
+    #transportV4CIDR:
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",

build/yamls/antrea.yml

@@ -3916,9 +3916,20 @@ data:
     #tlsMinVersion:
 
     # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes.
-    # If there are multiple IP addresses configured on the interface, the first one is used.
-    # The interface configured with Node IP is used if this parameter is not set.
+    # If there are multiple IP addresses configured on the interface, the first one is used. The order for
+    # configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP.
     #transportInterface:
+
+    # The network V4 CIDR of the interface on Node which is used for tunneling or routing the traffic across
+    # Nodes. If there are multiple interfaces configured the same network V4 CIDR, the first one is used. The
+    # order for configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+    # 1.TransportInterface
+    # 2.TransportV4CIDR
+    # 3.The Node IP
+    #transportV4CIDR:
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",

build/yamls/base/conf/antrea-agent.conf

@@ -161,6 +161,17 @@ wireGuard:
 #tlsMinVersion:
 
 # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes.
-# If there are multiple IP addresses configured on the interface, the first one is used.
-# The interface configured with Node IP is used if this parameter is not set.
+# If there are multiple IP addresses configured on the interface, the first one is used. The order for
+# configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+# 1.TransportInterface
+# 2.TransportV4CIDR
+# 3.The Node IP.
 #transportInterface:
+
+# The network V4 CIDR of the interface on Node which is used for tunneling or routing the traffic across
+# Nodes. If there are multiple interfaces configured the same network V4 CIDR, the first one is used. The
+# order for configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+# 1.TransportInterface
+# 2.TransportV4CIDR
+# 3.The Node IP
+#transportV4CIDR:

cmd/antrea-agent/agent.go

@@ -124,6 +124,7 @@ func run(o *Options) error {
 		TrafficEncapMode:      encapMode,
 		TrafficEncryptionMode: encryptionMode,
 		TransportIface:        o.config.TransportInterface,
+		TransportV4CIDR:       o.config.TransportV4CIDR,
 	}
 
 	wireguardConfig := &config.WireGuardConfig{

cmd/antrea-agent/config.go

@@ -159,9 +159,19 @@ type AgentConfig struct {
 	// TLS min version.
 	TLSMinVersion string `yaml:"tlsMinVersion,omitempty"`
 	// The name of the interface on Node which is used for tunneling or routing the traffic across Nodes.
-	// If there are multiple IP addresses configured on the interface, the first one is used.
-	// The interface configured with Node IP is used if this parameter is not set.
+	// If there are multiple IP addresses configured on the interface, the first one is used. The order for
+	// configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+	// 1.TransportInterface
+	// 2.TransportV4CIDR
+	// 3.The Node IP
 	TransportInterface string `yaml:"transportInterface,omitempty"`
+	// The network CIDR of the interface on Node which is used for tunneling or routing the traffic across
+	// Nodes. If there are multiple interfaces configured the same network CIDR, the first one is used. The
+	// order for configuring tunneling or routing the traffic across Nodes is (from highest to lowest):
+	// 1.TransportInterface
+	// 2.TransportV4CIDR
+	// 3.The Node IP
+	TransportV4CIDR string `yaml:"transportV4CIDR,omitempty"`
 }
 
 type WireGuardConfig struct {

pkg/agent/agent.go

@@ -61,11 +61,16 @@ const (
 	maxRetryForRoundNumSave = 5
 )
 
-// getIPNetDeviceFromIP is meant to be overridden for testing.
-var getIPNetDeviceFromIP = util.GetIPNetDeviceFromIP
+var (
+	// getIPNetDeviceFromIP is meant to be overridden for testing.
+	getIPNetDeviceFromIP = util.GetIPNetDeviceFromIP
 
-// getTransportIPNetDeviceByName is meant to be overridden for testing.
-var getTransportIPNetDeviceByName = GetTransportIPNetDeviceByName
+	// getIPNetDeviceByV4CIDR is meant to be overridden for testing.
+	getIPNetDeviceByV4CIDR = util.GetIPNetDeviceByV4CIDR
+
+	// getTransportIPNetDeviceByName is meant to be overridden for testing.
+	getTransportIPNetDeviceByName = GetTransportIPNetDeviceByName
+)
 
 // Initializer knows how to setup host networking, OpenVSwitch, and Openflow.
 type Initializer struct {
@@ -728,6 +733,16 @@ func (i *Initializer) initNodeLocalConfig() error {
 		if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, strings.Join(ips, ",")); err != nil {
 			return err
 		}
+	} else if i.networkConfig.TransportV4CIDR != "" {
+		transportIPv4Addr, localIntf, err = getIPNetDeviceByV4CIDR(i.networkConfig.TransportV4CIDR)
+		if err != nil {
+			return fmt.Errorf("failed to get local IPNet device with transport CIDR %s: %v", i.networkConfig.TransportV4CIDR, err)
+		}
+		ips := []string{transportIPv4Addr.IP.String()}
+		klog.InfoS("Updating Node transport addresses annotation")
+		if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, strings.Join(ips, ",")); err != nil {
+			return err
+		}
 	} else {
 		// Remove the existing annotation "transport-address" if transportInterface is not set in the configuration.
 		if node.Annotations[types.NodeTransportAddressAnnotationKey] != "" {

pkg/agent/agent_test.go

@@ -158,7 +158,8 @@ func TestInitNodeLocalConfig(t *testing.T) {
 	}
 	podCIDRStr := "172.16.10.0/24"
 	_, podCIDR, _ := net.ParseCIDR(podCIDRStr)
-	transportIP, transportIPNet, _ := net.ParseCIDR("172.16.100.7/24")
+	transportV4CIDRStr := "172.16.100.7/24"
+	transportIP, transportIPNet, _ := net.ParseCIDR(transportV4CIDRStr)
 	transportIPNet.IP = transportIP
 	transportIfaceMAC, _ := net.ParseMAC("00:0c:29:f5:e2:ce")
 	type testTransInterface struct {
@@ -177,6 +178,8 @@ func TestInitNodeLocalConfig(t *testing.T) {
 	tests := []struct {
 		name                      string
 		trafficEncapMode          config.TrafficEncapModeType
+		transportIfName           string
+		transportIfV4CIDR         string
 		transportInterface        *testTransInterface
 		tunnelType                ovsconfig.TunnelType
 		mtu                       int
@@ -221,6 +224,7 @@ func TestInitNodeLocalConfig(t *testing.T) {
 		{
 			name:                      "noencap mode with transportInterface",
 			trafficEncapMode:          config.TrafficEncapModeNoEncap,
+			transportIfName:           testTransportIface.iface.Name,
 			transportInterface:        testTransportIface,
 			mtu:                       0,
 			expectedNodeLocalIfaceMTU: 1500,
@@ -233,6 +237,7 @@ func TestInitNodeLocalConfig(t *testing.T) {
 		{
 			name:                      "hybrid mode with transportInterface",
 			trafficEncapMode:          config.TrafficEncapModeHybrid,
+			transportIfName:           testTransportIface.iface.Name,
 			transportInterface:        testTransportIface,
 			mtu:                       0,
 			expectedNodeLocalIfaceMTU: 1500,
@@ -245,6 +250,7 @@ func TestInitNodeLocalConfig(t *testing.T) {
 		{
 			name:                      "encap mode with transportInterface, geneve tunnel",
 			trafficEncapMode:          config.TrafficEncapModeEncap,
+			transportIfName:           testTransportIface.iface.Name,
 			transportInterface:        testTransportIface,
 			tunnelType:                ovsconfig.GeneveTunnel,
 			mtu:                       0,
@@ -257,6 +263,59 @@ func TestInitNodeLocalConfig(t *testing.T) {
 		{
 			name:                      "encap mode with transportInterface, mtu specified",
 			trafficEncapMode:          config.TrafficEncapModeEncap,
+			transportIfName:           testTransportIface.iface.Name,
+			transportInterface:        testTransportIface,
+			tunnelType:                ovsconfig.GeneveTunnel,
+			mtu:                       1400,
+			expectedNodeLocalIfaceMTU: 1500,
+			expectedMTU:               1400,
+			expectedNodeAnnotation: map[string]string{
+				types.NodeTransportAddressAnnotationKey: transportIP.String(),
+			},
+		},
+		{
+			name:                      "noencap mode with transportV4CIDR",
+			trafficEncapMode:          config.TrafficEncapModeNoEncap,
+			transportIfV4CIDR:         transportV4CIDRStr,
+			transportInterface:        testTransportIface,
+			mtu:                       0,
+			expectedNodeLocalIfaceMTU: 1500,
+			expectedMTU:               1500,
+			expectedNodeAnnotation: map[string]string{
+				types.NodeMACAddressAnnotationKey:       transportIfaceMAC.String(),
+				types.NodeTransportAddressAnnotationKey: transportIP.String(),
+			},
+		},
+		{
+			name:                      "hybrid mode with transportV4CIDR",
+			trafficEncapMode:          config.TrafficEncapModeHybrid,
+			transportIfV4CIDR:         transportV4CIDRStr,
+			transportInterface:        testTransportIface,
+			mtu:                       0,
+			expectedNodeLocalIfaceMTU: 1500,
+			expectedMTU:               1500,
+			expectedNodeAnnotation: map[string]string{
+				types.NodeMACAddressAnnotationKey:       transportIfaceMAC.String(),
+				types.NodeTransportAddressAnnotationKey: transportIP.String(),
+			},
+		},
+		{
+			name:                      "encap mode with transportV4CIDR, geneve tunnel",
+			trafficEncapMode:          config.TrafficEncapModeEncap,
+			transportIfV4CIDR:         transportV4CIDRStr,
+			transportInterface:        testTransportIface,
+			tunnelType:                ovsconfig.GeneveTunnel,
+			mtu:                       0,
+			expectedNodeLocalIfaceMTU: 1500,
+			expectedMTU:               1450,
+			expectedNodeAnnotation: map[string]string{
+				types.NodeTransportAddressAnnotationKey: transportIP.String(),
+			},
+		},
+		{
+			name:                      "encap mode with transportV4CIDR, mtu specified",
+			trafficEncapMode:          config.TrafficEncapModeEncap,
+			transportIfV4CIDR:         transportV4CIDRStr,
 			transportInterface:        testTransportIface,
 			tunnelType:                ovsconfig.GeneveTunnel,
 			mtu:                       1400,
@@ -309,10 +368,14 @@ func TestInitNodeLocalConfig(t *testing.T) {
 					TunnelType:       tt.tunnelType,
 				},
 			}
-			if tt.transportInterface != nil {
+			if tt.transportIfName != "" {
 				initializer.networkConfig.TransportIface = tt.transportInterface.iface.Name
 				expectedNodeConfig.NodeTransportIPv4Addr = tt.transportInterface.ipNet
 				defer mockGetTransportIPNetDeviceByName(tt.transportInterface.ipNet, tt.transportInterface.iface)()
+			} else if tt.transportIfV4CIDR != "" {
+				initializer.networkConfig.TransportV4CIDR = transportV4CIDRStr
+				expectedNodeConfig.NodeTransportIPv4Addr = tt.transportInterface.ipNet
+				defer mockGetIPNetDeviceByV4CIDR(tt.transportInterface.ipNet, tt.transportInterface.iface)()
 			}
 			defer mockGetIPNetDeviceFromIP(nodeIPNet, ipDevice)()
 			defer mockNodeNameEnv(nodeName)()
@@ -346,3 +409,11 @@ func mockGetTransportIPNetDeviceByName(ipNet *net.IPNet, ipDevice *net.Interface
 	}
 	return func() { getTransportIPNetDeviceByName = prevGetIPNetDeviceByName }
 }
+
+func mockGetIPNetDeviceByV4CIDR(ipNet *net.IPNet, ipDevice *net.Interface) func() {
+	prevGetIPNetDeviceByV4CIDR := getIPNetDeviceByV4CIDR
+	getIPNetDeviceByV4CIDR = func(ifName string) (*net.IPNet, *net.Interface, error) {
+		return ipNet, ipDevice, nil
+	}
+	return func() { getIPNetDeviceByV4CIDR = prevGetIPNetDeviceByV4CIDR }
+}

pkg/agent/config/node_config.go

@@ -129,6 +129,7 @@ type NetworkConfig struct {
 	TrafficEncryptionMode TrafficEncryptionModeType
 	IPSecPSK              string
 	TransportIface        string
+	TransportV4CIDR     string
 }
 
 // IsIPv4Enabled returns true if the cluster network supports IPv4.

pkg/agent/controller/noderoute/node_route_controller.go

@@ -739,8 +739,8 @@ func getNodeMAC(node *corev1.Node) (net.HardwareAddr, error) {
 }
 
 func (c *Controller) getNodeTransportAddrs(node *corev1.Node) (*utilip.DualStackIPs, error) {
-	var transportAddrs *utilip.DualStackIPs
-	if c.networkConfig.TransportIface != "" {
+	transportAddrs := &utilip.DualStackIPs{}
+	if c.networkConfig.TransportIface != "" || c.networkConfig.TransportV4CIDR != "" {
 		transportAddrsStr := node.Annotations[types.NodeTransportAddressAnnotationKey]
 		if transportAddrsStr != "" {
 			for _, addr := range strings.Split(transportAddrsStr, ",") {