diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 03064d039b9..ca78e300310 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -22,7 +22,6 @@ import ( "antrea.io/libOpenflow/openflow15" "antrea.io/libOpenflow/protocol" ofutil "antrea.io/libOpenflow/util" - v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/config" @@ -97,14 +96,13 @@ type Client interface { // InstallEndpointFlows. UninstallEndpointFlows(protocol binding.Protocol, endpoints []proxy.Endpoint) error - // InstallServiceFlows installs flows for accessing Service NodePort, LoadBalancer and ClusterIP. It installs the - // flow that uses the group/bucket to do service LB. If the affinityTimeout is not zero, it also installs the flow - // which has a learn action to maintain the LB decision. The group with the groupID must be installed before, - // otherwise the installation will fail. - // nodeLocalExternal represents if the externalTrafficPolicy is Local or not. This field is meaningful only when - // the svcType is NodePort or LoadBalancer. - // nested represents if the Service has the Endpoints which is other Service's ClusterIP. - InstallServiceFlows(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool, svcType v1.ServiceType, nested bool) error + // InstallServiceFlows installs flows for accessing Service NodePort, LoadBalancer, ExternalIP and ClusterIP. It + // installs the flow that uses the group/bucket to do Service LB. If the affinityTimeout is not zero, it also + // installs the flow which has a learn action to maintain the LB decision. The group with the groupID must be + // installed before, otherwise the installation will fail. + // externallyAccessible indicates that whether the Service is externally accessible, like NodePort, LoadBalancer and ExternalIP. + // nested indicates that whether the Service has an Endpoints which is other Service's ClusterIP. + InstallServiceFlows(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, externallyAccessible, nested bool) error // UninstallServiceFlows removes flows installed by InstallServiceFlows. UninstallServiceFlows(svcIP net.IP, svcPort uint16, protocol binding.Protocol) error @@ -754,15 +752,16 @@ func (c *client) UninstallEndpointFlows(protocol binding.Protocol, endpoints []p return c.deleteFlowsWithMultipleKeys(c.featureService.cachedFlows, flowCacheKeys) } -func (c *client) InstallServiceFlows(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool, svcType v1.ServiceType, nested bool) error { +func (c *client) InstallServiceFlows(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, externallyAccessible, nested bool) error { c.replayMutex.RLock() defer c.replayMutex.RUnlock() var flows []binding.Flow - flows = append(flows, c.featureService.serviceLBFlow(groupID, svcIP, svcPort, protocol, affinityTimeout != 0, nodeLocalExternal, svcType, nested)) + matchNodePortAddress := svcIP.Equal(config.VirtualNodePortDNATIPv4) || svcIP.Equal(config.VirtualNodePortDNATIPv6) + flows = append(flows, c.featureService.serviceLBFlow(groupID, svcIP, svcPort, protocol, affinityTimeout != 0, externallyAccessible, matchNodePortAddress, nested)) if affinityTimeout != 0 { - flows = append(flows, c.featureService.serviceLearnFlow(groupID, svcIP, svcPort, protocol, affinityTimeout, nodeLocalExternal, svcType)) + flows = append(flows, c.featureService.serviceLearnFlow(groupID, svcIP, svcPort, protocol, affinityTimeout, externallyAccessible, matchNodePortAddress)) } - if svcType == v1.ServiceTypeClusterIP && !nested { + if !externallyAccessible && !nested { flows = append(flows, c.featureService.endpointRedirectFlowForServiceIP(svcIP, svcPort, protocol, groupID)) } cacheKey := generateServicePortFlowCacheKey(svcIP, svcPort, protocol) diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index 0785773c3a1..bf2038bfc9b 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -30,7 +30,6 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/openflow/cookie" @@ -1074,32 +1073,29 @@ func Test_client_InstallServiceFlows(t *testing.T) { port := uint16(80) testCases := []struct { - name string - protocol binding.Protocol - svcIP net.IP - affinityTimeout uint16 - nodeLocalExternal bool - svcType corev1.ServiceType - expectedFlows []string - nested bool + name string + protocol binding.Protocol + svcIP net.IP + affinityTimeout uint16 + externallyAccessible bool + expectedFlows []string + nested bool }{ { name: "Service ClusterIP", protocol: binding.ProtocolTCP, svcIP: svcIPv4, - svcType: corev1.ServiceTypeClusterIP, expectedFlows: []string{ "cookie=0x1030000000000, table=EndpointDNAT, priority=210,tcp,reg3=0xa600064,reg4=0x1020050/0x107ffff actions=group:100", - "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x20000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x64->reg7,group:100", + "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x20000/0x70000->reg4,set_field:0x64->reg7,group:100", }, }, { name: "Service ClusterIP, nested", protocol: binding.ProtocolTCP, svcIP: svcIPv4, - svcType: corev1.ServiceTypeClusterIP, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x20000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x64->reg7,set_field:0x1000000/0x1000000->reg4,group:100", + "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x20000/0x70000->reg4,set_field:0x64->reg7,set_field:0x1000000/0x1000000->reg4,group:100", }, nested: true, }, @@ -1108,11 +1104,10 @@ func Test_client_InstallServiceFlows(t *testing.T) { protocol: binding.ProtocolTCP, svcIP: svcIPv4, affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeClusterIP, expectedFlows: []string{ "cookie=0x1030000000000, table=EndpointDNAT, priority=210,tcp,reg3=0xa600064,reg4=0x1020050/0x107ffff actions=group:100", - "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,tcp,reg4=0x30000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x800->NXM_OF_ETH_TYPE[],load:0x6->NXM_OF_IP_PROTO[],load:OXM_OF_TCP_DST[]->OXM_OF_TCP_DST[],load:NXM_OF_IP_DST[]->NXM_OF_IP_DST[],load:NXM_OF_IP_SRC[]->NXM_OF_IP_SRC[],NXM_NX_REG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,tcp,reg4=0x30000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x800,nw_proto=0x6,OXM_OF_TCP_DST[],NXM_OF_IP_DST[],NXM_OF_IP_SRC[],load:NXM_NX_REG3[]->NXM_NX_REG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, { @@ -1120,55 +1115,53 @@ func Test_client_InstallServiceFlows(t *testing.T) { protocol: binding.ProtocolTCPv6, svcIP: svcIPv6, affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeClusterIP, expectedFlows: []string{ "cookie=0x1030000000000, table=EndpointDNAT, priority=210,tcp6,reg4=0x1020050/0x107ffff actions=group:100", - "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp6,reg4=0x10000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,tcp6,reg4=0x30000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x86dd->NXM_OF_ETH_TYPE[],load:0x6->NXM_OF_IP_PROTO[],load:OXM_OF_TCP_DST[]->OXM_OF_TCP_DST[],load:NXM_NX_IPV6_DST[]->NXM_NX_IPV6_DST[],load:NXM_NX_IPV6_SRC[]->NXM_NX_IPV6_SRC[],NXM_NX_XXREG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", - }, + "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp6,reg4=0x10000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,tcp6,reg4=0x30000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x86dd,nw_proto=0x6,OXM_OF_TCP_DST[],NXM_NX_IPV6_DST[],NXM_NX_IPV6_SRC[],load:NXM_NX_XXREG3[]->NXM_NX_XXREG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT"}, }, { - name: "Service NodePort,SessionAffinity", - protocol: binding.ProtocolUDP, - svcIP: svcIPv4, - affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeNodePort, + name: "Service NodePort,SessionAffinity", + protocol: binding.ProtocolUDP, + svcIP: config.VirtualNodePortDNATIPv4, + affinityTimeout: uint16(100), + externallyAccessible: true, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,udp,reg4=0x90000/0xf0000,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,udp,reg4=0xb0000/0xf0000,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x800->NXM_OF_ETH_TYPE[],load:0x11->NXM_OF_IP_PROTO[],load:OXM_OF_UDP_DST[]->OXM_OF_UDP_DST[],reg4=0x1,load:NXM_OF_IP_DST[]->NXM_OF_IP_DST[],load:NXM_OF_IP_SRC[]->NXM_OF_IP_SRC[],NXM_NX_REG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,udp,reg4=0x90000/0xf0000,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,udp,reg4=0xb0000/0xf0000,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x800,nw_proto=0x11,OXM_OF_UDP_DST[],NXM_OF_IP_DST[],NXM_OF_IP_SRC[],load:NXM_NX_REG3[]->NXM_NX_REG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, { - name: "Service NodePort,ExternalPolicyLocal true,SessionAffinity", - protocol: binding.ProtocolUDPv6, - svcIP: svcIPv6, - affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeNodePort, + name: "Service NodePort,ExternalPolicyLocal true,SessionAffinity", + protocol: binding.ProtocolUDPv6, + svcIP: config.VirtualNodePortDNATIPv6, + affinityTimeout: uint16(100), + externallyAccessible: true, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,udp6,reg4=0x90000/0xf0000,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,udp6,reg4=0xb0000/0xf0000,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x86dd->NXM_OF_ETH_TYPE[],load:0x11->NXM_OF_IP_PROTO[],load:OXM_OF_UDP_DST[]->OXM_OF_UDP_DST[],reg4=0x1,load:NXM_NX_IPV6_DST[]->NXM_NX_IPV6_DST[],load:NXM_NX_IPV6_SRC[]->NXM_NX_IPV6_SRC[],NXM_NX_XXREG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,udp6,reg4=0x90000/0xf0000,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,udp6,reg4=0xb0000/0xf0000,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x86dd,nw_proto=0x11,OXM_OF_UDP_DST[],NXM_NX_IPV6_DST[],NXM_NX_IPV6_SRC[],load:NXM_NX_XXREG3[]->NXM_NX_XXREG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, { - name: "Service LoadBalancer,SessionAffinity", - protocol: binding.ProtocolSCTP, - svcIP: svcIPv4, - affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeLoadBalancer, + name: "Service LoadBalancer,SessionAffinity", + protocol: binding.ProtocolSCTP, + svcIP: svcIPv4, + affinityTimeout: uint16(100), + externallyAccessible: true, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,sctp,reg4=0x30000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x800->NXM_OF_ETH_TYPE[],load:0x84->NXM_OF_IP_PROTO[],load:OXM_OF_SCTP_DST[]->OXM_OF_SCTP_DST[],reg4=0x1,load:NXM_OF_IP_DST[]->NXM_OF_IP_DST[],load:NXM_OF_IP_SRC[]->NXM_OF_IP_SRC[],NXM_NX_REG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,sctp,reg4=0x30000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x800,nw_proto=0x84,OXM_OF_SCTP_DST[],NXM_OF_IP_DST[],NXM_OF_IP_SRC[],load:NXM_NX_REG3[]->NXM_NX_REG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, { - name: "Service LoadBalancer,ExternalPolicyLocal true,SessionAffinity", - protocol: binding.ProtocolSCTPv6, - svcIP: svcIPv6, - affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeLoadBalancer, + name: "Service LoadBalancer,ExternalPolicyLocal true,SessionAffinity", + protocol: binding.ProtocolSCTPv6, + svcIP: svcIPv6, + affinityTimeout: uint16(100), + externallyAccessible: true, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp6,reg4=0x10000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,sctp6,reg4=0x30000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x86dd->NXM_OF_ETH_TYPE[],load:0x84->NXM_OF_IP_PROTO[],load:OXM_OF_SCTP_DST[]->OXM_OF_SCTP_DST[],reg4=0x1,load:NXM_NX_IPV6_DST[]->NXM_NX_IPV6_DST[],load:NXM_NX_IPV6_SRC[]->NXM_NX_IPV6_SRC[],NXM_NX_XXREG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp6,reg4=0x10000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,sctp6,reg4=0x30000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x86dd,nw_proto=0x84,OXM_OF_SCTP_DST[],NXM_NX_IPV6_DST[],NXM_NX_IPV6_SRC[],load:NXM_NX_XXREG3[]->NXM_NX_XXREG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, } @@ -1186,7 +1179,7 @@ func Test_client_InstallServiceFlows(t *testing.T) { cacheKey := generateServicePortFlowCacheKey(tc.svcIP, port, tc.protocol) - assert.NoError(t, fc.InstallServiceFlows(groupID, tc.svcIP, port, tc.protocol, tc.affinityTimeout, tc.nodeLocalExternal, tc.svcType, tc.nested)) + assert.NoError(t, fc.InstallServiceFlows(groupID, tc.svcIP, port, tc.protocol, tc.affinityTimeout, tc.externallyAccessible, tc.nested)) fCacheI, ok := fc.featureService.cachedFlows.Load(cacheKey) require.True(t, ok) assert.ElementsMatch(t, tc.expectedFlows, getFlowStrings(fCacheI)) @@ -1216,12 +1209,12 @@ func Test_client_GetServiceFlowKeys(t *testing.T) { proxy.NewBaseEndpointInfo("10.10.0.12", "", "", 80, true, true, false, false, nil), } - assert.NoError(t, fc.InstallServiceFlows(groupID, svcIP, svcPort, bindingProtocol, 100, true, corev1.ServiceTypeLoadBalancer, false)) + assert.NoError(t, fc.InstallServiceFlows(groupID, svcIP, svcPort, bindingProtocol, 100, true, false)) assert.NoError(t, fc.InstallEndpointFlows(bindingProtocol, endpoints)) flowKeys := fc.GetServiceFlowKeys(svcIP, svcPort, bindingProtocol, endpoints) expectedFlowKeys := []string{ "table=11,tcp,tp_dst=0x50,nw_dst=10.96.0.224,reg4=0x10000/0x70000", - "table=11,tcp,reg4=0x30000/0x70000,nw_dst=10.96.0.224,tp_dst=0x50", + "table=11,tcp,tp_dst=0x50,nw_dst=10.96.0.224,reg4=0x30000/0x70000", "table=12,tcp,reg4=0x20050/0x7ffff,reg3=0xa0a000b", "table=12,tcp,reg4=0x20050/0x7ffff,reg3=0xa0a000c", "table=20,ip,nw_src=10.10.0.12,nw_dst=10.10.0.12,ct_state=+new+trk", diff --git a/pkg/agent/openflow/fields.go b/pkg/agent/openflow/fields.go index e0f8fa1702c..0efce6f5557 100644 --- a/pkg/agent/openflow/fields.go +++ b/pkg/agent/openflow/fields.go @@ -121,15 +121,12 @@ var ( EpUnionField = binding.NewRegField(4, 0, 18) // reg4[19]: Mark to indicate the Service type is NodePort. ToNodePortAddressRegMark = binding.NewOneBitRegMark(4, 19) - // reg4[16..19]: Field to store the union value of Endpoint state and the mark of whether Service type is NodePort. - NodePortUnionField = binding.NewRegField(4, 16, 19) // reg4[20]: Field to indicate whether the packet is from local Antrea IPAM Pod. NotAntreaFlexibleIPAMRegMark will // be used with RewriteMACRegMark, thus the reg id must not be same due to the limitation of ofnet library. AntreaFlexibleIPAMRegMark = binding.NewOneBitRegMark(4, 20) NotAntreaFlexibleIPAMRegMark = binding.NewOneBitZeroRegMark(4, 20) - // reg4[21]: Mark to indicate whether the packet is accessing a NodePort or a LoadBalancer IP of a Service whose - // externalTrafficPolicy is Cluster. - ToClusterServiceRegMark = binding.NewOneBitRegMark(4, 21) + // reg4[21]: Mark to indicate whether the Service is accessible externally, like NodePort, LoadBalancer or ExternalIP. + ExternallyAccessibleRegMark = binding.NewOneBitRegMark(4, 21) // reg4[22..23]: Field to store the action of a traffic control rule. Marks in this field include: TrafficControlActionField = binding.NewRegField(4, 22, 23) TrafficControlMirrorRegMark = binding.NewRegMark(TrafficControlActionField, 0b01) diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 87783408026..5e802406d1d 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -25,7 +25,6 @@ import ( "antrea.io/libOpenflow/openflow15" "antrea.io/libOpenflow/protocol" "antrea.io/ofnet/ofctrl" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "antrea.io/antrea/pkg/agent/config" @@ -2292,26 +2291,25 @@ func (f *featureService) serviceLearnFlow(groupID binding.GroupIDType, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, - nodeLocalExternal bool, - svcType v1.ServiceType) binding.Flow { + externalAddress bool, + nodePortAddress bool) binding.Flow { // Using unique cookie ID here to avoid learned flow cascade deletion. cookieID := f.cookieAllocator.RequestWithObjectID(f.category, uint32(groupID)).Raw() - var flowBuilder binding.FlowBuilder - if svcType == v1.ServiceTypeNodePort { - unionVal := (ToNodePortAddressRegMark.GetValue() << ServiceEPStateField.GetRange().Length()) + EpToLearnRegMark.GetValue() - flowBuilder = ServiceLBTable.ofTable.BuildFlow(priorityLow). - Cookie(cookieID). - MatchProtocol(protocol). - MatchRegFieldWithValue(NodePortUnionField, unionVal). - MatchDstPort(svcPort, nil) + flowBuilder := ServiceLBTable.ofTable.BuildFlow(priorityLow). + Cookie(cookieID). + MatchProtocol(protocol). + MatchDstPort(svcPort, nil) + + // EpToLearnRegMark is required to match the packets that have done Endpoint selection. + regMarksToMatch := []*binding.RegMark{EpToLearnRegMark} + // If "nodePortAddress" is true, which means that Service type is NodePort, "ToNodePortAddressRegMark" must be used + // to match packets. Otherwise, Service IP must be used to match packets. + if nodePortAddress { + regMarksToMatch = append(regMarksToMatch, ToNodePortAddressRegMark) } else { - flowBuilder = ServiceLBTable.ofTable.BuildFlow(priorityLow). - Cookie(cookieID). - MatchProtocol(protocol). - MatchRegMark(EpToLearnRegMark). - MatchDstIP(svcIP). - MatchDstPort(svcPort, nil) + flowBuilder = flowBuilder.MatchDstIP(svcIP) } + flowBuilder = flowBuilder.MatchRegMark(regMarksToMatch...) // affinityTimeout is used as the OpenFlow "hard timeout": learned flow will be removed from // OVS after that time regarding of whether traffic is still hitting the flow. This is the @@ -2334,10 +2332,14 @@ func (f *featureService) serviceLearnFlow(groupID binding.GroupIDType, case binding.ProtocolSCTPv6: learnFlowBuilderLearnAction = learnFlowBuilderLearnAction.MatchLearnedSCTPv6DstPort() } - // If externalTrafficPolicy of NodePort/LoadBalancer is Cluster, the learned flow which - // is used to match the first packet of NodePort/LoadBalancer also requires SNAT. - if (svcType == v1.ServiceTypeNodePort || svcType == v1.ServiceTypeLoadBalancer) && !nodeLocalExternal { - learnFlowBuilderLearnAction = learnFlowBuilderLearnAction.LoadRegMark(ToClusterServiceRegMark) + + // Loading the EpSelectedRegMark indicates that the Endpoint selection is complete. RewriteMACRegMark is required to + // load for Service packets + regMarksToLoad := []*binding.RegMark{EpSelectedRegMark, RewriteMACRegMark} + // If the Service is accessible externally, the ExternallyAccessibleRegMark should be loaded to determine whether SNAT + // is required for the connection. + if externalAddress { + regMarksToLoad = append(regMarksToLoad, ExternallyAccessibleRegMark) } ipProtocol := getIPProtocol(svcIP) @@ -2347,7 +2349,7 @@ func (f *featureService) serviceLearnFlow(groupID binding.GroupIDType, MatchLearnedSrcIP(). LoadFieldToField(EndpointIPField, EndpointIPField). LoadFieldToField(EndpointPortField, EndpointPortField). - LoadRegMark(EpSelectedRegMark, RewriteMACRegMark). + LoadRegMark(regMarksToLoad...). Done(). Action().LoadRegMark(EpSelectedRegMark). Action().NextTable(). @@ -2358,7 +2360,7 @@ func (f *featureService) serviceLearnFlow(groupID binding.GroupIDType, MatchLearnedSrcIPv6(). LoadXXRegToXXReg(EndpointIP6Field, EndpointIP6Field). LoadFieldToField(EndpointPortField, EndpointPortField). - LoadRegMark(EpSelectedRegMark, RewriteMACRegMark). + LoadRegMark(regMarksToLoad...). Done(). Action().LoadRegMark(EpSelectedRegMark). Action().NextTable(). @@ -2372,55 +2374,49 @@ func (f *featureService) serviceLBFlow(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, - withSessionAffinity, - nodeLocalExternal bool, - serviceType v1.ServiceType, + withSessionAffinity bool, + externalAddress bool, + nodePortAddress bool, nested bool) binding.Flow { - cookieID := f.cookieAllocator.Request(f.category).Raw() - var lbResultMark *binding.RegMark - if withSessionAffinity { - lbResultMark = EpToLearnRegMark + flowBuilder := ServiceLBTable.ofTable.BuildFlow(priorityNormal). + Cookie(f.cookieAllocator.Request(f.category).Raw()). + MatchProtocol(protocol). + MatchDstPort(svcPort, nil) + + // EpToSelectRegMark is required to match the packets that haven't undergone Endpoint selection yet. + regMarksToMatch := []*binding.RegMark{EpToSelectRegMark} + // If "nodePortAddress" is true, which means that Service type is NodePort, "ToNodePortAddressRegMark" must be used + // to match packets. Otherwise, Service IP must be used to match packets. + if nodePortAddress { + regMarksToMatch = append(regMarksToMatch, ToNodePortAddressRegMark) } else { - lbResultMark = EpSelectedRegMark - } - regMarksToLoad := []*binding.RegMark{lbResultMark, RewriteMACRegMark} - var flowBuilder binding.FlowBuilder - if serviceType == v1.ServiceTypeNodePort { - unionVal := (ToNodePortAddressRegMark.GetValue() << ServiceEPStateField.GetRange().Length()) + EpToSelectRegMark.GetValue() - // If externalTrafficPolicy of a NodePort Service is Cluster (nodeLocalExternal=false), it loads - // ToClusterServiceRegMark to indicate it. The mark will be checked later when determining if SNAT is required - // or not. - if !nodeLocalExternal { - regMarksToLoad = append(regMarksToLoad, ToClusterServiceRegMark) - } - flowBuilder = ServiceLBTable.ofTable.BuildFlow(priorityNormal). - Cookie(cookieID). - MatchProtocol(protocol). - MatchRegFieldWithValue(NodePortUnionField, unionVal). - MatchDstPort(svcPort, nil). - Action().LoadRegMark(regMarksToLoad...) + flowBuilder = flowBuilder.MatchDstIP(svcIP) + } + + // RewriteMACRegMark is required to load for Service packets. + regMarksToLoad := []*binding.RegMark{RewriteMACRegMark} + // If SessionAffinity is set, the result of Endpoint selection should be cached by loading the EpToLearnRegMark. + // If SessionAffinity is not set, loading the EpSelectedRegMark indicates that the Endpoint selection is complete. + if withSessionAffinity { + regMarksToLoad = append(regMarksToLoad, EpToLearnRegMark) } else { - // If externalTrafficPolicy of a LoadBalancer Service is Cluster (nodeLocalExternal=false), it loads - // ToClusterServiceRegMark to indicate it. The mark will be checked later when determining if SNAT is required - // or not. - if serviceType == v1.ServiceTypeLoadBalancer && !nodeLocalExternal { - regMarksToLoad = append(regMarksToLoad, ToClusterServiceRegMark) - } - flowBuilder = ServiceLBTable.ofTable.BuildFlow(priorityNormal). - Cookie(cookieID). - MatchProtocol(protocol). - MatchDstPort(svcPort, nil). - MatchDstIP(svcIP). - MatchRegMark(EpToSelectRegMark). - Action().LoadRegMark(regMarksToLoad...) + regMarksToLoad = append(regMarksToLoad, EpSelectedRegMark) + } + // If the Service is accessible externally, the ExternallyAccessibleRegMark should be loaded to determine whether SNAT + // is required for the connection. + if externalAddress { + regMarksToLoad = append(regMarksToLoad, ExternallyAccessibleRegMark) } if f.enableAntreaPolicy { - flowBuilder = flowBuilder.Action().LoadToRegField(ServiceGroupIDField, uint32(groupID)) + regMarksToLoad = append(regMarksToLoad, binding.NewRegMark(ServiceGroupIDField, uint32(groupID))) } if nested { - flowBuilder = flowBuilder.Action().LoadRegMark(NestedServiceRegMark) + regMarksToLoad = append(regMarksToLoad, NestedServiceRegMark) } - return flowBuilder.Action().Group(groupID).Done() + + return flowBuilder.MatchRegMark(regMarksToMatch...). + Action().LoadRegMark(regMarksToLoad...). + Action().Group(groupID).Done() } // endpointRedirectFlowForServiceIP generates the flow which uses the specific group for a Service's ClusterIP @@ -3011,15 +3007,15 @@ func (f *featureService) gatewaySNATFlows() []binding.Flow { pktDstRegMarks = append(pktDstRegMarks, ToUplinkRegMark) } for _, pktDstRegMark := range pktDstRegMarks { - // This generates the flow to match the first packet of NodePort / LoadBalancer connection initiated through the - // Antrea gateway and externalTrafficPolicy of the Service is Cluster, and the selected Endpoint is on a remote - // Node, then ConnSNATCTMark will be loaded in DNAT CT zone, indicating that SNAT is required for the connection. + // This generates the flow to match the first packets of externally-originated connections towards external + // addresses of the Service initiated through the Antrea gateway, and the selected Endpoint is on a remote Node, + // then ConnSNATCTMark will be loaded in DNAT CT zone, indicating that SNAT is required for the connection. flows = append(flows, SNATMarkTable.ofTable.BuildFlow(priorityNormal). Cookie(cookieID). MatchProtocol(ipProtocol). MatchCTStateNew(true). MatchCTStateTrk(true). - MatchRegMark(FromGatewayRegMark, pktDstRegMark, ToClusterServiceRegMark). + MatchRegMark(FromGatewayRegMark, pktDstRegMark, ExternallyAccessibleRegMark). Action().CT(true, SNATMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField). LoadToCtMark(ConnSNATCTMark). CTDone(). diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index 5209a6bb51e..f4b6d8bdd78 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -29,7 +29,6 @@ import ( protocol "antrea.io/libOpenflow/protocol" util "antrea.io/libOpenflow/util" gomock "github.com/golang/mock/gomock" - v1 "k8s.io/api/core/v1" net "net" reflect "reflect" ) @@ -467,17 +466,17 @@ func (mr *MockClientMockRecorder) InstallSNATMarkFlows(arg0, arg1 interface{}) * } // InstallServiceFlows mocks base method -func (m *MockClient) InstallServiceFlows(arg0 openflow.GroupIDType, arg1 net.IP, arg2 uint16, arg3 openflow.Protocol, arg4 uint16, arg5 bool, arg6 v1.ServiceType, arg7 bool) error { +func (m *MockClient) InstallServiceFlows(arg0 openflow.GroupIDType, arg1 net.IP, arg2 uint16, arg3 openflow.Protocol, arg4 uint16, arg5, arg6 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstallServiceFlows", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + ret := m.ctrl.Call(m, "InstallServiceFlows", arg0, arg1, arg2, arg3, arg4, arg5, arg6) ret0, _ := ret[0].(error) return ret0 } // InstallServiceFlows indicates an expected call of InstallServiceFlows -func (mr *MockClientMockRecorder) InstallServiceFlows(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) InstallServiceFlows(arg0, arg1, arg2, arg3, arg4, arg5, arg6 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallServiceFlows", reflect.TypeOf((*MockClient)(nil).InstallServiceFlows), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallServiceFlows", reflect.TypeOf((*MockClient)(nil).InstallServiceFlows), arg0, arg1, arg2, arg3, arg4, arg5, arg6) } // InstallServiceGroup mocks base method diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go index 1774915d96d..38f787b00af 100644 --- a/pkg/agent/proxy/proxier.go +++ b/pkg/agent/proxy/proxier.go @@ -189,12 +189,18 @@ func (p *proxier) removeServiceFlows(svcInfo *types.ServiceInfo) bool { klog.ErrorS(err, "Error when uninstalling ClusterIP flows for Service", "ServiceInfo", svcInfoStr) return false } - // Remove NodePort flows and configurations. + if p.proxyAll { + // Remove NodePort flows and configurations. if err := p.uninstallNodePortService(uint16(svcInfo.NodePort()), svcInfo.OFProtocol); err != nil { klog.ErrorS(err, "Error when uninstalling NodePort flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } + // Remove ExternalIP flows and configurations. + if err := p.uninstallExternalIPService(svcInfoStr, svcInfo.ExternalIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling ExternalIP flows and configurations for Service", "ServiceInfo", svcInfoStr) + return false + } } // Remove LoadBalancer flows and configurations. if p.proxyLoadBalancerIPs { @@ -325,7 +331,9 @@ func serviceIdentityChanged(svcInfo, pSvcInfo *types.ServiceInfo) bool { } func serviceExternalAddressesChanged(svcInfo, pSvcInfo *types.ServiceInfo) bool { - return svcInfo.NodePort() != pSvcInfo.NodePort() || !slices.Equal(svcInfo.LoadBalancerIPStrings(), pSvcInfo.LoadBalancerIPStrings()) + return svcInfo.NodePort() != pSvcInfo.NodePort() || + !slices.Equal(svcInfo.LoadBalancerIPStrings(), pSvcInfo.LoadBalancerIPStrings()) || + !slices.Equal(svcInfo.ExternalIPStrings(), pSvcInfo.ExternalIPStrings()) } // smallSliceDifference builds a slice which includes all the strings from s1 @@ -349,7 +357,7 @@ func smallSliceDifference(s1, s2 []string) []string { return diff } -func (p *proxier) installNodePortService(groupID binding.GroupIDType, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { +func (p *proxier) installNodePortService(groupID binding.GroupIDType, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16) error { if svcPort == 0 { return nil } @@ -357,7 +365,7 @@ func (p *proxier) installNodePortService(groupID binding.GroupIDType, svcPort ui if p.isIPv6 { svcIP = agentconfig.VirtualNodePortDNATIPv6 } - if err := p.ofClient.InstallServiceFlows(groupID, svcIP, svcPort, protocol, affinityTimeout, nodeLocalExternal, corev1.ServiceTypeNodePort, false); err != nil { + if err := p.ofClient.InstallServiceFlows(groupID, svcIP, svcPort, protocol, affinityTimeout, true, false); err != nil { return fmt.Errorf("failed to install NodePort load balancing flows: %w", err) } if err := p.routeClient.AddNodePort(p.nodePortAddresses, svcPort, protocol); err != nil { @@ -383,15 +391,45 @@ func (p *proxier) uninstallNodePortService(svcPort uint16, protocol binding.Prot return nil } -func (p *proxier) installLoadBalancerService(svcInfoStr string, groupID binding.GroupIDType, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { +func (p *proxier) installExternalIPService(svcInfoStr string, groupID binding.GroupIDType, externalIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16) error { + for _, externalIP := range externalIPStrings { + if externalIP != "" { + ip := net.ParseIP(externalIP) + if err := p.ofClient.InstallServiceFlows(groupID, ip, svcPort, protocol, affinityTimeout, true, false); err != nil { + return fmt.Errorf("failed to install ExternalIP load balancing flows: %w", err) + } + if err := p.addRouteForServiceIP(svcInfoStr, ip, p.routeClient.AddExternalIPRoute); err != nil { + return fmt.Errorf("failed to install ExternalIP traffic redirecting routes: %w", err) + } + } + } + return nil +} + +func (p *proxier) uninstallExternalIPService(svcInfoStr string, externalIPStrings []string, svcPort uint16, protocol binding.Protocol) error { + for _, externalIP := range externalIPStrings { + if externalIP != "" { + ip := net.ParseIP(externalIP) + if err := p.ofClient.UninstallServiceFlows(ip, svcPort, protocol); err != nil { + return fmt.Errorf("failed to remove ExternalIP load balancing flows: %w", err) + } + if err := p.deleteRouteForServiceIP(svcInfoStr, ip, p.routeClient.DeleteExternalIPRoute); err != nil { + return fmt.Errorf("failed to remove ExternalIP traffic redirecting routes: %w", err) + } + } + } + return nil +} + +func (p *proxier) installLoadBalancerService(svcInfoStr string, groupID binding.GroupIDType, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16) error { for _, ingress := range loadBalancerIPStrings { if ingress != "" { ip := net.ParseIP(ingress) - if err := p.ofClient.InstallServiceFlows(groupID, ip, svcPort, protocol, affinityTimeout, nodeLocalExternal, corev1.ServiceTypeLoadBalancer, false); err != nil { + if err := p.ofClient.InstallServiceFlows(groupID, ip, svcPort, protocol, affinityTimeout, true, false); err != nil { return fmt.Errorf("failed to install LoadBalancer load balancing flows: %w", err) } if p.proxyAll { - if err := p.addRouteForServiceIP(svcInfoStr, ip, p.routeClient.AddLoadBalancer); err != nil { + if err := p.addRouteForServiceIP(svcInfoStr, ip, p.routeClient.AddExternalIPRoute); err != nil { return fmt.Errorf("failed to install LoadBalancer traffic redirecting routes: %w", err) } } @@ -425,7 +463,7 @@ func (p *proxier) uninstallLoadBalancerService(svcInfoStr string, loadBalancerIP return fmt.Errorf("failed to remove LoadBalancer load balancing flows: %w", err) } if p.proxyAll { - if err := p.deleteRouteForServiceIP(svcInfoStr, ip, p.routeClient.DeleteLoadBalancer); err != nil { + if err := p.deleteRouteForServiceIP(svcInfoStr, ip, p.routeClient.DeleteExternalIPRoute); err != nil { return fmt.Errorf("failed to remove LoadBalancer traffic redirecting routes: %w", err) } } @@ -584,20 +622,24 @@ func (p *proxier) installServiceFlows(svcInfo *types.ServiceInfo, internalGroupI } // Install ClusterIP flows. - if err := p.ofClient.InstallServiceFlows(internalGroupID, svcInfo.ClusterIP(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal(), corev1.ServiceTypeClusterIP, isNestedService); err != nil { + if err := p.ofClient.InstallServiceFlows(internalGroupID, svcInfo.ClusterIP(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, false, isNestedService); err != nil { klog.ErrorS(err, "Error when installing ClusterIP flows for Service", "ServiceInfo", svcInfoStr) return false } - // Install NodePort flows and configurations. if p.proxyAll { - if err := p.installNodePortService(externalGroupID, uint16(svcInfo.NodePort()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + if err := p.installNodePortService(externalGroupID, uint16(svcInfo.NodePort()), svcInfo.OFProtocol, affinityTimeout); err != nil { klog.ErrorS(err, "Error when installing NodePort flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } + // Install ExternalIP flows and configurations. + if err := p.installExternalIPService(svcInfo.String(), externalGroupID, svcInfo.ExternalIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout); err != nil { + klog.ErrorS(err, "Error when installing ExternalIP flows and configurations for Service", "ServiceInfo", svcInfoStr) + return false + } } // Install LoadBalancer flows and configurations. if p.proxyLoadBalancerIPs { - if err := p.installLoadBalancerService(svcInfo.String(), externalGroupID, svcInfo.LoadBalancerIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + if err := p.installLoadBalancerService(svcInfo.String(), externalGroupID, svcInfo.LoadBalancerIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout); err != nil { klog.ErrorS(err, "Error when installing LoadBalancer flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } @@ -616,11 +658,21 @@ func (p *proxier) updateServiceExternalAddresses(pSvcInfo, svcInfo *types.Servic klog.ErrorS(err, "Error when uninstalling NodePort flows and configurations for Service", "ServiceInfo", pSvcInfoStr) return false } - if err := p.installNodePortService(externalGroupID, uint16(svcInfo.NodePort()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + if err := p.installNodePortService(externalGroupID, uint16(svcInfo.NodePort()), svcInfo.OFProtocol, affinityTimeout); err != nil { klog.ErrorS(err, "Error when installing NodePort flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } } + deletedExternalIPs := smallSliceDifference(pSvcInfo.ExternalIPStrings(), svcInfo.ExternalIPStrings()) + addedExternalIPs := smallSliceDifference(svcInfo.ExternalIPStrings(), pSvcInfo.ExternalIPStrings()) + if err := p.uninstallExternalIPService(pSvcInfoStr, deletedExternalIPs, uint16(pSvcInfo.Port()), pSvcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling ExternalIP flows and configurations for Service", "ServiceInfo", pSvcInfoStr) + return false + } + if err := p.installExternalIPService(svcInfoStr, externalGroupID, addedExternalIPs, uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout); err != nil { + klog.ErrorS(err, "Error when installing ExternalIP flows and configurations for Service", "ServiceInfo", svcInfoStr) + return false + } } if p.proxyLoadBalancerIPs { deletedLoadBalancerIPs := smallSliceDifference(pSvcInfo.LoadBalancerIPStrings(), svcInfo.LoadBalancerIPStrings()) @@ -629,7 +681,7 @@ func (p *proxier) updateServiceExternalAddresses(pSvcInfo, svcInfo *types.Servic klog.ErrorS(err, "Error when uninstalling LoadBalancer flows and configurations for Service", "ServiceInfo", pSvcInfoStr) return false } - if err := p.installLoadBalancerService(svcInfoStr, externalGroupID, addedLoadBalancerIPs, uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + if err := p.installLoadBalancerService(svcInfoStr, externalGroupID, addedLoadBalancerIPs, uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout); err != nil { klog.ErrorS(err, "Error when installing LoadBalancer flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } @@ -666,7 +718,7 @@ func compareEndpoints(endpointsCached map[string]k8sproxy.Endpoint, endpointsIns // endpoints or services resources are not synced. syncProxyRules is only called // through the Run method of the runner object, and all calls are serialized. // This method is the only one that changes internal state, but -// GetServiceFlowKeys(), which is called by the the "/ovsflows" API handler, +// GetServiceFlowKeys(), which is called by the "/ovsflows" API handler, // also reads service and endpoints maps, so serviceEndpointsMapsMutex is used // to protect these two maps. func (p *proxier) syncProxyRules() { diff --git a/pkg/agent/proxy/proxier_test.go b/pkg/agent/proxy/proxier_test.go index 5359a1e3ca7..0a2a0a55b13 100644 --- a/pkg/agent/proxy/proxier_test.go +++ b/pkg/agent/proxy/proxier_test.go @@ -61,6 +61,8 @@ var ( loadBalancerIPv6 = net.ParseIP("fec0::169:254:169:1") svcNodePortIPv4 = net.ParseIP("192.168.77.100") svcNodePortIPv6 = net.ParseIP("2001::192:168:77:100") + externalIPv4 = net.ParseIP("192.168.77.101") + externalIPv6 = net.ParseIP("2001::192:168:77:101") nodePortAddressesIPv4 = []net.IP{svcNodePortIPv4} nodePortAddressesIPv6 = []net.IP{svcNodePortIPv6} @@ -139,6 +141,7 @@ func makeTestEndpointSlice(namespace, name string, eps []discovery.Endpoint, por func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + externalIPs []net.IP, svcPort int32, protocol corev1.Protocol, affinitySeconds *int32, @@ -151,6 +154,11 @@ func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, Port: svcPort, Protocol: protocol, }} + for _, ip := range externalIPs { + if ip != nil { + svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip.String()) + } + } if internalTrafficPolicy != nil { svc.Spec.InternalTrafficPolicy = internalTrafficPolicy } @@ -170,6 +178,7 @@ func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + externalIPs []net.IP, svcPort, svcNodePort int32, protocol corev1.Protocol, @@ -185,6 +194,11 @@ func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, Port: svcPort, Protocol: protocol, }} + for _, ip := range externalIPs { + if ip != nil { + svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip.String()) + } + } svc.Spec.ExternalTrafficPolicy = externalTrafficPolicy svc.Spec.InternalTrafficPolicy = &internalTrafficPolicy if affinitySeconds != nil { @@ -200,6 +214,7 @@ func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, func makeTestLoadBalancerService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + externalIPs, loadBalancerIPs []net.IP, svcPort, svcNodePort int32, @@ -212,9 +227,16 @@ func makeTestLoadBalancerService(svcPortName *k8sproxy.ServicePortName, svc.Spec.Type = corev1.ServiceTypeLoadBalancer var ingress []corev1.LoadBalancerIngress for _, ip := range loadBalancerIPs { - ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip.String()}) + if ip != nil { + ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip.String()}) + } } svc.Status.LoadBalancer.Ingress = ingress + for _, ip := range externalIPs { + if ip != nil { + svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip.String()) + } + } svc.Spec.Ports = []corev1.ServicePort{{ NodePort: svcNodePort, Name: svcPortName.Port, @@ -355,6 +377,7 @@ func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodeP func testClusterIPAdd(t *testing.T, svcIP net.IP, + externalIP net.IP, ep1IP net.IP, ep2IP net.IP, isIPv6 bool, @@ -377,7 +400,11 @@ func testClusterIPAdd(t *testing.T, if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - allSvcs := append(extraSvcs, makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true)) + var externalIPs []net.IP + if externalIP != nil { + externalIPs = append(externalIPs, externalIP) + } + allSvcs := append(extraSvcs, makeTestClusterIPService(&svcPortName, svcIP, externalIPs, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true)) makeServiceMap(fp, allSvcs...) if !endpointSliceEnabled { @@ -402,8 +429,9 @@ func testClusterIPAdd(t *testing.T, nodeName = hostname serving = true } - expectedAllEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), nodeName, "", svcPort, true, true, serving, false, nil)} - if !nodeLocalInternal { + expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), nodeName, "", svcPort, true, true, serving, false, nil)} + expectedAllEps := expectedLocalEps + if !nodeLocalInternal || externalIP != nil { expectedAllEps = append(expectedAllEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, serving, false, nil)) } @@ -412,11 +440,28 @@ func testClusterIPAdd(t *testing.T, bindingProtocol = binding.ProtocolTCPv6 } - groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) - mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) - + if nodeLocalInternal == false { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, true).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } + } else { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedLocalEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, true).Times(1) + if externalIP != nil { + groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } + } + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) assert.Contains(t, fp.endpointsInstalledMap, svcPortName) @@ -425,6 +470,7 @@ func testClusterIPAdd(t *testing.T, func testLoadBalancerAdd(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, + externalIP net.IP, ep1IP net.IP, ep2IP net.IP, loadBalancerIP net.IP, @@ -454,7 +500,9 @@ func testLoadBalancerAdd(t *testing.T, if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - svc := makeTestLoadBalancerService(&svcPortName, svcIP, + svc := makeTestLoadBalancerService(&svcPortName, + svcIP, + []net.IP{externalIP}, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), @@ -511,29 +559,38 @@ func testLoadBalancerAdd(t *testing.T, } groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(clusterIPEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalExternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(nodePortEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) if proxyLoadBalancerIPs { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) } } else { nodeLocalVal := nodeLocalInternal && nodeLocalExternal groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalVal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) if proxyLoadBalancerIPs { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) } groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, !nodeLocalVal) mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) } if proxyLoadBalancerIPs { - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -543,6 +600,7 @@ func testLoadBalancerAdd(t *testing.T, func testNodePortAdd(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, + externalIP net.IP, ep1IP net.IP, ep2IP net.IP, isIPv6 bool, @@ -567,7 +625,9 @@ func testNodePortAdd(t *testing.T, if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - svc := makeTestNodePortService(&svcPortName, svcIP, + svc := makeTestNodePortService(&svcPortName, + svcIP, + []net.IP{externalIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, @@ -623,22 +683,30 @@ func testNodePortAdd(t *testing.T, } groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(clusterIPEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalExternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(nodePortEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } } else { nodeLocalVal := nodeLocalInternal && nodeLocalExternal groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalVal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) - + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, !nodeLocalVal) mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -649,36 +717,36 @@ func TestClusterIPAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) }) }) @@ -688,72 +756,72 @@ func TestLoadBalancerAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, false) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, true) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, false) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, true) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, true) }) }) }) @@ -872,18 +940,18 @@ func TestLoadBalancerServiceWithMultiplePorts(t *testing.T) { mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort80, remoteEndpointForPort80})).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, []k8sproxy.Endpoint{localEndpointForPort80}).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort80, remoteEndpointForPort80})).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30001Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30001Int32), binding.ProtocolTCP, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(port30001Int32), binding.ProtocolTCP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIPv4).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIPv4).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort443, remoteEndpointForPort443})).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, []k8sproxy.Endpoint{localEndpointForPort443}).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort443, remoteEndpointForPort443})).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30002Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30002Int32), binding.ProtocolTCP, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(port30002Int32), binding.ProtocolTCP).Times(1) fp.syncProxyRules() @@ -906,7 +974,7 @@ func TestLoadBalancerServiceWithMultiplePorts(t *testing.T) { mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(port30002Int32), binding.ProtocolTCP) // The route for the ClusterIP and the LoadBalancer IP should only be uninstalled once. - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIPv4) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIPv4) fp.syncProxyRules() @@ -917,60 +985,60 @@ func TestNodePortAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, false, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, false, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, true, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, false, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, false, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, true, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, false, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, false, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, true, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, false, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, false, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, true, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, true, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, false, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, false, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, true, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, false, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, false, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, true, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, false, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, false, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, true, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, false, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, false, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, true, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, true, true) }) }) }) @@ -988,8 +1056,8 @@ func TestClusterSkipServices(t *testing.T) { skippedServiceName := strings.Split(skippedServiceNN, "/")[1] svc1PortName := makeSvcPortName(skippedServiceNamespace, skippedServiceName, strconv.Itoa(svc1Port), corev1.ProtocolTCP) svc2PortName := makeSvcPortName("kube-system", "test", strconv.Itoa(svc2Port), corev1.ProtocolTCP) - svc1 := makeTestClusterIPService(&svc1PortName, svc1ClusterIP, int32(svc1Port), corev1.ProtocolTCP, nil, nil, false) - svc2 := makeTestClusterIPService(&svc2PortName, svc2ClusterIP, int32(svc2Port), corev1.ProtocolTCP, nil, nil, false) + svc1 := makeTestClusterIPService(&svc1PortName, svc1ClusterIP, nil, int32(svc1Port), corev1.ProtocolTCP, nil, nil, false) + svc2 := makeTestClusterIPService(&svc2PortName, svc2ClusterIP, nil, int32(svc2Port), corev1.ProtocolTCP, nil, nil, false) svcs := []*corev1.Service{svc1, svc2} epSubset := makeTestEndpointSubset(&svc1PortName, ep1IP, int32(svc1Port), corev1.ProtocolTCP, false) @@ -998,7 +1066,7 @@ func TestClusterSkipServices(t *testing.T) { ep2 := makeTestEndpoints(&svc2PortName, []corev1.EndpointSubset{*epSubset}) eps := []*corev1.Endpoints{ep1, ep2} - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, svcs, eps, false) + testClusterIPAdd(t, svc1IPv4, nil, ep1IPv4, ep2IPv4, false, false, svcs, eps, false) } func TestDualStackService(t *testing.T) { @@ -1038,11 +1106,11 @@ func TestDualStackService(t *testing.T) { mockOFClient.EXPECT().InstallServiceGroup(groupIDv4, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDv4, svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDv4, svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, false).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDv6, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCPv6, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDv6, svc1IPv6, uint16(svcPort), binding.ProtocolTCPv6, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDv6, svc1IPv6, uint16(svcPort), binding.ProtocolTCPv6, uint16(0), false, false).Times(1) fpv4.syncProxyRules() fpv6.syncProxyRules() @@ -1050,7 +1118,7 @@ func TestDualStackService(t *testing.T) { assert.Contains(t, fpv6.serviceInstalledMap, svcPortName) } -func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, nodeLocalInternal, endpointSliceEnabled bool) { +func testClusterIPRemove(t *testing.T, svcIP, externalIP, epIP net.IP, isIPv6 bool, nodeLocalInternal, endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOFClient, mockRouteClient := getMockClients(ctrl) @@ -1065,7 +1133,7 @@ func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, n if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true) + svc := makeTestClusterIPService(&svcPortName, svcIP, []net.IP{externalIP}, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true) makeServiceMap(fp, svc) var ep *corev1.Endpoints @@ -1085,17 +1153,39 @@ func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, n bindingProtocol = binding.ProtocolTCPv6 } - groupID := groupAllocator.Next() - mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) - if !nodeLocalInternal { + if nodeLocalInternal == false { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) - } - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) - mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - if !nodeLocalInternal { + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, true).Times(1) + mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + } + } else { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, true).Times(1) + mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + if externalIP != nil { + groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + + mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) + mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + } + } + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + mockRouteClient.EXPECT().DeleteExternalIPRoute(externalIP) } - mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1115,7 +1205,7 @@ func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, n assert.False(t, exists) } -func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, epIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { +func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP, externalIP, epIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOFClient, mockRouteClient := getMockClients(ctrl) @@ -1126,7 +1216,9 @@ func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, } fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, options...) - svc := makeTestNodePortService(&svcPortName, svcIP, + svc := makeTestNodePortService(&svcPortName, + svcIP, + []net.IP{externalIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, @@ -1159,15 +1251,23 @@ func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1185,7 +1285,7 @@ func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, assert.NotContains(t, fp.endpointsInstalledMap, svcPortName) } -func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, epIP net.IP, loadBalancerIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { +func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP, externalIP, epIP, loadBalancerIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOFClient, mockRouteClient := getMockClients(ctrl) @@ -1199,7 +1299,9 @@ func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net. externalTrafficPolicy := corev1.ServiceExternalTrafficPolicyTypeLocal internalTrafficPolicy := corev1.ServiceInternalTrafficPolicyCluster - svc := makeTestLoadBalancerService(&svcPortName, svcIP, + svc := makeTestLoadBalancerService(&svcPortName, + svcIP, + []net.IP{externalIP}, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), @@ -1233,12 +1335,16 @@ func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net. groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) @@ -1246,7 +1352,11 @@ func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net. mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1268,36 +1378,36 @@ func TestClusterIPRemove(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, false, false) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, false, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, true, false) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, false, true) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, false, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, true, true) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, true, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, false, false) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, false, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, true, false) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, false, true) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, false, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, true, true) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, true, true) }) }) }) @@ -1306,18 +1416,18 @@ func TestClusterIPRemove(t *testing.T) { func TestNodePortRemove(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, false, false) + testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, false, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, false, true) + testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, false, true) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, true, false) + testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, true, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, true, true) + testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, true, true) }) }) } @@ -1325,18 +1435,18 @@ func TestNodePortRemove(t *testing.T) { func TestLoadBalancerRemove(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPv4, false, false) + testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, loadBalancerIPv4, false, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPv4, false, true) + testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, loadBalancerIPv4, false, true) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPv6, true, false) + testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, loadBalancerIPv6, true, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPv6, true, true) + testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, loadBalancerIPv6, true, true) }) }) @@ -1349,19 +1459,19 @@ func testClusterIPNoEndpoint(t *testing.T, svcIP net.IP, isIPv6 bool) { groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) + svc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) makeServiceMap(fp, svc) makeEndpointSliceMap(fp) groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, []k8sproxy.Endpoint{}).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), gomock.Any(), uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), gomock.Any(), uint16(0), false, false).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), false, false).Times(1) fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) fp.syncProxyRules() } @@ -1382,14 +1492,18 @@ func testNodePortNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP net. groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) - svc := makeTestNodePortService(&svcPortName, svcIP, + svc := makeTestNodePortService(&svcPortName, + svcIP, + nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) - updatedSvc := makeTestNodePortService(&svcPortName, svcIP, + updatedSvc := makeTestNodePortService(&svcPortName, + svcIP, + nil, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, @@ -1408,16 +1522,16 @@ func testNodePortNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP net. groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) mockOFClient.EXPECT().InstallServiceGroup(groupIDCluster, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort), gomock.Any(), uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort), gomock.Any(), uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) fp.syncProxyRules() mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), gomock.Any()).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) fp.syncProxyRules() @@ -1442,7 +1556,9 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP internalTrafficPolicy := corev1.ServiceInternalTrafficPolicyCluster externalTrafficPolicy := corev1.ServiceExternalTrafficPolicyTypeLocal - svc := makeTestLoadBalancerService(&svcPortName, svcIP, + svc := makeTestLoadBalancerService(&svcPortName, + svcIP, + nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), @@ -1450,7 +1566,9 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP nil, &internalTrafficPolicy, externalTrafficPolicy) - updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, + updatedSvc := makeTestLoadBalancerService(&svcPortName, + svcIP, + nil, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), @@ -1470,23 +1588,23 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) mockOFClient.EXPECT().InstallServiceGroup(groupIDCluster, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort), gomock.Any(), uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), gomock.Any(), uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort), gomock.Any(), uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), gomock.Any(), uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) fp.syncProxyRules() mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), gomock.Any()).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort+1), gomock.Any(), uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort+1), gomock.Any(), uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) fp.syncProxyRules() } @@ -1510,8 +1628,8 @@ func testClusterIPRemoveSamePortEndpoint(t *testing.T, svcIP net.IP, epIP net.IP svcPortNameTCP := makeSvcPortName("ns", "svc-tcp", strconv.Itoa(svcPort), corev1.ProtocolTCP) svcPortNameUDP := makeSvcPortName("ns", "svc-udp", strconv.Itoa(svcPort), corev1.ProtocolUDP) - svcTCP := makeTestClusterIPService(&svcPortNameTCP, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - svcUDP := makeTestClusterIPService(&svcPortNameUDP, svcIP, int32(svcPort), corev1.ProtocolUDP, nil, nil, false) + svcTCP := makeTestClusterIPService(&svcPortNameTCP, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svcUDP := makeTestClusterIPService(&svcPortNameUDP, svcIP, nil, int32(svcPort), corev1.ProtocolUDP, nil, nil, false) makeServiceMap(fp, svcTCP, svcUDP) epTCP, epPortTCP := makeTestEndpointSliceEndpointAndPort(&svcPortNameTCP, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -1534,8 +1652,8 @@ func testClusterIPRemoveSamePortEndpoint(t *testing.T, svcIP net.IP, epIP net.IP mockOFClient.EXPECT().InstallServiceGroup(groupIDUDP, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(protocolTCP, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(protocolUDP, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), protocolTCP, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDUDP, svcIP, uint16(svcPort), protocolUDP, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), protocolTCP, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDUDP, svcIP, uint16(svcPort), protocolUDP, uint16(0), false, false).Times(1) fp.syncProxyRules() mockOFClient.EXPECT().InstallServiceGroup(groupIDUDP, false, gomock.Any()).Times(1) @@ -1565,7 +1683,7 @@ func testClusterIPRemoveEndpoints(t *testing.T, svcIP net.IP, epIP net.IP, isIPv groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) makeServiceMap(fp, svc) ep, epPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -1580,7 +1698,7 @@ func testClusterIPRemoveEndpoints(t *testing.T, svcIP net.IP, epIP net.IP, isIPv groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1649,7 +1767,7 @@ func testSessionAffinity(t *testing.T, svcIP net.IP, epIP net.IP, affinitySecond } else { expectedAffinity = uint16(affinitySeconds) } - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, expectedAffinity, false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, expectedAffinity, false, false).Times(1) fp.syncProxyRules() } @@ -1702,7 +1820,7 @@ func testSessionAffinityNoEndpoint(t *testing.T, svcExternalIPs net.IP, svcIP ne groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, true, []k8sproxy.Endpoint{}).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), gomock.Any(), uint16(10800), false, gomock.Any(), false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), gomock.Any(), uint16(10800), false, false).Times(1) fp.syncProxyRules() } @@ -1732,14 +1850,14 @@ func testServiceClusterIPUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, updatedSvcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, updatedSvcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, updatedSvcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, updatedSvcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, updatedSvcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, updatedSvcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -1759,29 +1877,29 @@ func testServiceClusterIPUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) s1 := mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, updatedSvcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, updatedSvcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) s2.After(s1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() @@ -1834,14 +1952,14 @@ func testServicePortUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -1861,31 +1979,31 @@ func testServicePortUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) s1 := mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort+1), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort+1), bindingProtocol, uint16(0), false, false).Times(1) s2.After(s1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) s1 = mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol) - s2 = mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort+1), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) + s2 = mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort+1), bindingProtocol, uint16(0), true, false).Times(1) s2.After(s1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1937,11 +2055,11 @@ func testServiceNodePortUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -1961,21 +2079,21 @@ func testServiceNodePortUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) s1 := mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort+1), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort+1), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort+1), bindingProtocol).Times(1) s2.After(s1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() @@ -2023,11 +2141,11 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeLocal) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeLocal) } makeServiceMap(fp, svc) @@ -2053,15 +2171,15 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -2071,14 +2189,14 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, expectedLocalEps).Times(1) s1 := mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) s2.After(s1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) @@ -2086,11 +2204,11 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, } if svcType == corev1.ServiceTypeLoadBalancer { s1 := mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) s2.After(s1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -2130,8 +2248,8 @@ func testServiceInternalTrafficPolicyUpdate(t *testing.T, internalTrafficPolicyCluster := corev1.ServiceInternalTrafficPolicyCluster internalTrafficPolicyLocal := corev1.ServiceInternalTrafficPolicyLocal - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyCluster, false) - updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyLocal, false) + svc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyCluster, false) + updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyLocal, false) makeServiceMap(fp, svc) remoteEp, remoteEpPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) @@ -2155,7 +2273,7 @@ func testServiceInternalTrafficPolicyUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) assert.Contains(t, fp.endpointsInstalledMap, svcPortName) @@ -2179,7 +2297,7 @@ func testServiceInternalTrafficPolicyUpdate(t *testing.T, mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, expectedLocalEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -2222,8 +2340,8 @@ func testServiceIngressIPsUpdate(t *testing.T, updatedLoadBalancerIPStrs = append(updatedLoadBalancerIPStrs, ip.String()) } - svc := makeTestLoadBalancerService(&svcPortName, svcIP, loadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, updatedLoadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, loadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, updatedLoadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) makeServiceMap(fp, svc) ep, epPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -2242,25 +2360,25 @@ func testServiceIngressIPsUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedEps)).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) for _, ip := range loadBalancerIPs { - mockOFClient.EXPECT().InstallServiceFlows(groupID, ip, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, ip, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) for _, ip := range loadBalancerIPs { - mockRouteClient.EXPECT().AddLoadBalancer(ip).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(ip).Times(1) } toDeleteLoadBalancerIPs := smallSliceDifference(loadBalancerIPStrs, updatedLoadBalancerIPStrs) toAddLoadBalancerIPs := smallSliceDifference(updatedLoadBalancerIPStrs, loadBalancerIPStrs) for _, ipStr := range toDeleteLoadBalancerIPs { mockOFClient.EXPECT().UninstallServiceFlows(net.ParseIP(ipStr), uint16(svcPort), bindingProtocol).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(net.ParseIP(ipStr)).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(net.ParseIP(ipStr)).Times(1) } for _, ipStr := range toAddLoadBalancerIPs { - mockOFClient.EXPECT().InstallServiceFlows(groupID, net.ParseIP(ipStr), uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(net.ParseIP(ipStr)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, net.ParseIP(ipStr), uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(net.ParseIP(ipStr)).Times(1) } fp.syncProxyRules() @@ -2307,14 +2425,14 @@ func testServiceStickyMaxAgeSecondsUpdate(t *testing.T, updatedAffinitySeconds := int32(100) switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -2334,25 +2452,25 @@ func testServiceStickyMaxAgeSecondsUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, true, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, false).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(updatedAffinitySeconds), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(updatedAffinitySeconds), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(updatedAffinitySeconds), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(updatedAffinitySeconds), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(updatedAffinitySeconds), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(updatedAffinitySeconds), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() @@ -2406,14 +2524,14 @@ func testServiceSessionAffinityTypeUpdate(t *testing.T, affinitySeconds := int32(100) switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -2433,29 +2551,29 @@ func testServiceSessionAffinityTypeUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, true, expectedEps).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), true, false).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol) - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), true, false).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() @@ -2501,8 +2619,8 @@ func TestServicesWithSameEndpoints(t *testing.T) { svcPortName1 := makeSvcPortName("ns", "svc1", strconv.Itoa(svcPort), corev1.ProtocolTCP) svcPortName2 := makeSvcPortName("ns", "svc2", strconv.Itoa(svcPort), corev1.ProtocolTCP) - svc1 := makeTestClusterIPService(&svcPortName1, svc1IPv4, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - svc2 := makeTestClusterIPService(&svcPortName2, svc2IPv4, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc1 := makeTestClusterIPService(&svcPortName1, svc1IPv4, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc2 := makeTestClusterIPService(&svcPortName2, svc2IPv4, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) makeServiceMap(fp, svc1, svc2) ep1, ep1Port := makeTestEndpointSliceEndpointAndPort(&svcPortName1, ep1IPv4, int32(svcPort), corev1.ProtocolTCP, false) @@ -2517,8 +2635,8 @@ func TestServicesWithSameEndpoints(t *testing.T) { mockOFClient.EXPECT().InstallServiceGroup(groupID2, false, gomock.Any()).Times(1) bindingProtocol := binding.ProtocolTCP mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID1, svc1IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID2, svc2IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID1, svc1IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID2, svc2IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svc1IPv4, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svc2IPv4, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(groupID1).Times(1) @@ -2569,7 +2687,7 @@ func TestMetrics(t *testing.T) { servicesInstallMetric = metrics.ServicesInstalledTotalV6.GaugeMetric } - testClusterIPAdd(t, tc.svcIP, tc.ep1IP, tc.ep2IP, tc.isIPv6, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, tc.svcIP, nil, tc.ep1IP, tc.ep2IP, tc.isIPv6, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) v, err := testutil.GetCounterMetricValue(endpointsUpdateTotalMetric) assert.NoError(t, err) assert.Equal(t, 0, int(v)) @@ -2583,7 +2701,7 @@ func TestMetrics(t *testing.T) { assert.Equal(t, 2, int(v)) assert.NoError(t, err) - testClusterIPRemove(t, tc.svcIP, tc.ep1IP, tc.isIPv6, false, false) + testClusterIPRemove(t, tc.svcIP, nil, tc.ep1IP, tc.isIPv6, false, false) v, err = testutil.GetCounterMetricValue(endpointsUpdateTotalMetric) assert.NoError(t, err) @@ -2608,6 +2726,7 @@ func TestGetServiceFlowKeys(t *testing.T) { groupAllocator := openflow.NewGroupAllocator(false) svc := makeTestNodePortService(&svcPortName, svc1IPv4, + nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, @@ -2670,8 +2789,8 @@ func TestGetServiceFlowKeys(t *testing.T) { mockRouteClient.EXPECT().AddNodePort(nodePortAddressesIPv4, uint16(svcNodePort), binding.ProtocolTCP).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), gomock.Any(), gomock.Any()).Times(2) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), gomock.Any(), uint16(svcNodePort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), gomock.Any(), uint16(svcNodePort), binding.ProtocolTCP, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, false).Times(1) fp.syncProxyRules() } diff --git a/pkg/agent/route/interfaces.go b/pkg/agent/route/interfaces.go index 8975c98f6bd..01ef219d456 100644 --- a/pkg/agent/route/interfaces.go +++ b/pkg/agent/route/interfaces.go @@ -65,11 +65,11 @@ type Interface interface { // DeleteNodePort deletes related configurations when a NodePort Service is deleted. DeleteNodePort(nodePortAddresses []net.IP, port uint16, protocol binding.Protocol) error - // AddLoadBalancer adds configurations when a LoadBalancer IP is added. - AddLoadBalancer(externalIP net.IP) error + // AddExternalIPRoute adds a route entry when an external IP is added. + AddExternalIPRoute(externalIP net.IP) error - // DeleteLoadBalancer deletes related configurations when a LoadBalancer IP is deleted. - DeleteLoadBalancer(externalIP net.IP) error + // DeleteExternalIPRoute deletes the related route entry when an external IP is deleted. + DeleteExternalIPRoute(externalIP net.IP) error // Run starts the sync loop. Run(stopCh <-chan struct{}) diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index 51497536112..586367dfc72 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -1405,11 +1405,11 @@ func (c *Client) addVirtualNodePortDNATIPRoute(isIPv6 bool) error { return nil } -// AddLoadBalancer is used to add routing entry which is used to route LoadBalancer ingress IP to Antrea -// gateway on host. -func (c *Client) AddLoadBalancer(svcIP net.IP) error { +// AddExternalIPRoute adds a route entry that forwards traffic destined for the external IP to the Antrea gateway interface. +func (c *Client) AddExternalIPRoute(externalIP net.IP) error { + externalIPStr := externalIP.String() linkIndex := c.nodeConfig.GatewayConfig.LinkIndex - isIPv6 := utilnet.IsIPv6(svcIP) + isIPv6 := utilnet.IsIPv6(externalIP) var gw net.IP var mask int if !isIPv6 { @@ -1420,35 +1420,32 @@ func (c *Client) AddLoadBalancer(svcIP net.IP) error { mask = net.IPv6len * 8 } - route := generateRoute(svcIP, mask, gw, linkIndex, netlink.SCOPE_UNIVERSE) + route := generateRoute(externalIP, mask, gw, linkIndex, netlink.SCOPE_UNIVERSE) if err := c.netlink.RouteReplace(route); err != nil { - return fmt.Errorf("failed to install route for LoadBalancer ingress IP %s: %w", svcIP.String(), err) + return fmt.Errorf("failed to install route for external IP %s: %w", externalIPStr, err) } - klog.V(4).InfoS("Added LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Store(svcIP.String(), route) - + klog.V(4).InfoS("Added route for external IP", "IP", externalIPStr) + c.serviceRoutes.Store(externalIP.String(), route) return nil } -// DeleteLoadBalancer is used to delete routing entry which is used to route LoadBalancer ingress IP to Antrea -// gateway on host. -func (c *Client) DeleteLoadBalancer(svcIP net.IP) error { - svcIPStr := svcIP.String() - route, found := c.serviceRoutes.Load(svcIPStr) +// DeleteExternalIPRoute deletes the route entry for the external IP. +func (c *Client) DeleteExternalIPRoute(externalIP net.IP) error { + externalIPStr := externalIP.String() + route, found := c.serviceRoutes.Load(externalIPStr) if !found { - klog.V(2).InfoS("Didn't find route for LoadBalancer ingress IP", "ingressIP", svcIPStr) + klog.V(2).InfoS("Didn't find route for external IP", "IP", externalIPStr) return nil } if err := c.netlink.RouteDel(route.(*netlink.Route)); err != nil { if err.Error() == "no such process" { - klog.InfoS("Failed to delete route for LoadBalancer ingress IP since it doesn't exist", "route", route) + klog.InfoS("Failed to delete route for external IP since it doesn't exist", "IP", externalIPStr) } else { - return fmt.Errorf("failed to delete route for LoadBalancer ingress IP %s: %w", svcIPStr, err) + return fmt.Errorf("failed to delete route for external IP %s: %w", externalIPStr, err) } } - klog.V(4).InfoS("Deleted LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Delete(svcIPStr) - + c.serviceRoutes.Delete(externalIPStr) + klog.V(4).InfoS("Deleted route for external IP", "IP", externalIPStr) return nil } diff --git a/pkg/agent/route/route_linux_test.go b/pkg/agent/route/route_linux_test.go index 6dab7a20e85..97a92b41384 100644 --- a/pkg/agent/route/route_linux_test.go +++ b/pkg/agent/route/route_linux_test.go @@ -1443,7 +1443,7 @@ func TestAddServiceCIDRRoute(t *testing.T) { } } -func TestAddLoadBalancer(t *testing.T) { +func TestAddExternalIPRoute(t *testing.T) { tests := []struct { name string externalIPs []string @@ -1487,13 +1487,13 @@ func TestAddLoadBalancer(t *testing.T) { tt.expectedCalls(mockNetlink.EXPECT()) for _, externalIP := range tt.externalIPs { - assert.NoError(t, c.AddLoadBalancer(net.ParseIP(externalIP))) + assert.NoError(t, c.AddExternalIPRoute(net.ParseIP(externalIP))) } }) } } -func TestDeleteLoadBalancer(t *testing.T) { +func TestDeleteExternalIPRoute(t *testing.T) { tests := []struct { name string serviceRoutes map[string]*netlink.Route @@ -1541,7 +1541,7 @@ func TestDeleteLoadBalancer(t *testing.T) { tt.expectedCalls(mockNetlink.EXPECT()) for _, externalIP := range tt.externalIPs { - assert.NoError(t, c.DeleteLoadBalancer(net.ParseIP(externalIP))) + assert.NoError(t, c.DeleteExternalIPRoute(net.ParseIP(externalIP))) } }) } diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 82257b9a995..d1e43216357 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -510,40 +510,43 @@ func (c *Client) DeleteNodePort(nodePortAddresses []net.IP, port uint16, protoco return nil } -func (c *Client) AddLoadBalancer(externalIP net.IP) error { - svcIPStr := externalIP.String() +// AddExternalIPRoute adds a route entry that forwards traffic destined for the external IP to the Antrea gateway interface. +func (c *Client) AddExternalIPRoute(externalIP net.IP) error { + externalIPStr := externalIP.String() linkIndex := c.nodeConfig.GatewayConfig.LinkIndex gw := config.VirtualServiceIPv4 metric := util.MetricHigh - _, svcIPNet, _ := net.ParseCIDR(svcIPStr) + _, svcIPNet, _ := net.ParseCIDR(externalIPStr) route := generateRoute(svcIPNet, gw, linkIndex, metric) if err := util.ReplaceNetRoute(route); err != nil { - return fmt.Errorf("failed to install route for LoadBalancer ingress IP %s: %w", svcIPStr, err) + return err } - klog.V(4).InfoS("Added LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Store(svcIPStr, route) - + if err := util.ReplaceNetRoute(route); err != nil { + return fmt.Errorf("failed to install route for external IP %s: %w", externalIPStr, err) + } + c.serviceRoutes.Store(externalIPStr, route) + klog.V(4).InfoS("Added route for external IP", "IP", externalIPStr) return nil } -func (c *Client) DeleteLoadBalancer(externalIP net.IP) error { - svcIPStr := externalIP.String() - route, found := c.serviceRoutes.Load(svcIPStr) +// DeleteExternalIPRoute deletes the route entry for the external IP. +func (c *Client) DeleteExternalIPRoute(externalIP net.IP) error { + externalIPStr := externalIP.String() + route, found := c.serviceRoutes.Load(externalIPStr) if !found { - klog.V(2).InfoS("Didn't find route for LoadBalancer ingress IP", "ingressIP", svcIPStr) + klog.V(2).InfoS("Didn't find route for external IP", "IP", externalIPStr) return nil } if err := util.RemoveNetRoute(route.(*util.Route)); err != nil { if strings.Contains(err.Error(), "No matching MSFT_NetRoute objects") { - klog.InfoS("Failed to delete route for LoadBalancer ingress IP since it doesn't exist", "route", route) + klog.InfoS("Failed to delete route for external IP since it doesn't exist", "IP", externalIPStr) } else { - return fmt.Errorf("failed to delete route for LoadBalancer ingress IP %s: %w", svcIPStr, err) + return fmt.Errorf("failed to delete route for external IP %s: %w", externalIPStr, err) } } - klog.V(4).InfoS("Deleted LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Delete(svcIPStr) - + c.serviceRoutes.Delete(externalIPStr) + klog.V(4).InfoS("Deleted route for external IP", "IP", externalIPStr) return nil } diff --git a/pkg/agent/route/testing/mock_route.go b/pkg/agent/route/testing/mock_route.go index 8d5a3256530..f1eb8bf66c9 100644 --- a/pkg/agent/route/testing/mock_route.go +++ b/pkg/agent/route/testing/mock_route.go @@ -50,18 +50,18 @@ func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { return m.recorder } -// AddLoadBalancer mocks base method -func (m *MockInterface) AddLoadBalancer(arg0 net.IP) error { +// AddExternalIPRoute mocks base method +func (m *MockInterface) AddExternalIPRoute(arg0 net.IP) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddLoadBalancer", arg0) + ret := m.ctrl.Call(m, "AddExternalIPRoute", arg0) ret0, _ := ret[0].(error) return ret0 } -// AddLoadBalancer indicates an expected call of AddLoadBalancer -func (mr *MockInterfaceMockRecorder) AddLoadBalancer(arg0 interface{}) *gomock.Call { +// AddExternalIPRoute indicates an expected call of AddExternalIPRoute +func (mr *MockInterfaceMockRecorder) AddExternalIPRoute(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLoadBalancer", reflect.TypeOf((*MockInterface)(nil).AddLoadBalancer), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddExternalIPRoute", reflect.TypeOf((*MockInterface)(nil).AddExternalIPRoute), arg0) } // AddLocalAntreaFlexibleIPAMPodRule mocks base method @@ -120,18 +120,18 @@ func (mr *MockInterfaceMockRecorder) AddSNATRule(arg0, arg1 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddSNATRule", reflect.TypeOf((*MockInterface)(nil).AddSNATRule), arg0, arg1) } -// DeleteLoadBalancer mocks base method -func (m *MockInterface) DeleteLoadBalancer(arg0 net.IP) error { +// DeleteExternalIPRoute mocks base method +func (m *MockInterface) DeleteExternalIPRoute(arg0 net.IP) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteLoadBalancer", arg0) + ret := m.ctrl.Call(m, "DeleteExternalIPRoute", arg0) ret0, _ := ret[0].(error) return ret0 } -// DeleteLoadBalancer indicates an expected call of DeleteLoadBalancer -func (mr *MockInterfaceMockRecorder) DeleteLoadBalancer(arg0 interface{}) *gomock.Call { +// DeleteExternalIPRoute indicates an expected call of DeleteExternalIPRoute +func (mr *MockInterfaceMockRecorder) DeleteExternalIPRoute(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancer", reflect.TypeOf((*MockInterface)(nil).DeleteLoadBalancer), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteExternalIPRoute", reflect.TypeOf((*MockInterface)(nil).DeleteExternalIPRoute), arg0) } // DeleteLocalAntreaFlexibleIPAMPodRule mocks base method diff --git a/pkg/ovs/openflow/testing/utils.go b/pkg/ovs/openflow/testing/utils.go index d5cad523f22..99ea462373a 100644 --- a/pkg/ovs/openflow/testing/utils.go +++ b/pkg/ovs/openflow/testing/utils.go @@ -84,6 +84,9 @@ func (m *fieldMetadata) getActionNickname() string { if strings.HasPrefix(name, "tun_ipv4_") { name = strings.Replace(name, "tun_ipv4_", "tun_", 1) } + if strings.HasPrefix(name, "ip_") && name != "ip_dscp" { + name = strings.Replace(name, "ip_", "nw_", 1) + } return name } @@ -804,14 +807,14 @@ func nxActionLearnToString(action openflow15.Action) string { if len(a.LearnSpecs) != 0 { for _, spec := range a.LearnSpecs { nBits := spec.Header.NBits - isLoad := spec.Header.Dst == false - isMatch := spec.Header.Dst == true + isLoad := spec.Header.Dst == true + isMatch := spec.Header.Dst == false //TODO: add isOutput if spec.SrcValue != nil { srcValueStr := strings.TrimLeft(fmt.Sprintf("%x", spec.SrcValue), "0") if isMatch { - dstFieldStr := getFieldNameString(spec.DstField.Field.Class, spec.DstField.Field.Field, 0, 0, true, false) + dstFieldStr := getFieldNameString(spec.DstField.Field.Class, spec.DstField.Field.Field, spec.DstField.Ofs, nBits, true, false) parts = append(parts, fmt.Sprintf("%s=0x%s", dstFieldStr, srcValueStr)) } else if isLoad { dstFieldStr := getFieldNameString(spec.DstField.Field.Class, spec.DstField.Field.Field, spec.DstField.Ofs, nBits, false, false) diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index c41e49e3705..7a6860d66b3 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -28,7 +28,6 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/component-base/metrics/legacyregistry" @@ -785,7 +784,7 @@ func installServiceFlows(t *testing.T, gid uint32, svc svcConfig, endpointList [ assert.NoError(t, err, "no error should return when installing flows for Endpoints") err = c.InstallServiceGroup(groupID, svc.withSessionAffinity, endpointList) assert.NoError(t, err, "no error should return when installing groups for Service") - err = c.InstallServiceFlows(groupID, svc.ip, svc.port, svc.protocol, stickyMaxAgeSeconds, false, v1.ServiceTypeClusterIP, false) + err = c.InstallServiceFlows(groupID, svc.ip, svc.port, svc.protocol, stickyMaxAgeSeconds, false, false) assert.NoError(t, err, "no error should return when installing flows for Service") } @@ -824,7 +823,7 @@ func expectedProxyServiceGroupAndFlows(gid uint32, svc svcConfig, endpointList [ svcFlows := expectTableFlows{tableName: "ServiceLB", flows: []*ofTestUtils.ExpectFlow{ { MatchStr: fmt.Sprintf("priority=200,%s,reg4=0x10000/0x70000,nw_dst=%s,tp_dst=%d", string(svc.protocol), svc.ip.String(), svc.port), - ActStr: fmt.Sprintf("set_field:0x%x/0x70000->reg4,set_field:0x200/0x200->reg0,%sgroup:%d", serviceLearnReg<<16, loadGourpID, gid), + ActStr: fmt.Sprintf("set_field:0x200/0x200->reg0,set_field:0x%x/0x70000->reg4,%sgroup:%d", serviceLearnReg<<16, loadGourpID, gid), }, { MatchStr: fmt.Sprintf("priority=190,%s,reg4=0x30000/0x70000,nw_dst=%s,tp_dst=%d", string(svc.protocol), svc.ip.String(), svc.port), diff --git a/third_party/proxy/types.go b/third_party/proxy/types.go index 385dcdd8ea9..6d2225b7c2d 100644 --- a/third_party/proxy/types.go +++ b/third_party/proxy/types.go @@ -138,12 +138,12 @@ type Endpoint interface { // This is only set when watching EndpointSlices. If using Endpoints, this is always // true since only ready endpoints are read from Endpoints. IsServing() bool - // IsTerminating retruns true if an endpoint is terminating. For pods, + // IsTerminating returns true if an endpoint is terminating. For pods, // that is any pod with a deletion timestamp. // This is only set when watching EndpointSlices. If using Endpoints, this is always // false since terminating endpoints are always excluded from Endpoints. IsTerminating() bool - // GetZoneHint returns the zone hint for the endpoint. This is based on + // GetZoneHints returns the zone hint for the endpoint. This is based on // endpoint.hints.forZones[0].name in the EndpointSlice API. GetZoneHints() sets.String // IP returns IP part of the endpoint.