From 07a0467bdda41d920d9e72c12dfde73d9492271b Mon Sep 17 00:00:00 2001 From: Hongliang Liu Date: Fri, 14 Apr 2023 22:03:37 +0800 Subject: [PATCH] Add support for ExternalIP in AntreaProxy This PR adds the ability to serve ExternalIP for AntreaProxy, allowing for external client accesses to Services running in Kubernetes. In Kubernetes, an ExternalIP is a feature that allows a Service to be accessed from outside the cluster using a static IP address. Signed-off-by: Hongliang Liu --- pkg/agent/openflow/client.go | 25 +- pkg/agent/openflow/client_test.go | 95 ++- pkg/agent/openflow/fields.go | 7 +- pkg/agent/openflow/pipeline.go | 132 ++--- pkg/agent/openflow/testing/mock_openflow.go | 9 +- pkg/agent/proxy/proxier.go | 82 ++- pkg/agent/proxy/proxier_test.go | 617 ++++++++++++-------- pkg/agent/route/interfaces.go | 8 +- pkg/agent/route/route_linux.go | 37 +- pkg/agent/route/route_linux_test.go | 8 +- pkg/agent/route/route_windows.go | 35 +- pkg/agent/route/testing/mock_route.go | 24 +- pkg/ovs/openflow/testing/utils.go | 9 +- test/integration/agent/openflow_test.go | 5 +- third_party/proxy/types.go | 4 +- 15 files changed, 627 insertions(+), 470 deletions(-) diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 03064d039b9..ca78e300310 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -22,7 +22,6 @@ import ( "antrea.io/libOpenflow/openflow15" "antrea.io/libOpenflow/protocol" ofutil "antrea.io/libOpenflow/util" - v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/config" @@ -97,14 +96,13 @@ type Client interface { // InstallEndpointFlows. UninstallEndpointFlows(protocol binding.Protocol, endpoints []proxy.Endpoint) error - // InstallServiceFlows installs flows for accessing Service NodePort, LoadBalancer and ClusterIP. It installs the - // flow that uses the group/bucket to do service LB. If the affinityTimeout is not zero, it also installs the flow - // which has a learn action to maintain the LB decision. The group with the groupID must be installed before, - // otherwise the installation will fail. - // nodeLocalExternal represents if the externalTrafficPolicy is Local or not. This field is meaningful only when - // the svcType is NodePort or LoadBalancer. - // nested represents if the Service has the Endpoints which is other Service's ClusterIP. - InstallServiceFlows(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool, svcType v1.ServiceType, nested bool) error + // InstallServiceFlows installs flows for accessing Service NodePort, LoadBalancer, ExternalIP and ClusterIP. It + // installs the flow that uses the group/bucket to do Service LB. If the affinityTimeout is not zero, it also + // installs the flow which has a learn action to maintain the LB decision. The group with the groupID must be + // installed before, otherwise the installation will fail. + // externallyAccessible indicates that whether the Service is externally accessible, like NodePort, LoadBalancer and ExternalIP. + // nested indicates that whether the Service has an Endpoints which is other Service's ClusterIP. + InstallServiceFlows(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, externallyAccessible, nested bool) error // UninstallServiceFlows removes flows installed by InstallServiceFlows. UninstallServiceFlows(svcIP net.IP, svcPort uint16, protocol binding.Protocol) error @@ -754,15 +752,16 @@ func (c *client) UninstallEndpointFlows(protocol binding.Protocol, endpoints []p return c.deleteFlowsWithMultipleKeys(c.featureService.cachedFlows, flowCacheKeys) } -func (c *client) InstallServiceFlows(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool, svcType v1.ServiceType, nested bool) error { +func (c *client) InstallServiceFlows(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, externallyAccessible, nested bool) error { c.replayMutex.RLock() defer c.replayMutex.RUnlock() var flows []binding.Flow - flows = append(flows, c.featureService.serviceLBFlow(groupID, svcIP, svcPort, protocol, affinityTimeout != 0, nodeLocalExternal, svcType, nested)) + matchNodePortAddress := svcIP.Equal(config.VirtualNodePortDNATIPv4) || svcIP.Equal(config.VirtualNodePortDNATIPv6) + flows = append(flows, c.featureService.serviceLBFlow(groupID, svcIP, svcPort, protocol, affinityTimeout != 0, externallyAccessible, matchNodePortAddress, nested)) if affinityTimeout != 0 { - flows = append(flows, c.featureService.serviceLearnFlow(groupID, svcIP, svcPort, protocol, affinityTimeout, nodeLocalExternal, svcType)) + flows = append(flows, c.featureService.serviceLearnFlow(groupID, svcIP, svcPort, protocol, affinityTimeout, externallyAccessible, matchNodePortAddress)) } - if svcType == v1.ServiceTypeClusterIP && !nested { + if !externallyAccessible && !nested { flows = append(flows, c.featureService.endpointRedirectFlowForServiceIP(svcIP, svcPort, protocol, groupID)) } cacheKey := generateServicePortFlowCacheKey(svcIP, svcPort, protocol) diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index 0785773c3a1..bf2038bfc9b 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -30,7 +30,6 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/openflow/cookie" @@ -1074,32 +1073,29 @@ func Test_client_InstallServiceFlows(t *testing.T) { port := uint16(80) testCases := []struct { - name string - protocol binding.Protocol - svcIP net.IP - affinityTimeout uint16 - nodeLocalExternal bool - svcType corev1.ServiceType - expectedFlows []string - nested bool + name string + protocol binding.Protocol + svcIP net.IP + affinityTimeout uint16 + externallyAccessible bool + expectedFlows []string + nested bool }{ { name: "Service ClusterIP", protocol: binding.ProtocolTCP, svcIP: svcIPv4, - svcType: corev1.ServiceTypeClusterIP, expectedFlows: []string{ "cookie=0x1030000000000, table=EndpointDNAT, priority=210,tcp,reg3=0xa600064,reg4=0x1020050/0x107ffff actions=group:100", - "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x20000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x64->reg7,group:100", + "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x20000/0x70000->reg4,set_field:0x64->reg7,group:100", }, }, { name: "Service ClusterIP, nested", protocol: binding.ProtocolTCP, svcIP: svcIPv4, - svcType: corev1.ServiceTypeClusterIP, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x20000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x64->reg7,set_field:0x1000000/0x1000000->reg4,group:100", + "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x20000/0x70000->reg4,set_field:0x64->reg7,set_field:0x1000000/0x1000000->reg4,group:100", }, nested: true, }, @@ -1108,11 +1104,10 @@ func Test_client_InstallServiceFlows(t *testing.T) { protocol: binding.ProtocolTCP, svcIP: svcIPv4, affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeClusterIP, expectedFlows: []string{ "cookie=0x1030000000000, table=EndpointDNAT, priority=210,tcp,reg3=0xa600064,reg4=0x1020050/0x107ffff actions=group:100", - "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,tcp,reg4=0x30000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x800->NXM_OF_ETH_TYPE[],load:0x6->NXM_OF_IP_PROTO[],load:OXM_OF_TCP_DST[]->OXM_OF_TCP_DST[],load:NXM_OF_IP_DST[]->NXM_OF_IP_DST[],load:NXM_OF_IP_SRC[]->NXM_OF_IP_SRC[],NXM_NX_REG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,tcp,reg4=0x30000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x800,nw_proto=0x6,OXM_OF_TCP_DST[],NXM_OF_IP_DST[],NXM_OF_IP_SRC[],load:NXM_NX_REG3[]->NXM_NX_REG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, { @@ -1120,55 +1115,53 @@ func Test_client_InstallServiceFlows(t *testing.T) { protocol: binding.ProtocolTCPv6, svcIP: svcIPv6, affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeClusterIP, expectedFlows: []string{ "cookie=0x1030000000000, table=EndpointDNAT, priority=210,tcp6,reg4=0x1020050/0x107ffff actions=group:100", - "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp6,reg4=0x10000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,tcp6,reg4=0x30000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x86dd->NXM_OF_ETH_TYPE[],load:0x6->NXM_OF_IP_PROTO[],load:OXM_OF_TCP_DST[]->OXM_OF_TCP_DST[],load:NXM_NX_IPV6_DST[]->NXM_NX_IPV6_DST[],load:NXM_NX_IPV6_SRC[]->NXM_NX_IPV6_SRC[],NXM_NX_XXREG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", - }, + "cookie=0x1030000000000, table=ServiceLB, priority=200,tcp6,reg4=0x10000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,tcp6,reg4=0x30000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x86dd,nw_proto=0x6,OXM_OF_TCP_DST[],NXM_NX_IPV6_DST[],NXM_NX_IPV6_SRC[],load:NXM_NX_XXREG3[]->NXM_NX_XXREG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT"}, }, { - name: "Service NodePort,SessionAffinity", - protocol: binding.ProtocolUDP, - svcIP: svcIPv4, - affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeNodePort, + name: "Service NodePort,SessionAffinity", + protocol: binding.ProtocolUDP, + svcIP: config.VirtualNodePortDNATIPv4, + affinityTimeout: uint16(100), + externallyAccessible: true, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,udp,reg4=0x90000/0xf0000,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,udp,reg4=0xb0000/0xf0000,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x800->NXM_OF_ETH_TYPE[],load:0x11->NXM_OF_IP_PROTO[],load:OXM_OF_UDP_DST[]->OXM_OF_UDP_DST[],reg4=0x1,load:NXM_OF_IP_DST[]->NXM_OF_IP_DST[],load:NXM_OF_IP_SRC[]->NXM_OF_IP_SRC[],NXM_NX_REG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,udp,reg4=0x90000/0xf0000,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,udp,reg4=0xb0000/0xf0000,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x800,nw_proto=0x11,OXM_OF_UDP_DST[],NXM_OF_IP_DST[],NXM_OF_IP_SRC[],load:NXM_NX_REG3[]->NXM_NX_REG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, { - name: "Service NodePort,ExternalPolicyLocal true,SessionAffinity", - protocol: binding.ProtocolUDPv6, - svcIP: svcIPv6, - affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeNodePort, + name: "Service NodePort,ExternalPolicyLocal true,SessionAffinity", + protocol: binding.ProtocolUDPv6, + svcIP: config.VirtualNodePortDNATIPv6, + affinityTimeout: uint16(100), + externallyAccessible: true, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,udp6,reg4=0x90000/0xf0000,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,udp6,reg4=0xb0000/0xf0000,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x86dd->NXM_OF_ETH_TYPE[],load:0x11->NXM_OF_IP_PROTO[],load:OXM_OF_UDP_DST[]->OXM_OF_UDP_DST[],reg4=0x1,load:NXM_NX_IPV6_DST[]->NXM_NX_IPV6_DST[],load:NXM_NX_IPV6_SRC[]->NXM_NX_IPV6_SRC[],NXM_NX_XXREG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,udp6,reg4=0x90000/0xf0000,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,udp6,reg4=0xb0000/0xf0000,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x86dd,nw_proto=0x11,OXM_OF_UDP_DST[],NXM_NX_IPV6_DST[],NXM_NX_IPV6_SRC[],load:NXM_NX_XXREG3[]->NXM_NX_XXREG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, { - name: "Service LoadBalancer,SessionAffinity", - protocol: binding.ProtocolSCTP, - svcIP: svcIPv4, - affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeLoadBalancer, + name: "Service LoadBalancer,SessionAffinity", + protocol: binding.ProtocolSCTP, + svcIP: svcIPv4, + affinityTimeout: uint16(100), + externallyAccessible: true, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,sctp,reg4=0x30000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x800->NXM_OF_ETH_TYPE[],load:0x84->NXM_OF_IP_PROTO[],load:OXM_OF_SCTP_DST[]->OXM_OF_SCTP_DST[],reg4=0x1,load:NXM_OF_IP_DST[]->NXM_OF_IP_DST[],load:NXM_OF_IP_SRC[]->NXM_OF_IP_SRC[],NXM_NX_REG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp,reg4=0x10000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,sctp,reg4=0x30000/0x70000,nw_dst=10.96.0.100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x800,nw_proto=0x84,OXM_OF_SCTP_DST[],NXM_OF_IP_DST[],NXM_OF_IP_SRC[],load:NXM_NX_REG3[]->NXM_NX_REG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, { - name: "Service LoadBalancer,ExternalPolicyLocal true,SessionAffinity", - protocol: binding.ProtocolSCTPv6, - svcIP: svcIPv6, - affinityTimeout: uint16(100), - svcType: corev1.ServiceTypeLoadBalancer, + name: "Service LoadBalancer,ExternalPolicyLocal true,SessionAffinity", + protocol: binding.ProtocolSCTPv6, + svcIP: svcIPv6, + affinityTimeout: uint16(100), + externallyAccessible: true, expectedFlows: []string{ - "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp6,reg4=0x10000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x30000/0x70000->reg4,set_field:0x200/0x200->reg0,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", - "cookie=0x1030000000064, table=ServiceLB, priority=190,sctp6,reg4=0x30000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,load:0x86dd->NXM_OF_ETH_TYPE[],load:0x84->NXM_OF_IP_PROTO[],load:OXM_OF_SCTP_DST[]->OXM_OF_SCTP_DST[],reg4=0x1,load:NXM_NX_IPV6_DST[]->NXM_NX_IPV6_DST[],load:NXM_NX_IPV6_SRC[]->NXM_NX_IPV6_SRC[],NXM_NX_XXREG3[],NXM_NX_REG4[0..15],reg4=0x2,reg0=0x1),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", + "cookie=0x1030000000000, table=ServiceLB, priority=200,sctp6,reg4=0x10000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=set_field:0x200/0x200->reg0,set_field:0x30000/0x70000->reg4,set_field:0x200000/0x200000->reg4,set_field:0x64->reg7,group:100", + "cookie=0x1030000000064, table=ServiceLB, priority=190,sctp6,reg4=0x30000/0x70000,ipv6_dst=fec0:10:96::100,tp_dst=80 actions=learn(table=SessionAffinity,hard_timeout=100,priority=200,delete_learned,cookie=0x1030000000064,eth_type=0x86dd,nw_proto=0x84,OXM_OF_SCTP_DST[],NXM_NX_IPV6_DST[],NXM_NX_IPV6_SRC[],load:NXM_NX_XXREG3[]->NXM_NX_XXREG3[],load:NXM_NX_REG4[0..15]->NXM_NX_REG4[0..15],load:0x2->NXM_NX_REG4[16..18],load:0x1->NXM_NX_REG0[9],load:0x1->NXM_NX_REG4[21]),set_field:0x20000/0x70000->reg4,goto_table:EndpointDNAT", }, }, } @@ -1186,7 +1179,7 @@ func Test_client_InstallServiceFlows(t *testing.T) { cacheKey := generateServicePortFlowCacheKey(tc.svcIP, port, tc.protocol) - assert.NoError(t, fc.InstallServiceFlows(groupID, tc.svcIP, port, tc.protocol, tc.affinityTimeout, tc.nodeLocalExternal, tc.svcType, tc.nested)) + assert.NoError(t, fc.InstallServiceFlows(groupID, tc.svcIP, port, tc.protocol, tc.affinityTimeout, tc.externallyAccessible, tc.nested)) fCacheI, ok := fc.featureService.cachedFlows.Load(cacheKey) require.True(t, ok) assert.ElementsMatch(t, tc.expectedFlows, getFlowStrings(fCacheI)) @@ -1216,12 +1209,12 @@ func Test_client_GetServiceFlowKeys(t *testing.T) { proxy.NewBaseEndpointInfo("10.10.0.12", "", "", 80, true, true, false, false, nil), } - assert.NoError(t, fc.InstallServiceFlows(groupID, svcIP, svcPort, bindingProtocol, 100, true, corev1.ServiceTypeLoadBalancer, false)) + assert.NoError(t, fc.InstallServiceFlows(groupID, svcIP, svcPort, bindingProtocol, 100, true, false)) assert.NoError(t, fc.InstallEndpointFlows(bindingProtocol, endpoints)) flowKeys := fc.GetServiceFlowKeys(svcIP, svcPort, bindingProtocol, endpoints) expectedFlowKeys := []string{ "table=11,tcp,tp_dst=0x50,nw_dst=10.96.0.224,reg4=0x10000/0x70000", - "table=11,tcp,reg4=0x30000/0x70000,nw_dst=10.96.0.224,tp_dst=0x50", + "table=11,tcp,tp_dst=0x50,nw_dst=10.96.0.224,reg4=0x30000/0x70000", "table=12,tcp,reg4=0x20050/0x7ffff,reg3=0xa0a000b", "table=12,tcp,reg4=0x20050/0x7ffff,reg3=0xa0a000c", "table=20,ip,nw_src=10.10.0.12,nw_dst=10.10.0.12,ct_state=+new+trk", diff --git a/pkg/agent/openflow/fields.go b/pkg/agent/openflow/fields.go index e0f8fa1702c..0efce6f5557 100644 --- a/pkg/agent/openflow/fields.go +++ b/pkg/agent/openflow/fields.go @@ -121,15 +121,12 @@ var ( EpUnionField = binding.NewRegField(4, 0, 18) // reg4[19]: Mark to indicate the Service type is NodePort. ToNodePortAddressRegMark = binding.NewOneBitRegMark(4, 19) - // reg4[16..19]: Field to store the union value of Endpoint state and the mark of whether Service type is NodePort. - NodePortUnionField = binding.NewRegField(4, 16, 19) // reg4[20]: Field to indicate whether the packet is from local Antrea IPAM Pod. NotAntreaFlexibleIPAMRegMark will // be used with RewriteMACRegMark, thus the reg id must not be same due to the limitation of ofnet library. AntreaFlexibleIPAMRegMark = binding.NewOneBitRegMark(4, 20) NotAntreaFlexibleIPAMRegMark = binding.NewOneBitZeroRegMark(4, 20) - // reg4[21]: Mark to indicate whether the packet is accessing a NodePort or a LoadBalancer IP of a Service whose - // externalTrafficPolicy is Cluster. - ToClusterServiceRegMark = binding.NewOneBitRegMark(4, 21) + // reg4[21]: Mark to indicate whether the Service is accessible externally, like NodePort, LoadBalancer or ExternalIP. + ExternallyAccessibleRegMark = binding.NewOneBitRegMark(4, 21) // reg4[22..23]: Field to store the action of a traffic control rule. Marks in this field include: TrafficControlActionField = binding.NewRegField(4, 22, 23) TrafficControlMirrorRegMark = binding.NewRegMark(TrafficControlActionField, 0b01) diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 87783408026..5e802406d1d 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -25,7 +25,6 @@ import ( "antrea.io/libOpenflow/openflow15" "antrea.io/libOpenflow/protocol" "antrea.io/ofnet/ofctrl" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "antrea.io/antrea/pkg/agent/config" @@ -2292,26 +2291,25 @@ func (f *featureService) serviceLearnFlow(groupID binding.GroupIDType, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, - nodeLocalExternal bool, - svcType v1.ServiceType) binding.Flow { + externalAddress bool, + nodePortAddress bool) binding.Flow { // Using unique cookie ID here to avoid learned flow cascade deletion. cookieID := f.cookieAllocator.RequestWithObjectID(f.category, uint32(groupID)).Raw() - var flowBuilder binding.FlowBuilder - if svcType == v1.ServiceTypeNodePort { - unionVal := (ToNodePortAddressRegMark.GetValue() << ServiceEPStateField.GetRange().Length()) + EpToLearnRegMark.GetValue() - flowBuilder = ServiceLBTable.ofTable.BuildFlow(priorityLow). - Cookie(cookieID). - MatchProtocol(protocol). - MatchRegFieldWithValue(NodePortUnionField, unionVal). - MatchDstPort(svcPort, nil) + flowBuilder := ServiceLBTable.ofTable.BuildFlow(priorityLow). + Cookie(cookieID). + MatchProtocol(protocol). + MatchDstPort(svcPort, nil) + + // EpToLearnRegMark is required to match the packets that have done Endpoint selection. + regMarksToMatch := []*binding.RegMark{EpToLearnRegMark} + // If "nodePortAddress" is true, which means that Service type is NodePort, "ToNodePortAddressRegMark" must be used + // to match packets. Otherwise, Service IP must be used to match packets. + if nodePortAddress { + regMarksToMatch = append(regMarksToMatch, ToNodePortAddressRegMark) } else { - flowBuilder = ServiceLBTable.ofTable.BuildFlow(priorityLow). - Cookie(cookieID). - MatchProtocol(protocol). - MatchRegMark(EpToLearnRegMark). - MatchDstIP(svcIP). - MatchDstPort(svcPort, nil) + flowBuilder = flowBuilder.MatchDstIP(svcIP) } + flowBuilder = flowBuilder.MatchRegMark(regMarksToMatch...) // affinityTimeout is used as the OpenFlow "hard timeout": learned flow will be removed from // OVS after that time regarding of whether traffic is still hitting the flow. This is the @@ -2334,10 +2332,14 @@ func (f *featureService) serviceLearnFlow(groupID binding.GroupIDType, case binding.ProtocolSCTPv6: learnFlowBuilderLearnAction = learnFlowBuilderLearnAction.MatchLearnedSCTPv6DstPort() } - // If externalTrafficPolicy of NodePort/LoadBalancer is Cluster, the learned flow which - // is used to match the first packet of NodePort/LoadBalancer also requires SNAT. - if (svcType == v1.ServiceTypeNodePort || svcType == v1.ServiceTypeLoadBalancer) && !nodeLocalExternal { - learnFlowBuilderLearnAction = learnFlowBuilderLearnAction.LoadRegMark(ToClusterServiceRegMark) + + // Loading the EpSelectedRegMark indicates that the Endpoint selection is complete. RewriteMACRegMark is required to + // load for Service packets + regMarksToLoad := []*binding.RegMark{EpSelectedRegMark, RewriteMACRegMark} + // If the Service is accessible externally, the ExternallyAccessibleRegMark should be loaded to determine whether SNAT + // is required for the connection. + if externalAddress { + regMarksToLoad = append(regMarksToLoad, ExternallyAccessibleRegMark) } ipProtocol := getIPProtocol(svcIP) @@ -2347,7 +2349,7 @@ func (f *featureService) serviceLearnFlow(groupID binding.GroupIDType, MatchLearnedSrcIP(). LoadFieldToField(EndpointIPField, EndpointIPField). LoadFieldToField(EndpointPortField, EndpointPortField). - LoadRegMark(EpSelectedRegMark, RewriteMACRegMark). + LoadRegMark(regMarksToLoad...). Done(). Action().LoadRegMark(EpSelectedRegMark). Action().NextTable(). @@ -2358,7 +2360,7 @@ func (f *featureService) serviceLearnFlow(groupID binding.GroupIDType, MatchLearnedSrcIPv6(). LoadXXRegToXXReg(EndpointIP6Field, EndpointIP6Field). LoadFieldToField(EndpointPortField, EndpointPortField). - LoadRegMark(EpSelectedRegMark, RewriteMACRegMark). + LoadRegMark(regMarksToLoad...). Done(). Action().LoadRegMark(EpSelectedRegMark). Action().NextTable(). @@ -2372,55 +2374,49 @@ func (f *featureService) serviceLBFlow(groupID binding.GroupIDType, svcIP net.IP, svcPort uint16, protocol binding.Protocol, - withSessionAffinity, - nodeLocalExternal bool, - serviceType v1.ServiceType, + withSessionAffinity bool, + externalAddress bool, + nodePortAddress bool, nested bool) binding.Flow { - cookieID := f.cookieAllocator.Request(f.category).Raw() - var lbResultMark *binding.RegMark - if withSessionAffinity { - lbResultMark = EpToLearnRegMark + flowBuilder := ServiceLBTable.ofTable.BuildFlow(priorityNormal). + Cookie(f.cookieAllocator.Request(f.category).Raw()). + MatchProtocol(protocol). + MatchDstPort(svcPort, nil) + + // EpToSelectRegMark is required to match the packets that haven't undergone Endpoint selection yet. + regMarksToMatch := []*binding.RegMark{EpToSelectRegMark} + // If "nodePortAddress" is true, which means that Service type is NodePort, "ToNodePortAddressRegMark" must be used + // to match packets. Otherwise, Service IP must be used to match packets. + if nodePortAddress { + regMarksToMatch = append(regMarksToMatch, ToNodePortAddressRegMark) } else { - lbResultMark = EpSelectedRegMark - } - regMarksToLoad := []*binding.RegMark{lbResultMark, RewriteMACRegMark} - var flowBuilder binding.FlowBuilder - if serviceType == v1.ServiceTypeNodePort { - unionVal := (ToNodePortAddressRegMark.GetValue() << ServiceEPStateField.GetRange().Length()) + EpToSelectRegMark.GetValue() - // If externalTrafficPolicy of a NodePort Service is Cluster (nodeLocalExternal=false), it loads - // ToClusterServiceRegMark to indicate it. The mark will be checked later when determining if SNAT is required - // or not. - if !nodeLocalExternal { - regMarksToLoad = append(regMarksToLoad, ToClusterServiceRegMark) - } - flowBuilder = ServiceLBTable.ofTable.BuildFlow(priorityNormal). - Cookie(cookieID). - MatchProtocol(protocol). - MatchRegFieldWithValue(NodePortUnionField, unionVal). - MatchDstPort(svcPort, nil). - Action().LoadRegMark(regMarksToLoad...) + flowBuilder = flowBuilder.MatchDstIP(svcIP) + } + + // RewriteMACRegMark is required to load for Service packets. + regMarksToLoad := []*binding.RegMark{RewriteMACRegMark} + // If SessionAffinity is set, the result of Endpoint selection should be cached by loading the EpToLearnRegMark. + // If SessionAffinity is not set, loading the EpSelectedRegMark indicates that the Endpoint selection is complete. + if withSessionAffinity { + regMarksToLoad = append(regMarksToLoad, EpToLearnRegMark) } else { - // If externalTrafficPolicy of a LoadBalancer Service is Cluster (nodeLocalExternal=false), it loads - // ToClusterServiceRegMark to indicate it. The mark will be checked later when determining if SNAT is required - // or not. - if serviceType == v1.ServiceTypeLoadBalancer && !nodeLocalExternal { - regMarksToLoad = append(regMarksToLoad, ToClusterServiceRegMark) - } - flowBuilder = ServiceLBTable.ofTable.BuildFlow(priorityNormal). - Cookie(cookieID). - MatchProtocol(protocol). - MatchDstPort(svcPort, nil). - MatchDstIP(svcIP). - MatchRegMark(EpToSelectRegMark). - Action().LoadRegMark(regMarksToLoad...) + regMarksToLoad = append(regMarksToLoad, EpSelectedRegMark) + } + // If the Service is accessible externally, the ExternallyAccessibleRegMark should be loaded to determine whether SNAT + // is required for the connection. + if externalAddress { + regMarksToLoad = append(regMarksToLoad, ExternallyAccessibleRegMark) } if f.enableAntreaPolicy { - flowBuilder = flowBuilder.Action().LoadToRegField(ServiceGroupIDField, uint32(groupID)) + regMarksToLoad = append(regMarksToLoad, binding.NewRegMark(ServiceGroupIDField, uint32(groupID))) } if nested { - flowBuilder = flowBuilder.Action().LoadRegMark(NestedServiceRegMark) + regMarksToLoad = append(regMarksToLoad, NestedServiceRegMark) } - return flowBuilder.Action().Group(groupID).Done() + + return flowBuilder.MatchRegMark(regMarksToMatch...). + Action().LoadRegMark(regMarksToLoad...). + Action().Group(groupID).Done() } // endpointRedirectFlowForServiceIP generates the flow which uses the specific group for a Service's ClusterIP @@ -3011,15 +3007,15 @@ func (f *featureService) gatewaySNATFlows() []binding.Flow { pktDstRegMarks = append(pktDstRegMarks, ToUplinkRegMark) } for _, pktDstRegMark := range pktDstRegMarks { - // This generates the flow to match the first packet of NodePort / LoadBalancer connection initiated through the - // Antrea gateway and externalTrafficPolicy of the Service is Cluster, and the selected Endpoint is on a remote - // Node, then ConnSNATCTMark will be loaded in DNAT CT zone, indicating that SNAT is required for the connection. + // This generates the flow to match the first packets of externally-originated connections towards external + // addresses of the Service initiated through the Antrea gateway, and the selected Endpoint is on a remote Node, + // then ConnSNATCTMark will be loaded in DNAT CT zone, indicating that SNAT is required for the connection. flows = append(flows, SNATMarkTable.ofTable.BuildFlow(priorityNormal). Cookie(cookieID). MatchProtocol(ipProtocol). MatchCTStateNew(true). MatchCTStateTrk(true). - MatchRegMark(FromGatewayRegMark, pktDstRegMark, ToClusterServiceRegMark). + MatchRegMark(FromGatewayRegMark, pktDstRegMark, ExternallyAccessibleRegMark). Action().CT(true, SNATMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField). LoadToCtMark(ConnSNATCTMark). CTDone(). diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index 5209a6bb51e..f4b6d8bdd78 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -29,7 +29,6 @@ import ( protocol "antrea.io/libOpenflow/protocol" util "antrea.io/libOpenflow/util" gomock "github.com/golang/mock/gomock" - v1 "k8s.io/api/core/v1" net "net" reflect "reflect" ) @@ -467,17 +466,17 @@ func (mr *MockClientMockRecorder) InstallSNATMarkFlows(arg0, arg1 interface{}) * } // InstallServiceFlows mocks base method -func (m *MockClient) InstallServiceFlows(arg0 openflow.GroupIDType, arg1 net.IP, arg2 uint16, arg3 openflow.Protocol, arg4 uint16, arg5 bool, arg6 v1.ServiceType, arg7 bool) error { +func (m *MockClient) InstallServiceFlows(arg0 openflow.GroupIDType, arg1 net.IP, arg2 uint16, arg3 openflow.Protocol, arg4 uint16, arg5, arg6 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstallServiceFlows", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + ret := m.ctrl.Call(m, "InstallServiceFlows", arg0, arg1, arg2, arg3, arg4, arg5, arg6) ret0, _ := ret[0].(error) return ret0 } // InstallServiceFlows indicates an expected call of InstallServiceFlows -func (mr *MockClientMockRecorder) InstallServiceFlows(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) InstallServiceFlows(arg0, arg1, arg2, arg3, arg4, arg5, arg6 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallServiceFlows", reflect.TypeOf((*MockClient)(nil).InstallServiceFlows), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallServiceFlows", reflect.TypeOf((*MockClient)(nil).InstallServiceFlows), arg0, arg1, arg2, arg3, arg4, arg5, arg6) } // InstallServiceGroup mocks base method diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go index 1774915d96d..38f787b00af 100644 --- a/pkg/agent/proxy/proxier.go +++ b/pkg/agent/proxy/proxier.go @@ -189,12 +189,18 @@ func (p *proxier) removeServiceFlows(svcInfo *types.ServiceInfo) bool { klog.ErrorS(err, "Error when uninstalling ClusterIP flows for Service", "ServiceInfo", svcInfoStr) return false } - // Remove NodePort flows and configurations. + if p.proxyAll { + // Remove NodePort flows and configurations. if err := p.uninstallNodePortService(uint16(svcInfo.NodePort()), svcInfo.OFProtocol); err != nil { klog.ErrorS(err, "Error when uninstalling NodePort flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } + // Remove ExternalIP flows and configurations. + if err := p.uninstallExternalIPService(svcInfoStr, svcInfo.ExternalIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling ExternalIP flows and configurations for Service", "ServiceInfo", svcInfoStr) + return false + } } // Remove LoadBalancer flows and configurations. if p.proxyLoadBalancerIPs { @@ -325,7 +331,9 @@ func serviceIdentityChanged(svcInfo, pSvcInfo *types.ServiceInfo) bool { } func serviceExternalAddressesChanged(svcInfo, pSvcInfo *types.ServiceInfo) bool { - return svcInfo.NodePort() != pSvcInfo.NodePort() || !slices.Equal(svcInfo.LoadBalancerIPStrings(), pSvcInfo.LoadBalancerIPStrings()) + return svcInfo.NodePort() != pSvcInfo.NodePort() || + !slices.Equal(svcInfo.LoadBalancerIPStrings(), pSvcInfo.LoadBalancerIPStrings()) || + !slices.Equal(svcInfo.ExternalIPStrings(), pSvcInfo.ExternalIPStrings()) } // smallSliceDifference builds a slice which includes all the strings from s1 @@ -349,7 +357,7 @@ func smallSliceDifference(s1, s2 []string) []string { return diff } -func (p *proxier) installNodePortService(groupID binding.GroupIDType, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { +func (p *proxier) installNodePortService(groupID binding.GroupIDType, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16) error { if svcPort == 0 { return nil } @@ -357,7 +365,7 @@ func (p *proxier) installNodePortService(groupID binding.GroupIDType, svcPort ui if p.isIPv6 { svcIP = agentconfig.VirtualNodePortDNATIPv6 } - if err := p.ofClient.InstallServiceFlows(groupID, svcIP, svcPort, protocol, affinityTimeout, nodeLocalExternal, corev1.ServiceTypeNodePort, false); err != nil { + if err := p.ofClient.InstallServiceFlows(groupID, svcIP, svcPort, protocol, affinityTimeout, true, false); err != nil { return fmt.Errorf("failed to install NodePort load balancing flows: %w", err) } if err := p.routeClient.AddNodePort(p.nodePortAddresses, svcPort, protocol); err != nil { @@ -383,15 +391,45 @@ func (p *proxier) uninstallNodePortService(svcPort uint16, protocol binding.Prot return nil } -func (p *proxier) installLoadBalancerService(svcInfoStr string, groupID binding.GroupIDType, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { +func (p *proxier) installExternalIPService(svcInfoStr string, groupID binding.GroupIDType, externalIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16) error { + for _, externalIP := range externalIPStrings { + if externalIP != "" { + ip := net.ParseIP(externalIP) + if err := p.ofClient.InstallServiceFlows(groupID, ip, svcPort, protocol, affinityTimeout, true, false); err != nil { + return fmt.Errorf("failed to install ExternalIP load balancing flows: %w", err) + } + if err := p.addRouteForServiceIP(svcInfoStr, ip, p.routeClient.AddExternalIPRoute); err != nil { + return fmt.Errorf("failed to install ExternalIP traffic redirecting routes: %w", err) + } + } + } + return nil +} + +func (p *proxier) uninstallExternalIPService(svcInfoStr string, externalIPStrings []string, svcPort uint16, protocol binding.Protocol) error { + for _, externalIP := range externalIPStrings { + if externalIP != "" { + ip := net.ParseIP(externalIP) + if err := p.ofClient.UninstallServiceFlows(ip, svcPort, protocol); err != nil { + return fmt.Errorf("failed to remove ExternalIP load balancing flows: %w", err) + } + if err := p.deleteRouteForServiceIP(svcInfoStr, ip, p.routeClient.DeleteExternalIPRoute); err != nil { + return fmt.Errorf("failed to remove ExternalIP traffic redirecting routes: %w", err) + } + } + } + return nil +} + +func (p *proxier) installLoadBalancerService(svcInfoStr string, groupID binding.GroupIDType, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16) error { for _, ingress := range loadBalancerIPStrings { if ingress != "" { ip := net.ParseIP(ingress) - if err := p.ofClient.InstallServiceFlows(groupID, ip, svcPort, protocol, affinityTimeout, nodeLocalExternal, corev1.ServiceTypeLoadBalancer, false); err != nil { + if err := p.ofClient.InstallServiceFlows(groupID, ip, svcPort, protocol, affinityTimeout, true, false); err != nil { return fmt.Errorf("failed to install LoadBalancer load balancing flows: %w", err) } if p.proxyAll { - if err := p.addRouteForServiceIP(svcInfoStr, ip, p.routeClient.AddLoadBalancer); err != nil { + if err := p.addRouteForServiceIP(svcInfoStr, ip, p.routeClient.AddExternalIPRoute); err != nil { return fmt.Errorf("failed to install LoadBalancer traffic redirecting routes: %w", err) } } @@ -425,7 +463,7 @@ func (p *proxier) uninstallLoadBalancerService(svcInfoStr string, loadBalancerIP return fmt.Errorf("failed to remove LoadBalancer load balancing flows: %w", err) } if p.proxyAll { - if err := p.deleteRouteForServiceIP(svcInfoStr, ip, p.routeClient.DeleteLoadBalancer); err != nil { + if err := p.deleteRouteForServiceIP(svcInfoStr, ip, p.routeClient.DeleteExternalIPRoute); err != nil { return fmt.Errorf("failed to remove LoadBalancer traffic redirecting routes: %w", err) } } @@ -584,20 +622,24 @@ func (p *proxier) installServiceFlows(svcInfo *types.ServiceInfo, internalGroupI } // Install ClusterIP flows. - if err := p.ofClient.InstallServiceFlows(internalGroupID, svcInfo.ClusterIP(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal(), corev1.ServiceTypeClusterIP, isNestedService); err != nil { + if err := p.ofClient.InstallServiceFlows(internalGroupID, svcInfo.ClusterIP(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, false, isNestedService); err != nil { klog.ErrorS(err, "Error when installing ClusterIP flows for Service", "ServiceInfo", svcInfoStr) return false } - // Install NodePort flows and configurations. if p.proxyAll { - if err := p.installNodePortService(externalGroupID, uint16(svcInfo.NodePort()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + if err := p.installNodePortService(externalGroupID, uint16(svcInfo.NodePort()), svcInfo.OFProtocol, affinityTimeout); err != nil { klog.ErrorS(err, "Error when installing NodePort flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } + // Install ExternalIP flows and configurations. + if err := p.installExternalIPService(svcInfo.String(), externalGroupID, svcInfo.ExternalIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout); err != nil { + klog.ErrorS(err, "Error when installing ExternalIP flows and configurations for Service", "ServiceInfo", svcInfoStr) + return false + } } // Install LoadBalancer flows and configurations. if p.proxyLoadBalancerIPs { - if err := p.installLoadBalancerService(svcInfo.String(), externalGroupID, svcInfo.LoadBalancerIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + if err := p.installLoadBalancerService(svcInfo.String(), externalGroupID, svcInfo.LoadBalancerIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout); err != nil { klog.ErrorS(err, "Error when installing LoadBalancer flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } @@ -616,11 +658,21 @@ func (p *proxier) updateServiceExternalAddresses(pSvcInfo, svcInfo *types.Servic klog.ErrorS(err, "Error when uninstalling NodePort flows and configurations for Service", "ServiceInfo", pSvcInfoStr) return false } - if err := p.installNodePortService(externalGroupID, uint16(svcInfo.NodePort()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + if err := p.installNodePortService(externalGroupID, uint16(svcInfo.NodePort()), svcInfo.OFProtocol, affinityTimeout); err != nil { klog.ErrorS(err, "Error when installing NodePort flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } } + deletedExternalIPs := smallSliceDifference(pSvcInfo.ExternalIPStrings(), svcInfo.ExternalIPStrings()) + addedExternalIPs := smallSliceDifference(svcInfo.ExternalIPStrings(), pSvcInfo.ExternalIPStrings()) + if err := p.uninstallExternalIPService(pSvcInfoStr, deletedExternalIPs, uint16(pSvcInfo.Port()), pSvcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling ExternalIP flows and configurations for Service", "ServiceInfo", pSvcInfoStr) + return false + } + if err := p.installExternalIPService(svcInfoStr, externalGroupID, addedExternalIPs, uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout); err != nil { + klog.ErrorS(err, "Error when installing ExternalIP flows and configurations for Service", "ServiceInfo", svcInfoStr) + return false + } } if p.proxyLoadBalancerIPs { deletedLoadBalancerIPs := smallSliceDifference(pSvcInfo.LoadBalancerIPStrings(), svcInfo.LoadBalancerIPStrings()) @@ -629,7 +681,7 @@ func (p *proxier) updateServiceExternalAddresses(pSvcInfo, svcInfo *types.Servic klog.ErrorS(err, "Error when uninstalling LoadBalancer flows and configurations for Service", "ServiceInfo", pSvcInfoStr) return false } - if err := p.installLoadBalancerService(svcInfoStr, externalGroupID, addedLoadBalancerIPs, uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout, svcInfo.ExternalPolicyLocal()); err != nil { + if err := p.installLoadBalancerService(svcInfoStr, externalGroupID, addedLoadBalancerIPs, uint16(svcInfo.Port()), svcInfo.OFProtocol, affinityTimeout); err != nil { klog.ErrorS(err, "Error when installing LoadBalancer flows and configurations for Service", "ServiceInfo", svcInfoStr) return false } @@ -666,7 +718,7 @@ func compareEndpoints(endpointsCached map[string]k8sproxy.Endpoint, endpointsIns // endpoints or services resources are not synced. syncProxyRules is only called // through the Run method of the runner object, and all calls are serialized. // This method is the only one that changes internal state, but -// GetServiceFlowKeys(), which is called by the the "/ovsflows" API handler, +// GetServiceFlowKeys(), which is called by the "/ovsflows" API handler, // also reads service and endpoints maps, so serviceEndpointsMapsMutex is used // to protect these two maps. func (p *proxier) syncProxyRules() { diff --git a/pkg/agent/proxy/proxier_test.go b/pkg/agent/proxy/proxier_test.go index 5359a1e3ca7..0a2a0a55b13 100644 --- a/pkg/agent/proxy/proxier_test.go +++ b/pkg/agent/proxy/proxier_test.go @@ -61,6 +61,8 @@ var ( loadBalancerIPv6 = net.ParseIP("fec0::169:254:169:1") svcNodePortIPv4 = net.ParseIP("192.168.77.100") svcNodePortIPv6 = net.ParseIP("2001::192:168:77:100") + externalIPv4 = net.ParseIP("192.168.77.101") + externalIPv6 = net.ParseIP("2001::192:168:77:101") nodePortAddressesIPv4 = []net.IP{svcNodePortIPv4} nodePortAddressesIPv6 = []net.IP{svcNodePortIPv6} @@ -139,6 +141,7 @@ func makeTestEndpointSlice(namespace, name string, eps []discovery.Endpoint, por func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + externalIPs []net.IP, svcPort int32, protocol corev1.Protocol, affinitySeconds *int32, @@ -151,6 +154,11 @@ func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, Port: svcPort, Protocol: protocol, }} + for _, ip := range externalIPs { + if ip != nil { + svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip.String()) + } + } if internalTrafficPolicy != nil { svc.Spec.InternalTrafficPolicy = internalTrafficPolicy } @@ -170,6 +178,7 @@ func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + externalIPs []net.IP, svcPort, svcNodePort int32, protocol corev1.Protocol, @@ -185,6 +194,11 @@ func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, Port: svcPort, Protocol: protocol, }} + for _, ip := range externalIPs { + if ip != nil { + svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip.String()) + } + } svc.Spec.ExternalTrafficPolicy = externalTrafficPolicy svc.Spec.InternalTrafficPolicy = &internalTrafficPolicy if affinitySeconds != nil { @@ -200,6 +214,7 @@ func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, func makeTestLoadBalancerService(svcPortName *k8sproxy.ServicePortName, clusterIP net.IP, + externalIPs, loadBalancerIPs []net.IP, svcPort, svcNodePort int32, @@ -212,9 +227,16 @@ func makeTestLoadBalancerService(svcPortName *k8sproxy.ServicePortName, svc.Spec.Type = corev1.ServiceTypeLoadBalancer var ingress []corev1.LoadBalancerIngress for _, ip := range loadBalancerIPs { - ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip.String()}) + if ip != nil { + ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip.String()}) + } } svc.Status.LoadBalancer.Ingress = ingress + for _, ip := range externalIPs { + if ip != nil { + svc.Spec.ExternalIPs = append(svc.Spec.ExternalIPs, ip.String()) + } + } svc.Spec.Ports = []corev1.ServicePort{{ NodePort: svcNodePort, Name: svcPortName.Port, @@ -355,6 +377,7 @@ func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodeP func testClusterIPAdd(t *testing.T, svcIP net.IP, + externalIP net.IP, ep1IP net.IP, ep2IP net.IP, isIPv6 bool, @@ -377,7 +400,11 @@ func testClusterIPAdd(t *testing.T, if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - allSvcs := append(extraSvcs, makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true)) + var externalIPs []net.IP + if externalIP != nil { + externalIPs = append(externalIPs, externalIP) + } + allSvcs := append(extraSvcs, makeTestClusterIPService(&svcPortName, svcIP, externalIPs, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true)) makeServiceMap(fp, allSvcs...) if !endpointSliceEnabled { @@ -402,8 +429,9 @@ func testClusterIPAdd(t *testing.T, nodeName = hostname serving = true } - expectedAllEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), nodeName, "", svcPort, true, true, serving, false, nil)} - if !nodeLocalInternal { + expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), nodeName, "", svcPort, true, true, serving, false, nil)} + expectedAllEps := expectedLocalEps + if !nodeLocalInternal || externalIP != nil { expectedAllEps = append(expectedAllEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, serving, false, nil)) } @@ -412,11 +440,28 @@ func testClusterIPAdd(t *testing.T, bindingProtocol = binding.ProtocolTCPv6 } - groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) - mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) - + if nodeLocalInternal == false { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, true).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } + } else { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedLocalEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, true).Times(1) + if externalIP != nil { + groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } + } + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) assert.Contains(t, fp.endpointsInstalledMap, svcPortName) @@ -425,6 +470,7 @@ func testClusterIPAdd(t *testing.T, func testLoadBalancerAdd(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, + externalIP net.IP, ep1IP net.IP, ep2IP net.IP, loadBalancerIP net.IP, @@ -454,7 +500,9 @@ func testLoadBalancerAdd(t *testing.T, if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - svc := makeTestLoadBalancerService(&svcPortName, svcIP, + svc := makeTestLoadBalancerService(&svcPortName, + svcIP, + []net.IP{externalIP}, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), @@ -511,29 +559,38 @@ func testLoadBalancerAdd(t *testing.T, } groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(clusterIPEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalExternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(nodePortEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) if proxyLoadBalancerIPs { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) } } else { nodeLocalVal := nodeLocalInternal && nodeLocalExternal groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalVal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) if proxyLoadBalancerIPs { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) } groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, !nodeLocalVal) mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) } if proxyLoadBalancerIPs { - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -543,6 +600,7 @@ func testLoadBalancerAdd(t *testing.T, func testNodePortAdd(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, + externalIP net.IP, ep1IP net.IP, ep2IP net.IP, isIPv6 bool, @@ -567,7 +625,9 @@ func testNodePortAdd(t *testing.T, if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - svc := makeTestNodePortService(&svcPortName, svcIP, + svc := makeTestNodePortService(&svcPortName, + svcIP, + []net.IP{externalIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, @@ -623,22 +683,30 @@ func testNodePortAdd(t *testing.T, } groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(clusterIPEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalExternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(nodePortEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } } else { nodeLocalVal := nodeLocalInternal && nodeLocalExternal groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalVal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort, false).Times(1) - + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + } groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, !nodeLocalVal) mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -649,36 +717,36 @@ func TestClusterIPAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) }) }) }) @@ -688,72 +756,72 @@ func TestLoadBalancerAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, false) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, false) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, true) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, true) + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, false) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, false) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, true) }) t.Run("No External IPs", func(t *testing.T) { - testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, true) + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, true) }) }) }) @@ -872,18 +940,18 @@ func TestLoadBalancerServiceWithMultiplePorts(t *testing.T) { mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort80, remoteEndpointForPort80})).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, []k8sproxy.Endpoint{localEndpointForPort80}).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort80, remoteEndpointForPort80})).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30001Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30001Int32), binding.ProtocolTCP, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(port30001Int32), binding.ProtocolTCP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIPv4).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIPv4).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort443, remoteEndpointForPort443})).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, []k8sproxy.Endpoint{localEndpointForPort443}).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort443, remoteEndpointForPort443})).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30002Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30002Int32), binding.ProtocolTCP, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(port30002Int32), binding.ProtocolTCP).Times(1) fp.syncProxyRules() @@ -906,7 +974,7 @@ func TestLoadBalancerServiceWithMultiplePorts(t *testing.T) { mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(port30002Int32), binding.ProtocolTCP) // The route for the ClusterIP and the LoadBalancer IP should only be uninstalled once. - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIPv4) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIPv4) fp.syncProxyRules() @@ -917,60 +985,60 @@ func TestNodePortAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, false, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, false, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, true, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, false, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, false, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, true, false) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, false, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, false, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, true, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, false, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, false, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, true, true) + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, ep2IPv4, false, true, true, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, false, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, false, false) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, true, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, true, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, false, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, false, false) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, true, false) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, false, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, false, true) }) t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, true, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, false, true, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, false, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, false, true) }) t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, true, true) + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, ep2IPv6, true, true, true, true) }) }) }) @@ -988,8 +1056,8 @@ func TestClusterSkipServices(t *testing.T) { skippedServiceName := strings.Split(skippedServiceNN, "/")[1] svc1PortName := makeSvcPortName(skippedServiceNamespace, skippedServiceName, strconv.Itoa(svc1Port), corev1.ProtocolTCP) svc2PortName := makeSvcPortName("kube-system", "test", strconv.Itoa(svc2Port), corev1.ProtocolTCP) - svc1 := makeTestClusterIPService(&svc1PortName, svc1ClusterIP, int32(svc1Port), corev1.ProtocolTCP, nil, nil, false) - svc2 := makeTestClusterIPService(&svc2PortName, svc2ClusterIP, int32(svc2Port), corev1.ProtocolTCP, nil, nil, false) + svc1 := makeTestClusterIPService(&svc1PortName, svc1ClusterIP, nil, int32(svc1Port), corev1.ProtocolTCP, nil, nil, false) + svc2 := makeTestClusterIPService(&svc2PortName, svc2ClusterIP, nil, int32(svc2Port), corev1.ProtocolTCP, nil, nil, false) svcs := []*corev1.Service{svc1, svc2} epSubset := makeTestEndpointSubset(&svc1PortName, ep1IP, int32(svc1Port), corev1.ProtocolTCP, false) @@ -998,7 +1066,7 @@ func TestClusterSkipServices(t *testing.T) { ep2 := makeTestEndpoints(&svc2PortName, []corev1.EndpointSubset{*epSubset}) eps := []*corev1.Endpoints{ep1, ep2} - testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, svcs, eps, false) + testClusterIPAdd(t, svc1IPv4, nil, ep1IPv4, ep2IPv4, false, false, svcs, eps, false) } func TestDualStackService(t *testing.T) { @@ -1038,11 +1106,11 @@ func TestDualStackService(t *testing.T) { mockOFClient.EXPECT().InstallServiceGroup(groupIDv4, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDv4, svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDv4, svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, false).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDv6, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCPv6, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDv6, svc1IPv6, uint16(svcPort), binding.ProtocolTCPv6, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDv6, svc1IPv6, uint16(svcPort), binding.ProtocolTCPv6, uint16(0), false, false).Times(1) fpv4.syncProxyRules() fpv6.syncProxyRules() @@ -1050,7 +1118,7 @@ func TestDualStackService(t *testing.T) { assert.Contains(t, fpv6.serviceInstalledMap, svcPortName) } -func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, nodeLocalInternal, endpointSliceEnabled bool) { +func testClusterIPRemove(t *testing.T, svcIP, externalIP, epIP net.IP, isIPv6 bool, nodeLocalInternal, endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOFClient, mockRouteClient := getMockClients(ctrl) @@ -1065,7 +1133,7 @@ func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, n if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true) + svc := makeTestClusterIPService(&svcPortName, svcIP, []net.IP{externalIP}, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy, true) makeServiceMap(fp, svc) var ep *corev1.Endpoints @@ -1085,17 +1153,39 @@ func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, n bindingProtocol = binding.ProtocolTCPv6 } - groupID := groupAllocator.Next() - mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) - if !nodeLocalInternal { + if nodeLocalInternal == false { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) - } - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, true).Times(1) - mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - if !nodeLocalInternal { + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, true).Times(1) + mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + } + } else { + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalInternal) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, true).Times(1) + mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + if externalIP != nil { + groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + + mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) + mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + } + } + if externalIP != nil { + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + mockRouteClient.EXPECT().DeleteExternalIPRoute(externalIP) } - mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1115,7 +1205,7 @@ func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool, n assert.False(t, exists) } -func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, epIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { +func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP, externalIP, epIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOFClient, mockRouteClient := getMockClients(ctrl) @@ -1126,7 +1216,9 @@ func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, } fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, options...) - svc := makeTestNodePortService(&svcPortName, svcIP, + svc := makeTestNodePortService(&svcPortName, + svcIP, + []net.IP{externalIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, @@ -1159,15 +1251,23 @@ func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1185,7 +1285,7 @@ func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, assert.NotContains(t, fp.endpointsInstalledMap, svcPortName) } -func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, epIP net.IP, loadBalancerIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { +func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP, externalIP, epIP, loadBalancerIP net.IP, isIPv6 bool, endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOFClient, mockRouteClient := getMockClients(ctrl) @@ -1199,7 +1299,9 @@ func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net. externalTrafficPolicy := corev1.ServiceExternalTrafficPolicyTypeLocal internalTrafficPolicy := corev1.ServiceInternalTrafficPolicyCluster - svc := makeTestLoadBalancerService(&svcPortName, svcIP, + svc := makeTestLoadBalancerService(&svcPortName, + svcIP, + []net.IP{externalIP}, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), @@ -1233,12 +1335,16 @@ func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net. groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, externalIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(externalIP) + } mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) @@ -1246,7 +1352,11 @@ func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net. mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + if externalIP != nil { + mockOFClient.EXPECT().UninstallServiceFlows(externalIP, uint16(svcPort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(externalIP) + } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1268,36 +1378,36 @@ func TestClusterIPRemove(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, false, false) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, false, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, true, false) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, false, true) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, false, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv4, ep1IPv4, false, true, true) + testClusterIPRemove(t, svc1IPv4, externalIPv4, ep1IPv4, false, true, true) }) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, false, false) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, false, false) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, true, false) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, true, false) }) }) t.Run("EndpointSlice", func(t *testing.T) { t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, false, true) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, false, true) }) t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIPRemove(t, svc1IPv6, ep1IPv6, true, true, true) + testClusterIPRemove(t, svc1IPv6, externalIPv6, ep1IPv6, true, true, true) }) }) }) @@ -1306,18 +1416,18 @@ func TestClusterIPRemove(t *testing.T) { func TestNodePortRemove(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, false, false) + testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, false, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, false, true) + testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, false, true) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, true, false) + testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, true, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, true, true) + testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, true, true) }) }) } @@ -1325,18 +1435,18 @@ func TestNodePortRemove(t *testing.T) { func TestLoadBalancerRemove(t *testing.T) { t.Run("IPv4", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPv4, false, false) + testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, loadBalancerIPv4, false, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPv4, false, true) + testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, externalIPv4, ep1IPv4, loadBalancerIPv4, false, true) }) }) t.Run("IPv6", func(t *testing.T) { t.Run("Endpoints", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPv6, true, false) + testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, loadBalancerIPv6, true, false) }) t.Run("EndpointSlice", func(t *testing.T) { - testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPv6, true, true) + testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, externalIPv6, ep1IPv6, loadBalancerIPv6, true, true) }) }) @@ -1349,19 +1459,19 @@ func testClusterIPNoEndpoint(t *testing.T, svcIP net.IP, isIPv6 bool) { groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) + svc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) makeServiceMap(fp, svc) makeEndpointSliceMap(fp) groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, []k8sproxy.Endpoint{}).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), gomock.Any(), uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), gomock.Any(), uint16(0), false, false).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), false, false).Times(1) fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) fp.syncProxyRules() } @@ -1382,14 +1492,18 @@ func testNodePortNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP net. groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) - svc := makeTestNodePortService(&svcPortName, svcIP, + svc := makeTestNodePortService(&svcPortName, + svcIP, + nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) - updatedSvc := makeTestNodePortService(&svcPortName, svcIP, + updatedSvc := makeTestNodePortService(&svcPortName, + svcIP, + nil, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, @@ -1408,16 +1522,16 @@ func testNodePortNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP net. groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) mockOFClient.EXPECT().InstallServiceGroup(groupIDCluster, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort), gomock.Any(), uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort), gomock.Any(), uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) fp.syncProxyRules() mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), gomock.Any()).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) fp.syncProxyRules() @@ -1442,7 +1556,9 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP internalTrafficPolicy := corev1.ServiceInternalTrafficPolicyCluster externalTrafficPolicy := corev1.ServiceExternalTrafficPolicyTypeLocal - svc := makeTestLoadBalancerService(&svcPortName, svcIP, + svc := makeTestLoadBalancerService(&svcPortName, + svcIP, + nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), @@ -1450,7 +1566,9 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP nil, &internalTrafficPolicy, externalTrafficPolicy) - updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, + updatedSvc := makeTestLoadBalancerService(&svcPortName, + svcIP, + nil, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), @@ -1470,23 +1588,23 @@ func testLoadBalancerNoEndpoint(t *testing.T, nodePortAddresses []net.IP, svcIP groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) mockOFClient.EXPECT().InstallServiceGroup(groupIDCluster, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort), gomock.Any(), uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), gomock.Any(), uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort), gomock.Any(), uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), gomock.Any(), uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) fp.syncProxyRules() mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), gomock.Any()).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort+1), gomock.Any(), uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDCluster, svcIP, uint16(svcPort+1), gomock.Any(), uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), gomock.Any(), uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort+1), gomock.Any(), uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), gomock.Any()).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) fp.syncProxyRules() } @@ -1510,8 +1628,8 @@ func testClusterIPRemoveSamePortEndpoint(t *testing.T, svcIP net.IP, epIP net.IP svcPortNameTCP := makeSvcPortName("ns", "svc-tcp", strconv.Itoa(svcPort), corev1.ProtocolTCP) svcPortNameUDP := makeSvcPortName("ns", "svc-udp", strconv.Itoa(svcPort), corev1.ProtocolUDP) - svcTCP := makeTestClusterIPService(&svcPortNameTCP, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - svcUDP := makeTestClusterIPService(&svcPortNameUDP, svcIP, int32(svcPort), corev1.ProtocolUDP, nil, nil, false) + svcTCP := makeTestClusterIPService(&svcPortNameTCP, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svcUDP := makeTestClusterIPService(&svcPortNameUDP, svcIP, nil, int32(svcPort), corev1.ProtocolUDP, nil, nil, false) makeServiceMap(fp, svcTCP, svcUDP) epTCP, epPortTCP := makeTestEndpointSliceEndpointAndPort(&svcPortNameTCP, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -1534,8 +1652,8 @@ func testClusterIPRemoveSamePortEndpoint(t *testing.T, svcIP net.IP, epIP net.IP mockOFClient.EXPECT().InstallServiceGroup(groupIDUDP, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(protocolTCP, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(protocolUDP, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), protocolTCP, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDUDP, svcIP, uint16(svcPort), protocolUDP, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), protocolTCP, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDUDP, svcIP, uint16(svcPort), protocolUDP, uint16(0), false, false).Times(1) fp.syncProxyRules() mockOFClient.EXPECT().InstallServiceGroup(groupIDUDP, false, gomock.Any()).Times(1) @@ -1565,7 +1683,7 @@ func testClusterIPRemoveEndpoints(t *testing.T, svcIP net.IP, epIP net.IP, isIPv groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) makeServiceMap(fp, svc) ep, epPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -1580,7 +1698,7 @@ func testClusterIPRemoveEndpoints(t *testing.T, svcIP net.IP, epIP net.IP, isIPv groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1649,7 +1767,7 @@ func testSessionAffinity(t *testing.T, svcIP net.IP, epIP net.IP, affinitySecond } else { expectedAffinity = uint16(affinitySeconds) } - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, expectedAffinity, false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, expectedAffinity, false, false).Times(1) fp.syncProxyRules() } @@ -1702,7 +1820,7 @@ func testSessionAffinityNoEndpoint(t *testing.T, svcExternalIPs net.IP, svcIP ne groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, true, []k8sproxy.Endpoint{}).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), gomock.Any(), uint16(10800), false, gomock.Any(), false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), gomock.Any(), uint16(10800), false, false).Times(1) fp.syncProxyRules() } @@ -1732,14 +1850,14 @@ func testServiceClusterIPUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, updatedSvcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, updatedSvcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, updatedSvcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, updatedSvcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, updatedSvcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, updatedSvcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -1759,29 +1877,29 @@ func testServiceClusterIPUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) s1 := mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, updatedSvcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, updatedSvcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) s2.After(s1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() @@ -1834,14 +1952,14 @@ func testServicePortUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort+1), corev1.ProtocolTCP, nil, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -1861,31 +1979,31 @@ func testServicePortUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) s1 := mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort+1), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort+1), bindingProtocol, uint16(0), false, false).Times(1) s2.After(s1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) s1 = mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol) - s2 = mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort+1), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) + s2 = mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort+1), bindingProtocol, uint16(0), true, false).Times(1) s2.After(s1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -1937,11 +2055,11 @@ func testServiceNodePortUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -1961,21 +2079,21 @@ func testServiceNodePortUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) s1 := mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort+1), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort+1), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort+1), bindingProtocol).Times(1) s2.After(s1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() @@ -2023,11 +2141,11 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, var svc, updatedSvc *corev1.Service switch svcType { case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeLocal) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeLocal) } makeServiceMap(fp, svc) @@ -2053,15 +2171,15 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -2071,14 +2189,14 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, expectedLocalEps).Times(1) s1 := mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) s2.After(s1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) @@ -2086,11 +2204,11 @@ func testServiceExternalTrafficPolicyUpdate(t *testing.T, } if svcType == corev1.ServiceTypeLoadBalancer { s1 := mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) - s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeLoadBalancer, false).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) s2.After(s1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -2130,8 +2248,8 @@ func testServiceInternalTrafficPolicyUpdate(t *testing.T, internalTrafficPolicyCluster := corev1.ServiceInternalTrafficPolicyCluster internalTrafficPolicyLocal := corev1.ServiceInternalTrafficPolicyLocal - svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyCluster, false) - updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyLocal, false) + svc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyCluster, false) + updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyLocal, false) makeServiceMap(fp, svc) remoteEp, remoteEpPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) @@ -2155,7 +2273,7 @@ func testServiceInternalTrafficPolicyUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) assert.Contains(t, fp.endpointsInstalledMap, svcPortName) @@ -2179,7 +2297,7 @@ func testServiceInternalTrafficPolicyUpdate(t *testing.T, mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, expectedLocalEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) fp.syncProxyRules() assert.Contains(t, fp.serviceInstalledMap, svcPortName) @@ -2222,8 +2340,8 @@ func testServiceIngressIPsUpdate(t *testing.T, updatedLoadBalancerIPStrs = append(updatedLoadBalancerIPStrs, ip.String()) } - svc := makeTestLoadBalancerService(&svcPortName, svcIP, loadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, updatedLoadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, loadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, nil, updatedLoadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) makeServiceMap(fp, svc) ep, epPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) @@ -2242,25 +2360,25 @@ func testServiceIngressIPsUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedEps)).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) for _, ip := range loadBalancerIPs { - mockOFClient.EXPECT().InstallServiceFlows(groupID, ip, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, ip, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) for _, ip := range loadBalancerIPs { - mockRouteClient.EXPECT().AddLoadBalancer(ip).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(ip).Times(1) } toDeleteLoadBalancerIPs := smallSliceDifference(loadBalancerIPStrs, updatedLoadBalancerIPStrs) toAddLoadBalancerIPs := smallSliceDifference(updatedLoadBalancerIPStrs, loadBalancerIPStrs) for _, ipStr := range toDeleteLoadBalancerIPs { mockOFClient.EXPECT().UninstallServiceFlows(net.ParseIP(ipStr), uint16(svcPort), bindingProtocol).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(net.ParseIP(ipStr)).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(net.ParseIP(ipStr)).Times(1) } for _, ipStr := range toAddLoadBalancerIPs { - mockOFClient.EXPECT().InstallServiceFlows(groupID, net.ParseIP(ipStr), uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(net.ParseIP(ipStr)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, net.ParseIP(ipStr), uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(net.ParseIP(ipStr)).Times(1) } fp.syncProxyRules() @@ -2307,14 +2425,14 @@ func testServiceStickyMaxAgeSecondsUpdate(t *testing.T, updatedAffinitySeconds := int32(100) switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -2334,25 +2452,25 @@ func testServiceStickyMaxAgeSecondsUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, true, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, false).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(updatedAffinitySeconds), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(updatedAffinitySeconds), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(updatedAffinitySeconds), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(updatedAffinitySeconds), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(updatedAffinitySeconds), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(updatedAffinitySeconds), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() @@ -2406,14 +2524,14 @@ func testServiceSessionAffinityTypeUpdate(t *testing.T, affinitySeconds := int32(100) switch svcType { case corev1.ServiceTypeClusterIP: - svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) + svc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, nil, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil, false) case corev1.ServiceTypeNodePort: - svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) case corev1.ServiceTypeLoadBalancer: - svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) - updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + svc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, nil, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } makeServiceMap(fp, svc) @@ -2433,29 +2551,29 @@ func testServiceSessionAffinityTypeUpdate(t *testing.T, groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupID, true, expectedEps).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, false).Times(1) if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, false).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeNodePort, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), true, false).Times(1) mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) } if svcType == corev1.ServiceTypeLoadBalancer { - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, false).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol) - mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeLoadBalancer, false).Times(1) - mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) - mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), true, false).Times(1) + mockRouteClient.EXPECT().DeleteExternalIPRoute(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddExternalIPRoute(loadBalancerIP).Times(1) } fp.syncProxyRules() @@ -2501,8 +2619,8 @@ func TestServicesWithSameEndpoints(t *testing.T) { svcPortName1 := makeSvcPortName("ns", "svc1", strconv.Itoa(svcPort), corev1.ProtocolTCP) svcPortName2 := makeSvcPortName("ns", "svc2", strconv.Itoa(svcPort), corev1.ProtocolTCP) - svc1 := makeTestClusterIPService(&svcPortName1, svc1IPv4, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) - svc2 := makeTestClusterIPService(&svcPortName2, svc2IPv4, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc1 := makeTestClusterIPService(&svcPortName1, svc1IPv4, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) + svc2 := makeTestClusterIPService(&svcPortName2, svc2IPv4, nil, int32(svcPort), corev1.ProtocolTCP, nil, nil, false) makeServiceMap(fp, svc1, svc2) ep1, ep1Port := makeTestEndpointSliceEndpointAndPort(&svcPortName1, ep1IPv4, int32(svcPort), corev1.ProtocolTCP, false) @@ -2517,8 +2635,8 @@ func TestServicesWithSameEndpoints(t *testing.T) { mockOFClient.EXPECT().InstallServiceGroup(groupID2, false, gomock.Any()).Times(1) bindingProtocol := binding.ProtocolTCP mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID1, svc1IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID2, svc2IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID1, svc1IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID2, svc2IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, false).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svc1IPv4, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceFlows(svc2IPv4, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(groupID1).Times(1) @@ -2569,7 +2687,7 @@ func TestMetrics(t *testing.T) { servicesInstallMetric = metrics.ServicesInstalledTotalV6.GaugeMetric } - testClusterIPAdd(t, tc.svcIP, tc.ep1IP, tc.ep2IP, tc.isIPv6, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + testClusterIPAdd(t, tc.svcIP, nil, tc.ep1IP, tc.ep2IP, tc.isIPv6, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) v, err := testutil.GetCounterMetricValue(endpointsUpdateTotalMetric) assert.NoError(t, err) assert.Equal(t, 0, int(v)) @@ -2583,7 +2701,7 @@ func TestMetrics(t *testing.T) { assert.Equal(t, 2, int(v)) assert.NoError(t, err) - testClusterIPRemove(t, tc.svcIP, tc.ep1IP, tc.isIPv6, false, false) + testClusterIPRemove(t, tc.svcIP, nil, tc.ep1IP, tc.isIPv6, false, false) v, err = testutil.GetCounterMetricValue(endpointsUpdateTotalMetric) assert.NoError(t, err) @@ -2608,6 +2726,7 @@ func TestGetServiceFlowKeys(t *testing.T) { groupAllocator := openflow.NewGroupAllocator(false) svc := makeTestNodePortService(&svcPortName, svc1IPv4, + nil, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, @@ -2670,8 +2789,8 @@ func TestGetServiceFlowKeys(t *testing.T) { mockRouteClient.EXPECT().AddNodePort(nodePortAddressesIPv4, uint16(svcNodePort), binding.ProtocolTCP).Times(1) mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), gomock.Any(), gomock.Any()).Times(2) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), gomock.Any(), uint16(svcNodePort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeNodePort, false).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeClusterIP, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), gomock.Any(), uint16(svcNodePort), binding.ProtocolTCP, uint16(0), true, false).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, false).Times(1) fp.syncProxyRules() } diff --git a/pkg/agent/route/interfaces.go b/pkg/agent/route/interfaces.go index 8975c98f6bd..01ef219d456 100644 --- a/pkg/agent/route/interfaces.go +++ b/pkg/agent/route/interfaces.go @@ -65,11 +65,11 @@ type Interface interface { // DeleteNodePort deletes related configurations when a NodePort Service is deleted. DeleteNodePort(nodePortAddresses []net.IP, port uint16, protocol binding.Protocol) error - // AddLoadBalancer adds configurations when a LoadBalancer IP is added. - AddLoadBalancer(externalIP net.IP) error + // AddExternalIPRoute adds a route entry when an external IP is added. + AddExternalIPRoute(externalIP net.IP) error - // DeleteLoadBalancer deletes related configurations when a LoadBalancer IP is deleted. - DeleteLoadBalancer(externalIP net.IP) error + // DeleteExternalIPRoute deletes the related route entry when an external IP is deleted. + DeleteExternalIPRoute(externalIP net.IP) error // Run starts the sync loop. Run(stopCh <-chan struct{}) diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index 51497536112..586367dfc72 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -1405,11 +1405,11 @@ func (c *Client) addVirtualNodePortDNATIPRoute(isIPv6 bool) error { return nil } -// AddLoadBalancer is used to add routing entry which is used to route LoadBalancer ingress IP to Antrea -// gateway on host. -func (c *Client) AddLoadBalancer(svcIP net.IP) error { +// AddExternalIPRoute adds a route entry that forwards traffic destined for the external IP to the Antrea gateway interface. +func (c *Client) AddExternalIPRoute(externalIP net.IP) error { + externalIPStr := externalIP.String() linkIndex := c.nodeConfig.GatewayConfig.LinkIndex - isIPv6 := utilnet.IsIPv6(svcIP) + isIPv6 := utilnet.IsIPv6(externalIP) var gw net.IP var mask int if !isIPv6 { @@ -1420,35 +1420,32 @@ func (c *Client) AddLoadBalancer(svcIP net.IP) error { mask = net.IPv6len * 8 } - route := generateRoute(svcIP, mask, gw, linkIndex, netlink.SCOPE_UNIVERSE) + route := generateRoute(externalIP, mask, gw, linkIndex, netlink.SCOPE_UNIVERSE) if err := c.netlink.RouteReplace(route); err != nil { - return fmt.Errorf("failed to install route for LoadBalancer ingress IP %s: %w", svcIP.String(), err) + return fmt.Errorf("failed to install route for external IP %s: %w", externalIPStr, err) } - klog.V(4).InfoS("Added LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Store(svcIP.String(), route) - + klog.V(4).InfoS("Added route for external IP", "IP", externalIPStr) + c.serviceRoutes.Store(externalIP.String(), route) return nil } -// DeleteLoadBalancer is used to delete routing entry which is used to route LoadBalancer ingress IP to Antrea -// gateway on host. -func (c *Client) DeleteLoadBalancer(svcIP net.IP) error { - svcIPStr := svcIP.String() - route, found := c.serviceRoutes.Load(svcIPStr) +// DeleteExternalIPRoute deletes the route entry for the external IP. +func (c *Client) DeleteExternalIPRoute(externalIP net.IP) error { + externalIPStr := externalIP.String() + route, found := c.serviceRoutes.Load(externalIPStr) if !found { - klog.V(2).InfoS("Didn't find route for LoadBalancer ingress IP", "ingressIP", svcIPStr) + klog.V(2).InfoS("Didn't find route for external IP", "IP", externalIPStr) return nil } if err := c.netlink.RouteDel(route.(*netlink.Route)); err != nil { if err.Error() == "no such process" { - klog.InfoS("Failed to delete route for LoadBalancer ingress IP since it doesn't exist", "route", route) + klog.InfoS("Failed to delete route for external IP since it doesn't exist", "IP", externalIPStr) } else { - return fmt.Errorf("failed to delete route for LoadBalancer ingress IP %s: %w", svcIPStr, err) + return fmt.Errorf("failed to delete route for external IP %s: %w", externalIPStr, err) } } - klog.V(4).InfoS("Deleted LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Delete(svcIPStr) - + c.serviceRoutes.Delete(externalIPStr) + klog.V(4).InfoS("Deleted route for external IP", "IP", externalIPStr) return nil } diff --git a/pkg/agent/route/route_linux_test.go b/pkg/agent/route/route_linux_test.go index 6dab7a20e85..97a92b41384 100644 --- a/pkg/agent/route/route_linux_test.go +++ b/pkg/agent/route/route_linux_test.go @@ -1443,7 +1443,7 @@ func TestAddServiceCIDRRoute(t *testing.T) { } } -func TestAddLoadBalancer(t *testing.T) { +func TestAddExternalIPRoute(t *testing.T) { tests := []struct { name string externalIPs []string @@ -1487,13 +1487,13 @@ func TestAddLoadBalancer(t *testing.T) { tt.expectedCalls(mockNetlink.EXPECT()) for _, externalIP := range tt.externalIPs { - assert.NoError(t, c.AddLoadBalancer(net.ParseIP(externalIP))) + assert.NoError(t, c.AddExternalIPRoute(net.ParseIP(externalIP))) } }) } } -func TestDeleteLoadBalancer(t *testing.T) { +func TestDeleteExternalIPRoute(t *testing.T) { tests := []struct { name string serviceRoutes map[string]*netlink.Route @@ -1541,7 +1541,7 @@ func TestDeleteLoadBalancer(t *testing.T) { tt.expectedCalls(mockNetlink.EXPECT()) for _, externalIP := range tt.externalIPs { - assert.NoError(t, c.DeleteLoadBalancer(net.ParseIP(externalIP))) + assert.NoError(t, c.DeleteExternalIPRoute(net.ParseIP(externalIP))) } }) } diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 82257b9a995..d1e43216357 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -510,40 +510,43 @@ func (c *Client) DeleteNodePort(nodePortAddresses []net.IP, port uint16, protoco return nil } -func (c *Client) AddLoadBalancer(externalIP net.IP) error { - svcIPStr := externalIP.String() +// AddExternalIPRoute adds a route entry that forwards traffic destined for the external IP to the Antrea gateway interface. +func (c *Client) AddExternalIPRoute(externalIP net.IP) error { + externalIPStr := externalIP.String() linkIndex := c.nodeConfig.GatewayConfig.LinkIndex gw := config.VirtualServiceIPv4 metric := util.MetricHigh - _, svcIPNet, _ := net.ParseCIDR(svcIPStr) + _, svcIPNet, _ := net.ParseCIDR(externalIPStr) route := generateRoute(svcIPNet, gw, linkIndex, metric) if err := util.ReplaceNetRoute(route); err != nil { - return fmt.Errorf("failed to install route for LoadBalancer ingress IP %s: %w", svcIPStr, err) + return err } - klog.V(4).InfoS("Added LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Store(svcIPStr, route) - + if err := util.ReplaceNetRoute(route); err != nil { + return fmt.Errorf("failed to install route for external IP %s: %w", externalIPStr, err) + } + c.serviceRoutes.Store(externalIPStr, route) + klog.V(4).InfoS("Added route for external IP", "IP", externalIPStr) return nil } -func (c *Client) DeleteLoadBalancer(externalIP net.IP) error { - svcIPStr := externalIP.String() - route, found := c.serviceRoutes.Load(svcIPStr) +// DeleteExternalIPRoute deletes the route entry for the external IP. +func (c *Client) DeleteExternalIPRoute(externalIP net.IP) error { + externalIPStr := externalIP.String() + route, found := c.serviceRoutes.Load(externalIPStr) if !found { - klog.V(2).InfoS("Didn't find route for LoadBalancer ingress IP", "ingressIP", svcIPStr) + klog.V(2).InfoS("Didn't find route for external IP", "IP", externalIPStr) return nil } if err := util.RemoveNetRoute(route.(*util.Route)); err != nil { if strings.Contains(err.Error(), "No matching MSFT_NetRoute objects") { - klog.InfoS("Failed to delete route for LoadBalancer ingress IP since it doesn't exist", "route", route) + klog.InfoS("Failed to delete route for external IP since it doesn't exist", "IP", externalIPStr) } else { - return fmt.Errorf("failed to delete route for LoadBalancer ingress IP %s: %w", svcIPStr, err) + return fmt.Errorf("failed to delete route for external IP %s: %w", externalIPStr, err) } } - klog.V(4).InfoS("Deleted LoadBalancer ingress IP route", "route", route) - c.serviceRoutes.Delete(svcIPStr) - + c.serviceRoutes.Delete(externalIPStr) + klog.V(4).InfoS("Deleted route for external IP", "IP", externalIPStr) return nil } diff --git a/pkg/agent/route/testing/mock_route.go b/pkg/agent/route/testing/mock_route.go index 8d5a3256530..f1eb8bf66c9 100644 --- a/pkg/agent/route/testing/mock_route.go +++ b/pkg/agent/route/testing/mock_route.go @@ -50,18 +50,18 @@ func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { return m.recorder } -// AddLoadBalancer mocks base method -func (m *MockInterface) AddLoadBalancer(arg0 net.IP) error { +// AddExternalIPRoute mocks base method +func (m *MockInterface) AddExternalIPRoute(arg0 net.IP) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddLoadBalancer", arg0) + ret := m.ctrl.Call(m, "AddExternalIPRoute", arg0) ret0, _ := ret[0].(error) return ret0 } -// AddLoadBalancer indicates an expected call of AddLoadBalancer -func (mr *MockInterfaceMockRecorder) AddLoadBalancer(arg0 interface{}) *gomock.Call { +// AddExternalIPRoute indicates an expected call of AddExternalIPRoute +func (mr *MockInterfaceMockRecorder) AddExternalIPRoute(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLoadBalancer", reflect.TypeOf((*MockInterface)(nil).AddLoadBalancer), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddExternalIPRoute", reflect.TypeOf((*MockInterface)(nil).AddExternalIPRoute), arg0) } // AddLocalAntreaFlexibleIPAMPodRule mocks base method @@ -120,18 +120,18 @@ func (mr *MockInterfaceMockRecorder) AddSNATRule(arg0, arg1 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddSNATRule", reflect.TypeOf((*MockInterface)(nil).AddSNATRule), arg0, arg1) } -// DeleteLoadBalancer mocks base method -func (m *MockInterface) DeleteLoadBalancer(arg0 net.IP) error { +// DeleteExternalIPRoute mocks base method +func (m *MockInterface) DeleteExternalIPRoute(arg0 net.IP) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteLoadBalancer", arg0) + ret := m.ctrl.Call(m, "DeleteExternalIPRoute", arg0) ret0, _ := ret[0].(error) return ret0 } -// DeleteLoadBalancer indicates an expected call of DeleteLoadBalancer -func (mr *MockInterfaceMockRecorder) DeleteLoadBalancer(arg0 interface{}) *gomock.Call { +// DeleteExternalIPRoute indicates an expected call of DeleteExternalIPRoute +func (mr *MockInterfaceMockRecorder) DeleteExternalIPRoute(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancer", reflect.TypeOf((*MockInterface)(nil).DeleteLoadBalancer), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteExternalIPRoute", reflect.TypeOf((*MockInterface)(nil).DeleteExternalIPRoute), arg0) } // DeleteLocalAntreaFlexibleIPAMPodRule mocks base method diff --git a/pkg/ovs/openflow/testing/utils.go b/pkg/ovs/openflow/testing/utils.go index d5cad523f22..99ea462373a 100644 --- a/pkg/ovs/openflow/testing/utils.go +++ b/pkg/ovs/openflow/testing/utils.go @@ -84,6 +84,9 @@ func (m *fieldMetadata) getActionNickname() string { if strings.HasPrefix(name, "tun_ipv4_") { name = strings.Replace(name, "tun_ipv4_", "tun_", 1) } + if strings.HasPrefix(name, "ip_") && name != "ip_dscp" { + name = strings.Replace(name, "ip_", "nw_", 1) + } return name } @@ -804,14 +807,14 @@ func nxActionLearnToString(action openflow15.Action) string { if len(a.LearnSpecs) != 0 { for _, spec := range a.LearnSpecs { nBits := spec.Header.NBits - isLoad := spec.Header.Dst == false - isMatch := spec.Header.Dst == true + isLoad := spec.Header.Dst == true + isMatch := spec.Header.Dst == false //TODO: add isOutput if spec.SrcValue != nil { srcValueStr := strings.TrimLeft(fmt.Sprintf("%x", spec.SrcValue), "0") if isMatch { - dstFieldStr := getFieldNameString(spec.DstField.Field.Class, spec.DstField.Field.Field, 0, 0, true, false) + dstFieldStr := getFieldNameString(spec.DstField.Field.Class, spec.DstField.Field.Field, spec.DstField.Ofs, nBits, true, false) parts = append(parts, fmt.Sprintf("%s=0x%s", dstFieldStr, srcValueStr)) } else if isLoad { dstFieldStr := getFieldNameString(spec.DstField.Field.Class, spec.DstField.Field.Field, spec.DstField.Ofs, nBits, false, false) diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index c41e49e3705..7a6860d66b3 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -28,7 +28,6 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/component-base/metrics/legacyregistry" @@ -785,7 +784,7 @@ func installServiceFlows(t *testing.T, gid uint32, svc svcConfig, endpointList [ assert.NoError(t, err, "no error should return when installing flows for Endpoints") err = c.InstallServiceGroup(groupID, svc.withSessionAffinity, endpointList) assert.NoError(t, err, "no error should return when installing groups for Service") - err = c.InstallServiceFlows(groupID, svc.ip, svc.port, svc.protocol, stickyMaxAgeSeconds, false, v1.ServiceTypeClusterIP, false) + err = c.InstallServiceFlows(groupID, svc.ip, svc.port, svc.protocol, stickyMaxAgeSeconds, false, false) assert.NoError(t, err, "no error should return when installing flows for Service") } @@ -824,7 +823,7 @@ func expectedProxyServiceGroupAndFlows(gid uint32, svc svcConfig, endpointList [ svcFlows := expectTableFlows{tableName: "ServiceLB", flows: []*ofTestUtils.ExpectFlow{ { MatchStr: fmt.Sprintf("priority=200,%s,reg4=0x10000/0x70000,nw_dst=%s,tp_dst=%d", string(svc.protocol), svc.ip.String(), svc.port), - ActStr: fmt.Sprintf("set_field:0x%x/0x70000->reg4,set_field:0x200/0x200->reg0,%sgroup:%d", serviceLearnReg<<16, loadGourpID, gid), + ActStr: fmt.Sprintf("set_field:0x200/0x200->reg0,set_field:0x%x/0x70000->reg4,%sgroup:%d", serviceLearnReg<<16, loadGourpID, gid), }, { MatchStr: fmt.Sprintf("priority=190,%s,reg4=0x30000/0x70000,nw_dst=%s,tp_dst=%d", string(svc.protocol), svc.ip.String(), svc.port), diff --git a/third_party/proxy/types.go b/third_party/proxy/types.go index 385dcdd8ea9..6d2225b7c2d 100644 --- a/third_party/proxy/types.go +++ b/third_party/proxy/types.go @@ -138,12 +138,12 @@ type Endpoint interface { // This is only set when watching EndpointSlices. If using Endpoints, this is always // true since only ready endpoints are read from Endpoints. IsServing() bool - // IsTerminating retruns true if an endpoint is terminating. For pods, + // IsTerminating returns true if an endpoint is terminating. For pods, // that is any pod with a deletion timestamp. // This is only set when watching EndpointSlices. If using Endpoints, this is always // false since terminating endpoints are always excluded from Endpoints. IsTerminating() bool - // GetZoneHint returns the zone hint for the endpoint. This is based on + // GetZoneHints returns the zone hint for the endpoint. This is based on // endpoint.hints.forZones[0].name in the EndpointSlice API. GetZoneHints() sets.String // IP returns IP part of the endpoint.