From 89655218850ae42d5694b2d48c3e89d439a089b8 Mon Sep 17 00:00:00 2001 From: wgrayson Date: Sun, 10 Jul 2022 02:49:26 -0700 Subject: [PATCH] Add ACNP appliedto Nodeport svc support Fixes #1580 With this PR users could use ACNP to control the external access of a Nodeport service. 1. Add `service` field in `appliedTo` of ACNP. 2. Add `Service` in `AppliedToGroup` and appliedToGroup with `Service` will span to all Nodes. 3. Use groupIDs of a Service to do destination matching. 4. Use `ct_nw_src` to do source matching. Signed-off-by: wgrayson --- .../antrea/crds/clusternetworkpolicy.yaml | 20 ++ build/yamls/antrea-aks.yml | 20 ++ build/yamls/antrea-crds.yml | 20 ++ build/yamls/antrea-eks.yml | 20 ++ build/yamls/antrea-gke.yml | 20 ++ build/yamls/antrea-ipsec.yml | 20 ++ build/yamls/antrea.yml | 20 ++ docs/antrea-network-policy.md | 66 ++++ pkg/agent/controller/networkpolicy/cache.go | 41 ++- .../controller/networkpolicy/cache_test.go | 202 ++++++++--- .../networkpolicy_controller_test.go | 26 +- .../controller/networkpolicy/reconciler.go | 101 ++++-- .../networkpolicy/reconciler_test.go | 18 +- pkg/agent/controller/networkpolicy/reject.go | 11 +- .../networkpolicy/status_controller_test.go | 4 +- pkg/agent/openflow/client.go | 1 + pkg/agent/openflow/network_policy.go | 83 +++++ pkg/agent/openflow/pipeline.go | 28 +- pkg/apis/controlplane/types.go | 3 + pkg/apis/controlplane/v1beta2/generated.pb.go | 313 ++++++++++-------- pkg/apis/controlplane/v1beta2/generated.proto | 4 + pkg/apis/controlplane/v1beta2/types.go | 3 + .../v1beta2/zz_generated.conversion.go | 2 + .../v1beta2/zz_generated.deepcopy.go | 5 + .../controlplane/zz_generated.deepcopy.go | 5 + pkg/apis/crd/v1alpha1/types.go | 7 + .../crd/v1alpha1/zz_generated.deepcopy.go | 5 + pkg/apiserver/openapi/zz_generated.openapi.go | 8 +- .../networkpolicy/clusternetworkpolicy.go | 2 + .../clusternetworkpolicy_test.go | 69 ++++ pkg/controller/networkpolicy/crd_utils.go | 25 ++ .../networkpolicy/networkpolicy_controller.go | 110 +++--- .../networkpolicy/store/appliedtogroup.go | 3 +- pkg/controller/networkpolicy/validate.go | 53 ++- pkg/controller/types/networkpolicy.go | 15 +- test/e2e/antreapolicy_test.go | 65 ++++ test/e2e/fixtures.go | 10 + test/e2e/proxy_test.go | 10 - test/e2e/utils/cnpspecbuilder.go | 17 +- 39 files changed, 1157 insertions(+), 298 deletions(-) diff --git a/build/charts/antrea/crds/clusternetworkpolicy.yaml b/build/charts/antrea/crds/clusternetworkpolicy.yaml index 54ea3f385ee..f49409092eb 100644 --- a/build/charts/antrea/crds/clusternetworkpolicy.yaml +++ b/build/charts/antrea/crds/clusternetworkpolicy.yaml @@ -117,6 +117,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace ingress: type: array items: @@ -190,6 +200,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values action: type: string diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index ffdbe7d4304..865d6430514 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -495,6 +495,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace ingress: type: array items: @@ -568,6 +578,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values action: type: string diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index 64f7a1a2b62..403a160c351 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -488,6 +488,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace ingress: type: array items: @@ -561,6 +571,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values action: type: string diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 7c64611d3a5..ccec300b81a 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -495,6 +495,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace ingress: type: array items: @@ -568,6 +578,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values action: type: string diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 3a5dbc75ac4..dd84db65bc7 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -495,6 +495,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace ingress: type: array items: @@ -568,6 +578,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values action: type: string diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 06a718aec3c..13d7fea057b 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -495,6 +495,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace ingress: type: array items: @@ -568,6 +578,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values action: type: string diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 69e328f1dd2..447275ec2b9 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -495,6 +495,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace ingress: type: array items: @@ -568,6 +578,16 @@ spec: required: - name - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values action: type: string diff --git a/docs/antrea-network-policy.md b/docs/antrea-network-policy.md index c014c8fdb4e..d007a1d5f24 100644 --- a/docs/antrea-network-policy.md +++ b/docs/antrea-network-policy.md @@ -39,6 +39,7 @@ - [Node Selector](#node-selector) - [toServices egress rules](#toservices-egress-rules) - [ServiceAccount based selection](#serviceaccount-based-selection) + - [ACNP appliedTo NodePort Service](#acnp-appliedto-nodeport-service) - [ClusterGroup](#clustergroup) - [ClusterGroup CRD](#clustergroup-crd) - [kubectl commands for ClusterGroup](#kubectl-commands-for-clustergroup) @@ -482,6 +483,10 @@ Specific Pods from specific Namespaces can be selected by providing both a `podSelector` and a `namespaceSelector` in the same `appliedTo` entry. The `appliedTo` field can also reference a ClusterGroup resource by setting the ClusterGroup's name in `group` field in place of the stand-alone selectors. +The `appliedTo` field can also reference a Service by setting the Service's name +and namespace in `service` field in place of the stand-alone selectors. Only a +NodePort Service can be referred by this field. More details can be found in the +[ACNPAppliedToNodePortService](#acnp-appliedto-nodeport-service) section. IPBlock cannot be set in the `appliedTo` field. An IPBlock ClusterGroup referenced in an `appliedTo` field will be ignored, and the policy will have no effect. @@ -1296,6 +1301,67 @@ Note: Antrea will use a reserved label key for internal processing `serviceAccou The reserved label looks like: `internal.antrea.io/service-account:[ServiceAccountName]`. Users should avoid using this label key in any entities no matter if a policy with `serviceAccount` is applied in the cluster. +### ACNP appliedTo NodePort Service + +Antrea ClusterNetworkPolicy features a `service` field in `appliedTo` field to enable the ACNP could be enforced +on the traffic from external client to a NodePort Service. + +`service` uses `namespace` and `name` to select the Service with a specific name under a specific namespace and +only a NodePort Service can be referred by `service` field. + +`service` field cannot be used with any other fields and a policy or a rule can't be applied to NodePort Service +and other peers at the same time. Both two examples below are **invalid**. + +```yaml +# `service` can't be used with other field +appliedTo: + - service: + name: svc + namespace: default + podSelector: + matchLabels: + app: client +--- +# Can't be applied to `service` and other peers at the same time +appliedTo: + - service: + name: svc + namespace: default + - podSelector: + matchLabels: + app: client +``` + +Since `service` field is used control the external access of a NodePort Service, then + +1. If a `appliedTo` with `service` is used at policy level, then this policy can only contain ingress rules. +2. If a `appliedTo` with `service` is used at rule level, then this rule can only be an ingress rule. +3. If an ingress rule is applied to a NodePort Service, then this ingress can only use `ipBlock` in its `from` field. + +An example policy using `service` in `appliedTo` could look like this: + +```yaml +apiVersion: crd.antrea.io/v1alpha1 +kind: ClusterNetworkPolicy +metadata: + name: acnp-deny-external-client-nodeport-svc-access +spec: + priority: 5 + tier: securityops + appliedTo: + - service: + name: svc-1 + namespace: ns-1 + ingress: + - action: Drop + from: + - ipBlock: + cidr: 1.1.1.1/24 +``` + +In this example, the policy will be applied to the NodePort Service `svc-1` in Namespace `ns-1` +and drop all packets from CIDR `1.1.1.1/24`. + ## ClusterGroup A ClusterGroup (CG) CRD is a specification of how workloads are grouped together. diff --git a/pkg/agent/controller/networkpolicy/cache.go b/pkg/agent/controller/networkpolicy/cache.go index 9f38fa375f6..bc9d1a522d2 100644 --- a/pkg/agent/controller/networkpolicy/cache.go +++ b/pkg/agent/controller/networkpolicy/cache.go @@ -19,6 +19,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "strings" "sync" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -450,13 +451,7 @@ func (c *ruleCache) processGroupIDUpdates() { for { select { case svcStr := <-c.groupIDUpdates: - toSvcRules, err := c.rules.ByIndex(toServicesIndex, svcStr) - if err != nil { - continue - } - for _, toSvcRule := range toSvcRules { - c.dirtyRuleHandler(toSvcRule.(*rule).ID) - } + c.processServiceGroupIDUpdate(svcStr) } } } @@ -911,3 +906,35 @@ func (c *ruleCache) unionAppliedToGroups(groupNames []string) (v1beta.GroupMembe } return set, anyExists } + +// processServiceGroupIDUpdate gets names of AppliedToGroup by Service NamespacedName. +func (c *ruleCache) processServiceGroupIDUpdate(svcStr string) { + c.appliedToSetLock.RLock() + defer c.appliedToSetLock.RUnlock() + + // Reprocess rules if the Service referred by this rule's ToServices has updated. + toSvcRules, err := c.rules.ByIndex(toServicesIndex, svcStr) + if err != nil { + return + } + for _, toSvcRule := range toSvcRules { + c.dirtyRuleHandler(toSvcRule.(*rule).ID) + } + + // Reprocess rules if the Service referred by rule's AppliedToGroup has updated. + strListSvcRef := strings.Split(svcStr, "/") + if len(strListSvcRef) != 2 { + return + } + member := &v1beta.GroupMember{ + Service: &v1beta.ServiceReference{ + Name: strListSvcRef[1], + Namespace: strListSvcRef[0], + }, + } + for group, memberSet := range c.appliedToSetByGroup { + if memberSet.Has(member) { + c.onAppliedToGroupUpdate(group) + } + } +} diff --git a/pkg/agent/controller/networkpolicy/cache_test.go b/pkg/agent/controller/networkpolicy/cache_test.go index 00c01d3ed25..349173a9070 100644 --- a/pkg/agent/controller/networkpolicy/cache_test.go +++ b/pkg/agent/controller/networkpolicy/cache_test.go @@ -183,10 +183,14 @@ func (r *dirtyRuleRecorder) Record(ruleID string) { r.eventCh <- ruleID } -func newAppliedToGroupMember(name, namespace string, containerPorts ...v1beta2.NamedPort) *v1beta2.GroupMember { +func newAppliedToGroupMemberPod(name, namespace string, containerPorts ...v1beta2.NamedPort) *v1beta2.GroupMember { return &v1beta2.GroupMember{Pod: &v1beta2.PodReference{Name: name, Namespace: namespace}, Ports: containerPorts} } +func newAppliedToGroupMemberService(name, namespace string) *v1beta2.GroupMember { + return &v1beta2.GroupMember{Service: &v1beta2.ServiceReference{Name: name, Namespace: namespace}} +} + func newAddressGroupMember(ips ...string) *v1beta2.GroupMember { ipAddrs := make([]v1beta2.IPAddress, len(ips)) for idx, ip := range ips { @@ -256,7 +260,7 @@ func TestRuleCacheAddAddressGroup(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() for _, rule := range tt.rules { c.rules.Add(rule) } @@ -274,12 +278,12 @@ func TestRuleCacheAddAddressGroup(t *testing.T) { } } -func newFakeRuleCache() (*ruleCache, *dirtyRuleRecorder, *channel.SubscribableChannel) { +func newFakeRuleCache() (*ruleCache, *dirtyRuleRecorder, *channel.SubscribableChannel, chan string) { recorder := newDirtyRuleRecorder() podUpdateChannel := channel.NewSubscribableChannel("PodUpdate", 100) - ch2 := make(chan string, 100) - c := newRuleCache(recorder.Record, podUpdateChannel, ch2) - return c, recorder, podUpdateChannel + serviceGroupIDUpdateChannel := make(chan string, 100) + c := newRuleCache(recorder.Record, podUpdateChannel, serviceGroupIDUpdateChannel) + return c, recorder, podUpdateChannel, serviceGroupIDUpdateChannel } func TestRuleCacheReplaceAppliedToGroups(t *testing.T) { @@ -302,7 +306,7 @@ func TestRuleCacheReplaceAppliedToGroups(t *testing.T) { { "stale-group-can-be-cleaned", []*rule{}, - map[string]v1beta2.GroupMemberSet{"group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"))}, + map[string]v1beta2.GroupMemberSet{"group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"))}, []*v1beta2.AppliedToGroup{}, map[string]v1beta2.GroupMemberSet{}, sets.NewString(), @@ -310,27 +314,27 @@ func TestRuleCacheReplaceAppliedToGroups(t *testing.T) { { "existing-group-can-be-updated", []*rule{rule1, rule2}, - map[string]v1beta2.GroupMemberSet{"group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"))}, + map[string]v1beta2.GroupMemberSet{"group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"))}, []*v1beta2.AppliedToGroup{ { ObjectMeta: metav1.ObjectMeta{Name: "group1"}, - GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1"), *newAppliedToGroupMember("pod2", "ns1")}, + GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1"), *newAppliedToGroupMemberPod("pod2", "ns1")}, }, }, - map[string]v1beta2.GroupMemberSet{"group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"), newAppliedToGroupMember("pod2", "ns1"))}, + map[string]v1beta2.GroupMemberSet{"group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"), newAppliedToGroupMemberPod("pod2", "ns1"))}, sets.NewString("rule1", "rule2"), }, { "unchanged-group-can-be-skipped", []*rule{rule1, rule2}, - map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"))}, + map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"))}, []*v1beta2.AppliedToGroup{ { ObjectMeta: metav1.ObjectMeta{Name: "group2"}, - GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")}, + GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")}, }, }, - map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"))}, + map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"))}, sets.NewString(), }, { @@ -340,16 +344,16 @@ func TestRuleCacheReplaceAppliedToGroups(t *testing.T) { []*v1beta2.AppliedToGroup{ { ObjectMeta: metav1.ObjectMeta{Name: "group2"}, - GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1"), *newAppliedToGroupMember("pod2", "ns1")}, + GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1"), *newAppliedToGroupMemberPod("pod2", "ns1")}, }, }, - map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"), newAppliedToGroupMember("pod2", "ns1"))}, + map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"), newAppliedToGroupMemberPod("pod2", "ns1"))}, sets.NewString("rule2"), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() for _, rule := range tt.rules { c.rules.Add(rule) } @@ -433,7 +437,7 @@ func TestRuleCacheReplaceAddressGroups(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() for _, rule := range tt.rules { c.rules.Add(rule) } @@ -519,7 +523,7 @@ func TestRuleCacheReplaceNetworkPolicies(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() for _, rule := range tt.rules { c.rules.Add(rule) c.policyMap[string(rule.PolicyUID)] = &v1beta2.NetworkPolicy{} @@ -565,9 +569,9 @@ func TestRuleCacheAddAppliedToGroup(t *testing.T) { []*rule{rule1, rule2}, &v1beta2.AppliedToGroup{ ObjectMeta: metav1.ObjectMeta{Name: "group2"}, - GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")}, + GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")}, }, - []*v1beta2.GroupMember{newAppliedToGroupMember("pod1", "ns1")}, + []*v1beta2.GroupMember{newAppliedToGroupMemberPod("pod1", "ns1")}, sets.NewString("rule2"), }, { @@ -575,15 +579,15 @@ func TestRuleCacheAddAppliedToGroup(t *testing.T) { []*rule{rule1, rule2}, &v1beta2.AppliedToGroup{ ObjectMeta: metav1.ObjectMeta{Name: "group1"}, - GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1"), *newAppliedToGroupMember("pod2", "ns1")}, + GroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1"), *newAppliedToGroupMemberPod("pod2", "ns1")}, }, - []*v1beta2.GroupMember{newAppliedToGroupMember("pod1", "ns1"), newAppliedToGroupMember("pod2", "ns1")}, + []*v1beta2.GroupMember{newAppliedToGroupMemberPod("pod1", "ns1"), newAppliedToGroupMemberPod("pod2", "ns1")}, sets.NewString("rule1", "rule2"), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() for _, rule := range tt.rules { c.rules.Add(rule) } @@ -684,7 +688,7 @@ func TestRuleCacheAddNetworkPolicy(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() c.AddNetworkPolicy(tt.args) actualRules := c.rules.List() if !assert.ElementsMatch(t, tt.expectedRules, actualRules) { @@ -747,7 +751,7 @@ func TestRuleCacheDeleteNetworkPolicy(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() for _, rule := range tt.rules { c.rules.Add(rule) } @@ -767,8 +771,8 @@ func TestRuleCacheDeleteNetworkPolicy(t *testing.T) { func TestRuleCacheGetCompletedRule(t *testing.T) { addressGroup1 := v1beta2.NewGroupMemberSet(newAddressGroupMember("1.1.1.1"), newAddressGroupMember("1.1.1.2")) addressGroup2 := v1beta2.NewGroupMemberSet(newAddressGroupMember("1.1.1.3"), newAddressGroupMember("1.1.1.2")) - appliedToGroup1 := v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"), newAppliedToGroupMember("pod2", "ns1")) - appliedToGroup2 := v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod3", "ns1"), newAppliedToGroupMember("pod2", "ns1")) + appliedToGroup1 := v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"), newAppliedToGroupMemberPod("pod2", "ns1")) + appliedToGroup2 := v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod3", "ns1"), newAppliedToGroupMemberPod("pod2", "ns1")) rule1 := &rule{ ID: "rule1", Direction: v1beta2.DirectionIn, @@ -866,7 +870,7 @@ func TestRuleCacheGetCompletedRule(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, _, _ := newFakeRuleCache() + c, _, _, _ := newFakeRuleCache() c.addressSetByGroup["addressGroup1"] = addressGroup1 c.addressSetByGroup["addressGroup2"] = addressGroup2 c.appliedToSetByGroup["appliedToGroup1"] = appliedToGroup1 @@ -915,7 +919,7 @@ func TestRuleCachePatchAppliedToGroup(t *testing.T) { nil, &v1beta2.AppliedToGroupPatch{ ObjectMeta: metav1.ObjectMeta{Name: "group0"}, - AddedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")}, + AddedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")}, }, nil, sets.NewString(), @@ -924,33 +928,33 @@ func TestRuleCachePatchAppliedToGroup(t *testing.T) { { "add-and-remove-pods-affecting-one-rule", []*rule{rule1, rule2}, - map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"))}, + map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"))}, &v1beta2.AppliedToGroupPatch{ ObjectMeta: metav1.ObjectMeta{Name: "group2"}, - AddedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod2", "ns1"), *newAppliedToGroupMember("pod3", "ns3")}, - RemovedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")}, + AddedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod2", "ns1"), *newAppliedToGroupMemberPod("pod3", "ns3")}, + RemovedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")}, }, - []*v1beta2.GroupMember{newAppliedToGroupMember("pod2", "ns1"), newAppliedToGroupMember("pod3", "ns3")}, + []*v1beta2.GroupMember{newAppliedToGroupMemberPod("pod2", "ns1"), newAppliedToGroupMemberPod("pod3", "ns3")}, sets.NewString("rule2"), false, }, { "add-and-remove-pods-affecting-two-rule", []*rule{rule1, rule2}, - map[string]v1beta2.GroupMemberSet{"group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"))}, + map[string]v1beta2.GroupMemberSet{"group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"))}, &v1beta2.AppliedToGroupPatch{ ObjectMeta: metav1.ObjectMeta{Name: "group1"}, - AddedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod2", "ns1"), *newAppliedToGroupMember("pod3", "ns3")}, - RemovedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")}, + AddedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod2", "ns1"), *newAppliedToGroupMemberPod("pod3", "ns3")}, + RemovedGroupMembers: []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")}, }, - []*v1beta2.GroupMember{newAppliedToGroupMember("pod2", "ns1"), newAppliedToGroupMember("pod3", "ns3")}, + []*v1beta2.GroupMember{newAppliedToGroupMemberPod("pod2", "ns1"), newAppliedToGroupMemberPod("pod3", "ns3")}, sets.NewString("rule1", "rule2"), false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() c.appliedToSetByGroup = tt.podSetByGroup for _, rule := range tt.rules { c.rules.Add(rule) @@ -1027,7 +1031,7 @@ func TestRuleCachePatchAddressGroup(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() c.addressSetByGroup = tt.addressSetByGroup for _, rule := range tt.rules { c.rules.Add(rule) @@ -1118,7 +1122,7 @@ func TestRuleCacheUpdateNetworkPolicy(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, _ := newFakeRuleCache() + c, recorder, _, _ := newFakeRuleCache() for _, rule := range tt.rules { c.rules.Add(rule) } @@ -1161,7 +1165,7 @@ func TestRuleCacheProcessPodUpdates(t *testing.T) { { "matching-one-group-affecting-one-rule", []*rule{rule1, rule2}, - map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"))}, + map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"))}, "ns1/pod1", sets.NewString("rule2"), }, @@ -1169,8 +1173,8 @@ func TestRuleCacheProcessPodUpdates(t *testing.T) { "matching-two-groups-affecting-two-rules", []*rule{rule1, rule2}, map[string]v1beta2.GroupMemberSet{ - "group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1")), - "group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1")), + "group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1")), + "group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1")), }, "ns1/pod1", sets.NewString("rule1", "rule2"), @@ -1178,7 +1182,7 @@ func TestRuleCacheProcessPodUpdates(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, recorder, podUpdateNotifier := newFakeRuleCache() + c, recorder, podUpdateNotifier, _ := newFakeRuleCache() c.appliedToSetByGroup = tt.podSetByGroup for _, rule := range tt.rules { c.rules.Add(rule) @@ -1210,6 +1214,114 @@ func TestRuleCacheProcessPodUpdates(t *testing.T) { } } +func TestRuleCacheProcessServiceGroupIDUpdateaas(t *testing.T) {} + +func TestRuleCacheProcessServiceGroupIDUpdates(t *testing.T) { + rule1 := &rule{ + ID: "rule1", + AppliedToGroups: []string{"group1"}, + } + rule2 := &rule{ + ID: "rule2", + AppliedToGroups: []string{"group1", "group2"}, + } + rule3 := &rule{ + ID: "rule3", + To: v1beta2.NetworkPolicyPeer{ + ToServices: []v1beta2.ServiceReference{ + { + Name: "svc1", + Namespace: "ns1", + }, + }, + }, + } + rule4 := &rule{ + ID: "rule4", + To: v1beta2.NetworkPolicyPeer{ + ToServices: []v1beta2.ServiceReference{ + { + Name: "svc1", + Namespace: "ns1", + }, + { + Name: "svc2", + Namespace: "ns2", + }, + }, + }, + } + tests := []struct { + name string + rules []*rule + svcSetByGroup map[string]v1beta2.GroupMemberSet + svcUpdate string + expectedDirtyRules sets.String + }{ + { + "non-matching", + []*rule{rule1, rule2, rule3, rule4}, + nil, + "bar/foo", + sets.NewString(), + }, + { + "matching-one-group-affecting-one-rule", + []*rule{rule1, rule2}, + map[string]v1beta2.GroupMemberSet{"group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberService("svc1", "ns1"))}, + "ns1/svc1", + sets.NewString("rule2"), + }, + { + "matching-two-groups-affecting-two-rules", + []*rule{rule1, rule2}, + map[string]v1beta2.GroupMemberSet{ + "group1": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberService("svc1", "ns1")), + "group2": v1beta2.NewGroupMemberSet(newAppliedToGroupMemberService("svc1", "ns1")), + }, + "ns1/svc1", + sets.NewString("rule1", "rule2"), + }, + { + "matching-one-to-service-affecting-one-rule", + []*rule{rule3, rule4}, + nil, + "ns2/svc2", + sets.NewString("rule4"), + }, + { + "matching-two-to-service-affecting-two-rules", + []*rule{rule3, rule4}, + nil, + "ns1/svc1", + sets.NewString("rule3", "rule4"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, recorder, _, svcUpdateChan := newFakeRuleCache() + c.appliedToSetByGroup = tt.svcSetByGroup + for _, rule := range tt.rules { + c.rules.Add(rule) + } + svcUpdateChan <- tt.svcUpdate + func() { + // Drain the channel with 10 ms timeout so we can know it's done. + for { + select { + case <-recorder.eventCh: + case <-time.After(time.Millisecond * 10): + return + } + } + }() + if !recorder.rules.Equal(tt.expectedDirtyRules) { + t.Errorf("Got dirty rules %v, expected %v", recorder.rules, tt.expectedDirtyRules) + } + }) + } +} + func BenchmarkRuleCacheUnionAddressGroups(b *testing.B) { var addressGroupMembers1, addressGroupMembers2 []*v1beta2.GroupMember // addressGroup1 includes 10K members. @@ -1224,7 +1336,7 @@ func BenchmarkRuleCacheUnionAddressGroups(b *testing.B) { addressGroupMembers2 = append(addressGroupMembers2, newAddressGroupMember(fmt.Sprintf("2.2.2.%d", i))) } addressGroup2 := v1beta2.NewGroupMemberSet(addressGroupMembers2...) - c, _, _ := newFakeRuleCache() + c, _, _, _ := newFakeRuleCache() c.addressSetByGroup["addressGroup1"] = addressGroup1 c.addressSetByGroup["addressGroup2"] = addressGroup2 diff --git a/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go b/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go index d22b643a305..999c7787176 100644 --- a/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go +++ b/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go @@ -231,7 +231,7 @@ func TestAddSingleGroupRule(t *testing.T) { rule: &rule{Direction: v1beta2.DirectionIn, Services: services}, FromAddresses: v1beta2.NewGroupMemberSet(newAddressGroupMember("1.1.1.1"), newAddressGroupMember("2.2.2.2")), ToAddresses: v1beta2.NewGroupMemberSet(), - TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1")), + TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1")), } stopCh := make(chan struct{}) defer close(stopCh) @@ -266,7 +266,7 @@ func TestAddSingleGroupRule(t *testing.T) { assert.Equal(t, 0, controller.GetAppliedToGroupNum()) // appliedToGroup1 comes, policy1 will be synced. - appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")})) + appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")})) appliedToGroupWatcher.Action(watch.Bookmark, nil) select { case ruleID := <-reconciler.updated: @@ -311,7 +311,7 @@ func TestAddMultipleGroupsRule(t *testing.T) { rule: &rule{Direction: v1beta2.DirectionIn, Services: services}, FromAddresses: v1beta2.NewGroupMemberSet(newAddressGroupMember("1.1.1.1"), newAddressGroupMember("2.2.2.2"), newAddressGroupMember("3.3.3.3")), ToAddresses: nil, - TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"), newAppliedToGroupMember("pod2", "ns2")), + TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"), newAppliedToGroupMemberPod("pod2", "ns2")), } stopCh := make(chan struct{}) defer close(stopCh) @@ -321,7 +321,7 @@ func TestAddMultipleGroupsRule(t *testing.T) { addressGroupWatcher.Add(newAddressGroup("addressGroup1", []v1beta2.GroupMember{*newAddressGroupMember("1.1.1.1"), *newAddressGroupMember("2.2.2.2")})) addressGroupWatcher.Action(watch.Bookmark, nil) // appliedToGroup1 comes, no rule will be synced. - appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")})) + appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")})) appliedToGroupWatcher.Action(watch.Bookmark, nil) // policy1 comes first, no rule will be synced due to missing addressGroup2 and appliedToGroup2. policy1 := newNetworkPolicy("policy1", "uid1", []string{"addressGroup1", "addressGroup2"}, []string{}, []string{"appliedToGroup1", "appliedToGroup2"}, services) @@ -348,7 +348,7 @@ func TestAddMultipleGroupsRule(t *testing.T) { assert.ElementsMatch(t, actualRule.Services, desiredRule.Services) assert.Equal(t, actualRule.FromAddresses, desiredRule.FromAddresses) assert.Equal(t, actualRule.ToAddresses, desiredRule.ToAddresses) - assert.Equal(t, actualRule.TargetMembers, v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1"))) + assert.Equal(t, actualRule.TargetMembers, v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1"))) case <-time.After(time.Millisecond * 100): t.Fatal("Expected one update, got none") } @@ -357,7 +357,7 @@ func TestAddMultipleGroupsRule(t *testing.T) { assert.Equal(t, 1, controller.GetAppliedToGroupNum()) // appliedToGroup2 comes, policy1 will be synced with the TargetMembers populated from appliedToGroup1 and appliedToGroup2. - appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup2", []v1beta2.GroupMember{*newAppliedToGroupMember("pod2", "ns2")})) + appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup2", []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod2", "ns2")})) select { case ruleID := <-reconciler.updated: actualRule, _ := reconciler.getLastRealized(ruleID) @@ -393,7 +393,7 @@ func TestDeleteRule(t *testing.T) { addressGroupWatcher.Add(newAddressGroup("addressGroup1", []v1beta2.GroupMember{*newAddressGroupMember("1.1.1.1"), *newAddressGroupMember("2.2.2.2")})) addressGroupWatcher.Action(watch.Bookmark, nil) - appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")})) + appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")})) appliedToGroupWatcher.Action(watch.Bookmark, nil) networkPolicyWatcher.Add(newNetworkPolicy("policy1", "uid1", []string{"addressGroup1"}, []string{}, []string{"appliedToGroup1"}, services)) networkPolicyWatcher.Action(watch.Bookmark, nil) @@ -439,13 +439,13 @@ func TestAddNetworkPolicyWithMultipleRules(t *testing.T) { rule: &rule{Direction: v1beta2.DirectionIn, Services: services}, FromAddresses: v1beta2.NewGroupMemberSet(newAddressGroupMember("1.1.1.1"), newAddressGroupMember("2.2.2.2")), ToAddresses: v1beta2.NewGroupMemberSet(), - TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1")), + TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1")), } desiredRule2 := &CompletedRule{ rule: &rule{Direction: v1beta2.DirectionOut, Services: services}, FromAddresses: v1beta2.NewGroupMemberSet(), ToAddresses: v1beta2.NewGroupMemberSet(newAddressGroupMember("3.3.3.3"), newAddressGroupMember("4.4.4.4")), - TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1")), + TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1")), } stopCh := make(chan struct{}) defer close(stopCh) @@ -458,7 +458,7 @@ func TestAddNetworkPolicyWithMultipleRules(t *testing.T) { addressGroupWatcher.Add(newAddressGroup("addressGroup1", []v1beta2.GroupMember{*newAddressGroupMember("1.1.1.1"), *newAddressGroupMember("2.2.2.2")})) addressGroupWatcher.Add(newAddressGroup("addressGroup2", []v1beta2.GroupMember{*newAddressGroupMember("3.3.3.3"), *newAddressGroupMember("4.4.4.4")})) addressGroupWatcher.Action(watch.Bookmark, nil) - appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")})) + appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")})) appliedToGroupWatcher.Action(watch.Bookmark, nil) for i := 0; i < 2; i++ { select { @@ -594,7 +594,7 @@ func TestNetworkPolicyMetrics(t *testing.T) { policy1 := newNetworkPolicy("policy1", "uid1", []string{"addressGroup1"}, []string{}, []string{"appliedToGroup1"}, services) addressGroupWatcher.Add(newAddressGroup("addressGroup1", []v1beta2.GroupMember{*newAddressGroupMember("1.1.1.1"), *newAddressGroupMember("2.2.2.2")})) addressGroupWatcher.Action(watch.Bookmark, nil) - appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")})) + appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")})) appliedToGroupWatcher.Action(watch.Bookmark, nil) networkPolicyWatcher.Add(policy1) networkPolicyWatcher.Action(watch.Bookmark, nil) @@ -605,7 +605,7 @@ func TestNetworkPolicyMetrics(t *testing.T) { policy2 := newNetworkPolicyWithMultipleRules("policy2", "uid2", []string{"addressGroup2"}, []string{"addressGroup2"}, []string{"appliedToGroup2"}, services) addressGroupWatcher.Add(newAddressGroup("addressGroup2", []v1beta2.GroupMember{*newAddressGroupMember("3.3.3.3"), *newAddressGroupMember("4.4.4.4")})) addressGroupWatcher.Action(watch.Bookmark, nil) - appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup2", []v1beta2.GroupMember{*newAppliedToGroupMember("pod2", "ns2")})) + appliedToGroupWatcher.Add(newAppliedToGroup("appliedToGroup2", []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod2", "ns2")})) appliedToGroupWatcher.Action(watch.Bookmark, nil) networkPolicyWatcher.Add(policy2) waitForReconcilerUpdated() @@ -674,7 +674,7 @@ func TestValidate(t *testing.T) { groups := v1beta2.GroupMemberSet{} groupAddress1, groupAddress2 := "225.1.2.3", "225.1.2.4" - groups["ns1/pod1"] = newAppliedToGroupMember("pod1", "ns1") + groups["ns1/pod1"] = newAppliedToGroupMemberPod("pod1", "ns1") controller.ruleCache.appliedToSetByGroup["appliedToGroup01"] = groups controller.ruleCache.rules.Add(rule1) controller.ruleCache.rules.Add(rule2) diff --git a/pkg/agent/controller/networkpolicy/reconciler.go b/pkg/agent/controller/networkpolicy/reconciler.go index 09fb5c1d191..bee9c0f28ee 100644 --- a/pkg/agent/controller/networkpolicy/reconciler.go +++ b/pkg/agent/controller/networkpolicy/reconciler.go @@ -162,9 +162,10 @@ type lastRealized struct { // the fqdn selector of this policy rule. It must be empty for policy rule // that is not egress and does not have toFQDN field. fqdnIPAddresses sets.String - // groupIDAddresses tracks the last realized set of groupIDs resolved for - // the toServices of this policy rule. It must be empty for policy rule - // that is not egress and does not have toServices field. + // groupIDAddresses tracks the last realized set of groupIDs resolved for the + // toServices of this policy rule or services of TargetMember of this policy rule. + // It must be empty for policy rule that is neither an egress rule with toServices + // field nor an ingress rule that is applied to Services. groupIDAddresses sets.Int64 // groupAddresses track the latest realized set of multicast groups for the multicast traffic groupAddresses sets.String @@ -541,18 +542,31 @@ func (r *reconciler) computeOFRulesForAdd(rule *CompletedRule, ofPriority *uint1 return ofRuleByServicesMap, lastRealized } if rule.Direction == v1beta2.DirectionIn { + isRuleAppliedToService := isRuleAppliedToService(rule.TargetMembers) // Addresses got from source GroupMembers' IPs. from1 := groupMembersToOFAddresses(rule.FromAddresses) // Get addresses that in From IPBlock but not in Except IPBlocks. - from2 := ipBlocksToOFAddresses(rule.From.IPBlocks, r.ipv4Enabled, r.ipv6Enabled) + from2 := ipBlocksToOFAddresses(rule.From.IPBlocks, r.ipv4Enabled, r.ipv6Enabled, isRuleAppliedToService) membersByServicesMap, servicesMap := groupMembersByServices(rule.Services, rule.TargetMembers) for svcKey, members := range membersByServicesMap { - ofPorts := r.getOFPorts(members) - lastRealized.podOFPorts[svcKey] = ofPorts + var toAddresses []types.Address + if isRuleAppliedToService { + addressSet := sets.NewInt64() + for _, member := range members.Items() { + curAddresses, curAddressSet := r.svcRefToOFAddresses(*member.Service) + toAddresses = append(toAddresses, curAddresses...) + addressSet = addressSet.Union(curAddressSet) + } + lastRealized.groupIDAddresses = addressSet + } else { + ofPorts := r.getOFPorts(members) + toAddresses = ofPortsToOFAddresses(ofPorts) + lastRealized.podOFPorts[svcKey] = ofPorts + } ofRuleByServicesMap[svcKey] = &types.PolicyRule{ Direction: v1beta2.DirectionIn, From: append(from1, from2...), - To: ofPortsToOFAddresses(ofPorts), + To: toAddresses, Service: filterUnresolvablePort(servicesMap[svcKey]), Action: rule.Action, Name: rule.Name, @@ -617,7 +631,7 @@ func (r *reconciler) computeOFRulesForAdd(rule *CompletedRule, ofPriority *uint1 } if len(rule.To.IPBlocks) > 0 { // Diff Addresses between To and Except of IPBlocks - to := ipBlocksToOFAddresses(rule.To.IPBlocks, r.ipv4Enabled, r.ipv6Enabled) + to := ipBlocksToOFAddresses(rule.To.IPBlocks, r.ipv4Enabled, r.ipv6Enabled, false) ofRule.To = append(ofRule.To, to...) } if r.fqdnController != nil && len(rule.To.FQDNs) > 0 { @@ -636,12 +650,9 @@ func (r *reconciler) computeOFRulesForAdd(rule *CompletedRule, ofPriority *uint1 var addresses []types.Address addressSet := sets.NewInt64() for _, svcRef := range rule.To.ToServices { - for _, groupCounter := range r.groupCounters { - for _, groupID := range groupCounter.GetAllGroupIDs(k8s.NamespacedName(svcRef.Namespace, svcRef.Name)) { - addresses = append(addresses, openflow.NewServiceGroupIDAddress(groupID)) - addressSet.Insert(int64(groupID)) - } - } + curAddresses, curAddressSet := r.svcRefToOFAddresses(svcRef) + addresses = append(addresses, curAddresses...) + addressSet = addressSet.Union(curAddressSet) } ofRule.To = append(ofRule.To, addresses...) // If the rule installation fails, this will be reset. @@ -742,8 +753,9 @@ func (r *reconciler) update(lastRealized *lastRealized, newRule *CompletedRule, // As rule identifier is calculated from the rule's content, the update can // only happen to Group members. if newRule.Direction == v1beta2.DirectionIn { + isRuleAppliedToService := isRuleAppliedToService(newRule.TargetMembers) from1 := groupMembersToOFAddresses(newRule.FromAddresses) - from2 := ipBlocksToOFAddresses(newRule.From.IPBlocks, r.ipv4Enabled, r.ipv6Enabled) + from2 := ipBlocksToOFAddresses(newRule.From.IPBlocks, r.ipv4Enabled, r.ipv6Enabled, isRuleAppliedToService) addedFrom := ipsToOFAddresses(newRule.FromAddresses.IPDifference(lastRealized.FromAddresses)) deletedFrom := ipsToOFAddresses(lastRealized.FromAddresses.IPDifference(newRule.FromAddresses)) @@ -776,9 +788,28 @@ func (r *reconciler) update(lastRealized *lastRealized, newRule *CompletedRule, } else { addedTo := ofPortsToOFAddresses(newOFPorts.Difference(lastRealized.podOFPorts[svcKey])) deletedTo := ofPortsToOFAddresses(lastRealized.podOFPorts[svcKey].Difference(newOFPorts)) + originalGroupIDAddressSet, newGroupIDAddressSet := sets.NewInt64(), sets.NewInt64() + if isRuleAppliedToService { + if lastRealized.groupIDAddresses != nil { + originalGroupIDAddressSet = lastRealized.groupIDAddresses + } + for _, member := range members.Items() { + _, groupIDSets := r.svcRefToOFAddresses(*member.Service) + newGroupIDAddressSet = newGroupIDAddressSet.Union(groupIDSets) + } + addedGroupIDAddress := newGroupIDAddressSet.Difference(originalGroupIDAddressSet) + removedGroupIDAddress := originalGroupIDAddressSet.Difference(newGroupIDAddressSet) + for a := range addedGroupIDAddress { + addedTo = append(addedTo, openflow.NewServiceGroupIDAddress(binding.GroupIDType(a))) + } + for r := range removedGroupIDAddress { + deletedTo = append(deletedTo, openflow.NewServiceGroupIDAddress(binding.GroupIDType(r))) + } + } if err := r.updateOFRule(ofID, addedFrom, addedTo, deletedFrom, deletedTo, ofPriority); err != nil { return err } + lastRealized.groupIDAddresses = newGroupIDAddressSet // Delete valid servicesKey from staleOFIDs. delete(staleOFIDs, svcKey) } @@ -822,7 +853,7 @@ func (r *reconciler) update(lastRealized *lastRealized, newRule *CompletedRule, // If the PolicyRule for the original services doesn't exist and IPBlocks is present, it means the // reconciler hasn't installed flows for IPBlocks, then it must be added to the new PolicyRule. if svcKey == originalSvcKey && len(newRule.To.IPBlocks) > 0 { - to := ipBlocksToOFAddresses(newRule.To.IPBlocks, r.ipv4Enabled, r.ipv6Enabled) + to := ipBlocksToOFAddresses(newRule.To.IPBlocks, r.ipv4Enabled, r.ipv6Enabled, false) ofRule.To = append(ofRule.To, to...) } err := r.idAllocator.allocateForRule(ofRule) @@ -862,11 +893,8 @@ func (r *reconciler) update(lastRealized *lastRealized, newRule *CompletedRule, } if len(newRule.To.ToServices) > 0 { for _, svcRef := range newRule.To.ToServices { - for _, groupCounter := range r.groupCounters { - for _, groupID := range groupCounter.GetAllGroupIDs(k8s.NamespacedName(svcRef.Namespace, svcRef.Name)) { - newGroupIDAddressSet.Insert(int64(groupID)) - } - } + _, groupIDSets := r.svcRefToOFAddresses(svcRef) + newGroupIDAddressSet = newGroupIDAddressSet.Union(groupIDSets) } addedGroupIDAddress := newGroupIDAddressSet.Difference(originalGroupIDAddressSet) removedGroupIDAddress := originalGroupIDAddressSet.Difference(newGroupIDAddressSet) @@ -1108,6 +1136,18 @@ func ofPortsToOFAddresses(ofPorts sets.Int32) []types.Address { return addresses } +func (r *reconciler) svcRefToOFAddresses(svcRef v1beta2.ServiceReference) ([]types.Address, sets.Int64) { + var addresses []types.Address + addressSet := sets.NewInt64() + for _, groupCounter := range r.groupCounters { + for _, groupID := range groupCounter.GetAllGroupIDs(k8s.NamespacedName(svcRef.Namespace, svcRef.Name)) { + addresses = append(addresses, openflow.NewServiceGroupIDAddress(groupID)) + addressSet.Insert(int64(groupID)) + } + } + return addresses, addressSet +} + func groupMembersToOFAddresses(groupMemberSet v1beta2.GroupMemberSet) []types.Address { // Must not return nil as it means not restricted by addresses in Openflow implementation. addresses := make([]types.Address, 0, len(groupMemberSet)) @@ -1119,7 +1159,7 @@ func groupMembersToOFAddresses(groupMemberSet v1beta2.GroupMemberSet) []types.Ad return addresses } -func ipBlocksToOFAddresses(ipBlocks []v1beta2.IPBlock, ipv4Enabled, ipv6Enabled bool) []types.Address { +func ipBlocksToOFAddresses(ipBlocks []v1beta2.IPBlock, ipv4Enabled, ipv6Enabled, ctMatch bool) []types.Address { // Must not return nil as it means not restricted by addresses in Openflow implementation. addresses := make([]types.Address, 0) for _, b := range ipBlocks { @@ -1140,7 +1180,11 @@ func ipBlocksToOFAddresses(ipBlocks []v1beta2.IPBlock, ipv4Enabled, ipv6Enabled continue } for _, d := range diffCIDRs { - addresses = append(addresses, openflow.NewIPNetAddress(*d)) + if ctMatch { + addresses = append(addresses, openflow.NewCTIPNetAddress(*d)) + } else { + addresses = append(addresses, openflow.NewIPNetAddress(*d)) + } } } @@ -1204,3 +1248,14 @@ func resolveService(service *v1beta2.Service, member *v1beta2.GroupMember) *v1be // The group members that cannot resolve it will be grouped together. return service } + +func isRuleAppliedToService(memberSet v1beta2.GroupMemberSet) bool { + for _, member := range memberSet.Items() { + if member.Service != nil { + return true + } else { + return false + } + } + return false +} diff --git a/pkg/agent/controller/networkpolicy/reconciler_test.go b/pkg/agent/controller/networkpolicy/reconciler_test.go index a25e4056e53..71daa4b775b 100644 --- a/pkg/agent/controller/networkpolicy/reconciler_test.go +++ b/pkg/agent/controller/networkpolicy/reconciler_test.go @@ -41,19 +41,19 @@ var ( ipv6AddressGroup1 = v1beta2.NewGroupMemberSet(newAddressGroupMember("2002:1a23:fb44::1")) dualAddressGroup1 = v1beta2.NewGroupMemberSet(newAddressGroupMember("1.1.1.1", "2002:1a23:fb44::1")) - appliedToGroup1 = v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1")) - appliedToGroup2 = v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod2", "ns1")) - appliedToGroup3 = v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod4", "ns1")) + appliedToGroup1 = v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1")) + appliedToGroup2 = v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod2", "ns1")) + appliedToGroup3 = v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod4", "ns1")) appliedToGroupWithSameContainerPort = v1beta2.NewGroupMemberSet( - newAppliedToGroupMember("pod1", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 80}), - newAppliedToGroupMember("pod3", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 80}), + newAppliedToGroupMemberPod("pod1", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 80}), + newAppliedToGroupMemberPod("pod3", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 80}), ) appliedToGroupWithDiffContainerPort = v1beta2.NewGroupMemberSet( - newAppliedToGroupMember("pod1", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 80}), - newAppliedToGroupMember("pod3", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 443}), + newAppliedToGroupMemberPod("pod1", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 80}), + newAppliedToGroupMemberPod("pod3", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 443}), ) appliedToGroupWithSingleContainerPort = v1beta2.NewGroupMemberSet( - newAppliedToGroupMember("pod1", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 80})) + newAppliedToGroupMemberPod("pod1", "ns1", v1beta2.NamedPort{Name: "http", Protocol: v1beta2.ProtocolTCP, Port: 80})) protocolTCP = v1beta2.ProtocolTCP @@ -611,7 +611,7 @@ func TestReconcileWithTransientError(t *testing.T) { }, }, ToAddresses: v1beta2.NewGroupMemberSet(member1, member2, member3), - TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMember("pod1", "ns1")), + TargetMembers: v1beta2.NewGroupMemberSet(newAppliedToGroupMemberPod("pod1", "ns1")), } controller := gomock.NewController(t) diff --git a/pkg/agent/controller/networkpolicy/reject.go b/pkg/agent/controller/networkpolicy/reject.go index 5c5e4690b18..3c354142c0f 100644 --- a/pkg/agent/controller/networkpolicy/reject.go +++ b/pkg/agent/controller/networkpolicy/reject.go @@ -74,6 +74,10 @@ const ( // Service traffic, when AntreaProxy is disabled. The EndpointPod is on a remote // Node and the dstPod of the reject response is on the local Node. RejectNoAPServiceRemoteToLocal + // RejectServiceRemoteToExternal represents this packetOut is used to reject + // Service traffic, when AntreaProxy is enabled. The EndpointPod is on a remote + // Node and the destination of the reject response is an external client. + RejectServiceRemoteToExternal // Unsupported indicates that Antrea couldn't generate packetOut for current // packetIn. Unsupported @@ -140,6 +144,9 @@ func (c *Controller) rejectRequest(pktIn *ofctrl.PacketIn) error { if packetOutType == Unsupported { return fmt.Errorf("error when generating reject response for the packet from: %s to %s: neither source nor destination are on this Node", dstIP, srcIP) } + if packetOutType == RejectServiceRemoteToExternal { + dstMAC = "aa:bb:cc:dd:ee:ff" + } // When in AntreaIPAM mode, even though srcPod and dstPod are on the same Node, MAC // will still be re-written in L3ForwardingTable. During rejection, the reject // response will be directly sent to the dst OF port without go through @@ -242,7 +249,7 @@ func getRejectType(isServiceTraffic, antreaProxyEnabled, srcIsLocal, dstIsLocal if dstIsLocal { return RejectServiceRemoteToLocal } - return Unsupported + return RejectServiceRemoteToExternal } // getRejectOFPorts returns the inPort and outPort of a packetOut based on the RejectType. @@ -268,6 +275,8 @@ func getRejectOFPorts(rejectType RejectType, sIface, dIface *interfacestore.Inte case RejectNoAPServiceRemoteToLocal: inPort = tunOFPort outPort = gwOFPort + case RejectServiceRemoteToExternal: + inPort = tunOFPort } return inPort, outPort } diff --git a/pkg/agent/controller/networkpolicy/status_controller_test.go b/pkg/agent/controller/networkpolicy/status_controller_test.go index d719e8baa10..b21fe872f04 100644 --- a/pkg/agent/controller/networkpolicy/status_controller_test.go +++ b/pkg/agent/controller/networkpolicy/status_controller_test.go @@ -106,7 +106,7 @@ func TestSyncStatusForNewPolicy(t *testing.T) { go statusController.Run(stopCh) ruleCache.AddNetworkPolicy(tt.policy) - ruleCache.AddAppliedToGroup(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")})) + ruleCache.AddAppliedToGroup(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")})) rules := ruleCache.getEffectiveRulesByNetworkPolicy(string(tt.policy.UID)) for i, rule := range rules { // Only make specified number of rules realized. @@ -128,7 +128,7 @@ func TestSyncStatusUpForUpdatedPolicy(t *testing.T) { defer close(stopCh) go statusController.Run(stopCh) - ruleCache.AddAppliedToGroup(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMember("pod1", "ns1")})) + ruleCache.AddAppliedToGroup(newAppliedToGroup("appliedToGroup1", []v1beta2.GroupMember{*newAppliedToGroupMemberPod("pod1", "ns1")})) policy := newNetworkPolicy("policy1", "uid1", []string{"addressGroup1"}, []string{}, []string{"appliedToGroup1"}, nil) policy.Generation = 1 ruleCache.AddNetworkPolicy(policy) diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index d4995f2117d..346b6bc7393 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -777,6 +777,7 @@ func (c *client) generatePipelines() { c.enableDenyTracking, c.enableAntreaPolicy, c.enableMulticast, + c.proxyAll, c.connectUplinkToBridge) c.activatedFeatures = append(c.activatedFeatures, c.featureNetworkPolicy) c.traceableFeatures = append(c.traceableFeatures, c.featureNetworkPolicy) diff --git a/pkg/agent/openflow/network_policy.go b/pkg/agent/openflow/network_policy.go index 3235a54af92..20624b95e2f 100644 --- a/pkg/agent/openflow/network_policy.go +++ b/pkg/agent/openflow/network_policy.go @@ -38,10 +38,18 @@ var ( MatchSrcIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "nw_src") MatchDstIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_dst") MatchSrcIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "nw_src") + MatchCTDstIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "ct_nw_dst") + MatchCTSrcIP = types.NewMatchKey(binding.ProtocolIP, types.IPAddr, "ct_nw_src") + MatchCTDstIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "ct_nw_dst") + MatchCTSrcIPNet = types.NewMatchKey(binding.ProtocolIP, types.IPNetAddr, "ct_nw_src") MatchDstIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_dst") MatchSrcIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ipv6_src") MatchDstIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_dst") MatchSrcIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ipv6_src") + MatchCTDstIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ct_ipv6_dst") + MatchCTSrcIPv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPAddr, "ct_ipv6_src") + MatchCTDstIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ct_ipv6_dst") + MatchCTSrcIPNetv6 = types.NewMatchKey(binding.ProtocolIPv6, types.IPNetAddr, "ct_ipv6_src") MatchDstOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "reg1[0..31]") MatchSrcOFPort = types.NewMatchKey(binding.ProtocolIP, types.OFPortAddr, "in_port") MatchTCPDstPort = types.NewMatchKey(binding.ProtocolTCP, types.L4PortAddr, "tp_dst") @@ -192,6 +200,78 @@ func NewServiceGroupIDAddress(groupID binding.GroupIDType) *ServiceGroupIDAddres return &a } +// CT IP address calculated from Pod's address. +type CTIPAddress net.IP + +func (a *CTIPAddress) GetMatchKey(addrType types.AddressType) *types.MatchKey { + ipArr := net.IP(*a) + switch addrType { + case types.SrcAddress: + if ipArr.To4() != nil { + return MatchCTSrcIP + } + return MatchCTSrcIPv6 + case types.DstAddress: + if ipArr.To4() != nil { + return MatchCTDstIP + } + return MatchCTDstIPv6 + default: + klog.Errorf("Unknown AddressType %d in CTIPAddress", addrType) + return Unsupported + } +} + +func (a *CTIPAddress) GetMatchValue() string { + addr := net.IP(*a) + return addr.String() +} + +func (a *CTIPAddress) GetValue() interface{} { + return net.IP(*a) +} + +func NewCTIPAddress(addr net.IP) *CTIPAddress { + cia := CTIPAddress(addr) + return &cia +} + +// CT IP block calculated from Pod's address. +type CTIPNetAddress net.IPNet + +func (a *CTIPNetAddress) GetMatchKey(addrType types.AddressType) *types.MatchKey { + ipAddr := net.IPNet(*a) + switch addrType { + case types.SrcAddress: + if ipAddr.IP.To4() != nil { + return MatchCTSrcIPNet + } + return MatchCTSrcIPNetv6 + case types.DstAddress: + if ipAddr.IP.To4() != nil { + return MatchCTDstIPNet + } + return MatchCTDstIPNetv6 + default: + klog.Errorf("Unknown AddressType %d in CTIPNetAddress", addrType) + return Unsupported + } +} + +func (a *CTIPNetAddress) GetMatchValue() string { + addr := net.IPNet(*a) + return addr.String() +} + +func (a *CTIPNetAddress) GetValue() interface{} { + return net.IPNet(*a) +} + +func NewCTIPNetAddress(addr net.IPNet) *CTIPNetAddress { + ia := CTIPNetAddress(addr) + return &ia +} + // ConjunctionNotFound is an error response when the specified policyRuleConjunction is not found from the local cache. type ConjunctionNotFound uint32 @@ -1892,6 +1972,7 @@ type featureNetworkPolicy struct { enableDenyTracking bool enableAntreaPolicy bool enableMulticast bool + proxyAll bool ctZoneSrcField *binding.RegField // deterministic represents whether to generate flows deterministically. // For example, if a flow has multiple actions, setting it to true can get consistent flow. @@ -1913,6 +1994,7 @@ func newFeatureNetworkPolicy( enableDenyTracking, enableAntreaPolicy bool, enableMulticast bool, + proxyAll bool, connectUplinkToBridge bool) *featureNetworkPolicy { return &featureNetworkPolicy{ cookieAllocator: cookieAllocator, @@ -1924,6 +2006,7 @@ func newFeatureNetworkPolicy( ovsMetersAreSupported: ovsMetersAreSupported, enableDenyTracking: enableDenyTracking, enableAntreaPolicy: enableAntreaPolicy, + proxyAll: proxyAll, category: cookie.NetworkPolicy, ctZoneSrcField: getZoneSrcField(connectUplinkToBridge), } diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 3fdb03c5f69..8bafa971833 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -2005,6 +2005,14 @@ func (f *featureNetworkPolicy) addFlowMatch(fb binding.FlowBuilder, matchKey *ty fallthrough case MatchDstIPNetv6: fb = fb.MatchProtocol(matchKey.GetOFProtocol()).MatchDstIPNet(matchValue.(net.IPNet)) + case MatchCTDstIP: + fallthrough + case MatchCTDstIPv6: + fb = fb.MatchCTStateNew(true).MatchProtocol(matchKey.GetOFProtocol()).MatchCTDstIP(matchValue.(net.IP)) + case MatchCTDstIPNet: + fallthrough + case MatchCTDstIPNetv6: + fb = fb.MatchCTStateNew(true).MatchProtocol(matchKey.GetOFProtocol()).MatchCTDstIPNet(matchValue.(net.IPNet)) case MatchSrcIP: fallthrough case MatchSrcIPv6: @@ -2013,6 +2021,14 @@ func (f *featureNetworkPolicy) addFlowMatch(fb binding.FlowBuilder, matchKey *ty fb = fb.MatchProtocol(matchKey.GetOFProtocol()).MatchSrcIPNet(matchValue.(net.IPNet)) case MatchSrcIPNetv6: fb = fb.MatchProtocol(matchKey.GetOFProtocol()).MatchSrcIPNet(matchValue.(net.IPNet)) + case MatchCTSrcIP: + fallthrough + case MatchCTSrcIPv6: + fb = fb.MatchCTStateNew(true).MatchProtocol(matchKey.GetOFProtocol()).MatchCTSrcIP(matchValue.(net.IP)) + case MatchCTSrcIPNet: + fallthrough + case MatchCTSrcIPNetv6: + fb = fb.MatchCTStateNew(true).MatchProtocol(matchKey.GetOFProtocol()).MatchCTSrcIPNet(matchValue.(net.IPNet)) case MatchTCPDstPort: fallthrough case MatchTCPv6DstPort: @@ -2185,7 +2201,7 @@ func (f *featurePodConnectivity) localProbeFlows() []binding.Flow { // tables within stageIngressSecurity. func (f *featureNetworkPolicy) ingressClassifierFlows() []binding.Flow { cookieID := f.cookieAllocator.Request(f.category).Raw() - return []binding.Flow{ + flows := []binding.Flow{ // This generates the flow to match the packets to the Antrea gateway and forward them to IngressMetricTable. IngressSecurityClassifierTable.ofTable.BuildFlow(priorityNormal). Cookie(cookieID). @@ -2205,6 +2221,16 @@ func (f *featureNetworkPolicy) ingressClassifierFlows() []binding.Flow { Action().GotoTable(IngressMetricTable.GetID()). Done(), } + if f.proxyAll { + // This generates the flow to match the NodePort Service packets and forward them to AntreaPolicyIngressRuleTable. + // Policies applied on NodePort Service will be enforced in AntreaPolicyIngressRuleTable. + flows = append(flows, IngressSecurityClassifierTable.ofTable.BuildFlow(priorityNormal+1). + Cookie(cookieID). + MatchRegMark(ToNodePortAddressRegMark). + Action().GotoTable(AntreaPolicyIngressRuleTable.GetID()). + Done()) + } + return flows } // snatSkipNodeFlow generates the flow to skip SNAT for connection destined for the transport IP of a remote Node. diff --git a/pkg/apis/controlplane/types.go b/pkg/apis/controlplane/types.go index 3d2351e4007..5cfcfc829c4 100644 --- a/pkg/apis/controlplane/types.go +++ b/pkg/apis/controlplane/types.go @@ -88,6 +88,9 @@ type GroupMember struct { IPs []IPAddress // Ports is the list NamedPort of the GroupMember. Ports []NamedPort + // Service is the reference to the Service. It can only be used in an AppliedTo + // Group and only a NodePort type Service can be referred by this field. + Service *ServiceReference } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/controlplane/v1beta2/generated.pb.go b/pkg/apis/controlplane/v1beta2/generated.pb.go index f5b8501e7a0..1240b68ebf0 100644 --- a/pkg/apis/controlplane/v1beta2/generated.pb.go +++ b/pkg/apis/controlplane/v1beta2/generated.pb.go @@ -983,143 +983,143 @@ func init() { } var fileDescriptor_fbaa7d016762fa1d = []byte{ - // 2161 bytes of a gzipped FileDescriptorProto + // 2176 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x1a, 0xcd, 0x6f, 0x1b, 0x59, 0xbd, 0xe3, 0x8f, 0x24, 0xfe, 0xd9, 0x69, 0x9d, 0x97, 0x96, 0x9a, 0xa5, 0xd8, 0xdd, 0x59, 0x40, 0x3d, 0xc0, 0x78, 0x13, 0xda, 0x6d, 0x60, 0xb7, 0x0b, 0x71, 0x92, 0x46, 0x96, 0xda, 0xd4, 0xbc, - 0x64, 0x55, 0x09, 0xe8, 0xb2, 0x93, 0x99, 0x67, 0x67, 0x88, 0x3d, 0x6f, 0x98, 0x79, 0x0e, 0x8d, - 0x90, 0xd0, 0x22, 0xe0, 0xb0, 0x80, 0x04, 0x37, 0xc4, 0x0d, 0x4e, 0x5c, 0xf8, 0x0b, 0xb8, 0x71, - 0xab, 0x38, 0xed, 0x0a, 0x21, 0xf6, 0x64, 0x51, 0x23, 0x40, 0x1c, 0xe0, 0x0f, 0x08, 0x17, 0xf4, + 0x64, 0x55, 0x09, 0xe8, 0xb2, 0x93, 0x99, 0x67, 0xe7, 0x11, 0x7b, 0x66, 0x98, 0x79, 0x0e, 0x8d, + 0x90, 0xd0, 0x22, 0xe0, 0xb0, 0x80, 0x04, 0x37, 0xc4, 0x0d, 0x4e, 0x5c, 0xf8, 0x0b, 0xf6, 0xc6, + 0xad, 0xe2, 0xb4, 0x08, 0x21, 0xf6, 0x64, 0x51, 0x23, 0x40, 0x1c, 0xe0, 0x0f, 0x08, 0x17, 0xf4, 0xde, 0xbc, 0x99, 0x79, 0x63, 0x27, 0x4d, 0x9d, 0xa4, 0x41, 0x82, 0x3d, 0xc5, 0x7e, 0xbf, 0xef, - 0xf7, 0xfb, 0x7e, 0x0e, 0xbc, 0x69, 0xba, 0xcc, 0x27, 0xa6, 0xe1, 0xd0, 0x7a, 0xf8, 0xa9, 0xee, - 0xed, 0x76, 0xea, 0xa6, 0xe7, 0x04, 0x75, 0x8b, 0xba, 0xcc, 0xa7, 0x5d, 0xaf, 0x6b, 0xba, 0xa4, - 0xbe, 0xb7, 0xb0, 0x4d, 0x98, 0xb9, 0x58, 0xef, 0x10, 0x97, 0xf8, 0x26, 0x23, 0xb6, 0xe1, 0xf9, - 0x94, 0x51, 0x64, 0x84, 0x54, 0xdf, 0x70, 0xa8, 0xfc, 0x64, 0x78, 0xbb, 0x1d, 0x83, 0xd3, 0x1b, - 0x2a, 0xbd, 0x21, 0xe9, 0x5f, 0x5a, 0x3a, 0x5a, 0x5e, 0xc0, 0x4c, 0x16, 0xd4, 0xf7, 0x16, 0xcc, - 0xae, 0xb7, 0x63, 0x2e, 0x8c, 0x4a, 0x7a, 0xe9, 0x73, 0x1d, 0x87, 0xed, 0xf4, 0xb7, 0x0d, 0x8b, - 0xf6, 0xea, 0x1d, 0xda, 0xa1, 0x75, 0x71, 0xbc, 0xdd, 0x6f, 0x8b, 0x6f, 0xe2, 0x8b, 0xf8, 0x24, - 0xd1, 0x6f, 0xee, 0x2e, 0x05, 0x42, 0x8a, 0xe7, 0xf4, 0x4c, 0x6b, 0xc7, 0x71, 0x89, 0xbf, 0x9f, - 0xc8, 0xea, 0x11, 0x66, 0xd6, 0xf7, 0xc6, 0x85, 0xd4, 0x8f, 0xa2, 0xf2, 0xfb, 0x2e, 0x73, 0x7a, - 0x64, 0x8c, 0xe0, 0xb5, 0xe3, 0x08, 0x02, 0x6b, 0x87, 0xf4, 0xcc, 0x31, 0xba, 0xcf, 0x1f, 0x45, - 0xd7, 0x67, 0x4e, 0xb7, 0xee, 0xb8, 0x2c, 0x60, 0xfe, 0x28, 0x91, 0xfe, 0x77, 0x0d, 0x4a, 0xcb, - 0xb6, 0xed, 0x93, 0x20, 0x58, 0xf7, 0x69, 0xdf, 0x43, 0xef, 0xc0, 0x0c, 0xb7, 0xc4, 0x36, 0x99, - 0x59, 0xd1, 0xae, 0x6b, 0x37, 0x8a, 0x8b, 0xaf, 0x1a, 0x21, 0x63, 0x43, 0x65, 0x9c, 0xf8, 0x84, - 0x63, 0x1b, 0x7b, 0x0b, 0xc6, 0x83, 0xed, 0x6f, 0x12, 0x8b, 0xdd, 0x27, 0xcc, 0x6c, 0xa0, 0x27, - 0x83, 0xda, 0x85, 0xe1, 0xa0, 0x06, 0xc9, 0x19, 0x8e, 0xb9, 0xa2, 0x3e, 0x94, 0x3a, 0x5c, 0xd4, - 0x7d, 0xd2, 0xdb, 0x26, 0x7e, 0x50, 0xc9, 0x5c, 0xcf, 0xde, 0x28, 0x2e, 0xbe, 0x3e, 0xa1, 0xdb, - 0x8d, 0xf5, 0x84, 0x47, 0xe3, 0xb2, 0x14, 0x58, 0x52, 0x0e, 0x03, 0x9c, 0x12, 0xa3, 0xff, 0x41, - 0x83, 0xb2, 0x6a, 0xe9, 0x3d, 0x27, 0x60, 0xe8, 0xeb, 0x63, 0xd6, 0x1a, 0xcf, 0x67, 0x2d, 0xa7, - 0x16, 0xb6, 0x96, 0xa5, 0xe8, 0x99, 0xe8, 0x44, 0xb1, 0xd4, 0x84, 0xbc, 0xc3, 0x48, 0x2f, 0x32, - 0xf1, 0x8d, 0x49, 0x4d, 0x54, 0xd5, 0x6d, 0xcc, 0x4a, 0x41, 0xf9, 0x26, 0x67, 0x89, 0x43, 0xce, - 0xfa, 0x7b, 0x59, 0x98, 0x53, 0xd1, 0x5a, 0x26, 0xb3, 0x76, 0xce, 0xc1, 0x89, 0x3f, 0xd0, 0x60, - 0xce, 0xb4, 0x6d, 0x62, 0xaf, 0x9f, 0xb1, 0x2b, 0x3f, 0x2e, 0xc5, 0x72, 0xab, 0xd2, 0xdc, 0xf1, - 0xb8, 0x40, 0xf4, 0x23, 0x0d, 0xe6, 0x7d, 0xd2, 0xa3, 0x7b, 0x23, 0x8a, 0x64, 0x4f, 0xaf, 0xc8, + 0xf7, 0xfb, 0x7e, 0x0e, 0xbc, 0x69, 0x3a, 0xcc, 0x27, 0xa6, 0x41, 0xdd, 0x7a, 0xf8, 0xa9, 0xee, + 0xed, 0x76, 0xea, 0xa6, 0x47, 0x83, 0xba, 0xe5, 0x3a, 0xcc, 0x77, 0xbb, 0x5e, 0xd7, 0x74, 0x48, + 0x7d, 0x6f, 0x61, 0x9b, 0x30, 0x73, 0xb1, 0xde, 0x21, 0x0e, 0xf1, 0x4d, 0x46, 0x6c, 0xc3, 0xf3, + 0x5d, 0xe6, 0x22, 0x23, 0xa4, 0xfa, 0x06, 0x75, 0xe5, 0x27, 0xc3, 0xdb, 0xed, 0x18, 0x9c, 0xde, + 0x50, 0xe9, 0x0d, 0x49, 0xff, 0xd2, 0xd2, 0xd1, 0xf2, 0x02, 0x66, 0xb2, 0xa0, 0xbe, 0xb7, 0x60, + 0x76, 0xbd, 0x1d, 0x73, 0x61, 0x54, 0xd2, 0x4b, 0x9f, 0xeb, 0x50, 0xb6, 0xd3, 0xdf, 0x36, 0x2c, + 0xb7, 0x57, 0xef, 0xb8, 0x1d, 0xb7, 0x2e, 0x8e, 0xb7, 0xfb, 0x6d, 0xf1, 0x4d, 0x7c, 0x11, 0x9f, + 0x24, 0xfa, 0xcd, 0xdd, 0xa5, 0x40, 0x48, 0xf1, 0x68, 0xcf, 0xb4, 0x76, 0xa8, 0x43, 0xfc, 0xfd, + 0x44, 0x56, 0x8f, 0x30, 0xb3, 0xbe, 0x37, 0x2e, 0xa4, 0x7e, 0x14, 0x95, 0xdf, 0x77, 0x18, 0xed, + 0x91, 0x31, 0x82, 0xd7, 0x8e, 0x23, 0x08, 0xac, 0x1d, 0xd2, 0x33, 0xc7, 0xe8, 0x3e, 0x7f, 0x14, + 0x5d, 0x9f, 0xd1, 0x6e, 0x9d, 0x3a, 0x2c, 0x60, 0xfe, 0x28, 0x91, 0xfe, 0x77, 0x0d, 0x4a, 0xcb, + 0xb6, 0xed, 0x93, 0x20, 0x58, 0xf7, 0xdd, 0xbe, 0x87, 0xde, 0x81, 0x19, 0x6e, 0x89, 0x6d, 0x32, + 0xb3, 0xa2, 0x5d, 0xd7, 0x6e, 0x14, 0x17, 0x5f, 0x35, 0x42, 0xc6, 0x86, 0xca, 0x38, 0xf1, 0x09, + 0xc7, 0x36, 0xf6, 0x16, 0x8c, 0x07, 0xdb, 0xdf, 0x24, 0x16, 0xbb, 0x4f, 0x98, 0xd9, 0x40, 0x4f, + 0x06, 0xb5, 0x0b, 0xc3, 0x41, 0x0d, 0x92, 0x33, 0x1c, 0x73, 0x45, 0x7d, 0x28, 0x75, 0xb8, 0xa8, + 0xfb, 0xa4, 0xb7, 0x4d, 0xfc, 0xa0, 0x92, 0xb9, 0x9e, 0xbd, 0x51, 0x5c, 0x7c, 0x7d, 0x42, 0xb7, + 0x1b, 0xeb, 0x09, 0x8f, 0xc6, 0x65, 0x29, 0xb0, 0xa4, 0x1c, 0x06, 0x38, 0x25, 0x46, 0xff, 0x83, + 0x06, 0x65, 0xd5, 0xd2, 0x7b, 0x34, 0x60, 0xe8, 0xeb, 0x63, 0xd6, 0x1a, 0xcf, 0x67, 0x2d, 0xa7, + 0x16, 0xb6, 0x96, 0xa5, 0xe8, 0x99, 0xe8, 0x44, 0xb1, 0xd4, 0x84, 0x3c, 0x65, 0xa4, 0x17, 0x99, + 0xf8, 0xc6, 0xa4, 0x26, 0xaa, 0xea, 0x36, 0x66, 0xa5, 0xa0, 0x7c, 0x93, 0xb3, 0xc4, 0x21, 0x67, + 0xfd, 0xbd, 0x2c, 0xcc, 0xa9, 0x68, 0x2d, 0x93, 0x59, 0x3b, 0xe7, 0xe0, 0xc4, 0x1f, 0x68, 0x30, + 0x67, 0xda, 0x36, 0xb1, 0xd7, 0xcf, 0xd8, 0x95, 0x1f, 0x97, 0x62, 0xb9, 0x55, 0x69, 0xee, 0x78, + 0x5c, 0x20, 0xfa, 0x91, 0x06, 0xf3, 0x3e, 0xe9, 0xb9, 0x7b, 0x23, 0x8a, 0x64, 0x4f, 0xaf, 0xc8, 0x27, 0xa4, 0x22, 0xf3, 0x78, 0x9c, 0x3f, 0x3e, 0x4c, 0xa8, 0xfe, 0x0f, 0x0d, 0x2e, 0x2e, 0x7b, - 0x5e, 0xd7, 0x21, 0xf6, 0x16, 0xfd, 0x1f, 0xcf, 0xa6, 0x3f, 0x69, 0x80, 0xd2, 0xb6, 0x9e, 0x43, + 0x5e, 0x97, 0x12, 0x7b, 0xcb, 0xfd, 0x1f, 0xcf, 0xa6, 0x3f, 0x69, 0x80, 0xd2, 0xb6, 0x9e, 0x43, 0x3e, 0x59, 0xe9, 0x7c, 0x7a, 0x73, 0xe2, 0x7c, 0x4a, 0x29, 0x7c, 0x44, 0x46, 0xfd, 0x38, 0x0b, 0xf3, 0x69, 0xc4, 0x8f, 0x72, 0xea, 0xbf, 0x97, 0x53, 0xbf, 0xcc, 0xc1, 0xfc, 0x4a, 0xb7, 0x1f, - 0x30, 0xe2, 0xa7, 0x94, 0x7c, 0xf1, 0xde, 0xf8, 0x9e, 0x06, 0x65, 0xd2, 0x6e, 0x13, 0x8b, 0x39, - 0x7b, 0xe4, 0x0c, 0x9d, 0x51, 0x91, 0x52, 0xcb, 0x6b, 0x23, 0xcc, 0xf1, 0x98, 0x38, 0xf4, 0x5d, - 0x98, 0x8b, 0xcf, 0x9a, 0xad, 0x46, 0x97, 0x5a, 0xbb, 0x91, 0x1f, 0x6e, 0x4d, 0xaa, 0x43, 0xb3, - 0xb5, 0x41, 0x58, 0x12, 0x0a, 0x6b, 0xa3, 0x7c, 0xf1, 0xb8, 0x28, 0xb4, 0x04, 0x25, 0x46, 0x99, - 0xd9, 0x8d, 0xcc, 0xcf, 0x5d, 0xd7, 0x6e, 0x64, 0x93, 0xfa, 0xb0, 0xa5, 0xc0, 0x70, 0x0a, 0x13, - 0x2d, 0x02, 0x88, 0xef, 0x2d, 0xb3, 0x43, 0x82, 0x4a, 0x5e, 0xd0, 0xc5, 0xf7, 0xbd, 0x15, 0x43, - 0xb0, 0x82, 0x85, 0x6e, 0x41, 0xd1, 0xea, 0xfb, 0x3e, 0x71, 0x19, 0xff, 0x5e, 0x99, 0x12, 0x44, - 0xf3, 0x92, 0xa8, 0xb8, 0x92, 0x80, 0xb0, 0x8a, 0xa7, 0xff, 0x4d, 0x83, 0xe2, 0x5a, 0xe7, 0xff, - 0x60, 0x82, 0xf9, 0x40, 0x83, 0x4b, 0x8a, 0xa1, 0xe7, 0x50, 0x70, 0xdf, 0x49, 0x17, 0xdc, 0x89, - 0x2d, 0x54, 0xb4, 0x3d, 0xa2, 0xda, 0xfe, 0x24, 0x0b, 0x65, 0x05, 0x2b, 0x2c, 0xb5, 0x36, 0x00, - 0x8d, 0xef, 0xfd, 0x4c, 0x7d, 0xa8, 0xf0, 0xfd, 0xa8, 0xdc, 0x1e, 0x52, 0x6e, 0xbb, 0x70, 0x75, - 0xed, 0x31, 0x23, 0xbe, 0x6b, 0x76, 0xd7, 0x5c, 0xe6, 0xb0, 0x7d, 0x4c, 0xda, 0xc4, 0x27, 0xae, - 0x45, 0xd0, 0x75, 0xc8, 0xb9, 0x66, 0x8f, 0x08, 0x77, 0x14, 0x1a, 0x25, 0xc9, 0x3a, 0xb7, 0x61, - 0xf6, 0x08, 0x16, 0x10, 0x54, 0x87, 0x02, 0xff, 0x1b, 0x78, 0xa6, 0x45, 0x2a, 0x19, 0x81, 0x36, - 0x27, 0xd1, 0x0a, 0x1b, 0x11, 0x00, 0x27, 0x38, 0xfa, 0xbf, 0x35, 0x28, 0x0b, 0xf1, 0xcb, 0x41, - 0x40, 0x2d, 0xc7, 0x64, 0x0e, 0x75, 0xcf, 0xa7, 0xcf, 0x96, 0x4d, 0x29, 0x51, 0xda, 0x7f, 0xe2, - 0x91, 0x42, 0x50, 0xc7, 0x97, 0x94, 0x14, 0xf7, 0xe5, 0x11, 0xfe, 0x78, 0x4c, 0xa2, 0xfe, 0x41, - 0x16, 0x8a, 0xca, 0xe5, 0xa3, 0x87, 0x90, 0xf5, 0xa8, 0x2d, 0x6d, 0x9e, 0x78, 0x57, 0x68, 0x51, - 0x3b, 0x51, 0x63, 0x7a, 0x38, 0xa8, 0x65, 0xf9, 0x09, 0xe7, 0x88, 0xbe, 0xaf, 0xc1, 0x45, 0x92, - 0xf2, 0xaa, 0xf0, 0x4e, 0x71, 0x71, 0x7d, 0xe2, 0x7c, 0x3e, 0x3c, 0x36, 0x1a, 0x68, 0x38, 0xa8, - 0x5d, 0x1c, 0x01, 0x8e, 0x88, 0x44, 0x9f, 0x81, 0xac, 0xe3, 0x85, 0x61, 0x5d, 0x6a, 0x5c, 0xe6, - 0x0a, 0x36, 0x5b, 0xc1, 0xc1, 0xa0, 0x56, 0x68, 0xb6, 0xe4, 0x02, 0x83, 0x39, 0x02, 0x7a, 0x1b, - 0xf2, 0x1e, 0xf5, 0x19, 0x6f, 0x36, 0xdc, 0x23, 0x5f, 0x98, 0x54, 0x47, 0x1e, 0x69, 0x76, 0x8b, - 0xfa, 0x2c, 0xa9, 0x38, 0xfc, 0x5b, 0x80, 0x43, 0xb6, 0xe8, 0x6b, 0x90, 0x73, 0xa9, 0x4d, 0x44, - 0x4f, 0x2a, 0x2e, 0xde, 0x99, 0x98, 0x3d, 0xb5, 0x49, 0x62, 0xf8, 0x8c, 0x48, 0x01, 0x7e, 0x24, - 0x98, 0xea, 0xbf, 0xd6, 0xe0, 0x62, 0x3a, 0x24, 0xd2, 0x59, 0xa1, 0x1d, 0x9f, 0x15, 0x71, 0xa2, - 0x65, 0x8e, 0x4c, 0xb4, 0x06, 0x64, 0xfb, 0x8e, 0x5d, 0xc9, 0x0a, 0x84, 0x57, 0x25, 0x42, 0xf6, - 0xad, 0xe6, 0xea, 0xc1, 0xa0, 0xf6, 0xf2, 0x51, 0xaf, 0x00, 0x6c, 0xdf, 0x23, 0x81, 0xf1, 0x56, - 0x73, 0x15, 0x73, 0x62, 0xfd, 0x77, 0x1a, 0x4c, 0xcb, 0x3e, 0x8f, 0x1e, 0x42, 0xce, 0x72, 0x6c, - 0x5f, 0x86, 0xde, 0x09, 0x27, 0x8b, 0x58, 0xd1, 0x95, 0xe6, 0x2a, 0xc6, 0x82, 0x21, 0x7a, 0x04, - 0x53, 0xe4, 0xb1, 0x45, 0x3c, 0x26, 0xd3, 0xeb, 0x84, 0xac, 0x2f, 0x4a, 0xd6, 0x53, 0x6b, 0x82, - 0x19, 0x96, 0x4c, 0xf5, 0x36, 0xe4, 0x05, 0x02, 0x7a, 0x05, 0x32, 0x8e, 0x27, 0xd4, 0x2f, 0x35, - 0xe6, 0x87, 0x83, 0x5a, 0xa6, 0xd9, 0x4a, 0x47, 0x56, 0xc6, 0xf1, 0xf8, 0x30, 0xe3, 0xf9, 0xa4, - 0xed, 0x3c, 0xbe, 0x47, 0xdc, 0x0e, 0xdb, 0x11, 0xf7, 0x9b, 0x4f, 0x1a, 0x6f, 0x4b, 0x81, 0xe1, - 0x14, 0xa6, 0xfe, 0x0b, 0x0d, 0xd0, 0xfd, 0x7e, 0x97, 0x39, 0x96, 0x19, 0x30, 0xe1, 0xde, 0xa6, - 0xdb, 0xa6, 0xe8, 0x15, 0xc8, 0x8b, 0xfe, 0x2c, 0xbd, 0x1a, 0x87, 0x5b, 0x18, 0x00, 0x21, 0x0c, - 0xbd, 0x0d, 0x39, 0x8f, 0xda, 0x27, 0x7e, 0x02, 0x48, 0xa5, 0x75, 0x7c, 0xc5, 0x2d, 0x6a, 0x07, - 0x58, 0xf0, 0xd5, 0xdf, 0xd3, 0xa0, 0x10, 0x87, 0x3c, 0x8f, 0x1d, 0x1e, 0xe5, 0x42, 0xa3, 0xbc, - 0x8a, 0xef, 0x33, 0x2c, 0x20, 0xcf, 0x11, 0x5d, 0x4b, 0x30, 0x23, 0xde, 0x86, 0x2c, 0xda, 0x95, - 0x21, 0x76, 0x2d, 0x1a, 0x11, 0x5a, 0xf2, 0xfc, 0x40, 0xf9, 0x8c, 0x63, 0x6c, 0xfd, 0x9f, 0x59, - 0x98, 0xdd, 0x20, 0xec, 0xdb, 0xd4, 0xdf, 0x6d, 0xd1, 0xae, 0x63, 0xed, 0x9f, 0x43, 0x31, 0x6f, - 0x43, 0xde, 0xef, 0x77, 0x49, 0x74, 0xc1, 0xcb, 0x13, 0xe7, 0xb3, 0xaa, 0x2f, 0xee, 0x77, 0x49, - 0xe2, 0x47, 0xfe, 0x2d, 0xc0, 0x21, 0x7b, 0x74, 0x07, 0x2e, 0x99, 0xa9, 0xad, 0x30, 0x2c, 0x65, - 0x05, 0x11, 0x6f, 0x97, 0xd2, 0x0b, 0x63, 0x80, 0x47, 0x71, 0xd1, 0x0d, 0x7e, 0xa9, 0x0e, 0xf5, - 0x79, 0xf1, 0xe5, 0x53, 0xb4, 0xd6, 0x28, 0x85, 0x17, 0x1a, 0x9e, 0xe1, 0x18, 0x8a, 0x6e, 0x42, - 0x89, 0x39, 0xc4, 0x8f, 0x20, 0xa2, 0x4e, 0xe5, 0x1b, 0x65, 0x31, 0x6f, 0x2b, 0xe7, 0x38, 0x85, - 0x85, 0x02, 0x28, 0x04, 0xb4, 0xef, 0x5b, 0xbc, 0x36, 0x89, 0xc9, 0xb9, 0xb8, 0x78, 0xf7, 0x74, - 0x57, 0x11, 0x47, 0xdd, 0x2c, 0xaf, 0x54, 0x9b, 0x11, 0x73, 0x9c, 0xc8, 0xd1, 0xff, 0xa8, 0xc1, - 0x5c, 0x8a, 0xe8, 0x1c, 0x46, 0xd2, 0xed, 0xf4, 0x48, 0x7a, 0xe7, 0x54, 0x46, 0x1e, 0x31, 0x94, - 0x7e, 0x07, 0xae, 0xa6, 0xd0, 0x78, 0x81, 0xdf, 0x64, 0x26, 0xeb, 0x07, 0xe8, 0xb3, 0x30, 0xc3, - 0x0b, 0xfd, 0x46, 0x32, 0x09, 0xc5, 0xca, 0x6e, 0xc8, 0x73, 0x1c, 0x63, 0xf0, 0x2d, 0x48, 0x3e, - 0xb8, 0x3a, 0xd4, 0x15, 0x29, 0xa7, 0x6c, 0x41, 0xeb, 0x31, 0x04, 0x2b, 0x58, 0xfa, 0xef, 0x33, - 0x23, 0x97, 0xda, 0x22, 0xc4, 0x47, 0xb7, 0x61, 0xd6, 0x54, 0x9e, 0xf9, 0x82, 0x8a, 0x26, 0x82, - 0x6f, 0x6e, 0x38, 0xa8, 0xcd, 0xaa, 0xef, 0x7f, 0x01, 0x4e, 0xe3, 0x21, 0x02, 0x33, 0x8e, 0x27, - 0x37, 0xc7, 0xf0, 0xca, 0x6e, 0x4f, 0x5e, 0x84, 0x05, 0x7d, 0x62, 0x69, 0xbc, 0x32, 0xc6, 0xac, - 0x51, 0x0d, 0xf2, 0xed, 0x6f, 0xd9, 0x6e, 0x94, 0x14, 0x05, 0x7e, 0xa7, 0x77, 0xbf, 0xb2, 0xba, - 0x11, 0xe0, 0xf0, 0x1c, 0x31, 0xbe, 0x10, 0x6e, 0x12, 0x7f, 0xcf, 0xb1, 0x48, 0xd4, 0xdb, 0xbf, - 0x3c, 0xa9, 0x26, 0x92, 0x5e, 0x19, 0x3c, 0x92, 0x95, 0x32, 0xe2, 0x8d, 0x15, 0x39, 0x7c, 0x37, - 0xfc, 0xd8, 0xe1, 0x61, 0x8d, 0x6e, 0x41, 0x8e, 0xb7, 0x44, 0xe9, 0xc5, 0x97, 0xa3, 0x42, 0xb8, - 0xb5, 0xef, 0x91, 0x83, 0x41, 0x2d, 0xed, 0x02, 0x7e, 0x88, 0x05, 0xfa, 0xc4, 0x43, 0x6e, 0x5c, - 0x70, 0xb3, 0xc7, 0xb5, 0xf3, 0xdc, 0x69, 0xda, 0xf9, 0xaf, 0xf2, 0x23, 0x51, 0xc3, 0x8b, 0x17, - 0x7a, 0x03, 0x0a, 0xb6, 0xe3, 0xf3, 0xa5, 0x9e, 0xba, 0xd2, 0xd0, 0x6a, 0xa4, 0xec, 0x6a, 0x04, - 0x38, 0x50, 0xbf, 0xe0, 0x84, 0x00, 0x59, 0x90, 0x6b, 0xfb, 0xb4, 0x27, 0x87, 0xc5, 0xd3, 0x55, - 0x56, 0x1e, 0xc4, 0x89, 0xf1, 0x77, 0x7d, 0xda, 0xc3, 0x82, 0x39, 0x7a, 0x04, 0x19, 0x46, 0xc5, - 0xe5, 0x9c, 0x89, 0x08, 0x90, 0x22, 0x32, 0x5b, 0x14, 0x67, 0x18, 0xe5, 0xe1, 0x1f, 0xa4, 0x83, - 0xee, 0xf6, 0x09, 0x83, 0x2e, 0x09, 0xff, 0x38, 0xd2, 0x62, 0xd6, 0xbc, 0x2c, 0x78, 0x23, 0x05, - 0x3b, 0xe9, 0x99, 0x63, 0x25, 0xfe, 0x21, 0x4c, 0x99, 0xa1, 0x4f, 0xa6, 0x84, 0x4f, 0xbe, 0xc4, - 0x67, 0x9b, 0xe5, 0xc8, 0x19, 0x0b, 0xcf, 0xf8, 0xfd, 0xcc, 0xb7, 0xe3, 0x5f, 0xb3, 0x0c, 0xee, - 0xe1, 0x90, 0x08, 0x4b, 0x76, 0xe8, 0x75, 0x98, 0x25, 0xae, 0xb9, 0xdd, 0x25, 0xf7, 0x68, 0xa7, - 0xe3, 0xb8, 0x9d, 0xca, 0xf4, 0x75, 0xed, 0xc6, 0x4c, 0xe3, 0x8a, 0xd4, 0x65, 0x76, 0x4d, 0x05, - 0xe2, 0x34, 0xee, 0x61, 0x1d, 0x6e, 0x66, 0x82, 0x0e, 0x17, 0xc5, 0x79, 0xe1, 0xa8, 0x38, 0xd7, - 0x7f, 0x9a, 0x05, 0x94, 0xf2, 0x18, 0xaf, 0xa9, 0x01, 0x5f, 0x4f, 0x66, 0x5d, 0xf5, 0x58, 0x76, - 0x8d, 0xb3, 0xea, 0x5f, 0xb1, 0xf5, 0x69, 0x78, 0x5a, 0x26, 0xf2, 0xa0, 0xc4, 0x7c, 0xb3, 0xdd, - 0x76, 0x2c, 0xa1, 0x95, 0x0c, 0xfa, 0xd7, 0x9e, 0xa1, 0x83, 0xf8, 0x71, 0xd1, 0x88, 0xdd, 0xb1, - 0xa5, 0x50, 0x2b, 0x4f, 0x64, 0xca, 0x29, 0x4e, 0x49, 0x40, 0xef, 0x6a, 0x50, 0xe6, 0xb3, 0x85, - 0x8a, 0x22, 0xb7, 0xfe, 0x2f, 0x3e, 0xbf, 0x58, 0x3c, 0xc2, 0x21, 0x59, 0x41, 0x47, 0x21, 0x78, - 0x4c, 0x9a, 0xfe, 0x57, 0x0d, 0xe6, 0xc7, 0x3c, 0xd2, 0x3f, 0x8f, 0xd7, 0xd5, 0x2e, 0xe4, 0x79, - 0x97, 0x8c, 0x7a, 0xd2, 0xfa, 0xa9, 0x7c, 0x9d, 0xf4, 0xe7, 0xa4, 0xa1, 0xf3, 0xb3, 0x00, 0x87, - 0x42, 0xf4, 0x05, 0x98, 0x4d, 0xed, 0x6d, 0xc7, 0x3f, 0x66, 0xe8, 0xbf, 0xcd, 0x43, 0x39, 0xe2, - 0x1b, 0x6c, 0xf6, 0x7b, 0x3d, 0xd3, 0x3f, 0x8f, 0x71, 0xf6, 0x87, 0x1a, 0x5c, 0x52, 0x03, 0xd3, - 0x89, 0xaf, 0xa8, 0x71, 0xaa, 0x2b, 0x0a, 0x63, 0xe3, 0xaa, 0x94, 0x7d, 0x69, 0x23, 0x2d, 0x02, - 0x8f, 0xca, 0x44, 0xbf, 0xd1, 0xe0, 0x5a, 0x28, 0x45, 0xbe, 0xbe, 0x8f, 0x50, 0xc8, 0x40, 0x3d, - 0x0b, 0xa5, 0x3e, 0x25, 0x95, 0xba, 0xb6, 0xfc, 0x0c, 0x79, 0xf8, 0x99, 0xda, 0xa0, 0x9f, 0x6b, - 0x70, 0x25, 0x44, 0x18, 0xd5, 0x33, 0x77, 0x66, 0x7a, 0x7e, 0x52, 0xea, 0x79, 0x65, 0xf9, 0x30, - 0x41, 0xf8, 0x70, 0xf9, 0x7c, 0x30, 0xef, 0x45, 0xab, 0x63, 0x25, 0x7f, 0x32, 0x65, 0xc6, 0x77, - 0xcf, 0x64, 0xe6, 0x88, 0x61, 0x38, 0x91, 0xa3, 0x3f, 0x82, 0xcb, 0x2d, 0xb3, 0xe3, 0xb8, 0x62, - 0xa2, 0x5c, 0x27, 0xec, 0x81, 0xc7, 0x3f, 0x88, 0x1a, 0xed, 0x99, 0x9d, 0x30, 0xec, 0xb3, 0xca, - 0x7a, 0x68, 0x76, 0x08, 0x16, 0x10, 0xbe, 0xd3, 0x76, 0x9d, 0x9e, 0xc3, 0xe4, 0xb0, 0x1a, 0xa7, - 0xd3, 0x3d, 0x7e, 0x88, 0x43, 0x98, 0x6e, 0x42, 0x49, 0xdd, 0x4b, 0x5f, 0xc4, 0xd3, 0xe0, 0xbf, - 0x32, 0x30, 0x2d, 0xfb, 0x2c, 0xba, 0xa9, 0x2c, 0xa4, 0xa1, 0x88, 0xca, 0xf1, 0xcb, 0x28, 0xda, - 0x90, 0xab, 0x70, 0xe6, 0x98, 0x3c, 0xed, 0x33, 0xa7, 0x6b, 0x84, 0xff, 0x1d, 0x61, 0x34, 0x5d, - 0xf6, 0xc0, 0xdf, 0x64, 0xbe, 0xe3, 0x76, 0xc2, 0xa7, 0x1d, 0x65, 0x71, 0xfe, 0x34, 0x4c, 0x13, - 0x57, 0x6c, 0xd9, 0x62, 0x5a, 0xc9, 0x37, 0x8a, 0xc3, 0x41, 0x6d, 0x7a, 0x2d, 0x3c, 0xc2, 0x11, - 0x8c, 0x2f, 0x7a, 0x8e, 0xd5, 0xf3, 0xf8, 0xc4, 0x28, 0x26, 0xba, 0x7c, 0xb8, 0xe8, 0x35, 0x57, - 0xee, 0xb7, 0xc4, 0x14, 0x19, 0x43, 0x23, 0xcc, 0x95, 0xe8, 0x31, 0x4a, 0xc1, 0xe4, 0x67, 0x38, - 0x86, 0x0a, 0xcc, 0x8e, 0xe4, 0x39, 0xa5, 0x60, 0xae, 0xc7, 0x3c, 0x25, 0x14, 0x2d, 0xc9, 0x5f, - 0x26, 0xe4, 0x4a, 0x20, 0xfa, 0x7f, 0x61, 0xe4, 0xc7, 0x85, 0xe8, 0x4d, 0x24, 0x85, 0xa9, 0x13, - 0x28, 0x8f, 0x4e, 0xd7, 0x2f, 0xc0, 0xaf, 0x8d, 0xad, 0x27, 0x4f, 0xab, 0x17, 0xde, 0x7f, 0x5a, - 0xbd, 0xf0, 0xe1, 0xd3, 0xea, 0x85, 0x77, 0x87, 0x55, 0xed, 0xc9, 0xb0, 0xaa, 0xbd, 0x3f, 0xac, - 0x6a, 0x1f, 0x0e, 0xab, 0xda, 0x9f, 0x87, 0x55, 0xed, 0x67, 0x7f, 0xa9, 0x5e, 0xf8, 0xaa, 0x31, - 0xd9, 0xbf, 0x10, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x31, 0xcc, 0xfd, 0xfc, 0x73, 0x24, 0x00, - 0x00, + 0x30, 0xe2, 0xa7, 0x94, 0x7c, 0xf1, 0xde, 0xf8, 0x9e, 0x06, 0x65, 0xd2, 0x6e, 0x13, 0x8b, 0xd1, + 0x3d, 0x72, 0x86, 0xce, 0xa8, 0x48, 0xa9, 0xe5, 0xb5, 0x11, 0xe6, 0x78, 0x4c, 0x1c, 0xfa, 0x2e, + 0xcc, 0xc5, 0x67, 0xcd, 0x56, 0xa3, 0xeb, 0x5a, 0xbb, 0x91, 0x1f, 0x6e, 0x4d, 0xaa, 0x43, 0xb3, + 0xb5, 0x41, 0x58, 0x12, 0x0a, 0x6b, 0xa3, 0x7c, 0xf1, 0xb8, 0x28, 0xb4, 0x04, 0x25, 0xe6, 0x32, + 0xb3, 0x1b, 0x99, 0x9f, 0xbb, 0xae, 0xdd, 0xc8, 0x26, 0xf5, 0x61, 0x4b, 0x81, 0xe1, 0x14, 0x26, + 0x5a, 0x04, 0x10, 0xdf, 0x5b, 0x66, 0x87, 0x04, 0x95, 0xbc, 0xa0, 0x8b, 0xef, 0x7b, 0x2b, 0x86, + 0x60, 0x05, 0x0b, 0xdd, 0x82, 0xa2, 0xd5, 0xf7, 0x7d, 0xe2, 0x30, 0xfe, 0xbd, 0x32, 0x25, 0x88, + 0xe6, 0x25, 0x51, 0x71, 0x25, 0x01, 0x61, 0x15, 0x4f, 0xff, 0x9b, 0x06, 0xc5, 0xb5, 0xce, 0xff, + 0xc1, 0x04, 0xf3, 0x7b, 0x0d, 0x2e, 0x29, 0x86, 0x9e, 0x43, 0xc1, 0x7d, 0x27, 0x5d, 0x70, 0x27, + 0xb6, 0x50, 0xd1, 0xf6, 0x88, 0x6a, 0xfb, 0x93, 0x2c, 0x94, 0x15, 0xac, 0xb0, 0xd4, 0xda, 0x00, + 0x6e, 0x7c, 0xef, 0x67, 0xea, 0x43, 0x85, 0xef, 0x47, 0xe5, 0xf6, 0x90, 0x72, 0xdb, 0x85, 0xab, + 0x6b, 0x8f, 0x19, 0xf1, 0x1d, 0xb3, 0xbb, 0xe6, 0x30, 0xca, 0xf6, 0x31, 0x69, 0x13, 0x9f, 0x38, + 0x16, 0x41, 0xd7, 0x21, 0xe7, 0x98, 0x3d, 0x22, 0xdc, 0x51, 0x68, 0x94, 0x24, 0xeb, 0xdc, 0x86, + 0xd9, 0x23, 0x58, 0x40, 0x50, 0x1d, 0x0a, 0xfc, 0x6f, 0xe0, 0x99, 0x16, 0xa9, 0x64, 0x04, 0xda, + 0x9c, 0x44, 0x2b, 0x6c, 0x44, 0x00, 0x9c, 0xe0, 0xe8, 0xff, 0xd6, 0xa0, 0x2c, 0xc4, 0x2f, 0x07, + 0x81, 0x6b, 0x51, 0x93, 0x51, 0xd7, 0x39, 0x9f, 0x3e, 0x5b, 0x36, 0xa5, 0x44, 0x69, 0xff, 0x89, + 0x47, 0x0a, 0x41, 0x1d, 0x5f, 0x52, 0x52, 0xdc, 0x97, 0x47, 0xf8, 0xe3, 0x31, 0x89, 0xfa, 0xfb, + 0x39, 0x28, 0x2a, 0x97, 0x8f, 0x1e, 0x42, 0xd6, 0x73, 0x6d, 0x69, 0xf3, 0xc4, 0xbb, 0x42, 0xcb, + 0xb5, 0x13, 0x35, 0xa6, 0x87, 0x83, 0x5a, 0x96, 0x9f, 0x70, 0x8e, 0xe8, 0xfb, 0x1a, 0x5c, 0x24, + 0x29, 0xaf, 0x0a, 0xef, 0x14, 0x17, 0xd7, 0x27, 0xce, 0xe7, 0xc3, 0x63, 0xa3, 0x81, 0x86, 0x83, + 0xda, 0xc5, 0x11, 0xe0, 0x88, 0x48, 0xf4, 0x19, 0xc8, 0x52, 0x2f, 0x0c, 0xeb, 0x52, 0xe3, 0x32, + 0x57, 0xb0, 0xd9, 0x0a, 0x0e, 0x06, 0xb5, 0x42, 0xb3, 0x25, 0x17, 0x18, 0xcc, 0x11, 0xd0, 0xdb, + 0x90, 0xf7, 0x5c, 0x9f, 0xf1, 0x66, 0xc3, 0x3d, 0xf2, 0x85, 0x49, 0x75, 0xe4, 0x91, 0x66, 0xb7, + 0x5c, 0x9f, 0x25, 0x15, 0x87, 0x7f, 0x0b, 0x70, 0xc8, 0x16, 0x7d, 0x0d, 0x72, 0x8e, 0x6b, 0x13, + 0xd1, 0x93, 0x8a, 0x8b, 0x77, 0x26, 0x66, 0xef, 0xda, 0x24, 0x31, 0x7c, 0x46, 0xa4, 0x00, 0x3f, + 0x12, 0x4c, 0x51, 0x07, 0xa6, 0x03, 0xe2, 0xef, 0x51, 0x2b, 0x6c, 0x5f, 0xc5, 0xc5, 0x2f, 0x4f, + 0xca, 0x7f, 0x33, 0x24, 0x4f, 0x44, 0x14, 0x87, 0x83, 0xda, 0x74, 0x74, 0x1a, 0x71, 0xd7, 0x7f, + 0xad, 0xc1, 0xc5, 0x74, 0xec, 0xa5, 0xd3, 0x4f, 0x3b, 0x3e, 0xfd, 0xe2, 0x8c, 0xce, 0x1c, 0x99, + 0xd1, 0x0d, 0xc8, 0xf6, 0xa9, 0x5d, 0xc9, 0x0a, 0x84, 0x57, 0x25, 0x42, 0xf6, 0xad, 0xe6, 0xea, + 0xc1, 0xa0, 0xf6, 0xf2, 0x51, 0xcf, 0x0d, 0x6c, 0xdf, 0x23, 0x81, 0xf1, 0x56, 0x73, 0x15, 0x73, + 0x62, 0xfd, 0xb7, 0x1a, 0x4c, 0xcb, 0x81, 0x02, 0x3d, 0x84, 0x9c, 0x45, 0x6d, 0x5f, 0xc6, 0xf8, + 0x09, 0x47, 0x98, 0x58, 0xd1, 0x95, 0xe6, 0x2a, 0xc6, 0x82, 0x21, 0x7a, 0x04, 0x53, 0xe4, 0xb1, + 0x45, 0x3c, 0x26, 0xf3, 0xf8, 0x84, 0xac, 0x2f, 0x4a, 0xd6, 0x53, 0x6b, 0x82, 0x19, 0x96, 0x4c, + 0xf5, 0x36, 0xe4, 0x05, 0x02, 0x7a, 0x05, 0x32, 0xd4, 0x13, 0xea, 0x97, 0x1a, 0xf3, 0xc3, 0x41, + 0x2d, 0xd3, 0x6c, 0xa5, 0x43, 0x38, 0x43, 0x3d, 0x3e, 0x35, 0x79, 0x3e, 0x69, 0xd3, 0xc7, 0xf7, + 0x88, 0xd3, 0x61, 0x3b, 0xe2, 0x7e, 0xf3, 0x49, 0x87, 0x6f, 0x29, 0x30, 0x9c, 0xc2, 0xd4, 0x7f, + 0xa1, 0x01, 0xba, 0xdf, 0xef, 0x32, 0x6a, 0x99, 0x01, 0x13, 0xee, 0x6d, 0x3a, 0x6d, 0x17, 0xbd, + 0x02, 0x79, 0x31, 0x08, 0x48, 0xaf, 0xc6, 0x71, 0x1d, 0x06, 0x40, 0x08, 0x43, 0x6f, 0x43, 0xce, + 0x73, 0xed, 0x13, 0xbf, 0x35, 0xa4, 0xea, 0x47, 0x7c, 0xc5, 0x2d, 0xd7, 0x0e, 0xb0, 0xe0, 0xab, + 0xbf, 0xa7, 0x41, 0x21, 0xce, 0x2d, 0x1e, 0x3b, 0x3c, 0x9d, 0x84, 0x46, 0x79, 0x15, 0xdf, 0x67, + 0x58, 0x40, 0x9e, 0x23, 0xba, 0x96, 0x60, 0x46, 0x3c, 0x42, 0x59, 0x6e, 0x57, 0x86, 0xd8, 0xb5, + 0x68, 0x16, 0x69, 0xc9, 0xf3, 0x03, 0xe5, 0x33, 0x8e, 0xb1, 0xf5, 0x7f, 0x66, 0x61, 0x76, 0x83, + 0xb0, 0x6f, 0xbb, 0xfe, 0x6e, 0xcb, 0xed, 0x52, 0x6b, 0xff, 0x1c, 0xba, 0x46, 0x1b, 0xf2, 0x7e, + 0xbf, 0x4b, 0xa2, 0x0b, 0x5e, 0x9e, 0xb8, 0x70, 0xa8, 0xfa, 0xe2, 0x7e, 0x97, 0x24, 0x7e, 0xe4, + 0xdf, 0x02, 0x1c, 0xb2, 0x47, 0x77, 0xe0, 0x92, 0x99, 0x5a, 0x3f, 0xc3, 0x9a, 0x59, 0x10, 0xf1, + 0x76, 0x29, 0xbd, 0x99, 0x06, 0x78, 0x14, 0x17, 0xdd, 0xe0, 0x97, 0x4a, 0x5d, 0x9f, 0x57, 0x79, + 0x3e, 0xae, 0x6b, 0x8d, 0x52, 0x78, 0xa1, 0xe1, 0x19, 0x8e, 0xa1, 0xe8, 0x26, 0x94, 0x18, 0x25, + 0x7e, 0x04, 0x11, 0x05, 0x31, 0xdf, 0x28, 0x8b, 0xc1, 0x5e, 0x39, 0xc7, 0x29, 0x2c, 0x14, 0x40, + 0x21, 0x70, 0xfb, 0xbe, 0xa8, 0x50, 0xb2, 0xc6, 0xdd, 0x3d, 0xdd, 0x55, 0xc4, 0x51, 0x37, 0xcb, + 0x2b, 0xd5, 0x66, 0xc4, 0x1c, 0x27, 0x72, 0xf4, 0x3f, 0x6a, 0x30, 0x97, 0x22, 0x3a, 0x87, 0xd9, + 0x77, 0x3b, 0x3d, 0xfb, 0xde, 0x39, 0x95, 0x91, 0x47, 0x4c, 0xbf, 0xdf, 0x81, 0xab, 0x29, 0x34, + 0xde, 0x49, 0x36, 0x99, 0xc9, 0xfa, 0x01, 0xfa, 0x2c, 0xcc, 0xf0, 0x8e, 0xb2, 0x91, 0x8c, 0x5c, + 0xb1, 0xb2, 0x1b, 0xf2, 0x1c, 0xc7, 0x18, 0x7c, 0xdd, 0x92, 0x2f, 0xbb, 0xd4, 0x75, 0x44, 0xca, + 0x29, 0xeb, 0xd6, 0x7a, 0x0c, 0xc1, 0x0a, 0x96, 0xfe, 0xbb, 0xcc, 0xc8, 0xa5, 0xb6, 0x08, 0xf1, + 0xd1, 0x6d, 0x98, 0x35, 0x95, 0xf7, 0xc4, 0xa0, 0xa2, 0x89, 0xe0, 0x9b, 0x1b, 0x0e, 0x6a, 0xb3, + 0xea, 0x43, 0x63, 0x80, 0xd3, 0x78, 0x88, 0xc0, 0x0c, 0xf5, 0xe4, 0x8a, 0x1a, 0x5e, 0xd9, 0xed, + 0xc9, 0x8b, 0xb0, 0xa0, 0x4f, 0x2c, 0x8d, 0x77, 0xd3, 0x98, 0x35, 0xaa, 0x41, 0xbe, 0xfd, 0x2d, + 0xdb, 0x89, 0x92, 0xa2, 0xc0, 0xef, 0xf4, 0xee, 0x57, 0x56, 0x37, 0x02, 0x1c, 0x9e, 0x23, 0xc6, + 0x37, 0x4f, 0xd9, 0x2f, 0xa3, 0x21, 0xe2, 0xf4, 0x5d, 0x58, 0xd9, 0x5d, 0x23, 0xde, 0x58, 0x91, + 0xc3, 0x97, 0xd0, 0x8f, 0x1d, 0x1e, 0xd6, 0xe8, 0x16, 0xe4, 0x78, 0x4b, 0x94, 0x5e, 0x7c, 0x39, + 0x2a, 0x84, 0x5b, 0xfb, 0x1e, 0x39, 0x18, 0xd4, 0xd2, 0x2e, 0xe0, 0x87, 0x58, 0xa0, 0x4f, 0x3c, + 0x4d, 0xc7, 0x05, 0x37, 0x7b, 0x5c, 0x3b, 0xcf, 0x9d, 0xa6, 0x9d, 0xff, 0x2a, 0x3f, 0x12, 0x35, + 0xbc, 0x78, 0xa1, 0x37, 0xa0, 0x60, 0x53, 0x9f, 0x58, 0x22, 0xfc, 0x42, 0x43, 0xab, 0x91, 0xb2, + 0xab, 0x11, 0xe0, 0x40, 0xfd, 0x82, 0x13, 0x02, 0x64, 0x41, 0xae, 0xed, 0xbb, 0x3d, 0x39, 0x95, + 0x9e, 0xae, 0xb2, 0xf2, 0x20, 0x4e, 0x8c, 0xbf, 0xeb, 0xbb, 0x3d, 0x2c, 0x98, 0xa3, 0x47, 0x90, + 0x61, 0xae, 0xb8, 0x9c, 0x33, 0x11, 0x01, 0x52, 0x44, 0x66, 0xcb, 0xc5, 0x19, 0xe6, 0xf2, 0xf0, + 0x0f, 0xd2, 0x41, 0x77, 0xfb, 0x84, 0x41, 0x97, 0x84, 0x7f, 0x1c, 0x69, 0x31, 0x6b, 0x5e, 0x16, + 0xbc, 0x91, 0x82, 0x9d, 0xf4, 0xcc, 0xb1, 0x12, 0xff, 0x10, 0xa6, 0xcc, 0xd0, 0x27, 0x53, 0xc2, + 0x27, 0x5f, 0xe2, 0xb3, 0xcd, 0x72, 0xe4, 0x8c, 0x85, 0x67, 0xfc, 0x50, 0xe7, 0xdb, 0xf1, 0xcf, + 0x66, 0x06, 0xf7, 0x70, 0x48, 0x84, 0x25, 0x3b, 0xf4, 0x3a, 0xcc, 0x12, 0xc7, 0xdc, 0xee, 0x92, + 0x7b, 0x6e, 0xa7, 0x43, 0x9d, 0x4e, 0x65, 0xfa, 0xba, 0x76, 0x63, 0xa6, 0x71, 0x45, 0xea, 0x32, + 0xbb, 0xa6, 0x02, 0x71, 0x1a, 0xf7, 0xb0, 0x0e, 0x37, 0x33, 0x41, 0x87, 0x8b, 0xe2, 0xbc, 0x70, + 0x54, 0x9c, 0xeb, 0x3f, 0xcd, 0x02, 0x4a, 0x79, 0x8c, 0xd7, 0xd4, 0x80, 0xef, 0x41, 0xb3, 0x8e, + 0x7a, 0x2c, 0xbb, 0xc6, 0x59, 0xf5, 0xaf, 0xd8, 0xfa, 0x34, 0x3c, 0x2d, 0x13, 0x79, 0x50, 0x62, + 0xbe, 0xd9, 0x6e, 0x53, 0x4b, 0x68, 0x25, 0x83, 0xfe, 0xb5, 0x67, 0xe8, 0x20, 0x7e, 0xc5, 0x34, + 0x62, 0x77, 0x6c, 0x29, 0xd4, 0xca, 0x5b, 0x9c, 0x72, 0x8a, 0x53, 0x12, 0xd0, 0xbb, 0x1a, 0x94, + 0xf9, 0x6c, 0xa1, 0xa2, 0xc8, 0xe7, 0x85, 0x2f, 0x3e, 0xbf, 0x58, 0x3c, 0xc2, 0x21, 0xd9, 0x75, + 0x47, 0x21, 0x78, 0x4c, 0x9a, 0xfe, 0x57, 0x0d, 0xe6, 0xc7, 0x3c, 0xd2, 0x3f, 0x8f, 0x67, 0xdc, + 0x2e, 0xe4, 0x79, 0x97, 0x8c, 0x7a, 0xd2, 0xfa, 0xa9, 0x7c, 0x9d, 0xf4, 0xe7, 0xa4, 0xa1, 0xf3, + 0xb3, 0x00, 0x87, 0x42, 0xf4, 0x05, 0x98, 0x4d, 0x2d, 0x88, 0xc7, 0xbf, 0x9a, 0xe8, 0xef, 0xe7, + 0xa1, 0x1c, 0xf1, 0x0d, 0x36, 0xfb, 0xbd, 0x9e, 0xe9, 0x9f, 0xc7, 0x38, 0xfb, 0x43, 0x0d, 0x2e, + 0xa9, 0x81, 0x49, 0xe3, 0x2b, 0x6a, 0x9c, 0xea, 0x8a, 0xc2, 0xd8, 0xb8, 0x2a, 0x65, 0x5f, 0xda, + 0x48, 0x8b, 0xc0, 0xa3, 0x32, 0xd1, 0x6f, 0x34, 0xb8, 0x16, 0x4a, 0x91, 0xcf, 0xfc, 0x23, 0x14, + 0x32, 0x50, 0xcf, 0x42, 0xa9, 0x4f, 0x49, 0xa5, 0xae, 0x2d, 0x3f, 0x43, 0x1e, 0x7e, 0xa6, 0x36, + 0xe8, 0xe7, 0x1a, 0x5c, 0x09, 0x11, 0x46, 0xf5, 0xcc, 0x9d, 0x99, 0x9e, 0x9f, 0x94, 0x7a, 0x5e, + 0x59, 0x3e, 0x4c, 0x10, 0x3e, 0x5c, 0x3e, 0x1f, 0xcc, 0x7b, 0xd1, 0xea, 0x58, 0xc9, 0x9f, 0x4c, + 0x99, 0xf1, 0xdd, 0x33, 0x99, 0x39, 0x62, 0x18, 0x4e, 0xe4, 0xe8, 0x8f, 0xe0, 0x72, 0xcb, 0xec, + 0x50, 0x47, 0x4c, 0x94, 0xeb, 0x84, 0x3d, 0xf0, 0xf8, 0x07, 0x51, 0xa3, 0x3d, 0xb3, 0x13, 0x86, + 0x7d, 0x56, 0x59, 0x0f, 0xcd, 0x0e, 0xc1, 0x02, 0xc2, 0x77, 0xda, 0x2e, 0xed, 0x51, 0x26, 0x87, + 0xd5, 0x38, 0x9d, 0xee, 0xf1, 0x43, 0x1c, 0xc2, 0x74, 0x13, 0x4a, 0xea, 0x5e, 0xfa, 0x22, 0xde, + 0x20, 0xff, 0x95, 0x81, 0xe8, 0x75, 0x05, 0xdd, 0x54, 0x16, 0xd2, 0x50, 0x44, 0xe5, 0xf8, 0x65, + 0x14, 0x6d, 0xc8, 0x55, 0x38, 0x73, 0x4c, 0x9e, 0xf6, 0x19, 0xed, 0x1a, 0xe1, 0xbf, 0x61, 0x18, + 0x4d, 0x87, 0x3d, 0xf0, 0x37, 0x99, 0x4f, 0x9d, 0x4e, 0xf8, 0x86, 0xa4, 0x2c, 0xce, 0x9f, 0x86, + 0x69, 0xe2, 0x88, 0x2d, 0x5b, 0x4c, 0x2b, 0xf9, 0xf0, 0x05, 0x68, 0x2d, 0x3c, 0xc2, 0x11, 0x8c, + 0x2f, 0x7a, 0xd4, 0xea, 0x79, 0x7c, 0x62, 0x14, 0x13, 0x5d, 0x3e, 0x5c, 0xf4, 0x9a, 0x2b, 0xf7, + 0x5b, 0x62, 0x8a, 0x8c, 0xa1, 0x11, 0xe6, 0x4a, 0xf4, 0xea, 0xa5, 0x60, 0xf2, 0x33, 0x1c, 0x43, + 0x05, 0x66, 0x47, 0xf2, 0x9c, 0x52, 0x30, 0xd7, 0x63, 0x9e, 0x12, 0x8a, 0x96, 0xe4, 0x4f, 0x20, + 0x72, 0x25, 0x10, 0xfd, 0xbf, 0x30, 0xf2, 0x2b, 0x46, 0xf4, 0x26, 0x92, 0xc2, 0xd4, 0x09, 0x94, + 0x47, 0xa7, 0xeb, 0x17, 0xe0, 0xd7, 0xc6, 0xd6, 0x93, 0xa7, 0xd5, 0x0b, 0x1f, 0x3c, 0xad, 0x5e, + 0xf8, 0xf0, 0x69, 0xf5, 0xc2, 0xbb, 0xc3, 0xaa, 0xf6, 0x64, 0x58, 0xd5, 0x3e, 0x18, 0x56, 0xb5, + 0x0f, 0x87, 0x55, 0xed, 0xcf, 0xc3, 0xaa, 0xf6, 0xb3, 0xbf, 0x54, 0x2f, 0x7c, 0xd5, 0x98, 0xec, + 0x7f, 0x95, 0xfe, 0x13, 0x00, 0x00, 0xff, 0xff, 0xca, 0x4d, 0x24, 0x0a, 0xdc, 0x24, 0x00, 0x00, } func (m *AddressGroup) Marshal() (dAtA []byte, err error) { @@ -1757,6 +1757,18 @@ func (m *GroupMember) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Service != nil { + { + size, err := m.Service.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } if m.Node != nil { { size, err := m.Node.MarshalToSizedBuffer(dAtA[:i]) @@ -3029,6 +3041,10 @@ func (m *GroupMember) Size() (n int) { l = m.Node.Size() n += 1 + l + sovGenerated(uint64(l)) } + if m.Service != nil { + l = m.Service.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -3641,6 +3657,7 @@ func (this *GroupMember) String() string { `IPs:` + fmt.Sprintf("%v", this.IPs) + `,`, `Ports:` + repeatedStringForPorts + `,`, `Node:` + strings.Replace(this.Node.String(), "NodeReference", "NodeReference", 1) + `,`, + `Service:` + strings.Replace(this.Service.String(), "ServiceReference", "ServiceReference", 1) + `,`, `}`, }, "") return s @@ -5749,6 +5766,42 @@ func (m *GroupMember) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Service", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Service == nil { + m.Service = &ServiceReference{} + } + if err := m.Service.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/controlplane/v1beta2/generated.proto b/pkg/apis/controlplane/v1beta2/generated.proto index 60584192abf..1729daf44ca 100644 --- a/pkg/apis/controlplane/v1beta2/generated.proto +++ b/pkg/apis/controlplane/v1beta2/generated.proto @@ -146,6 +146,10 @@ message GroupMember { // Node maintains the reference to the Node. optional NodeReference node = 5; + + // Service is the reference to the Service. It can only be used in an AppliedTo + // Group and only a NodePort type Service can be referred by this field. + optional ServiceReference service = 6; } message GroupReference { diff --git a/pkg/apis/controlplane/v1beta2/types.go b/pkg/apis/controlplane/v1beta2/types.go index 4c87f3385ce..d05d7062e51 100644 --- a/pkg/apis/controlplane/v1beta2/types.go +++ b/pkg/apis/controlplane/v1beta2/types.go @@ -88,6 +88,9 @@ type GroupMember struct { Ports []NamedPort `json:"ports,omitempty" protobuf:"bytes,4,rep,name=ports"` // Node maintains the reference to the Node. Node *NodeReference `json:"node,omitempty" protobuf:"bytes,5,opt,name=node"` + // Service is the reference to the Service. It can only be used in an AppliedTo + // Group and only a NodePort type Service can be referred by this field. + Service *ServiceReference `json:"service,omitempty" protobuf:"bytes,6,opt,name=service"` } // +genclient diff --git a/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go b/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go index 8be066a1445..3b0efc79e4b 100644 --- a/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go +++ b/pkg/apis/controlplane/v1beta2/zz_generated.conversion.go @@ -911,6 +911,7 @@ func autoConvert_v1beta2_GroupMember_To_controlplane_GroupMember(in *GroupMember out.IPs = *(*[]controlplane.IPAddress)(unsafe.Pointer(&in.IPs)) out.Ports = *(*[]controlplane.NamedPort)(unsafe.Pointer(&in.Ports)) out.Node = (*controlplane.NodeReference)(unsafe.Pointer(in.Node)) + out.Service = (*controlplane.ServiceReference)(unsafe.Pointer(in.Service)) return nil } @@ -925,6 +926,7 @@ func autoConvert_controlplane_GroupMember_To_v1beta2_GroupMember(in *controlplan out.Node = (*NodeReference)(unsafe.Pointer(in.Node)) out.IPs = *(*[]IPAddress)(unsafe.Pointer(&in.IPs)) out.Ports = *(*[]NamedPort)(unsafe.Pointer(&in.Ports)) + out.Service = (*ServiceReference)(unsafe.Pointer(in.Service)) return nil } diff --git a/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go b/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go index 03bbcaa5cc2..961fef5ded8 100644 --- a/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/controlplane/v1beta2/zz_generated.deepcopy.go @@ -465,6 +465,11 @@ func (in *GroupMember) DeepCopyInto(out *GroupMember) { *out = new(NodeReference) **out = **in } + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = new(ServiceReference) + **out = **in + } return } diff --git a/pkg/apis/controlplane/zz_generated.deepcopy.go b/pkg/apis/controlplane/zz_generated.deepcopy.go index ffb95f1b042..85c5d46379f 100644 --- a/pkg/apis/controlplane/zz_generated.deepcopy.go +++ b/pkg/apis/controlplane/zz_generated.deepcopy.go @@ -465,6 +465,11 @@ func (in *GroupMember) DeepCopyInto(out *GroupMember) { *out = make([]NamedPort, len(*in)) copy(*out, *in) } + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = new(ServiceReference) + **out = **in + } return } diff --git a/pkg/apis/crd/v1alpha1/types.go b/pkg/apis/crd/v1alpha1/types.go index c17d5d7b9d8..698c9642ef2 100644 --- a/pkg/apis/crd/v1alpha1/types.go +++ b/pkg/apis/crd/v1alpha1/types.go @@ -450,6 +450,13 @@ type NetworkPolicyPeer struct { // A NodeSelector cannot be set in AppliedTo field or set with any other selector. // +optional NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"` + // Select a certain Service which matches the NamespacedName. + // A Service can only be set in either policy level AppliedTo field in a policy + // that only has ingress rules or rule level AppliedTo field in an ingress rule. + // Only a NodePort Service can be referred by this field. + // Cannot be set with any other selector. + // +optional + Service *NamespacedName `json:"service,omitempty"` } type PeerNamespaces struct { diff --git a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go index 6435f0eff9d..93d6d20dafa 100644 --- a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go @@ -371,6 +371,11 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { *out = new(v1.LabelSelector) (*in).DeepCopyInto(*out) } + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = new(NamespacedName) + **out = **in + } return } diff --git a/pkg/apiserver/openapi/zz_generated.openapi.go b/pkg/apiserver/openapi/zz_generated.openapi.go index 5d8a1f885b7..dc25b9f05d1 100644 --- a/pkg/apiserver/openapi/zz_generated.openapi.go +++ b/pkg/apiserver/openapi/zz_generated.openapi.go @@ -1027,11 +1027,17 @@ func schema_pkg_apis_controlplane_v1beta2_GroupMember(ref common.ReferenceCallba Ref: ref("antrea.io/antrea/pkg/apis/controlplane/v1beta2.NodeReference"), }, }, + "service": { + SchemaProps: spec.SchemaProps{ + Description: "Service is the reference to the Service. It can only be used in an AppliedTo Group and only a NodePort type Service can be referred by this field.", + Ref: ref("antrea.io/antrea/pkg/apis/controlplane/v1beta2.ServiceReference"), + }, + }, }, }, }, Dependencies: []string{ - "antrea.io/antrea/pkg/apis/controlplane/v1beta2.ExternalEntityReference", "antrea.io/antrea/pkg/apis/controlplane/v1beta2.NamedPort", "antrea.io/antrea/pkg/apis/controlplane/v1beta2.NodeReference", "antrea.io/antrea/pkg/apis/controlplane/v1beta2.PodReference"}, + "antrea.io/antrea/pkg/apis/controlplane/v1beta2.ExternalEntityReference", "antrea.io/antrea/pkg/apis/controlplane/v1beta2.NamedPort", "antrea.io/antrea/pkg/apis/controlplane/v1beta2.NodeReference", "antrea.io/antrea/pkg/apis/controlplane/v1beta2.PodReference", "antrea.io/antrea/pkg/apis/controlplane/v1beta2.ServiceReference"}, } } diff --git a/pkg/controller/networkpolicy/clusternetworkpolicy.go b/pkg/controller/networkpolicy/clusternetworkpolicy.go index 52c67775b93..967cc9c4e67 100644 --- a/pkg/controller/networkpolicy/clusternetworkpolicy.go +++ b/pkg/controller/networkpolicy/clusternetworkpolicy.go @@ -510,6 +510,8 @@ func (n *NetworkPolicyController) processClusterAppliedTo(appliedTo []crdv1alpha var atg string if at.Group != "" { atg = n.processAppliedToGroupForCG(at.Group) + } else if at.Service != nil { + atg = n.createAppliedToGroupForService(at.Service) } else if at.ServiceAccount != nil { atg = n.createAppliedToGroup(at.ServiceAccount.Namespace, serviceAccountNameToPodSelector(at.ServiceAccount.Name), nil, nil) } else { diff --git a/pkg/controller/networkpolicy/clusternetworkpolicy_test.go b/pkg/controller/networkpolicy/clusternetworkpolicy_test.go index cc74ddbc866..b093c0c5e00 100644 --- a/pkg/controller/networkpolicy/clusternetworkpolicy_test.go +++ b/pkg/controller/networkpolicy/clusternetworkpolicy_test.go @@ -27,6 +27,7 @@ import ( crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" crdv1alpha3 "antrea.io/antrea/pkg/apis/crd/v1alpha3" antreatypes "antrea.io/antrea/pkg/controller/types" + "antrea.io/antrea/pkg/util/k8s" ) func TestProcessClusterNetworkPolicy(t *testing.T) { @@ -74,6 +75,8 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { }, } + ipA := "1.1.1.1" + allowAction := crdv1alpha1.RuleActionAllow dropAction := crdv1alpha1.RuleActionDrop protocolTCP := controlplane.ProtocolTCP @@ -1376,6 +1379,72 @@ func TestProcessClusterNetworkPolicy(t *testing.T) { expectedAppliedToGroups: 1, expectedAddressGroups: 0, }, + { + name: "appliedTo-service", + inputPolicy: &crdv1alpha1.ClusterNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpM", UID: "uidM"}, + Spec: crdv1alpha1.ClusterNetworkPolicySpec{ + AppliedTo: nil, + Priority: p10, + Ingress: []crdv1alpha1.Rule{ + { + AppliedTo: []crdv1alpha1.NetworkPolicyPeer{ + { + Service: &crdv1alpha1.NamespacedName{ + Name: svcA.Name, + Namespace: svcA.Namespace, + }, + }, + }, + From: []crdv1alpha1.NetworkPolicyPeer{ + { + IPBlock: &crdv1alpha1.IPBlock{ + CIDR: ipA + "/32", + }, + }, + }, + Action: &allowAction, + }, + }, + }, + }, + expectedPolicy: &antreatypes.NetworkPolicy{ + UID: "uidM", + Name: "uidM", + SourceRef: &controlplane.NetworkPolicyReference{ + Type: controlplane.AntreaClusterNetworkPolicy, + Name: "cnpM", + UID: "uidM", + }, + Priority: &p10, + TierPriority: &DefaultTierPriority, + Rules: []controlplane.NetworkPolicyRule{ + { + Direction: controlplane.DirectionIn, + AppliedToGroups: []string{getNormalizedUID(k8s.NamespacedName(svcA.Namespace, svcA.Name))}, + From: controlplane.NetworkPolicyPeer{ + IPBlocks: []controlplane.IPBlock{ + { + CIDR: controlplane.IPNet{ + IP: controlplane.IPAddress(net.ParseIP(ipA)), + PrefixLength: 32, + }, + Except: []controlplane.IPNet{}, + }, + }, + }, + Priority: 0, + Action: &allowAction, + }, + }, + AppliedToGroups: []string{ + getNormalizedUID(k8s.NamespacedName(svcA.Namespace, svcA.Name)), + }, + AppliedToPerRule: true, + }, + expectedAppliedToGroups: 1, + expectedAddressGroups: 0, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/controller/networkpolicy/crd_utils.go b/pkg/controller/networkpolicy/crd_utils.go index 5e434354b03..454ef74f832 100644 --- a/pkg/controller/networkpolicy/crd_utils.go +++ b/pkg/controller/networkpolicy/crd_utils.go @@ -18,6 +18,7 @@ import ( "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/klog/v2" @@ -25,6 +26,7 @@ import ( "antrea.io/antrea/pkg/apis/crd/v1alpha1" "antrea.io/antrea/pkg/controller/networkpolicy/store" antreatypes "antrea.io/antrea/pkg/controller/types" + "antrea.io/antrea/pkg/util/k8s" ) var ( @@ -202,6 +204,29 @@ func (n *NetworkPolicyController) createAppliedToGroupForClusterGroupCRD(intGrp return key } +// createAppliedToGroupForService creates an AppliedToGroup object corresponding to a Service if it is not created already. +func (n *NetworkPolicyController) createAppliedToGroupForService(service *v1alpha1.NamespacedName) string { + key := getNormalizedUID(k8s.NamespacedName(service.Namespace, service.Name)) + // Check to see if the AppliedToGroup already exists + _, found, _ := n.appliedToGroupStore.Get(key) + if found { + return key + } + // Create an AppliedToGroup object for this Service. + appliedToGroup := &antreatypes.AppliedToGroup{ + UID: types.UID(key), + Name: key, + Service: &controlplane.ServiceReference{ + Namespace: service.Namespace, + Name: service.Name, + }, + } + klog.V(2).Infof("Creating new AppliedToGroup %v corresponding to a Service %s", appliedToGroup.UID, k8s.NamespacedName(service.Namespace, service.Name)) + n.appliedToGroupStore.Create(appliedToGroup) + n.enqueueAppliedToGroup(key) + return key +} + // createAddressGroupForClusterGroupCRD creates an AddressGroup object corresponding to a // ClusterGroup spec. If the AddressGroup already exists, it returns the key // otherwise it copies the ClusterGroup CRD contents to an AddressGroup resource and returns diff --git a/pkg/controller/networkpolicy/networkpolicy_controller.go b/pkg/controller/networkpolicy/networkpolicy_controller.go index 35cb67a0efd..6e31d859a7b 100644 --- a/pkg/controller/networkpolicy/networkpolicy_controller.go +++ b/pkg/controller/networkpolicy/networkpolicy_controller.go @@ -18,6 +18,7 @@ package networkpolicy import ( + "context" "fmt" "net" "reflect" @@ -468,7 +469,7 @@ func (n *NetworkPolicyController) createAppliedToGroup(npNsName string, pSel, nS newAppliedToGroup := &antreatypes.AppliedToGroup{ Name: appliedToGroupUID, UID: types.UID(appliedToGroupUID), - Selector: *groupSelector, + Selector: groupSelector, } klog.V(2).Infof("Creating new AppliedToGroup %s with selector (%s)", newAppliedToGroup.Name, newAppliedToGroup.Selector.NormalizedName) n.appliedToGroupStore.Create(newAppliedToGroup) @@ -1194,6 +1195,15 @@ func nodeToGroupMember(node *v1.Node) (member *controlplane.GroupMember) { return } +func serviceToGroupMember(serviceReference *controlplane.ServiceReference) (member *controlplane.GroupMember) { + return &controlplane.GroupMember{ + Service: &controlplane.ServiceReference{ + Namespace: serviceReference.Namespace, + Name: serviceReference.Name, + }, + } +} + func externalEntityToGroupMember(ee *v1alpha2.ExternalEntity) *controlplane.GroupMember { memberEntity := &controlplane.GroupMember{} namedPorts := make([]controlplane.NamedPort, len(ee.Spec.Ports)) @@ -1229,56 +1239,76 @@ func (n *NetworkPolicyController) syncAppliedToGroup(key string) error { metrics.DurationAppliedToGroupSyncing.Observe(float64(d.Milliseconds())) klog.V(2).Infof("Finished syncing AppliedToGroup %s. (%v)", key, d) }() - var pods []*v1.Pod appGroupNodeNames := sets.String{} appliedToGroupObj, found, _ := n.appliedToGroupStore.Get(key) if !found { klog.V(2).Infof("AppliedToGroup %s not found.", key) return nil } - memberSetByNode := make(map[string]controlplane.GroupMemberSet) - scheduledPodNum, scheduledExtEntityNum := 0, 0 appliedToGroup := appliedToGroupObj.(*antreatypes.AppliedToGroup) - pods, externalEntities := n.getAppliedToWorkloads(appliedToGroup) - for _, pod := range pods { - if pod.Spec.NodeName == "" || pod.Spec.HostNetwork == true { - // No need to process Pod when it's not scheduled. - // HostNetwork Pods will not be applied to by policies. - continue + memberSetByNode := make(map[string]controlplane.GroupMemberSet) + var updatedAppliedToGroup *antreatypes.AppliedToGroup + if appliedToGroup.Service != nil { + // AppliedToGroup for NodePort Service span to all Nodes. + nodeList, err := n.kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("unable to list Nodes") } - scheduledPodNum++ - podSet := memberSetByNode[pod.Spec.NodeName] - if podSet == nil { - podSet = controlplane.GroupMemberSet{} + for _, node := range nodeList.Items { + appGroupNodeNames.Insert(node.Name) + memberSetByNode[node.Name] = controlplane.NewGroupMemberSet(serviceToGroupMember(appliedToGroup.Service)) } - podSet.Insert(podToGroupMember(pod, false)) - // Update the Pod references by Node. - memberSetByNode[pod.Spec.NodeName] = podSet - // Update the NodeNames in order to set the SpanMeta for AppliedToGroup. - appGroupNodeNames.Insert(pod.Spec.NodeName) - } - for _, extEntity := range externalEntities { - if extEntity.Spec.ExternalNode == "" { - continue + updatedAppliedToGroup = &antreatypes.AppliedToGroup{ + UID: appliedToGroup.UID, + Name: appliedToGroup.Name, + Service: appliedToGroup.Service, + GroupMemberByNode: memberSetByNode, + SpanMeta: antreatypes.SpanMeta{NodeNames: appGroupNodeNames}, + } + klog.V(2).Infof("Updating existing AppliedToGroup %s on %d Nodes", key, appGroupNodeNames.Len()) + } else { + scheduledPodNum, scheduledExtEntityNum := 0, 0 + pods, externalEntities := n.getAppliedToWorkloads(appliedToGroup) + for _, pod := range pods { + if pod.Spec.NodeName == "" || pod.Spec.HostNetwork == true { + // No need to process Pod when it's not scheduled. + // HostNetwork Pods will not be applied to by policies. + continue + } + scheduledPodNum++ + podSet := memberSetByNode[pod.Spec.NodeName] + if podSet == nil { + podSet = controlplane.GroupMemberSet{} + } + podSet.Insert(podToGroupMember(pod, false)) + // Update the Pod references by Node. + memberSetByNode[pod.Spec.NodeName] = podSet + // Update the NodeNames in order to set the SpanMeta for AppliedToGroup. + appGroupNodeNames.Insert(pod.Spec.NodeName) } - scheduledExtEntityNum++ - entitySet := memberSetByNode[extEntity.Spec.ExternalNode] - if entitySet == nil { - entitySet = controlplane.GroupMemberSet{} + for _, extEntity := range externalEntities { + if extEntity.Spec.ExternalNode == "" { + continue + } + scheduledExtEntityNum++ + entitySet := memberSetByNode[extEntity.Spec.ExternalNode] + if entitySet == nil { + entitySet = controlplane.GroupMemberSet{} + } + entitySet.Insert(externalEntityToGroupMember(extEntity)) + memberSetByNode[extEntity.Spec.ExternalNode] = entitySet + appGroupNodeNames.Insert(extEntity.Spec.ExternalNode) } - entitySet.Insert(externalEntityToGroupMember(extEntity)) - memberSetByNode[extEntity.Spec.ExternalNode] = entitySet - appGroupNodeNames.Insert(extEntity.Spec.ExternalNode) - } - updatedAppliedToGroup := &antreatypes.AppliedToGroup{ - UID: appliedToGroup.UID, - Name: appliedToGroup.Name, - Selector: appliedToGroup.Selector, - GroupMemberByNode: memberSetByNode, - SpanMeta: antreatypes.SpanMeta{NodeNames: appGroupNodeNames}, - } - klog.V(2).Infof("Updating existing AppliedToGroup %s with %d Pods and %d External Entities on %d Nodes", - key, scheduledPodNum, scheduledExtEntityNum, appGroupNodeNames.Len()) + updatedAppliedToGroup = &antreatypes.AppliedToGroup{ + UID: appliedToGroup.UID, + Name: appliedToGroup.Name, + Selector: appliedToGroup.Selector, + GroupMemberByNode: memberSetByNode, + SpanMeta: antreatypes.SpanMeta{NodeNames: appGroupNodeNames}, + } + klog.V(2).Infof("Updating existing AppliedToGroup %s with %d Pods and %d External Entities on %d Nodes", + key, scheduledPodNum, scheduledExtEntityNum, appGroupNodeNames.Len()) + } n.appliedToGroupStore.Update(updatedAppliedToGroup) // Get all internal NetworkPolicy objects that refers this AppliedToGroup. // Note that this must be executed after storing the result, to ensure that diff --git a/pkg/controller/networkpolicy/store/appliedtogroup.go b/pkg/controller/networkpolicy/store/appliedtogroup.go index af0a671112f..4a336a9bfe8 100644 --- a/pkg/controller/networkpolicy/store/appliedtogroup.go +++ b/pkg/controller/networkpolicy/store/appliedtogroup.go @@ -170,9 +170,10 @@ func NewAppliedToGroupStore() storage.Interface { indexers := cache.Indexers{ cache.NamespaceIndex: func(obj interface{}) ([]string, error) { atg, ok := obj.(*types.AppliedToGroup) - if !ok { + if !ok || atg.Selector == nil { return []string{}, nil } + return []string{atg.Selector.Namespace}, nil }, } diff --git a/pkg/controller/networkpolicy/validate.go b/pkg/controller/networkpolicy/validate.go index ead48aaddd2..33778b48bbc 100644 --- a/pkg/controller/networkpolicy/validate.go +++ b/pkg/controller/networkpolicy/validate.go @@ -415,6 +415,10 @@ func (v *antreaPolicyValidator) createValidate(curObj interface{}, userInfo auth if !allowed { return reason, allowed } + reason, allowed = v.validateAppliedToServiceIngressPeer(specAppliedTo, ingress) + if !allowed { + return reason, allowed + } reason, allowed = v.validateFQDNSelectors(egress) if !allowed { return reason, allowed @@ -472,7 +476,14 @@ func (v *antreaPolicyValidator) validateAppliedTo(ingress, egress []crdv1alpha1. return "appliedTo field should either be set in all rules or in none of them", false } - checkAppliedTo := func(appliedTo []crdv1alpha1.NetworkPolicyPeer) (string, bool) { + var ( + appliedToPolicy = 0 + appliedToIngressRule = 1 + appliedToEgressRule = 2 + ) + + checkAppliedTo := func(appliedTo []crdv1alpha1.NetworkPolicyPeer, appliedToScope int) (string, bool) { + appliedToSvcNum := 0 for _, eachAppliedTo := range appliedTo { appliedToFieldsNum := numFieldsSetInPeer(eachAppliedTo) if eachAppliedTo.Group != "" && appliedToFieldsNum > 1 { @@ -481,26 +492,38 @@ func (v *antreaPolicyValidator) validateAppliedTo(ingress, egress []crdv1alpha1. if eachAppliedTo.ServiceAccount != nil && appliedToFieldsNum > 1 { return "serviceAccount cannot be set with other peers in appliedTo", false } + if eachAppliedTo.Service != nil { + if appliedToFieldsNum > 1 { + return "service cannot be set with other peers in appliedTo", false + } + if appliedToScope == appliedToEgressRule || appliedToScope == appliedToPolicy && len(egress) > 0 { + return "egress rule cannot be applied to Services", false + } + appliedToSvcNum++ + } if reason, allowed := checkSelectorsLabels(eachAppliedTo.PodSelector, eachAppliedTo.NamespaceSelector, eachAppliedTo.ExternalEntitySelector); !allowed { return reason, allowed } } + if appliedToSvcNum > 0 && appliedToSvcNum < len(appliedTo) { + return "a rule/policy cannot be applied to Services and other peers at the same time", false + } return "", true } - reason, allowed := checkAppliedTo(specAppliedTo) + reason, allowed := checkAppliedTo(specAppliedTo, appliedToPolicy) if !allowed { return reason, allowed } for _, eachIngress := range ingress { - reason, allowed = checkAppliedTo(eachIngress.AppliedTo) + reason, allowed = checkAppliedTo(eachIngress.AppliedTo, appliedToIngressRule) if !allowed { return reason, allowed } } for _, eachEgress := range egress { - reason, allowed = checkAppliedTo(eachEgress.AppliedTo) + reason, allowed = checkAppliedTo(eachEgress.AppliedTo, appliedToEgressRule) if !allowed { return reason, allowed } @@ -555,6 +578,28 @@ func (v *antreaPolicyValidator) validatePeers(ingress, egress []crdv1alpha1.Rule return "", true } +// validateAppliedToServiceIngressPeer ensures that if a policy or an ingress rule +// is applied to Services, the ingress rule can only use ipBlock to select workloads. +func (v *antreaPolicyValidator) validateAppliedToServiceIngressPeer(specAppliedTo []crdv1alpha1.NetworkPolicyPeer, ingress []crdv1alpha1.Rule) (string, bool) { + isAppliedToService := func(peers []crdv1alpha1.NetworkPolicyPeer) bool { + if len(peers) > 0 { + return peers[0].Service != nil + } + return false + } + policyAppliedToService := isAppliedToService(specAppliedTo) + for _, rule := range ingress { + if policyAppliedToService || isAppliedToService(rule.AppliedTo) { + for _, peer := range rule.From { + if peer.IPBlock == nil || numFieldsSetInPeer(peer) > 1 { + return "a rule/policy that is applied to Services can only use ipBlock to select workloads", false + } + } + } + } + return "", true +} + // numFieldsSetInPeer returns the number of fields in use of a peer. func numFieldsSetInPeer(peer crdv1alpha1.NetworkPolicyPeer) int { num := 0 diff --git a/pkg/controller/types/networkpolicy.go b/pkg/controller/types/networkpolicy.go index f468e87aa02..f4e34692208 100644 --- a/pkg/controller/types/networkpolicy.go +++ b/pkg/controller/types/networkpolicy.go @@ -38,17 +38,22 @@ func (meta *SpanMeta) Has(nodeName string) bool { return meta.NodeNames.Has(nodeName) } -// AppliedToGroup describes a set of GroupMembers to apply Network Policies to. +// AppliedToGroup describes a set of GroupMembers or a Service to apply Network Policies to. type AppliedToGroup struct { SpanMeta - // UID is generated from the hash value of GroupSelector.NormalizedName. - // In case the AppliedToGroup is created for a ClusterGroup, the UID is - // that of the corresponding ClusterGroup. + // If the AppliedToGroup is created from GroupSelector, UID is generated from the hash value of GroupSelector.NormalizedName. + // If the AppliedToGroup is created for a ClusterGroup, the UID is that of the corresponding ClusterGroup. + // If the AppliedToGroup is created for a Service, the UID is generated from the hash value of NamespacedName of the Service. UID types.UID // Name of this group, currently it's same as UID. Name string // Selector describes how the group selects pods. - Selector GroupSelector + // Selector can't be used with Service. + Selector *GroupSelector + // Service refers to the Service this group selects. Only a NodePort type Service + // can be referred by this field. + // Service can't be used with Selector. + Service *controlplane.ServiceReference // GroupMemberByNode is a mapping from nodeName to a set of GroupMembers on the Node, // either GroupMembers or ExternalEntity on the external node. // It will be converted to a slice of GroupMember for transferring according diff --git a/test/e2e/antreapolicy_test.go b/test/e2e/antreapolicy_test.go index 98f45b124c5..819ff0a2a4c 100644 --- a/test/e2e/antreapolicy_test.go +++ b/test/e2e/antreapolicy_test.go @@ -2974,6 +2974,70 @@ func testACNPICMPSupport(t *testing.T, data *TestData) { failOnError(k8sUtils.DeleteACNP(builder.Name), t) } +func testACNPNodePortServiceSupport(t *testing.T, data *TestData) { + skipIfProxyAllDisabled(t, data) + + // Create a client on Node 0 and a NodePort Service whose Endpoint is on Node 0. + // Initiate traffic from this client to Service NodePort on Node 1 to simulate the + // traffic from external client to NodePort. + clientName := "agnhost-client" + failOnError(data.createAgnhostPodOnNode(clientName, data.testNamespace, nodeName(0), true), t) + defer data.deletePodAndWait(defaultTimeout, clientName, data.testNamespace) + ips, err := data.podWaitForIPs(defaultTimeout, clientName, data.testNamespace) + failOnError(err, t) + + var cidr string + if clusterInfo.podV4NetworkCIDR != "" { + cidr = ips.ipv4.String() + } else { + cidr = ips.ipv6.String() + } + cidr += "/32" + + svc, cleanup := data.createAgnhostServiceAndBackendPods(t, "svc", data.testNamespace, nodeName(0), v1.ServiceTypeNodePort) + defer cleanup() + + builder := &ClusterNetworkPolicySpecBuilder{} + builder = builder.SetName("test-acnp-nodeport-svc"). + SetPriority(1.0). + SetAppliedToGroup([]ACNPAppliedToSpec{ + { + Service: &crdv1alpha1.NamespacedName{ + Name: svc.Name, + Namespace: svc.Namespace, + }, + }, + }) + builder.AddIngress(ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, &cidr, nil, nil, + nil, nil, false, nil, crdv1alpha1.RuleActionDrop, "", "", nil) + + testcases := []podToAddrTestStep{ + { + Pod(fmt.Sprintf("%s/%s", data.testNamespace, clientName)), + nodeIP(1), + svc.Spec.Ports[0].NodePort, + Dropped, + }, + } + + acnp, err := k8sUtils.CreateOrUpdateACNP(builder.Get()) + failOnError(err, t) + failOnError(waitForResourceReady(t, timeout, acnp), t) + for _, tc := range testcases { + log.Tracef("Probing: %s -> %s:%d", cidr, tc.destAddr, tc.destPort) + connectivity, err := k8sUtils.ProbeAddr(tc.clientPod.Namespace(), "antrea-e2e", tc.clientPod.PodName(), tc.destAddr, tc.destPort, ProtocolTCP) + if err != nil { + t.Errorf("failure -- could not complete probe: %v", err) + } + if connectivity != tc.expectedConnectivity { + t.Errorf("failure -- wrong results for probe: Source %s --> Dest %s:%d connectivity: %v, expected: %v", + cidr, tc.destAddr, tc.destPort, connectivity, tc.expectedConnectivity) + } + } + // cleanup test resources + failOnError(k8sUtils.DeleteACNP(builder.Name), t) +} + func testACNPIGMPQueryAllow(t *testing.T, data *TestData) { testACNPIGMPQuery(t, data, "test-acnp-igmp-query-allow", "testMulticastIGMPQueryAllow", "224.3.4.13", crdv1alpha1.RuleActionAllow) } @@ -3469,6 +3533,7 @@ func TestAntreaPolicy(t *testing.T) { t.Run("Case=ACNPNodeSelectorEgress", func(t *testing.T) { testACNPNodeSelectorEgress(t) }) t.Run("Case=ACNPNodeSelectorIngress", func(t *testing.T) { testACNPNodeSelectorIngress(t, data) }) t.Run("Case=ACNPICMPSupport", func(t *testing.T) { testACNPICMPSupport(t, data) }) + t.Run("Case=ACNPNodePortServiceSupport", func(t *testing.T) { testACNPNodePortServiceSupport(t, data) }) }) // print results for reachability tests printResults() diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 4c5b3e62b8e..7d54f90ba68 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -163,6 +163,16 @@ func skipIfProxyDisabled(t *testing.T) { skipIfFeatureDisabled(t, features.AntreaProxy, true /* checkAgent */, false /* checkController */) } +func skipIfProxyAllDisabled(t *testing.T, data *TestData) { + isProxyAll, err := data.isProxyAll() + if err != nil { + t.Fatalf("Error getting option antreaProxy.proxyAll value") + } + if !isProxyAll { + t.Skipf("Skipping test because option antreaProxy.proxyAll is not enabled") + } +} + func ensureAntreaRunning(data *TestData) error { log.Println("Applying Antrea YAML") if err := data.deployAntrea(deployAntreaDefault); err != nil { diff --git a/test/e2e/proxy_test.go b/test/e2e/proxy_test.go index db9ac888078..01f73858252 100644 --- a/test/e2e/proxy_test.go +++ b/test/e2e/proxy_test.go @@ -69,16 +69,6 @@ func testProxyServiceSessionAffinityCase(t *testing.T, data *TestData) { } } -func skipIfProxyAllDisabled(t *testing.T, data *TestData) { - isProxyAll, err := data.isProxyAll() - if err != nil { - t.Fatalf("Error getting option antreaProxy.proxyAll value") - } - if !isProxyAll { - t.Skipf("Skipping test because option antreaProxy.proxyAll is not enabled") - } -} - func skipIfKubeProxyEnabled(t *testing.T, data *TestData) { _, err := data.clientset.AppsV1().DaemonSets(kubeNamespace).Get(context.TODO(), "kube-proxy", metav1.GetOptions{}) if err == nil { diff --git a/test/e2e/utils/cnpspecbuilder.go b/test/e2e/utils/cnpspecbuilder.go index 12aed6a65e2..7f0bf9eff4d 100644 --- a/test/e2e/utils/cnpspecbuilder.go +++ b/test/e2e/utils/cnpspecbuilder.go @@ -32,6 +32,7 @@ type ACNPAppliedToSpec struct { PodSelectorMatchExp []metav1.LabelSelectorRequirement NSSelectorMatchExp []metav1.LabelSelectorRequirement Group string + Service *crdv1alpha1.NamespacedName } func (b *ClusterNetworkPolicySpecBuilder) Get() *crdv1alpha1.ClusterNetworkPolicy { @@ -66,7 +67,7 @@ func (b *ClusterNetworkPolicySpecBuilder) SetTier(tier string) *ClusterNetworkPo func (b *ClusterNetworkPolicySpecBuilder) SetAppliedToGroup(specs []ACNPAppliedToSpec) *ClusterNetworkPolicySpecBuilder { for _, spec := range specs { - appliedToPeer := b.GetAppliedToPeer(spec.PodSelector, spec.NSSelector, spec.PodSelectorMatchExp, spec.NSSelectorMatchExp, spec.Group) + appliedToPeer := b.GetAppliedToPeer(spec.PodSelector, spec.NSSelector, spec.PodSelectorMatchExp, spec.NSSelectorMatchExp, spec.Group, spec.Service) b.Spec.AppliedTo = append(b.Spec.AppliedTo, appliedToPeer) } return b @@ -76,7 +77,8 @@ func (b *ClusterNetworkPolicySpecBuilder) GetAppliedToPeer(podSelector map[strin nsSelector map[string]string, podSelectorMatchExp []metav1.LabelSelectorRequirement, nsSelectorMatchExp []metav1.LabelSelectorRequirement, - appliedToCG string) crdv1alpha1.NetworkPolicyPeer { + appliedToCG string, + service *crdv1alpha1.NamespacedName) crdv1alpha1.NetworkPolicyPeer { var ps *metav1.LabelSelector var ns *metav1.LabelSelector @@ -100,6 +102,9 @@ func (b *ClusterNetworkPolicySpecBuilder) GetAppliedToPeer(podSelector map[strin if appliedToCG != "" { peer.Group = appliedToCG } + if service != nil { + peer.Service = service + } return peer } @@ -143,7 +148,7 @@ func (b *ClusterNetworkPolicySpecBuilder) AddIngress(protoc AntreaPolicyProtocol } } for _, at := range ruleAppliedToSpecs { - appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group)) + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group, at.Service)) } // An empty From/To in ACNP rules evaluates to match all addresses. policyPeer := make([]crdv1alpha1.NetworkPolicyPeer, 0) @@ -197,7 +202,7 @@ func (b *ClusterNetworkPolicySpecBuilder) AddNodeSelectorRule(nodeSelector *meta ruleAppliedToSpecs []ACNPAppliedToSpec, action crdv1alpha1.RuleAction, isEgress bool) *ClusterNetworkPolicySpecBuilder { var appliedTos []crdv1alpha1.NetworkPolicyPeer for _, at := range ruleAppliedToSpecs { - appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group)) + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group, at.Service)) } policyPeer := []crdv1alpha1.NetworkPolicyPeer{{NodeSelector: nodeSelector}} k8sProtocol, _ := AntreaPolicyProtocolToK8sProtocol(protoc) @@ -224,7 +229,7 @@ func (b *ClusterNetworkPolicySpecBuilder) AddFQDNRule(fqdn string, ruleAppliedToSpecs []ACNPAppliedToSpec, action crdv1alpha1.RuleAction) *ClusterNetworkPolicySpecBuilder { var appliedTos []crdv1alpha1.NetworkPolicyPeer for _, at := range ruleAppliedToSpecs { - appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group)) + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group, at.Service)) } policyPeer := []crdv1alpha1.NetworkPolicyPeer{{FQDN: fqdn}} ports, _ := GenPortsOrProtocols(protoc, port, portName, endPort, nil, nil, nil, nil) @@ -243,7 +248,7 @@ func (b *ClusterNetworkPolicySpecBuilder) AddToServicesRule(svcRefs []crdv1alpha name string, ruleAppliedToSpecs []ACNPAppliedToSpec, action crdv1alpha1.RuleAction) *ClusterNetworkPolicySpecBuilder { var appliedTos []crdv1alpha1.NetworkPolicyPeer for _, at := range ruleAppliedToSpecs { - appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group)) + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.NSSelector, at.PodSelectorMatchExp, at.NSSelectorMatchExp, at.Group, at.Service)) } newRule := crdv1alpha1.Rule{ To: make([]crdv1alpha1.NetworkPolicyPeer, 0),