From 7cbe8eed31908323d9d5b79bb0a0231139f00d26 Mon Sep 17 00:00:00 2001 From: Qiyue Yao Date: Thu, 16 Mar 2023 04:21:15 -0700 Subject: [PATCH] Add Log Label to Audit Logging Antrea native policies support firewall rule logs. Adding a rule label to the log allows convenient post-processing of the logs like grepping the same log labels. This feature adds a field "logSetting" to the crd, which has two fields "enabled" and "logLabel". "logSetting" and "enableLogging" cannot be used at the same time. "enabled" must always be specified when used. "logLabel" defaults to empty value. Fixes #4652 Signed-off-by: Qiyue Yao --- .../antrea/crds/clusternetworkpolicy.yaml | 20 + build/charts/antrea/crds/networkpolicy.yaml | 20 + build/yamls/antrea-aks.yml | 40 ++ build/yamls/antrea-crds.yml | 40 ++ build/yamls/antrea-eks.yml | 40 ++ build/yamls/antrea-gke.yml | 40 ++ build/yamls/antrea-ipsec.yml | 40 ++ build/yamls/antrea.yml | 40 ++ docs/antrea-network-policy.md | 100 ++- .../antrea-multicluster-leader-global.yml | 76 ++- ...cluster.crd.antrea.io_resourceexports.yaml | 38 +- ...cluster.crd.antrea.io_resourceimports.yaml | 38 +- .../controller/networkpolicy/audit_logging.go | 27 +- .../networkpolicy/audit_logging_test.go | 26 +- pkg/agent/controller/networkpolicy/cache.go | 8 + .../networkpolicy/networkpolicy_controller.go | 4 +- .../controller/networkpolicy/reconciler.go | 13 +- .../networkpolicy/reconciler_test.go | 17 +- .../connections/conntrack_connections_test.go | 21 +- pkg/agent/openflow/client.go | 2 +- pkg/agent/openflow/network_policy.go | 41 +- pkg/agent/openflow/network_policy_test.go | 48 +- pkg/agent/openflow/testing/mock_openflow.go | 5 +- pkg/agent/types/networkpolicy.go | 6 + pkg/apis/controlplane/types.go | 8 + pkg/apis/controlplane/v1beta2/generated.pb.go | 576 ++++++++++++------ pkg/apis/controlplane/v1beta2/generated.proto | 11 +- pkg/apis/controlplane/v1beta2/types.go | 8 + .../v1beta2/zz_generated.conversion.go | 32 + .../v1beta2/zz_generated.deepcopy.go | 21 + .../controlplane/zz_generated.deepcopy.go | 21 + pkg/apis/crd/v1alpha1/types.go | 14 + .../crd/v1alpha1/zz_generated.deepcopy.go | 21 + pkg/apiserver/openapi/zz_generated.openapi.go | 29 +- .../networkpolicy/antreanetworkpolicy.go | 2 + .../networkpolicy/antreanetworkpolicy_test.go | 66 ++ .../networkpolicy/clusternetworkpolicy.go | 1 + .../clusternetworkpolicy_test.go | 65 ++ pkg/controller/networkpolicy/crd_utils.go | 9 + .../networkpolicy/crd_utils_test.go | 29 + .../networkpolicy/networkpolicy_controller.go | 28 +- .../networkpolicy_controller_test.go | 43 +- pkg/controller/networkpolicy/validate.go | 18 + pkg/controller/networkpolicy/validate_test.go | 26 + test/e2e/antreapolicy_test.go | 3 +- test/e2e/utils/anp_spec_builder.go | 7 +- test/e2e/utils/cnp_spec_builder.go | 7 +- 47 files changed, 1451 insertions(+), 344 deletions(-) diff --git a/build/charts/antrea/crds/clusternetworkpolicy.yaml b/build/charts/antrea/crds/clusternetworkpolicy.yaml index 2f9ac6017c7..b57a437f897 100644 --- a/build/charts/antrea/crds/clusternetworkpolicy.yaml +++ b/build/charts/antrea/crds/clusternetworkpolicy.yaml @@ -383,6 +383,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -645,6 +655,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: diff --git a/build/charts/antrea/crds/networkpolicy.yaml b/build/charts/antrea/crds/networkpolicy.yaml index 617100f054f..a65da3d85b9 100644 --- a/build/charts/antrea/crds/networkpolicy.yaml +++ b/build/charts/antrea/crds/networkpolicy.yaml @@ -326,6 +326,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -559,6 +569,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 2b5cbf397dd..7db5c484a8d 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -761,6 +761,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -1023,6 +1033,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: @@ -1857,6 +1877,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -2090,6 +2120,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index 86f6099c28b..726be13f379 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -754,6 +754,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -1016,6 +1026,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: @@ -1838,6 +1858,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -2071,6 +2101,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 1d7bd09b0a4..3c43a552b0d 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -761,6 +761,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -1023,6 +1033,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: @@ -1857,6 +1877,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -2090,6 +2120,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 332e1bc4969..50286a0db7d 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -761,6 +761,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -1023,6 +1033,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: @@ -1857,6 +1877,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -2090,6 +2120,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 8f320edc824..afc067a87ef 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -761,6 +761,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -1023,6 +1033,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: @@ -1857,6 +1877,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -2090,6 +2120,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 5ec4e750882..016af1895a5 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -761,6 +761,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -1023,6 +1033,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: @@ -1857,6 +1877,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" egress: type: array items: @@ -2090,6 +2120,16 @@ spec: type: string enableLogging: type: boolean + logSetting: + type: object + required: + - enabled + properties: + enabled: + type: boolean + logLabel: + type: string + pattern: "^.{0,32}$" status: type: object properties: diff --git a/docs/antrea-network-policy.md b/docs/antrea-network-policy.md index 7f184de34d9..c58b60c8ac3 100644 --- a/docs/antrea-network-policy.md +++ b/docs/antrea-network-policy.md @@ -20,6 +20,7 @@ - [ACNP for IGMP traffic](#acnp-for-igmp-traffic) - [ACNP for multicast egress traffic](#acnp-for-multicast-egress-traffic) - [ACNP for HTTP traffic](#acnp-for-http-traffic) + - [ACNP with log setting](#acnp-with-log-setting) - [Behavior of to and from selectors](#behavior-of-to-and-from-selectors) - [Key differences from K8s NetworkPolicy](#key-differences-from-k8s-networkpolicy) - [kubectl commands for Antrea ClusterNetworkPolicy](#kubectl-commands-for-antrea-clusternetworkpolicy) @@ -535,6 +536,50 @@ spec: host: "*.bar.com" # not be considered. ``` +#### ACNP with log setting + +```yaml +apiVersion: crd.antrea.io/v1alpha1 +kind: ClusterNetworkPolicy +metadata: + name: acnp-with-log-setting +spec: + priority: 5 + tier: securityops + appliedTo: + - podSelector: + matchLabels: + role: db + - namespaceSelector: + matchLabels: + env: prod + ingress: + - action: Allow + from: + - podSelector: + matchLabels: + role: frontend + namespaceSelector: + matchLabels: + role: db + name: AllowFromFrontend + logSetting: + enabled: false + logLabel: "frontend-allowed" + egress: + - action: Drop + to: + - ipBlock: + cidr: 10.0.10.0/24 + ports: + - protocol: TCP + port: 5978 + name: DropToThirdParty + logSetting: + enabled: true + logLabel: "third-party-blocked" +``` + Please refer to [Antrea Layer 7 NetworkPolicy](antrea-l7-network-policy.md) for extra information. **spec**: The ClusterNetworkPolicy `spec` has all the information needed to @@ -696,30 +741,39 @@ to select Pods. More details can be found in the [ServiceAccountSelector](#servi **Note**: The order in which the egress rules are specified matters, i.e., rules will be enforced in the order in which they are written. -**enableLogging**: Antrea-native policy ingress or egress rules can be -audited by enabling its logging field. When `enableLogging` field is set to -true, the first packet of any connection that matches this rule will be logged -to a separate file (`/var/log/antrea/networkpolicy/np.log`) on the Node on -which the rule is applied. These log files can then be retrieved for further -analysis. By default, rules are not logged. The example policy logs all -traffic that matches the "DropToThirdParty" egress rule, while the rule -"AllowFromFrontend" is not logged. Specifically for drop and reject rules, -deduplication is applied to reduce duplicated logs, and duplication buffer -length is set to 1 second. If a rule name is not provided, an identifiable -name will be generated for the rule and displayed in the log. For rules in layer -7 NetworkPolicy, packets are logged with action `Redirect` prior to analysis by -the layer 7 engine, more details are available in the corresponding engine logs. -The rules are logged in the following format: +**logSetting** and **enableLogging**: Antrea-native policy ingress or egress rules +can be audited by enabling its logging field. Since v0.11.0, `enableLogging` +became available and `logSetting` is introduced in v1.12.0. We will be deprecating +`enableLogging` for `logSetting` so they should not be set at the same time. +`logSetting.enabled` and `enableLogging` are equivalent. When `logSetting.enabled` +or `enableLogging` field is set to true, the first packet of any connection that +matches this rule will be logged to a separate file (`/var/log/antrea/networkpolicy/np.log`) +on the Node on which the rule is applied. These log files can then be retrieved for +further analysis. By default, rules are not logged. When using `logSetting`, +`enabled` must be specified, if `enabled` is true and `logLabel` is provided, the +label will be displayed in the log, otherwise it defaults to empty value. + +The [first example](#acnp-with-stand-alone-selectors) policy logs all traffic that +matches the "DropToThirdParty" egress rule, while the rule "AllowFromFrontend" is +not logged. The [eleventh example](#acnp-with-log-setting) policy logs all traffic +that matches the "DropToThirdParty" egress rule with "third-party-blocked", but does +not log "AllowFromFrontend" which is not enabled even when a label is provided. +Specifically for drop and reject rules, deduplication is applied to reduce duplicated +logs, and duplication buffer length is set to 1 second. If a rule name is not +provided, an identifiable name will be generated for the rule and displayed in the log. +For rules in layer 7 NetworkPolicy, packets are logged with action `Redirect` prior +to analysis by the layer 7 engine, more details are available in the corresponding +engine logs. The rules are logged in the following format: ```text -