Skip to content

Commit

Permalink
Support L7 NetworkPolicy Logging
Browse files Browse the repository at this point in the history 
Antrea-native policy now supports layer 7 NetworkPolicy. To provide
more information for users, logging for this feature is introduced.

Antrea-native policy is not accurate enough in reporting packet status
before sending to l7 engine. Logs are fixed to reflect "Redirect" action.
Audit logging UT are updated to cover more cases.

L7 engine provides its own logs. Currently, Suricata is used as L7 engine.
Configuration is updated to generate two log files, fast.log and eve.json
Both files locates at /var/log/antrea/networkpolicy/. Documentation is updated.

Signed-off-by: Qiyue Yao <yaoq@vmware.com>
  • Loading branch information
@qiyueyao
qiyueyao committed Mar 9, 2023
1 parent bd683b0 commit 09a9fc0
Show file tree
Hide file tree
Showing 11 changed files with 273 additions and 58 deletions.
9 changes: 9 additions & 0 deletions docs/antrea-l7-network-policy.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
- [Usage](#usage)
- [HTTP](#http)
- [More examples](#more-examples)
- [Logs](#logs)
- [Limitations](#limitations)
<!-- /toc -->

Expand Down Expand Up @@ -200,6 +201,14 @@ spec:
- http: {} # automatically dropped, and subsequent rules will not be considered.
```

### Logs

Layer 7 traffic that match the NetworkPolicy will be logged in both an event
triggered log file (`/var/log/antrea/networkpolicy/eve.json`) and a single line
based fast log file (`/var/log/antrea/networkpolicy/fast.log`). Currently,
events include `pass` and `deny`. If `enableLogging` is set for the rule,
packets that matches the rule will also be logged in addition to the event.

## Limitations

This feature is currently only supported for Nodes running Linux.
4 changes: 3 additions & 1 deletion docs/antrea-network-policy.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -706,7 +706,9 @@ traffic that matches the "DropToThirdParty" egress rule, while the rule
"AllowFromFrontend" is not logged. Specifically for drop and reject rules,
deduplication is applied to reduce duplicated logs, and duplication buffer
length is set to 1 second. If a rule name is not provided, an identifiable
name will be generated for the rule and displayed in the log.
name will be generated for the rule and displayed in the log. For rules in layer
7 NetworkPolicy, packets are logged with action `Redirect` prior to analysis by
the layer 7 engine, more details are available in the corresponding engine logs.
The rules are logged in the following format:

```text
Expand Down
10 changes: 10 additions & 0 deletions pkg/agent/controller/networkpolicy/audit_logging.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,16 @@ func getNetworkPolicyInfo(pktIn *ofctrl.PacketIn, c *Controller, ob *logInfo) er
}
ob.disposition = openflow.DispositionToString[disposition]

// Get traffic control action, if traffic is redirected, disposition log should be overwritten.
match = getMatchRegField(matchers, openflow.TrafficControlActionField)
trafficControl, err := getInfoInReg(match, openflow.TrafficControlActionField.GetRange().ToNXRange())
if err != nil {
return fmt.Errorf("received error while unloading traffic control action from reg: %v", err)
}
if trafficControl == openflow.TrafficControlRedirect {
ob.disposition = "Redirect"
}

// Set match to corresponding ingress/egress reg according to disposition.
match = getMatch(matchers, tableID, disposition)

Expand Down
199 changes: 167 additions & 32 deletions pkg/agent/controller/networkpolicy/audit_logging_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ import (

"antrea.io/libOpenflow/openflow15"
"antrea.io/ofnet/ofctrl"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"antrea.io/antrea/pkg/agent/openflow"
openflowtest "antrea.io/antrea/pkg/agent/openflow/testing"
"antrea.io/antrea/pkg/apis/controlplane/v1beta2"
binding "antrea.io/antrea/pkg/ovs/openflow"
"antrea.io/antrea/pkg/util/ip"
)
Expand All @@ -41,6 +41,12 @@ const (
testBufferLength time.Duration = 100 * time.Millisecond
)

var (
actionAllow = openflow.DispositionToString[openflow.DispositionAllow]
actionDrop = openflow.DispositionToString[openflow.DispositionDrop]
actionRedirect = "Redirect"
)

// mockLogger implements io.Writer.
type mockLogger struct {
mu sync.Mutex
Expand Down Expand Up @@ -122,7 +128,7 @@ func newTestAntreaPolicyLogger(bufferLength time.Duration, clock Clock) (*Antrea

func newLogInfo(disposition string) (*logInfo, string) {
testLogInfo := &logInfo{
tableName: "AntreaPolicyIngressRule",
tableName: openflow.AntreaPolicyIngressRuleTable.GetName(),
npRef: "AntreaNetworkPolicy:default/test",
ruleName: "test-rule",
ofPriority: "0",
Expand All @@ -146,7 +152,7 @@ func expectedLogWithCount(msg string, count int) string {

func TestAllowPacketLog(t *testing.T) {
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, &realClock{})
ob, expected := newLogInfo("Allow")
ob, expected := newLogInfo(actionAllow)

antreaLogger.LogDedupPacket(ob)
actual := <-mockAnpLogger.logged
Expand All @@ -155,7 +161,7 @@ func TestAllowPacketLog(t *testing.T) {

func TestDropPacketLog(t *testing.T) {
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, &realClock{})
ob, expected := newLogInfo("Drop")
ob, expected := newLogInfo(actionDrop)

antreaLogger.LogDedupPacket(ob)
actual := <-mockAnpLogger.logged
Expand All @@ -166,7 +172,7 @@ func TestDropPacketDedupLog(t *testing.T) {
clock := NewVirtualClock(time.Now())
defer clock.Stop()
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, clock)
ob, expected := newLogInfo("Drop")
ob, expected := newLogInfo(actionDrop)
// Add the additional log info for duplicate packets.
expected = expectedLogWithCount(expected, 2)

Expand All @@ -187,7 +193,7 @@ func TestDropPacketMultiDedupLog(t *testing.T) {
clock := NewVirtualClock(time.Now())
defer clock.Stop()
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, clock)
ob, expected := newLogInfo("Drop")
ob, expected := newLogInfo(actionDrop)

consumeLog := func() (int, error) {
select {
Expand Down Expand Up @@ -231,33 +237,158 @@ func TestDropPacketMultiDedupLog(t *testing.T) {
assert.Equal(t, 1, c2)
}

func TestRedirectPacketLog(t *testing.T) {
antreaLogger, mockAnpLogger := newTestAntreaPolicyLogger(testBufferLength, &realClock{})
ob, expected := newLogInfo(actionRedirect)

antreaLogger.LogDedupPacket(ob)
actual := <-mockAnpLogger.logged
assert.Contains(t, actual, expected)
}

func TestGetNetworkPolicyInfo(t *testing.T) {
openflow.InitMockTables(
map[*openflow.Table]uint8{
openflow.AntreaPolicyEgressRuleTable: uint8(5),
openflow.EgressRuleTable: uint8(6),
openflow.EgressDefaultTable: uint8(7),
openflow.AntreaPolicyIngressRuleTable: uint8(12),
openflow.IngressRuleTable: uint8(13),
openflow.IngressDefaultTable: uint8(14),
})
c := &Controller{ofClient: &openflowtest.MockClient{}}
ob := new(logInfo)
regID := openflow.APDispositionField.GetRegID()
dispositionMatch := openflow15.MatchField{
Class: openflow15.OXM_CLASS_PACKET_REGS,
Field: uint8(regID / 2),
HasMask: false,
Value: &openflow15.ByteArrayField{Data: []byte{1, 1, 1, 1}},
prepareMockOFTablesWithCache()
generateMatch := func(regID int, data []byte) openflow15.MatchField {
return openflow15.MatchField{
Class: openflow15.OXM_CLASS_PACKET_REGS,
Field: uint8(regID / 2),
HasMask: false,
Value: &openflow15.ByteArrayField{Data: data},
}
}
testANPRef := "AntreaNetworkPolicy:default/test-anp"
testK8sRef := "AntreaNetworkPolicy:default/test-anp"
testPriority, testRule := "61800", "test-rule"
allowDispositionData := []byte{0x11, 0x11, 0x00, 0x11}
dropDispositionData := []byte{0x11, 0x11, 0x08, 0x11}
noTrafficControlData := []byte{0x11, 0x00, 0x11, 0x11}
redirectTrafficControlData := []byte{0x11, 0x80, 0x11, 0x11}
ingressData := []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}
tests := []struct {
name string
tableID uint8
expectedCalls func(mockClient *openflowtest.MockClientMockRecorder)
dispositionData []byte
trafficControlData []byte
ob *logInfo
wantOb *logInfo
wantErr error
}{
{
name: "ANP Allow",
tableID: openflow.AntreaPolicyIngressRuleTable.GetID(),
expectedCalls: func(mockClient *openflowtest.MockClientMockRecorder) {
mockClient.GetPolicyInfoFromConjunction(gomock.Any()).Return(
testANPRef, testPriority, testRule)
},
dispositionData: allowDispositionData,
trafficControlData: noTrafficControlData,
wantOb: &logInfo{
tableName: openflow.AntreaPolicyIngressRuleTable.GetName(),
disposition: actionAllow,
npRef: testANPRef,
ofPriority: testPriority,
ruleName: testRule,
},
},
{
name: "K8s Allow",
tableID: openflow.IngressRuleTable.GetID(),
expectedCalls: func(mockClient *openflowtest.MockClientMockRecorder) {
mockClient.GetPolicyInfoFromConjunction(gomock.Any()).Return(
testK8sRef, testPriority, "")
},
dispositionData: allowDispositionData,
trafficControlData: noTrafficControlData,
wantOb: &logInfo{
tableName: openflow.IngressRuleTable.GetName(),
disposition: actionAllow,
npRef: testK8sRef,
ofPriority: testPriority,
ruleName: "<nil>",
},
},
{
name: "ANP Drop",
tableID: openflow.AntreaPolicyIngressRuleTable.GetID(),
expectedCalls: func(mockClient *openflowtest.MockClientMockRecorder) {
mockClient.GetPolicyInfoFromConjunction(gomock.Any()).Return(
testANPRef, testPriority, testRule)
},
dispositionData: dropDispositionData,
trafficControlData: noTrafficControlData,
wantOb: &logInfo{
tableName: openflow.AntreaPolicyIngressRuleTable.GetName(),
disposition: actionDrop,
npRef: testANPRef,
ofPriority: testPriority,
ruleName: testRule,
},
},
{
name: "K8s Drop",
tableID: openflow.IngressDefaultTable.GetID(),
dispositionData: dropDispositionData,
trafficControlData: noTrafficControlData,
wantOb: &logInfo{
tableName: openflow.IngressDefaultTable.GetName(),
disposition: actionDrop,
npRef: "K8sNetworkPolicy",
ofPriority: "<nil>",
ruleName: "<nil>",
},
},
{
name: "ANP Redirect",
tableID: openflow.AntreaPolicyIngressRuleTable.GetID(),
expectedCalls: func(mockClient *openflowtest.MockClientMockRecorder) {
mockClient.GetPolicyInfoFromConjunction(gomock.Any()).Return(
testANPRef, testPriority, testRule)
},
dispositionData: allowDispositionData,
trafficControlData: redirectTrafficControlData,
wantOb: &logInfo{
tableName: openflow.AntreaPolicyIngressRuleTable.GetName(),
disposition: actionRedirect,
npRef: testANPRef,
ofPriority: testPriority,
ruleName: testRule,
},
},
}
matchers := []openflow15.MatchField{dispositionMatch}
pktIn := &ofctrl.PacketIn{TableId: 17, Match: openflow15.Match{Fields: matchers}}

err := getNetworkPolicyInfo(pktIn, c, ob)
assert.Equal(t, string(v1beta2.K8sNetworkPolicy), ob.npRef)
assert.Equal(t, "<nil>", ob.ofPriority)
assert.Equal(t, "<nil>", ob.ruleName)
require.NoError(t, err)
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
// Inject disposition match.
dispositionMatch := generateMatch(openflow.APDispositionField.GetRegID(), tc.dispositionData)
matchers := []openflow15.MatchField{dispositionMatch}
// Inject traffic control action match.
trafficControlMatch := generateMatch(openflow.TrafficControlActionField.GetRegID(), tc.trafficControlData)
matchers = append(matchers, trafficControlMatch)
// Inject ingress/egress match when case is not K8s default drop.
if tc.expectedCalls != nil {
regID := openflow.TFIngressConjIDField.GetRegID()
if tc.wantOb.disposition == actionDrop {
regID = openflow.CNPConjIDField.GetRegID()
}
ingressMatch := generateMatch(regID, ingressData)
matchers = append(matchers, ingressMatch)
}
pktIn := &ofctrl.PacketIn{TableId: tc.tableID, Match: openflow15.Match{Fields: matchers}}

ctrl := gomock.NewController(t)
defer ctrl.Finish()
testClientInterface := openflowtest.NewMockClient(ctrl)
if tc.expectedCalls != nil {
tc.expectedCalls(testClientInterface.EXPECT())
}
c := &Controller{ofClient: testClientInterface}
tc.ob = new(logInfo)
gotErr := getNetworkPolicyInfo(pktIn, c, tc.ob)
assert.Equal(t, tc.wantOb, tc.ob)
assert.Equal(t, tc.wantErr, gotErr)
})
}
}

func TestGetPacketInfo(t *testing.T) {
Expand All @@ -277,7 +408,6 @@ func TestGetPacketInfo(t *testing.T) {
SourcePort: 35402,
DestinationPort: 80,
},
ob: new(logInfo),
wantOb: &logInfo{
srcIP: "0.0.0.0",
srcPort: "35402",
Expand All @@ -295,7 +425,6 @@ func TestGetPacketInfo(t *testing.T) {
IPLength: 60,
IPProto: ip.ICMPProtocol,
},
ob: new(logInfo),
wantOb: &logInfo{
srcIP: "0.0.0.0",
srcPort: "<nil>",
Expand All @@ -308,7 +437,13 @@ func TestGetPacketInfo(t *testing.T) {
}

for _, tc := range tests {
tc.ob = new(logInfo)
getPacketInfo(tc.packet, tc.ob)
assert.Equal(t, tc.wantOb, tc.ob)
}
}

func prepareMockOFTablesWithCache() {
openflow.InitMockTables(mockOFTables)
openflow.InitOFTableCache(mockOFTables)
}
Loading

0 comments on commit 09a9fc0

Please sign in to comment.