From 71734d1b8f41ea3d6e64ab1b9fbd500ba41b4015 Mon Sep 17 00:00:00 2001 From: Yongming Ding Date: Mon, 24 Oct 2022 12:11:12 -0700 Subject: [PATCH] Enclose Pod labels with single quote in s3 uploader In this commit, we updated the s3 uploader inside Flow Aggregator to enclose Pod label fields with single quote for snowflake usecases. Because commas are used to separate different columns in CSV and Pod labels json string contains commas, we enclose it with single quote to avoid them being separated into multiple columns in Snowflake. Signed-off-by: Yongming Ding --- pkg/flowaggregator/s3uploader/s3uploader.go | 6 ++++-- pkg/flowaggregator/s3uploader/s3uploader_test.go | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/flowaggregator/s3uploader/s3uploader.go b/pkg/flowaggregator/s3uploader/s3uploader.go index 2f6d98417ea..406475a331b 100644 --- a/pkg/flowaggregator/s3uploader/s3uploader.go +++ b/pkg/flowaggregator/s3uploader/s3uploader.go @@ -459,9 +459,11 @@ func writeRecord(w io.Writer, r *flowrecord.FlowRecord, clusterUUID string) { io.WriteString(w, ",") io.WriteString(w, fmt.Sprintf("%d", r.FlowType)) io.WriteString(w, ",") - io.WriteString(w, r.SourcePodLabels) + // Enclose Pod labels with single quote because commas are used to separate + // different columns in CSV and Pod labels json string contains commas. + io.WriteString(w, fmt.Sprintf("'%s'", r.SourcePodLabels)) io.WriteString(w, ",") - io.WriteString(w, r.DestinationPodLabels) + io.WriteString(w, fmt.Sprintf("'%s'", r.DestinationPodLabels)) io.WriteString(w, ",") io.WriteString(w, fmt.Sprintf("%d", r.Throughput)) io.WriteString(w, ",") diff --git a/pkg/flowaggregator/s3uploader/s3uploader_test.go b/pkg/flowaggregator/s3uploader/s3uploader_test.go index a98016bd02d..03cb49b8f85 100644 --- a/pkg/flowaggregator/s3uploader/s3uploader_test.go +++ b/pkg/flowaggregator/s3uploader/s3uploader_test.go @@ -38,8 +38,8 @@ import ( var ( fakeClusterUUID = uuid.New().String() - recordStrIPv4 = "1637706961,1637706973,1637706974,1637706975,3,10.10.0.79,10.10.0.80,44752,5201,6,823188,30472817041,241333,8982624938,471111,24500996,136211,7083284,perftest-a,antrea-test,k8s-node-control-plane,perftest-b,antrea-test-b,k8s-node-control-plane-b,10.10.1.10,5202,perftest,test-flow-aggregator-networkpolicy-ingress-allow,antrea-test-ns,test-flow-aggregator-networkpolicy-rule,2,1,test-flow-aggregator-networkpolicy-egress-allow,antrea-test-ns-e,test-flow-aggregator-networkpolicy-rule-e,5,4,TIME_WAIT,11,{\"antrea-e2e\":\"perftest-a\",\"app\":\"perftool\"},{\"antrea-e2e\":\"perftest-b\",\"app\":\"perftool\"},15902813472,12381344,15902813473,15902813474,12381345,12381346," + fakeClusterUUID - recordStrIPv6 = "1637706961,1637706973,1637706974,1637706975,3,2001:0:3238:dfe1:63::fefb,2001:0:3238:dfe1:63::fefc,44752,5201,6,823188,30472817041,241333,8982624938,471111,24500996,136211,7083284,perftest-a,antrea-test,k8s-node-control-plane,perftest-b,antrea-test-b,k8s-node-control-plane-b,2001:0:3238:dfe1:64::a,5202,perftest,test-flow-aggregator-networkpolicy-ingress-allow,antrea-test-ns,test-flow-aggregator-networkpolicy-rule,2,1,test-flow-aggregator-networkpolicy-egress-allow,antrea-test-ns-e,test-flow-aggregator-networkpolicy-rule-e,5,4,TIME_WAIT,11,{\"antrea-e2e\":\"perftest-a\",\"app\":\"perftool\"},{\"antrea-e2e\":\"perftest-b\",\"app\":\"perftool\"},15902813472,12381344,15902813473,15902813474,12381345,12381346," + fakeClusterUUID + recordStrIPv4 = "1637706961,1637706973,1637706974,1637706975,3,10.10.0.79,10.10.0.80,44752,5201,6,823188,30472817041,241333,8982624938,471111,24500996,136211,7083284,perftest-a,antrea-test,k8s-node-control-plane,perftest-b,antrea-test-b,k8s-node-control-plane-b,10.10.1.10,5202,perftest,test-flow-aggregator-networkpolicy-ingress-allow,antrea-test-ns,test-flow-aggregator-networkpolicy-rule,2,1,test-flow-aggregator-networkpolicy-egress-allow,antrea-test-ns-e,test-flow-aggregator-networkpolicy-rule-e,5,4,TIME_WAIT,11,'{\"antrea-e2e\":\"perftest-a\",\"app\":\"perftool\"}','{\"antrea-e2e\":\"perftest-b\",\"app\":\"perftool\"}',15902813472,12381344,15902813473,15902813474,12381345,12381346," + fakeClusterUUID + recordStrIPv6 = "1637706961,1637706973,1637706974,1637706975,3,2001:0:3238:dfe1:63::fefb,2001:0:3238:dfe1:63::fefc,44752,5201,6,823188,30472817041,241333,8982624938,471111,24500996,136211,7083284,perftest-a,antrea-test,k8s-node-control-plane,perftest-b,antrea-test-b,k8s-node-control-plane-b,2001:0:3238:dfe1:64::a,5202,perftest,test-flow-aggregator-networkpolicy-ingress-allow,antrea-test-ns,test-flow-aggregator-networkpolicy-rule,2,1,test-flow-aggregator-networkpolicy-egress-allow,antrea-test-ns-e,test-flow-aggregator-networkpolicy-rule-e,5,4,TIME_WAIT,11,'{\"antrea-e2e\":\"perftest-a\",\"app\":\"perftool\"}','{\"antrea-e2e\":\"perftest-b\",\"app\":\"perftool\"}',15902813472,12381344,15902813473,15902813474,12381345,12381346," + fakeClusterUUID ) const seed = 1