From 1ddfc725f9647f540990a4697855374b95eba7b9 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Sat, 7 Nov 2020 18:46:48 +0800 Subject: [PATCH 01/17] Implement anonymizer's main program Signed-off-by: Ashmita Bohara --- .gitignore | 2 + Makefile | 4 + cmd/anonymizer/app/anonymizer/anonymizer.go | 6 +- cmd/anonymizer/app/flags.go | 73 ++++++++++++++ cmd/anonymizer/app/options.go | 14 +++ cmd/anonymizer/app/writer/writer.go | 4 +- cmd/anonymizer/main.go | 101 +++++++++++++++++++- cmd/opentelemetry/go.sum | 11 +++ 8 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 cmd/anonymizer/app/flags.go create mode 100644 cmd/anonymizer/app/options.go diff --git a/.gitignore b/.gitignore index 3ab8672ab4b..471f3c91849 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ examples/memstore-plugin/memstore-plugin cmd/all-in-one/all-in-one-* cmd/agent/agent cmd/agent/agent-* +cmd/anonymizer/anonymizer +cmd/anonymizer/anonymizer-* cmd/collector/collector cmd/collector/collector-* cmd/ingester/ingester diff --git a/Makefile b/Makefile index 61bc175cd59..469697ead79 100644 --- a/Makefile +++ b/Makefile @@ -250,6 +250,10 @@ build-all-in-one build-all-in-one-debug: build-ui elasticsearch-mappings build-agent build-agent-debug: $(GOBUILD) $(DISABLE_OPTIMIZATIONS) -o ./cmd/agent/agent$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/agent/main.go +.PHONY: build-anonymizer +build-anonymizer: + $(GOBUILD) $(DISABLE_OPTIMIZATIONS) -o ./cmd/anonymizer/anonymizer$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/anonymizer/main.go + .PHONY: build-query build-query-debug build-query build-query-debug: build-ui $(GOBUILD) $(DISABLE_OPTIMIZATIONS) -tags ui -o ./cmd/query/query$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/query/main.go diff --git a/cmd/anonymizer/app/anonymizer/anonymizer.go b/cmd/anonymizer/app/anonymizer/anonymizer.go index 9996900b518..47bc67eaa2b 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer.go @@ -65,7 +65,7 @@ type Anonymizer struct { // New creates new Anonymizer. The mappingFile stores the mapping from original to // obfuscated strings, in case later investigations require looking at the original traces. -func New(mappingFile string, logger *zap.Logger) *Anonymizer { +func New(mappingFile string, logger *zap.Logger, hashStandardTags, hashCustomTags, hashLogs, hashProcess bool) *Anonymizer { a := &Anonymizer{ mappingFile: mappingFile, logger: logger, @@ -73,6 +73,10 @@ func New(mappingFile string, logger *zap.Logger) *Anonymizer { Services: make(map[string]string), Operations: make(map[string]string), }, + hashStandardTags: hashStandardTags, + hashCustomTags: hashCustomTags, + hashLogs: hashLogs, + hashProcess: hashProcess, } if _, err := os.Stat(filepath.Clean(mappingFile)); err == nil { dat, err := ioutil.ReadFile(filepath.Clean(mappingFile)) diff --git a/cmd/anonymizer/app/flags.go b/cmd/anonymizer/app/flags.go new file mode 100644 index 00000000000..5c3c1717f48 --- /dev/null +++ b/cmd/anonymizer/app/flags.go @@ -0,0 +1,73 @@ +package app + +import ( + "github.com/spf13/cobra" +) + +var QueryGRPCPort, MaxSpansCount int +var QueryGRPCHost, TraceID, OutputDir string +var HashStandardTags, HashCustomTags, HashLogs, HashProcess bool + +const ( + queryGRPCHostFlag = "query.host" + queryGRPCPortFlag = "query.port" + outputDirFlag = "output.dir" + traceIDFlag = "trace.id" + hashStandardTagsFlag = "hash.standardtags" + hashCustomTagsFlag = "hash.customtags" + hashLogsFlag = "hash.logs" + hashProcessFlag = "hash.process" + maxSpansCount = "max.spanscount" +) + +// AddFlags adds flags for anonymizer main program +func AddFlags(command *cobra.Command) { + command.Flags().StringVar( + &QueryGRPCHost, + queryGRPCHostFlag, + DefaultQueryGRPCHost, + "hostname of the jaeger-query endpoint") + command.Flags().IntVar( + &QueryGRPCPort, + queryGRPCPortFlag, + DefaultQueryGRPCPort, + "port of the jaeger-query endpoint") + command.Flags().StringVar( + &OutputDir, + outputDirFlag, + DefaultOutputDir, + "directory to store the anonymized trace") + command.Flags().StringVar( + &TraceID, + traceIDFlag, + "", + "trace-id of trace to anonymize") + command.Flags().BoolVar( + &HashStandardTags, + hashStandardTagsFlag, + DefaultHashStandardTags, + "whether to hash standard tags") + command.Flags().BoolVar( + &HashCustomTags, + hashCustomTagsFlag, + DefaultHashCustomTags, + "whether to hash custom tags") + command.Flags().BoolVar( + &HashLogs, + hashLogsFlag, + DefaultHashLogs, + "whether to hash logs") + command.Flags().BoolVar( + &HashProcess, + hashProcessFlag, + DefaultHashProcess, + "whether to hash process") + command.Flags().IntVar( + &MaxSpansCount, + maxSpansCount, + DefaultMaxSpansCount, + "maximum number of spans to anonymize") + + // mark traceid flag as mandatory + command.MarkFlagRequired(traceIDFlag) +} \ No newline at end of file diff --git a/cmd/anonymizer/app/options.go b/cmd/anonymizer/app/options.go new file mode 100644 index 00000000000..6a87540a0f9 --- /dev/null +++ b/cmd/anonymizer/app/options.go @@ -0,0 +1,14 @@ +package app + +import "github.com/jaegertracing/jaeger/ports" + +const ( + DefaultQueryGRPCHost = "localhost" + DefaultQueryGRPCPort = ports.QueryHTTP + DefaultOutputDir = "/tmp" + DefaultHashStandardTags = true + DefaultHashCustomTags = false + DefaultHashLogs = false + DefaultHashProcess = false + DefaultMaxSpansCount = -1 +) diff --git a/cmd/anonymizer/app/writer/writer.go b/cmd/anonymizer/app/writer/writer.go index 5e36f8a4db2..be9cb53b8ad 100644 --- a/cmd/anonymizer/app/writer/writer.go +++ b/cmd/anonymizer/app/writer/writer.go @@ -48,7 +48,7 @@ type Writer struct { } // New creates an Writer -func New(config Config, logger *zap.Logger) (*Writer, error) { +func New(config Config, logger *zap.Logger, hashStandardTags, hashCustomTags, hashLogs, hashProcess bool) (*Writer, error) { wd, err := os.Getwd() if err != nil { return nil, err @@ -80,7 +80,7 @@ func New(config Config, logger *zap.Logger) (*Writer, error) { logger: logger, capturedFile: cf, anonymizedFile: af, - anonymizer: anonymizer.New(config.MappingFile, logger), + anonymizer: anonymizer.New(config.MappingFile, logger, hashStandardTags, hashCustomTags, hashLogs, hashProcess), }, nil } diff --git a/cmd/anonymizer/main.go b/cmd/anonymizer/main.go index 221bdc04eb8..5182d433926 100644 --- a/cmd/anonymizer/main.go +++ b/cmd/anonymizer/main.go @@ -15,13 +15,106 @@ package main import ( + "context" + "fmt" + "os" + "strconv" + "time" + "go.uber.org/zap" + "google.golang.org/grpc" + "github.com/jaegertracing/jaeger/model" + "github.com/jaegertracing/jaeger/pkg/version" + "github.com/jaegertracing/jaeger/proto-gen/api_v2" + "github.com/jaegertracing/jaeger/storage/spanstore" + "github.com/spf13/cobra" - "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" + app "github.com/jaegertracing/jaeger/cmd/anonymizer/app" + writer "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" ) +var logger, _ = zap.NewDevelopment() + +type grpcClient struct { + api_v2.QueryServiceClient + conn *grpc.ClientConn +} + +func newGRPCClient(addr string) *grpcClient { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure()) + if err != nil { + logger.Fatal("failed to connect with the jaeger-query service", zap.Error(err)) + } + + return &grpcClient{ + QueryServiceClient: api_v2.NewQueryServiceClient(conn), + conn: conn, + } +} + func main() { - // TODO - _, _ = writer.New(writer.Config{}, zap.NewNop()) - println("not implemented") + var command = &cobra.Command{ + Use: "jaeger-anonymizer", + Short: "Jaeger anonymizer hashes fields of a trace for easy sharing", + Long: `Jaeger anonymizer queries Jaeger query for a trace, anonymizes fields, and store in file`, + Run: func(cmd *cobra.Command, args []string) { + prefix := app.OutputDir + "/" + app.TraceID + conf := writer.Config{ + MaxSpansCount: app.MaxSpansCount, + CapturedFile: prefix + ".orig", + AnonymizedFile: prefix + ".anon", + MappingFile: prefix + ".map", + } + + writer, err := writer.New(conf, logger, app.HashStandardTags, app.HashCustomTags, app.HashLogs, app.HashProcess) + if err != nil { + logger.Error("error while creating writer object", zap.Error(err)) + } + + queryEndpoint := app.QueryGRPCHost + ":" + strconv.Itoa(app.QueryGRPCPort) + logger.Info(queryEndpoint) + + client := newGRPCClient(queryEndpoint) + defer client.conn.Close() + + traceID, err := model.TraceIDFromString(app.TraceID) + if err != nil { + logger.Fatal("failed to convert the provided trace id", zap.Error(err)) + } + logger.Info(app.TraceID) + + response, err := client.GetTrace(context.Background(), &api_v2.GetTraceRequest{ + TraceID: traceID, + }) + if err != nil { + logger.Fatal("failed to fetch the provided trace id", zap.Error(err)) + } + + spanResponseChunk, err := response.Recv() + if err == spanstore.ErrTraceNotFound { + logger.Fatal("failed to find the provided trace id", zap.Error(err)) + } + if err != nil { + logger.Fatal("failed to fetch spans of provided trace id", zap.Error(err)) + } + + spans := spanResponseChunk.GetSpans() + + for _, span := range spans { + writer.WriteSpan(&span) + } + }, + } + + app.AddFlags(command) + + command.AddCommand(version.Command()) + + if error := command.Execute(); error != nil { + fmt.Println(error.Error()) + os.Exit(1) + } } diff --git a/cmd/opentelemetry/go.sum b/cmd/opentelemetry/go.sum index a127a1d938e..6c432918d50 100644 --- a/cmd/opentelemetry/go.sum +++ b/cmd/opentelemetry/go.sum @@ -189,6 +189,7 @@ github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -213,6 +214,8 @@ github.com/denis-tingajkin/go-header v0.3.1 h1:ymEpSiFjeItCy1FOP+x0M2KdCELdEAHUs github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5na0fBt4/CRe7Lrji0= github.com/dgraph-io/badger v1.5.3 h1:5oWIuRvwn93cie+OSt1zSnkaIQ1JFQM8bGlIv6O6Sts= github.com/dgraph-io/badger v1.5.3/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -238,6 +241,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -1063,6 +1068,7 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -1073,6 +1079,7 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= @@ -1127,6 +1134,7 @@ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6 github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA= github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= @@ -1195,6 +1203,7 @@ go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1318,6 +1327,7 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1335,6 +1345,7 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From ccf970e507e01b296a21526dbf7c889587ad826f Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Sun, 8 Nov 2020 23:27:31 +0800 Subject: [PATCH 02/17] Ran make fmt Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/anonymizer/anonymizer.go | 6 +++--- cmd/anonymizer/app/flags.go | 16 +++++++++++++++- cmd/anonymizer/app/options.go | 14 ++++++++++++++ cmd/anonymizer/main.go | 18 +++++++++--------- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/cmd/anonymizer/app/anonymizer/anonymizer.go b/cmd/anonymizer/app/anonymizer/anonymizer.go index 47bc67eaa2b..a9f3c6dac76 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer.go @@ -74,9 +74,9 @@ func New(mappingFile string, logger *zap.Logger, hashStandardTags, hashCustomTag Operations: make(map[string]string), }, hashStandardTags: hashStandardTags, - hashCustomTags: hashCustomTags, - hashLogs: hashLogs, - hashProcess: hashProcess, + hashCustomTags: hashCustomTags, + hashLogs: hashLogs, + hashProcess: hashProcess, } if _, err := os.Stat(filepath.Clean(mappingFile)); err == nil { dat, err := ioutil.ReadFile(filepath.Clean(mappingFile)) diff --git a/cmd/anonymizer/app/flags.go b/cmd/anonymizer/app/flags.go index 5c3c1717f48..129dda03322 100644 --- a/cmd/anonymizer/app/flags.go +++ b/cmd/anonymizer/app/flags.go @@ -1,3 +1,17 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package app import ( @@ -70,4 +84,4 @@ func AddFlags(command *cobra.Command) { // mark traceid flag as mandatory command.MarkFlagRequired(traceIDFlag) -} \ No newline at end of file +} diff --git a/cmd/anonymizer/app/options.go b/cmd/anonymizer/app/options.go index 6a87540a0f9..b3f8a3d94e7 100644 --- a/cmd/anonymizer/app/options.go +++ b/cmd/anonymizer/app/options.go @@ -1,3 +1,17 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package app import "github.com/jaegertracing/jaeger/ports" diff --git a/cmd/anonymizer/main.go b/cmd/anonymizer/main.go index 5182d433926..d5bbc00a04f 100644 --- a/cmd/anonymizer/main.go +++ b/cmd/anonymizer/main.go @@ -21,16 +21,16 @@ import ( "strconv" "time" + "github.com/spf13/cobra" "go.uber.org/zap" "google.golang.org/grpc" + + app "github.com/jaegertracing/jaeger/cmd/anonymizer/app" + writer "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" "github.com/jaegertracing/jaeger/model" "github.com/jaegertracing/jaeger/pkg/version" "github.com/jaegertracing/jaeger/proto-gen/api_v2" "github.com/jaegertracing/jaeger/storage/spanstore" - "github.com/spf13/cobra" - - app "github.com/jaegertracing/jaeger/cmd/anonymizer/app" - writer "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" ) var logger, _ = zap.NewDevelopment() @@ -57,16 +57,16 @@ func newGRPCClient(addr string) *grpcClient { func main() { var command = &cobra.Command{ - Use: "jaeger-anonymizer", + Use: "jaeger-anonymizer", Short: "Jaeger anonymizer hashes fields of a trace for easy sharing", - Long: `Jaeger anonymizer queries Jaeger query for a trace, anonymizes fields, and store in file`, + Long: `Jaeger anonymizer queries Jaeger query for a trace, anonymizes fields, and store in file`, Run: func(cmd *cobra.Command, args []string) { prefix := app.OutputDir + "/" + app.TraceID conf := writer.Config{ - MaxSpansCount: app.MaxSpansCount, - CapturedFile: prefix + ".orig", + MaxSpansCount: app.MaxSpansCount, + CapturedFile: prefix + ".orig", AnonymizedFile: prefix + ".anon", - MappingFile: prefix + ".map", + MappingFile: prefix + ".map", } writer, err := writer.New(conf, logger, app.HashStandardTags, app.HashCustomTags, app.HashLogs, app.HashProcess) From f16f9183424b1a4764a31434ed83af08791a41d7 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Mon, 9 Nov 2020 22:55:20 +0800 Subject: [PATCH 03/17] Feedbacks Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/anonymizer/anonymizer.go | 36 ++++---- cmd/anonymizer/app/flags.go | 52 +++++++----- cmd/anonymizer/app/query/query.go | 91 +++++++++++++++++++++ cmd/anonymizer/app/writer/writer.go | 24 ++++-- cmd/anonymizer/main.go | 75 ++++------------- 5 files changed, 176 insertions(+), 102 deletions(-) create mode 100644 cmd/anonymizer/app/query/query.go diff --git a/cmd/anonymizer/app/anonymizer/anonymizer.go b/cmd/anonymizer/app/anonymizer/anonymizer.go index a9f3c6dac76..18765c02544 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer.go @@ -53,19 +53,24 @@ type mapping struct { // // The mapping from original to obfuscated strings is stored in a file and can be reused between runs. type Anonymizer struct { - mappingFile string - logger *zap.Logger - lock sync.Mutex - mapping mapping - hashStandardTags bool - hashCustomTags bool - hashLogs bool - hashProcess bool + mappingFile string + logger *zap.Logger + lock sync.Mutex + mapping mapping + anonymizerOptions *AnonymizerOptions +} + +// AnonymizerOptions represents the various options with which the anonymizer can be configured. +type AnonymizerOptions struct { + HashStandardTags bool + HashCustomTags bool + HashLogs bool + HashProcess bool } // New creates new Anonymizer. The mappingFile stores the mapping from original to // obfuscated strings, in case later investigations require looking at the original traces. -func New(mappingFile string, logger *zap.Logger, hashStandardTags, hashCustomTags, hashLogs, hashProcess bool) *Anonymizer { +func New(mappingFile string, options *AnonymizerOptions, logger *zap.Logger) *Anonymizer { a := &Anonymizer{ mappingFile: mappingFile, logger: logger, @@ -73,10 +78,7 @@ func New(mappingFile string, logger *zap.Logger, hashStandardTags, hashCustomTag Services: make(map[string]string), Operations: make(map[string]string), }, - hashStandardTags: hashStandardTags, - hashCustomTags: hashCustomTags, - hashLogs: hashLogs, - hashProcess: hashProcess, + anonymizerOptions: options, } if _, err := os.Stat(filepath.Clean(mappingFile)); err == nil { dat, err := ioutil.ReadFile(filepath.Clean(mappingFile)) @@ -146,18 +148,18 @@ func (a *Anonymizer) AnonymizeSpan(span *model.Span) *uimodel.Span { outputTags := filterStandardTags(span.Tags) // when true, the allowedTags are hashed and when false they are preserved as it is - if a.hashStandardTags { + if a.anonymizerOptions.HashStandardTags { outputTags = hashTags(outputTags) } // when true, all tags other than allowedTags are hashed, when false they are dropped - if a.hashCustomTags { + if a.anonymizerOptions.HashCustomTags { customTags := hashTags(filterCustomTags(span.Tags)) outputTags = append(outputTags, customTags...) } span.Tags = outputTags // when true, logs are hashed, when false, they are dropped - if a.hashLogs { + if a.anonymizerOptions.HashLogs { for _, log := range span.Logs { log.Fields = hashTags(log.Fields) } @@ -168,7 +170,7 @@ func (a *Anonymizer) AnonymizeSpan(span *model.Span) *uimodel.Span { span.Process.ServiceName = a.mapServiceName(service) // when true, process tags are hashed, when false they are dropped - if a.hashProcess { + if a.anonymizerOptions.HashProcess { span.Process.Tags = hashTags(span.Process.Tags) } else { span.Process.Tags = nil diff --git a/cmd/anonymizer/app/flags.go b/cmd/anonymizer/app/flags.go index 129dda03322..27148ad318f 100644 --- a/cmd/anonymizer/app/flags.go +++ b/cmd/anonymizer/app/flags.go @@ -18,66 +18,74 @@ import ( "github.com/spf13/cobra" ) -var QueryGRPCPort, MaxSpansCount int -var QueryGRPCHost, TraceID, OutputDir string -var HashStandardTags, HashCustomTags, HashLogs, HashProcess bool +type Options struct { + QueryGRPCPort int + QueryGRPCHost string + MaxSpansCount int + TraceID string + OutputDir string + HashStandardTags bool + HashCustomTags bool + HashLogs bool + HashProcess bool +} const ( - queryGRPCHostFlag = "query.host" - queryGRPCPortFlag = "query.port" - outputDirFlag = "output.dir" - traceIDFlag = "trace.id" - hashStandardTagsFlag = "hash.standardtags" - hashCustomTagsFlag = "hash.customtags" - hashLogsFlag = "hash.logs" - hashProcessFlag = "hash.process" - maxSpansCount = "max.spanscount" + queryGRPCHostFlag = "query-host" + queryGRPCPortFlag = "query-port" + outputDirFlag = "output-dir" + traceIDFlag = "trace-id" + hashStandardTagsFlag = "hash-standard-tags" + hashCustomTagsFlag = "hash-custom-tags" + hashLogsFlag = "hash-logs" + hashProcessFlag = "hash-process" + maxSpansCount = "max-spans-count" ) // AddFlags adds flags for anonymizer main program -func AddFlags(command *cobra.Command) { +func (o *Options) AddFlags(command *cobra.Command) { command.Flags().StringVar( - &QueryGRPCHost, + &o.QueryGRPCHost, queryGRPCHostFlag, DefaultQueryGRPCHost, "hostname of the jaeger-query endpoint") command.Flags().IntVar( - &QueryGRPCPort, + &o.QueryGRPCPort, queryGRPCPortFlag, DefaultQueryGRPCPort, "port of the jaeger-query endpoint") command.Flags().StringVar( - &OutputDir, + &o.OutputDir, outputDirFlag, DefaultOutputDir, "directory to store the anonymized trace") command.Flags().StringVar( - &TraceID, + &o.TraceID, traceIDFlag, "", "trace-id of trace to anonymize") command.Flags().BoolVar( - &HashStandardTags, + &o.HashStandardTags, hashStandardTagsFlag, DefaultHashStandardTags, "whether to hash standard tags") command.Flags().BoolVar( - &HashCustomTags, + &o.HashCustomTags, hashCustomTagsFlag, DefaultHashCustomTags, "whether to hash custom tags") command.Flags().BoolVar( - &HashLogs, + &o.HashLogs, hashLogsFlag, DefaultHashLogs, "whether to hash logs") command.Flags().BoolVar( - &HashProcess, + &o.HashProcess, hashProcessFlag, DefaultHashProcess, "whether to hash process") command.Flags().IntVar( - &MaxSpansCount, + &o.MaxSpansCount, maxSpansCount, DefaultMaxSpansCount, "maximum number of spans to anonymize") diff --git a/cmd/anonymizer/app/query/query.go b/cmd/anonymizer/app/query/query.go new file mode 100644 index 00000000000..f84a347020a --- /dev/null +++ b/cmd/anonymizer/app/query/query.go @@ -0,0 +1,91 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "time" + + "go.uber.org/zap" + "google.golang.org/grpc" + + "github.com/jaegertracing/jaeger/model" + "github.com/jaegertracing/jaeger/proto-gen/api_v2" + "github.com/jaegertracing/jaeger/storage/spanstore" +) + +// GrpcClient represents a grpc-client to jaeger-query +type GrpcClient struct { + api_v2.QueryServiceClient + conn *grpc.ClientConn +} + +// Query represents a jaeger-query's query for trace-id +type Query struct { + grpcClient *GrpcClient + logger *zap.Logger +} + +// newGRPCClient returns a new GrpcClient +func newGRPCClient(addr string, logger *zap.Logger) *GrpcClient { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure()) + if err != nil { + logger.Fatal("failed to connect with the jaeger-query service", zap.Error(err)) + } + + return &GrpcClient{ + QueryServiceClient: api_v2.NewQueryServiceClient(conn), + conn: conn, + } +} + +// New creates a Query object +func New(addr string, logger *zap.Logger) (*Query, error) { + client := newGRPCClient(addr, logger) + + return &Query{ + grpcClient: client, + logger: logger, + }, nil +} + +// QueryTrace queries for a trace and returns all spans inside it +func (q *Query) QueryTrace(traceId string) []model.Span { + traceID, err := model.TraceIDFromString(traceId) + if err != nil { + q.logger.Fatal("failed to convert the provided trace id", zap.Error(err)) + } + + response, err := q.grpcClient.GetTrace(context.Background(), &api_v2.GetTraceRequest{ + TraceID: traceID, + }) + if err != nil { + q.logger.Fatal("failed to fetch the provided trace id", zap.Error(err)) + } + + spanResponseChunk, err := response.Recv() + if err == spanstore.ErrTraceNotFound { + q.logger.Fatal("failed to find the provided trace id", zap.Error(err)) + } + if err != nil { + q.logger.Fatal("failed to fetch spans of provided trace id", zap.Error(err)) + } + + spans := spanResponseChunk.GetSpans() + return spans +} diff --git a/cmd/anonymizer/app/writer/writer.go b/cmd/anonymizer/app/writer/writer.go index be9cb53b8ad..9b5f6f4c2ef 100644 --- a/cmd/anonymizer/app/writer/writer.go +++ b/cmd/anonymizer/app/writer/writer.go @@ -30,10 +30,14 @@ import ( // Config contains parameters to NewWriter. type Config struct { - MaxSpansCount int `yaml:"max_spans_count" name:"max_spans_count"` - CapturedFile string `yaml:"captured_file" name:"captured_file"` - AnonymizedFile string `yaml:"anonymized_file" name:"anonymized_file"` - MappingFile string `yaml:"mapping_file" name:"mapping_file"` + MaxSpansCount int `yaml:"max_spans_count" name:"max_spans_count"` + CapturedFile string `yaml:"captured_file" name:"captured_file"` + AnonymizedFile string `yaml:"anonymized_file" name:"anonymized_file"` + MappingFile string `yaml:"mapping_file" name:"mapping_file"` + HashStandardTags bool `yaml:"hash_standard_tags" name:"hash_standard_tags"` + HashCustomTags bool `yaml:"hash_custom_tags" name:"hash_custom_tags"` + HashLogs bool `yaml:"hash_logs" name:"hash_logs"` + HashProcess bool `yaml:"hash_process" name:"hash_process"` } // Writer is a span Writer that obfuscates the span and writes it to a JSON file. @@ -48,7 +52,7 @@ type Writer struct { } // New creates an Writer -func New(config Config, logger *zap.Logger, hashStandardTags, hashCustomTags, hashLogs, hashProcess bool) (*Writer, error) { +func New(config Config, logger *zap.Logger) (*Writer, error) { wd, err := os.Getwd() if err != nil { return nil, err @@ -75,12 +79,20 @@ func New(config Config, logger *zap.Logger, hashStandardTags, hashCustomTags, ha if err != nil { return nil, fmt.Errorf("cannot write tp output file: %w", err) } + + options := &anonymizer.AnonymizerOptions{ + HashStandardTags: config.HashStandardTags, + HashCustomTags: config.HashCustomTags, + HashLogs: config.HashLogs, + HashProcess: config.HashProcess, + } + return &Writer{ config: config, logger: logger, capturedFile: cf, anonymizedFile: af, - anonymizer: anonymizer.New(config.MappingFile, logger, hashStandardTags, hashCustomTags, hashLogs, hashProcess), + anonymizer: anonymizer.New(config.MappingFile, options, logger), }, nil } diff --git a/cmd/anonymizer/main.go b/cmd/anonymizer/main.go index d5bbc00a04f..e31b6c683c0 100644 --- a/cmd/anonymizer/main.go +++ b/cmd/anonymizer/main.go @@ -15,101 +15,62 @@ package main import ( - "context" "fmt" "os" "strconv" - "time" "github.com/spf13/cobra" "go.uber.org/zap" - "google.golang.org/grpc" app "github.com/jaegertracing/jaeger/cmd/anonymizer/app" + query "github.com/jaegertracing/jaeger/cmd/anonymizer/app/query" writer "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" - "github.com/jaegertracing/jaeger/model" "github.com/jaegertracing/jaeger/pkg/version" - "github.com/jaegertracing/jaeger/proto-gen/api_v2" - "github.com/jaegertracing/jaeger/storage/spanstore" ) var logger, _ = zap.NewDevelopment() -type grpcClient struct { - api_v2.QueryServiceClient - conn *grpc.ClientConn -} - -func newGRPCClient(addr string) *grpcClient { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure()) - if err != nil { - logger.Fatal("failed to connect with the jaeger-query service", zap.Error(err)) - } - - return &grpcClient{ - QueryServiceClient: api_v2.NewQueryServiceClient(conn), - conn: conn, - } -} - func main() { + var options = app.Options{} + var command = &cobra.Command{ Use: "jaeger-anonymizer", Short: "Jaeger anonymizer hashes fields of a trace for easy sharing", Long: `Jaeger anonymizer queries Jaeger query for a trace, anonymizes fields, and store in file`, Run: func(cmd *cobra.Command, args []string) { - prefix := app.OutputDir + "/" + app.TraceID + prefix := options.OutputDir + "/" + options.TraceID conf := writer.Config{ - MaxSpansCount: app.MaxSpansCount, - CapturedFile: prefix + ".orig", - AnonymizedFile: prefix + ".anon", - MappingFile: prefix + ".map", + MaxSpansCount: options.MaxSpansCount, + CapturedFile: prefix + ".original", + AnonymizedFile: prefix + ".anonymized", + MappingFile: prefix + ".mapping", + HashStandardTags: options.HashStandardTags, + HashCustomTags: options.HashCustomTags, + HashLogs: options.HashLogs, + HashProcess: options.HashProcess, } - writer, err := writer.New(conf, logger, app.HashStandardTags, app.HashCustomTags, app.HashLogs, app.HashProcess) + writer, err := writer.New(conf, logger) if err != nil { logger.Error("error while creating writer object", zap.Error(err)) } - queryEndpoint := app.QueryGRPCHost + ":" + strconv.Itoa(app.QueryGRPCPort) + queryEndpoint := options.QueryGRPCHost + ":" + strconv.Itoa(options.QueryGRPCPort) logger.Info(queryEndpoint) - client := newGRPCClient(queryEndpoint) - defer client.conn.Close() - - traceID, err := model.TraceIDFromString(app.TraceID) + query, err := query.New(queryEndpoint, logger) if err != nil { - logger.Fatal("failed to convert the provided trace id", zap.Error(err)) + logger.Error("error while creating query object", zap.Error(err)) } - logger.Info(app.TraceID) - - response, err := client.GetTrace(context.Background(), &api_v2.GetTraceRequest{ - TraceID: traceID, - }) - if err != nil { - logger.Fatal("failed to fetch the provided trace id", zap.Error(err)) - } - - spanResponseChunk, err := response.Recv() - if err == spanstore.ErrTraceNotFound { - logger.Fatal("failed to find the provided trace id", zap.Error(err)) - } - if err != nil { - logger.Fatal("failed to fetch spans of provided trace id", zap.Error(err)) - } - - spans := spanResponseChunk.GetSpans() + spans := query.QueryTrace(options.TraceID) for _, span := range spans { writer.WriteSpan(&span) } }, } - app.AddFlags(command) + options.AddFlags(command) command.AddCommand(version.Command()) From 20ee0504fb14ac08c7ba0bd1e439a6bd3aef34a8 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Mon, 9 Nov 2020 22:57:27 +0800 Subject: [PATCH 04/17] Revert otlm go.sum changes Signed-off-by: Ashmita Bohara --- cmd/opentelemetry/go.sum | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cmd/opentelemetry/go.sum b/cmd/opentelemetry/go.sum index 6c432918d50..a127a1d938e 100644 --- a/cmd/opentelemetry/go.sum +++ b/cmd/opentelemetry/go.sum @@ -189,7 +189,6 @@ github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -214,8 +213,6 @@ github.com/denis-tingajkin/go-header v0.3.1 h1:ymEpSiFjeItCy1FOP+x0M2KdCELdEAHUs github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5na0fBt4/CRe7Lrji0= github.com/dgraph-io/badger v1.5.3 h1:5oWIuRvwn93cie+OSt1zSnkaIQ1JFQM8bGlIv6O6Sts= github.com/dgraph-io/badger v1.5.3/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= -github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= -github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -241,8 +238,6 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -1068,7 +1063,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -1079,7 +1073,6 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= @@ -1134,7 +1127,6 @@ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6 github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA= github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= @@ -1203,7 +1195,6 @@ go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1327,7 +1318,6 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1345,7 +1335,6 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From ff919e3ad3991202b31147684e5910d22e2a61d3 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Mon, 9 Nov 2020 23:00:36 +0800 Subject: [PATCH 05/17] Added comment Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/flags.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/anonymizer/app/flags.go b/cmd/anonymizer/app/flags.go index 27148ad318f..0903998473f 100644 --- a/cmd/anonymizer/app/flags.go +++ b/cmd/anonymizer/app/flags.go @@ -18,6 +18,7 @@ import ( "github.com/spf13/cobra" ) +// Options represent configurable parameters for jaeger-anonymizer type Options struct { QueryGRPCPort int QueryGRPCHost string From 67b391411d11a24901ba2eb56734b063e68563df Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Mon, 9 Nov 2020 23:43:54 +0800 Subject: [PATCH 06/17] Fix testing test Signed-off-by: Ashmita Bohara --- .../app/anonymizer/anonymizer_test.go | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cmd/anonymizer/app/anonymizer/anonymizer_test.go b/cmd/anonymizer/app/anonymizer/anonymizer_test.go index d89279b37fb..332e0dfe4e7 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer_test.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer_test.go @@ -98,15 +98,18 @@ func TestAnonymizer_Hash(t *testing.T) { } func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) { + options := &AnonymizerOptions{ + HashStandardTags: true, + HashCustomTags: true, + HashProcess: true, + HashLogs: true, + } anonymizer := &Anonymizer{ mapping: mapping{ Services: make(map[string]string), Operations: make(map[string]string), }, - hashStandardTags: true, - hashCustomTags: true, - hashProcess: true, - hashLogs: true, + anonymizerOptions: options, } _ = anonymizer.AnonymizeSpan(span1) assert.Equal(t, 3, len(span1.Tags)) @@ -115,15 +118,18 @@ func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) { } func TestAnonymizer_AnonymizeSpan_AllFalse(t *testing.T) { + options := &AnonymizerOptions{ + HashStandardTags: false, + HashCustomTags: false, + HashProcess: false, + HashLogs: false, + } anonymizer := &Anonymizer{ mapping: mapping{ Services: make(map[string]string), Operations: make(map[string]string), }, - hashStandardTags: false, - hashCustomTags: false, - hashProcess: false, - hashLogs: false, + anonymizerOptions: options, } _ = anonymizer.AnonymizeSpan(span2) assert.Equal(t, 2, len(span2.Tags)) From b278eaddf4adc0476fb5e467e95133e96de32b4b Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 10 Nov 2020 00:44:50 +0800 Subject: [PATCH 07/17] Fix for lint failure Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/anonymizer/anonymizer.go | 18 +++++++++--------- .../app/anonymizer/anonymizer_test.go | 8 ++++---- cmd/anonymizer/app/options.go | 8 ++++++++ cmd/anonymizer/app/query/query.go | 6 +++--- cmd/anonymizer/app/writer/writer.go | 2 +- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/cmd/anonymizer/app/anonymizer/anonymizer.go b/cmd/anonymizer/app/anonymizer/anonymizer.go index 18765c02544..7777c3b5628 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer.go @@ -57,11 +57,11 @@ type Anonymizer struct { logger *zap.Logger lock sync.Mutex mapping mapping - anonymizerOptions *AnonymizerOptions + options *Options } -// AnonymizerOptions represents the various options with which the anonymizer can be configured. -type AnonymizerOptions struct { +// Options represents the various options with which the anonymizer can be configured. +type Options struct { HashStandardTags bool HashCustomTags bool HashLogs bool @@ -70,7 +70,7 @@ type AnonymizerOptions struct { // New creates new Anonymizer. The mappingFile stores the mapping from original to // obfuscated strings, in case later investigations require looking at the original traces. -func New(mappingFile string, options *AnonymizerOptions, logger *zap.Logger) *Anonymizer { +func New(mappingFile string, options *Options, logger *zap.Logger) *Anonymizer { a := &Anonymizer{ mappingFile: mappingFile, logger: logger, @@ -78,7 +78,7 @@ func New(mappingFile string, options *AnonymizerOptions, logger *zap.Logger) *An Services: make(map[string]string), Operations: make(map[string]string), }, - anonymizerOptions: options, + options: options, } if _, err := os.Stat(filepath.Clean(mappingFile)); err == nil { dat, err := ioutil.ReadFile(filepath.Clean(mappingFile)) @@ -148,18 +148,18 @@ func (a *Anonymizer) AnonymizeSpan(span *model.Span) *uimodel.Span { outputTags := filterStandardTags(span.Tags) // when true, the allowedTags are hashed and when false they are preserved as it is - if a.anonymizerOptions.HashStandardTags { + if a.options.HashStandardTags { outputTags = hashTags(outputTags) } // when true, all tags other than allowedTags are hashed, when false they are dropped - if a.anonymizerOptions.HashCustomTags { + if a.options.HashCustomTags { customTags := hashTags(filterCustomTags(span.Tags)) outputTags = append(outputTags, customTags...) } span.Tags = outputTags // when true, logs are hashed, when false, they are dropped - if a.anonymizerOptions.HashLogs { + if a.options.HashLogs { for _, log := range span.Logs { log.Fields = hashTags(log.Fields) } @@ -170,7 +170,7 @@ func (a *Anonymizer) AnonymizeSpan(span *model.Span) *uimodel.Span { span.Process.ServiceName = a.mapServiceName(service) // when true, process tags are hashed, when false they are dropped - if a.anonymizerOptions.HashProcess { + if a.options.HashProcess { span.Process.Tags = hashTags(span.Process.Tags) } else { span.Process.Tags = nil diff --git a/cmd/anonymizer/app/anonymizer/anonymizer_test.go b/cmd/anonymizer/app/anonymizer/anonymizer_test.go index 332e0dfe4e7..b76ba64b857 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer_test.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer_test.go @@ -98,7 +98,7 @@ func TestAnonymizer_Hash(t *testing.T) { } func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) { - options := &AnonymizerOptions{ + options := &Options{ HashStandardTags: true, HashCustomTags: true, HashProcess: true, @@ -109,7 +109,7 @@ func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) { Services: make(map[string]string), Operations: make(map[string]string), }, - anonymizerOptions: options, + options: options, } _ = anonymizer.AnonymizeSpan(span1) assert.Equal(t, 3, len(span1.Tags)) @@ -118,7 +118,7 @@ func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) { } func TestAnonymizer_AnonymizeSpan_AllFalse(t *testing.T) { - options := &AnonymizerOptions{ + options := &Options{ HashStandardTags: false, HashCustomTags: false, HashProcess: false, @@ -129,7 +129,7 @@ func TestAnonymizer_AnonymizeSpan_AllFalse(t *testing.T) { Services: make(map[string]string), Operations: make(map[string]string), }, - anonymizerOptions: options, + options: options, } _ = anonymizer.AnonymizeSpan(span2) assert.Equal(t, 2, len(span2.Tags)) diff --git a/cmd/anonymizer/app/options.go b/cmd/anonymizer/app/options.go index b3f8a3d94e7..4cf93a6c2e4 100644 --- a/cmd/anonymizer/app/options.go +++ b/cmd/anonymizer/app/options.go @@ -17,12 +17,20 @@ package app import "github.com/jaegertracing/jaeger/ports" const ( + // DefaultQueryGRPCHost is the default host for jaeger-query endpoint DefaultQueryGRPCHost = "localhost" + // DefaultQueryGRPCPort is the default port for jaeger-query endpoint DefaultQueryGRPCPort = ports.QueryHTTP + // DefaultOutputDir is the default output directory for spans DefaultOutputDir = "/tmp" + // DefaultHashStandardTags is the default flag for whether to hash standard tags DefaultHashStandardTags = true + // DefaultHashCustomTags is the default flag for whether to hash custom tags DefaultHashCustomTags = false + // DefaultHashLogs is the default flag for whether to hash logs DefaultHashLogs = false + // DefaultHashProcess is the default flag for whether to hash process DefaultHashProcess = false + // DefaultMaxSpansCount is the default value of maximum number of spans DefaultMaxSpansCount = -1 ) diff --git a/cmd/anonymizer/app/query/query.go b/cmd/anonymizer/app/query/query.go index f84a347020a..e9b7e3e9328 100644 --- a/cmd/anonymizer/app/query/query.go +++ b/cmd/anonymizer/app/query/query.go @@ -65,14 +65,14 @@ func New(addr string, logger *zap.Logger) (*Query, error) { } // QueryTrace queries for a trace and returns all spans inside it -func (q *Query) QueryTrace(traceId string) []model.Span { - traceID, err := model.TraceIDFromString(traceId) +func (q *Query) QueryTrace(traceID string) []model.Span { + mTraceID, err := model.TraceIDFromString(traceID) if err != nil { q.logger.Fatal("failed to convert the provided trace id", zap.Error(err)) } response, err := q.grpcClient.GetTrace(context.Background(), &api_v2.GetTraceRequest{ - TraceID: traceID, + TraceID: mTraceID, }) if err != nil { q.logger.Fatal("failed to fetch the provided trace id", zap.Error(err)) diff --git a/cmd/anonymizer/app/writer/writer.go b/cmd/anonymizer/app/writer/writer.go index 9b5f6f4c2ef..26eb0746be4 100644 --- a/cmd/anonymizer/app/writer/writer.go +++ b/cmd/anonymizer/app/writer/writer.go @@ -80,7 +80,7 @@ func New(config Config, logger *zap.Logger) (*Writer, error) { return nil, fmt.Errorf("cannot write tp output file: %w", err) } - options := &anonymizer.AnonymizerOptions{ + options := &anonymizer.Options{ HashStandardTags: config.HashStandardTags, HashCustomTags: config.HashCustomTags, HashLogs: config.HashLogs, From c8e2860ed493561791f180f4f041dc400f5d03a1 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 10 Nov 2020 00:47:13 +0800 Subject: [PATCH 08/17] Ran make fmt Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/anonymizer/anonymizer.go | 10 +++++----- cmd/anonymizer/app/options.go | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/anonymizer/app/anonymizer/anonymizer.go b/cmd/anonymizer/app/anonymizer/anonymizer.go index 7777c3b5628..a3beea7d72b 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer.go @@ -53,11 +53,11 @@ type mapping struct { // // The mapping from original to obfuscated strings is stored in a file and can be reused between runs. type Anonymizer struct { - mappingFile string - logger *zap.Logger - lock sync.Mutex - mapping mapping - options *Options + mappingFile string + logger *zap.Logger + lock sync.Mutex + mapping mapping + options *Options } // Options represents the various options with which the anonymizer can be configured. diff --git a/cmd/anonymizer/app/options.go b/cmd/anonymizer/app/options.go index 4cf93a6c2e4..431f0937f34 100644 --- a/cmd/anonymizer/app/options.go +++ b/cmd/anonymizer/app/options.go @@ -18,19 +18,19 @@ import "github.com/jaegertracing/jaeger/ports" const ( // DefaultQueryGRPCHost is the default host for jaeger-query endpoint - DefaultQueryGRPCHost = "localhost" + DefaultQueryGRPCHost = "localhost" // DefaultQueryGRPCPort is the default port for jaeger-query endpoint - DefaultQueryGRPCPort = ports.QueryHTTP + DefaultQueryGRPCPort = ports.QueryHTTP // DefaultOutputDir is the default output directory for spans - DefaultOutputDir = "/tmp" + DefaultOutputDir = "/tmp" // DefaultHashStandardTags is the default flag for whether to hash standard tags DefaultHashStandardTags = true // DefaultHashCustomTags is the default flag for whether to hash custom tags - DefaultHashCustomTags = false + DefaultHashCustomTags = false // DefaultHashLogs is the default flag for whether to hash logs - DefaultHashLogs = false + DefaultHashLogs = false // DefaultHashProcess is the default flag for whether to hash process - DefaultHashProcess = false + DefaultHashProcess = false // DefaultMaxSpansCount is the default value of maximum number of spans - DefaultMaxSpansCount = -1 + DefaultMaxSpansCount = -1 ) From e7c6e34e49238c595b9bd99aeb87b844893a85bc Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 10 Nov 2020 22:01:58 +0800 Subject: [PATCH 09/17] Feedbacks fixes Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/anonymizer/anonymizer.go | 4 +- .../app/anonymizer/anonymizer_test.go | 26 +++---- cmd/anonymizer/app/flags.go | 73 +++++++++---------- cmd/anonymizer/app/options.go | 36 --------- cmd/anonymizer/app/query/.nocover | 1 + cmd/anonymizer/app/query/query.go | 45 ++++-------- cmd/anonymizer/app/writer/writer.go | 23 +++--- cmd/anonymizer/main.go | 31 ++++---- 8 files changed, 91 insertions(+), 148 deletions(-) delete mode 100644 cmd/anonymizer/app/options.go create mode 100644 cmd/anonymizer/app/query/.nocover diff --git a/cmd/anonymizer/app/anonymizer/anonymizer.go b/cmd/anonymizer/app/anonymizer/anonymizer.go index a3beea7d72b..3a806f4d063 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer.go @@ -57,7 +57,7 @@ type Anonymizer struct { logger *zap.Logger lock sync.Mutex mapping mapping - options *Options + options Options } // Options represents the various options with which the anonymizer can be configured. @@ -70,7 +70,7 @@ type Options struct { // New creates new Anonymizer. The mappingFile stores the mapping from original to // obfuscated strings, in case later investigations require looking at the original traces. -func New(mappingFile string, options *Options, logger *zap.Logger) *Anonymizer { +func New(mappingFile string, options Options, logger *zap.Logger) *Anonymizer { a := &Anonymizer{ mappingFile: mappingFile, logger: logger, diff --git a/cmd/anonymizer/app/anonymizer/anonymizer_test.go b/cmd/anonymizer/app/anonymizer/anonymizer_test.go index b76ba64b857..94e35f7903f 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer_test.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer_test.go @@ -98,18 +98,17 @@ func TestAnonymizer_Hash(t *testing.T) { } func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) { - options := &Options{ - HashStandardTags: true, - HashCustomTags: true, - HashProcess: true, - HashLogs: true, - } anonymizer := &Anonymizer{ mapping: mapping{ Services: make(map[string]string), Operations: make(map[string]string), }, - options: options, + options: Options{ + HashStandardTags: true, + HashCustomTags: true, + HashProcess: true, + HashLogs: true, + }, } _ = anonymizer.AnonymizeSpan(span1) assert.Equal(t, 3, len(span1.Tags)) @@ -118,18 +117,17 @@ func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) { } func TestAnonymizer_AnonymizeSpan_AllFalse(t *testing.T) { - options := &Options{ - HashStandardTags: false, - HashCustomTags: false, - HashProcess: false, - HashLogs: false, - } anonymizer := &Anonymizer{ mapping: mapping{ Services: make(map[string]string), Operations: make(map[string]string), }, - options: options, + options: Options{ + HashStandardTags: false, + HashCustomTags: false, + HashProcess: false, + HashLogs: false, + }, } _ = anonymizer.AnonymizeSpan(span2) assert.Equal(t, 2, len(span2.Tags)) diff --git a/cmd/anonymizer/app/flags.go b/cmd/anonymizer/app/flags.go index 0903998473f..91382c15a47 100644 --- a/cmd/anonymizer/app/flags.go +++ b/cmd/anonymizer/app/flags.go @@ -20,76 +20,69 @@ import ( // Options represent configurable parameters for jaeger-anonymizer type Options struct { - QueryGRPCPort int - QueryGRPCHost string - MaxSpansCount int - TraceID string - OutputDir string - HashStandardTags bool - HashCustomTags bool - HashLogs bool - HashProcess bool + QueryGRPCHostPort string + MaxSpansCount int + TraceID string + OutputDir string + HashStandardTags bool + HashCustomTags bool + HashLogs bool + HashProcess bool } const ( - queryGRPCHostFlag = "query-host" - queryGRPCPortFlag = "query-port" - outputDirFlag = "output-dir" - traceIDFlag = "trace-id" - hashStandardTagsFlag = "hash-standard-tags" - hashCustomTagsFlag = "hash-custom-tags" - hashLogsFlag = "hash-logs" - hashProcessFlag = "hash-process" - maxSpansCount = "max-spans-count" + queryGRPCHostPortFlag = "query-host-port" + outputDirFlag = "output-dir" + traceIDFlag = "trace-id" + hashStandardTagsFlag = "hash-standard-tags" + hashCustomTagsFlag = "hash-custom-tags" + hashLogsFlag = "hash-logs" + hashProcessFlag = "hash-process" + maxSpansCount = "max-spans-count" ) // AddFlags adds flags for anonymizer main program func (o *Options) AddFlags(command *cobra.Command) { command.Flags().StringVar( - &o.QueryGRPCHost, - queryGRPCHostFlag, - DefaultQueryGRPCHost, - "hostname of the jaeger-query endpoint") - command.Flags().IntVar( - &o.QueryGRPCPort, - queryGRPCPortFlag, - DefaultQueryGRPCPort, - "port of the jaeger-query endpoint") + &o.QueryGRPCHostPort, + queryGRPCHostPortFlag, + "localhost:16686", + "The host:port of the jaeger-query endpoint") command.Flags().StringVar( &o.OutputDir, outputDirFlag, - DefaultOutputDir, - "directory to store the anonymized trace") + "/tmp", + "The directory to store the anonymized trace") command.Flags().StringVar( &o.TraceID, traceIDFlag, "", - "trace-id of trace to anonymize") + "The trace-id of trace to anonymize") command.Flags().BoolVar( &o.HashStandardTags, hashStandardTagsFlag, - DefaultHashStandardTags, - "whether to hash standard tags") + false, + "Whether to hash standard tags") command.Flags().BoolVar( &o.HashCustomTags, hashCustomTagsFlag, - DefaultHashCustomTags, - "whether to hash custom tags") + false, + "Whether to hash custom tags") command.Flags().BoolVar( &o.HashLogs, hashLogsFlag, - DefaultHashLogs, - "whether to hash logs") + false, + "Whether to hash logs") command.Flags().BoolVar( &o.HashProcess, hashProcessFlag, - DefaultHashProcess, - "whether to hash process") + false, + "Whether to hash process") command.Flags().IntVar( &o.MaxSpansCount, maxSpansCount, - DefaultMaxSpansCount, - "maximum number of spans to anonymize") + -1, + "The maximum number of spans to anonymize") // mark traceid flag as mandatory command.MarkFlagRequired(traceIDFlag) diff --git a/cmd/anonymizer/app/options.go b/cmd/anonymizer/app/options.go deleted file mode 100644 index 431f0937f34..00000000000 --- a/cmd/anonymizer/app/options.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2020 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package app - -import "github.com/jaegertracing/jaeger/ports" - -const ( - // DefaultQueryGRPCHost is the default host for jaeger-query endpoint - DefaultQueryGRPCHost = "localhost" - // DefaultQueryGRPCPort is the default port for jaeger-query endpoint - DefaultQueryGRPCPort = ports.QueryHTTP - // DefaultOutputDir is the default output directory for spans - DefaultOutputDir = "/tmp" - // DefaultHashStandardTags is the default flag for whether to hash standard tags - DefaultHashStandardTags = true - // DefaultHashCustomTags is the default flag for whether to hash custom tags - DefaultHashCustomTags = false - // DefaultHashLogs is the default flag for whether to hash logs - DefaultHashLogs = false - // DefaultHashProcess is the default flag for whether to hash process - DefaultHashProcess = false - // DefaultMaxSpansCount is the default value of maximum number of spans - DefaultMaxSpansCount = -1 -) diff --git a/cmd/anonymizer/app/query/.nocover b/cmd/anonymizer/app/query/.nocover new file mode 100644 index 00000000000..5b583b79e93 --- /dev/null +++ b/cmd/anonymizer/app/query/.nocover @@ -0,0 +1 @@ +non-critical test utility diff --git a/cmd/anonymizer/app/query/query.go b/cmd/anonymizer/app/query/query.go index e9b7e3e9328..93c8a0ebd16 100644 --- a/cmd/anonymizer/app/query/query.go +++ b/cmd/anonymizer/app/query/query.go @@ -16,6 +16,7 @@ package query import ( "context" + "fmt" "time" "go.uber.org/zap" @@ -26,66 +27,52 @@ import ( "github.com/jaegertracing/jaeger/storage/spanstore" ) -// GrpcClient represents a grpc-client to jaeger-query -type GrpcClient struct { - api_v2.QueryServiceClient - conn *grpc.ClientConn -} - // Query represents a jaeger-query's query for trace-id type Query struct { - grpcClient *GrpcClient - logger *zap.Logger + api_v2.QueryServiceClient + conn *grpc.ClientConn + logger *zap.Logger } -// newGRPCClient returns a new GrpcClient -func newGRPCClient(addr string, logger *zap.Logger) *GrpcClient { +// New creates a Query object +func New(addr string, logger *zap.Logger) (*Query, error) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure()) if err != nil { - logger.Fatal("failed to connect with the jaeger-query service", zap.Error(err)) + return nil, fmt.Errorf("failed to connect with the jaeger-query service: %w", err) } - return &GrpcClient{ + return &Query{ QueryServiceClient: api_v2.NewQueryServiceClient(conn), conn: conn, - } -} - -// New creates a Query object -func New(addr string, logger *zap.Logger) (*Query, error) { - client := newGRPCClient(addr, logger) - - return &Query{ - grpcClient: client, - logger: logger, + logger: logger, }, nil } // QueryTrace queries for a trace and returns all spans inside it -func (q *Query) QueryTrace(traceID string) []model.Span { +func (q *Query) QueryTrace(traceID string) ([]model.Span, error) { mTraceID, err := model.TraceIDFromString(traceID) if err != nil { - q.logger.Fatal("failed to convert the provided trace id", zap.Error(err)) + return nil, fmt.Errorf("failed to convert the provided trace id: %w", err) } - response, err := q.grpcClient.GetTrace(context.Background(), &api_v2.GetTraceRequest{ + response, err := q.GetTrace(context.Background(), &api_v2.GetTraceRequest{ TraceID: mTraceID, }) if err != nil { - q.logger.Fatal("failed to fetch the provided trace id", zap.Error(err)) + return nil, fmt.Errorf("failed to fetch the provided trace id: %w", err) } spanResponseChunk, err := response.Recv() if err == spanstore.ErrTraceNotFound { - q.logger.Fatal("failed to find the provided trace id", zap.Error(err)) + return nil, fmt.Errorf("failed to find the provided trace id: %w", err) } if err != nil { - q.logger.Fatal("failed to fetch spans of provided trace id", zap.Error(err)) + return nil, fmt.Errorf("failed to fetch spans of provided trace id: %w", err) } spans := spanResponseChunk.GetSpans() - return spans + return spans, nil } diff --git a/cmd/anonymizer/app/writer/writer.go b/cmd/anonymizer/app/writer/writer.go index 26eb0746be4..6193fff77d8 100644 --- a/cmd/anonymizer/app/writer/writer.go +++ b/cmd/anonymizer/app/writer/writer.go @@ -30,14 +30,11 @@ import ( // Config contains parameters to NewWriter. type Config struct { - MaxSpansCount int `yaml:"max_spans_count" name:"max_spans_count"` - CapturedFile string `yaml:"captured_file" name:"captured_file"` - AnonymizedFile string `yaml:"anonymized_file" name:"anonymized_file"` - MappingFile string `yaml:"mapping_file" name:"mapping_file"` - HashStandardTags bool `yaml:"hash_standard_tags" name:"hash_standard_tags"` - HashCustomTags bool `yaml:"hash_custom_tags" name:"hash_custom_tags"` - HashLogs bool `yaml:"hash_logs" name:"hash_logs"` - HashProcess bool `yaml:"hash_process" name:"hash_process"` + MaxSpansCount int `yaml:"max_spans_count" name:"max_spans_count"` + CapturedFile string `yaml:"captured_file" name:"captured_file"` + AnonymizedFile string `yaml:"anonymized_file" name:"anonymized_file"` + MappingFile string `yaml:"mapping_file" name:"mapping_file"` + AnonymizerOpts anonymizer.Options } // Writer is a span Writer that obfuscates the span and writes it to a JSON file. @@ -80,11 +77,11 @@ func New(config Config, logger *zap.Logger) (*Writer, error) { return nil, fmt.Errorf("cannot write tp output file: %w", err) } - options := &anonymizer.Options{ - HashStandardTags: config.HashStandardTags, - HashCustomTags: config.HashCustomTags, - HashLogs: config.HashLogs, - HashProcess: config.HashProcess, + options := anonymizer.Options{ + HashStandardTags: config.AnonymizerOpts.HashStandardTags, + HashCustomTags: config.AnonymizerOpts.HashCustomTags, + HashLogs: config.AnonymizerOpts.HashLogs, + HashProcess: config.AnonymizerOpts.HashProcess, } return &Writer{ diff --git a/cmd/anonymizer/main.go b/cmd/anonymizer/main.go index e31b6c683c0..fb23c328d3f 100644 --- a/cmd/anonymizer/main.go +++ b/cmd/anonymizer/main.go @@ -17,12 +17,12 @@ package main import ( "fmt" "os" - "strconv" "github.com/spf13/cobra" "go.uber.org/zap" app "github.com/jaegertracing/jaeger/cmd/anonymizer/app" + "github.com/jaegertracing/jaeger/cmd/anonymizer/app/anonymizer" query "github.com/jaegertracing/jaeger/cmd/anonymizer/app/query" writer "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" "github.com/jaegertracing/jaeger/pkg/version" @@ -40,14 +40,16 @@ func main() { Run: func(cmd *cobra.Command, args []string) { prefix := options.OutputDir + "/" + options.TraceID conf := writer.Config{ - MaxSpansCount: options.MaxSpansCount, - CapturedFile: prefix + ".original", - AnonymizedFile: prefix + ".anonymized", - MappingFile: prefix + ".mapping", - HashStandardTags: options.HashStandardTags, - HashCustomTags: options.HashCustomTags, - HashLogs: options.HashLogs, - HashProcess: options.HashProcess, + MaxSpansCount: options.MaxSpansCount, + CapturedFile: prefix + ".original", + AnonymizedFile: prefix + ".anonymized", + MappingFile: prefix + ".mapping", + AnonymizerOpts: anonymizer.Options{ + HashStandardTags: options.HashStandardTags, + HashCustomTags: options.HashCustomTags, + HashLogs: options.HashLogs, + HashProcess: options.HashProcess, + }, } writer, err := writer.New(conf, logger) @@ -55,15 +57,16 @@ func main() { logger.Error("error while creating writer object", zap.Error(err)) } - queryEndpoint := options.QueryGRPCHost + ":" + strconv.Itoa(options.QueryGRPCPort) - logger.Info(queryEndpoint) - - query, err := query.New(queryEndpoint, logger) + query, err := query.New(options.QueryGRPCHostPort, logger) if err != nil { logger.Error("error while creating query object", zap.Error(err)) } - spans := query.QueryTrace(options.TraceID) + spans, err := query.QueryTrace(options.TraceID) + if err != nil { + logger.Error("error while querying for trace", zap.Error(err)) + } + for _, span := range spans { writer.WriteSpan(&span) } From 4f9ed2f99a9027ec0b7545991214d38eed0a8bfb Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 10 Nov 2020 23:38:26 +0800 Subject: [PATCH 10/17] Added empty_test.go Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/query/empty_test.go | 15 +++++++++++++++ cmd/anonymizer/app/writer/empty_test.go | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 cmd/anonymizer/app/query/empty_test.go create mode 100644 cmd/anonymizer/app/writer/empty_test.go diff --git a/cmd/anonymizer/app/query/empty_test.go b/cmd/anonymizer/app/query/empty_test.go new file mode 100644 index 00000000000..54500b058f4 --- /dev/null +++ b/cmd/anonymizer/app/query/empty_test.go @@ -0,0 +1,15 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query diff --git a/cmd/anonymizer/app/writer/empty_test.go b/cmd/anonymizer/app/writer/empty_test.go new file mode 100644 index 00000000000..04b3865ffdf --- /dev/null +++ b/cmd/anonymizer/app/writer/empty_test.go @@ -0,0 +1,15 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package writer From 13fbf94c632f313117639b5f4e5c97e2f95ea2cc Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 10 Nov 2020 23:42:08 +0800 Subject: [PATCH 11/17] Added empty_test.go Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/empty_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 cmd/anonymizer/app/empty_test.go diff --git a/cmd/anonymizer/app/empty_test.go b/cmd/anonymizer/app/empty_test.go new file mode 100644 index 00000000000..31115ecbf4c --- /dev/null +++ b/cmd/anonymizer/app/empty_test.go @@ -0,0 +1,15 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app From b7102d9ec7438f5ba947ddbf12f475044c795b48 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Wed, 11 Nov 2020 01:09:49 +0800 Subject: [PATCH 12/17] Added flags_test.go Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/empty_test.go | 15 -------- cmd/anonymizer/app/flags_test.go | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 15 deletions(-) delete mode 100644 cmd/anonymizer/app/empty_test.go create mode 100644 cmd/anonymizer/app/flags_test.go diff --git a/cmd/anonymizer/app/empty_test.go b/cmd/anonymizer/app/empty_test.go deleted file mode 100644 index 31115ecbf4c..00000000000 --- a/cmd/anonymizer/app/empty_test.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2020 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package app diff --git a/cmd/anonymizer/app/flags_test.go b/cmd/anonymizer/app/flags_test.go new file mode 100644 index 00000000000..ae659259b92 --- /dev/null +++ b/cmd/anonymizer/app/flags_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func TestOptionsWithDefaultFlags(t *testing.T) { + o := Options{} + c := cobra.Command{} + o.AddFlags(&c) + + assert.Equal(t, "localhost:16686", o.QueryGRPCHostPort) + assert.Equal(t, "/tmp", o.OutputDir) + assert.Equal(t, false, o.HashStandardTags) + assert.Equal(t, false, o.HashCustomTags) + assert.Equal(t, false, o.HashLogs) + assert.Equal(t, false, o.HashProcess) + assert.Equal(t, -1, o.MaxSpansCount) +} + +func TestOptionsWithFlags(t *testing.T) { + o := Options{} + c := cobra.Command{} + + o.AddFlags(&c) + c.ParseFlags([]string{ + "--query-host-port=192.168.1.10:16686", + "--output-dir=/data", + "--trace-id=6ef2debb698f2f7c", + "--hash-standard-tags", + "--hash-custom-tags", + "--hash-logs", + "--hash-process", + "--max-spans-count=100", + }) + + assert.Equal(t, "192.168.1.10:16686", o.QueryGRPCHostPort) + assert.Equal(t, "/data", o.OutputDir) + assert.Equal(t, true, o.HashStandardTags) + assert.Equal(t, true, o.HashCustomTags) + assert.Equal(t, true, o.HashLogs) + assert.Equal(t, true, o.HashProcess) + assert.Equal(t, 100, o.MaxSpansCount) +} From f660de3808662c124d1aac1920e9cd869dd4dfd3 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Wed, 11 Nov 2020 01:12:21 +0800 Subject: [PATCH 13/17] Updated flags_test.go Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/flags_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/anonymizer/app/flags_test.go b/cmd/anonymizer/app/flags_test.go index ae659259b92..80790f15c15 100644 --- a/cmd/anonymizer/app/flags_test.go +++ b/cmd/anonymizer/app/flags_test.go @@ -53,6 +53,7 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, "192.168.1.10:16686", o.QueryGRPCHostPort) assert.Equal(t, "/data", o.OutputDir) + assert.Equal(t, "6ef2debb698f2f7c", o.TraceID) assert.Equal(t, true, o.HashStandardTags) assert.Equal(t, true, o.HashCustomTags) assert.Equal(t, true, o.HashLogs) From ba246185c65c21c47167d23d132c1f8ada2cd44b Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Wed, 11 Nov 2020 07:29:58 +0800 Subject: [PATCH 14/17] Remove empty_test.go files Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/query/empty_test.go | 15 --------------- cmd/anonymizer/app/writer/empty_test.go | 15 --------------- 2 files changed, 30 deletions(-) delete mode 100644 cmd/anonymizer/app/query/empty_test.go delete mode 100644 cmd/anonymizer/app/writer/empty_test.go diff --git a/cmd/anonymizer/app/query/empty_test.go b/cmd/anonymizer/app/query/empty_test.go deleted file mode 100644 index 54500b058f4..00000000000 --- a/cmd/anonymizer/app/query/empty_test.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2020 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query diff --git a/cmd/anonymizer/app/writer/empty_test.go b/cmd/anonymizer/app/writer/empty_test.go deleted file mode 100644 index 04b3865ffdf..00000000000 --- a/cmd/anonymizer/app/writer/empty_test.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2020 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package writer From cf23b2cce46c5e580b7784336cb7a80a94c90f61 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Wed, 11 Nov 2020 20:38:20 +0800 Subject: [PATCH 15/17] Feedbacks fixes Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/anonymizer/anonymizer.go | 8 ++--- cmd/anonymizer/app/query/query.go | 36 ++++++++++++--------- cmd/anonymizer/app/writer/writer.go | 16 ++++++--- cmd/anonymizer/main.go | 9 +++--- 4 files changed, 41 insertions(+), 28 deletions(-) diff --git a/cmd/anonymizer/app/anonymizer/anonymizer.go b/cmd/anonymizer/app/anonymizer/anonymizer.go index 3a806f4d063..13d8dd17737 100644 --- a/cmd/anonymizer/app/anonymizer/anonymizer.go +++ b/cmd/anonymizer/app/anonymizer/anonymizer.go @@ -62,10 +62,10 @@ type Anonymizer struct { // Options represents the various options with which the anonymizer can be configured. type Options struct { - HashStandardTags bool - HashCustomTags bool - HashLogs bool - HashProcess bool + HashStandardTags bool `yaml:"hash_standard_tags" name:"hash_standard_tags"` + HashCustomTags bool `yaml:"hash_custom_tags" name:"hash_custom_tags"` + HashLogs bool `yaml:"hash_logs" name:"hash_logs"` + HashProcess bool `yaml:"hash_process" name:"hash_process"` } // New creates new Anonymizer. The mappingFile stores the mapping from original to diff --git a/cmd/anonymizer/app/query/query.go b/cmd/anonymizer/app/query/query.go index 93c8a0ebd16..a4f12959bb3 100644 --- a/cmd/anonymizer/app/query/query.go +++ b/cmd/anonymizer/app/query/query.go @@ -17,10 +17,11 @@ package query import ( "context" "fmt" + "io" "time" - "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/status" "github.com/jaegertracing/jaeger/model" "github.com/jaegertracing/jaeger/proto-gen/api_v2" @@ -29,13 +30,12 @@ import ( // Query represents a jaeger-query's query for trace-id type Query struct { - api_v2.QueryServiceClient + client api_v2.QueryServiceClient conn *grpc.ClientConn - logger *zap.Logger } // New creates a Query object -func New(addr string, logger *zap.Logger) (*Query, error) { +func New(addr string) (*Query, error) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() @@ -45,9 +45,8 @@ func New(addr string, logger *zap.Logger) (*Query, error) { } return &Query{ - QueryServiceClient: api_v2.NewQueryServiceClient(conn), - conn: conn, - logger: logger, + client: api_v2.NewQueryServiceClient(conn), + conn: conn, }, nil } @@ -58,21 +57,28 @@ func (q *Query) QueryTrace(traceID string) ([]model.Span, error) { return nil, fmt.Errorf("failed to convert the provided trace id: %w", err) } - response, err := q.GetTrace(context.Background(), &api_v2.GetTraceRequest{ + stream, err := q.client.GetTrace(context.Background(), &api_v2.GetTraceRequest{ TraceID: mTraceID, }) if err != nil { return nil, fmt.Errorf("failed to fetch the provided trace id: %w", err) } - spanResponseChunk, err := response.Recv() - if err == spanstore.ErrTraceNotFound { - return nil, fmt.Errorf("failed to find the provided trace id: %w", err) - } - if err != nil { - return nil, fmt.Errorf("failed to fetch spans of provided trace id: %w", err) + var spans []model.Span + for received, err := stream.Recv(); err != io.EOF; received, err = stream.Recv() { + if err != nil { + if s, _ := status.FromError(err); s != nil { + if s.Message() == spanstore.ErrTraceNotFound.Error() { + return nil, spanstore.ErrTraceNotFound + } + } + return nil, fmt.Errorf("grpc stream error: %w", err) + } + + for i := range received.Spans { + spans = append(spans, received.Spans[i]) + } } - spans := spanResponseChunk.GetSpans() return spans, nil } diff --git a/cmd/anonymizer/app/writer/writer.go b/cmd/anonymizer/app/writer/writer.go index 6193fff77d8..327e6214d53 100644 --- a/cmd/anonymizer/app/writer/writer.go +++ b/cmd/anonymizer/app/writer/writer.go @@ -30,11 +30,11 @@ import ( // Config contains parameters to NewWriter. type Config struct { - MaxSpansCount int `yaml:"max_spans_count" name:"max_spans_count"` - CapturedFile string `yaml:"captured_file" name:"captured_file"` - AnonymizedFile string `yaml:"anonymized_file" name:"anonymized_file"` - MappingFile string `yaml:"mapping_file" name:"mapping_file"` - AnonymizerOpts anonymizer.Options + MaxSpansCount int `yaml:"max_spans_count" name:"max_spans_count"` + CapturedFile string `yaml:"captured_file" name:"captured_file"` + AnonymizedFile string `yaml:"anonymized_file" name:"anonymized_file"` + MappingFile string `yaml:"mapping_file" name:"mapping_file"` + AnonymizerOpts anonymizer.Options `yaml:"anonymizer_opts" name:"anonymizer_opts"` } // Writer is a span Writer that obfuscates the span and writes it to a JSON file. @@ -139,3 +139,9 @@ func (w *Writer) WriteSpan(msg *model.Span) error { return nil } + +// Close closes the captured and anonymized files. +func (w *Writer) Close() { + w.capturedFile.Close() + w.anonymizedFile.Close() +} diff --git a/cmd/anonymizer/main.go b/cmd/anonymizer/main.go index fb23c328d3f..a4dd00e4a9f 100644 --- a/cmd/anonymizer/main.go +++ b/cmd/anonymizer/main.go @@ -54,22 +54,23 @@ func main() { writer, err := writer.New(conf, logger) if err != nil { - logger.Error("error while creating writer object", zap.Error(err)) + logger.Fatal("error while creating writer object", zap.Error(err)) } - query, err := query.New(options.QueryGRPCHostPort, logger) + query, err := query.New(options.QueryGRPCHostPort) if err != nil { - logger.Error("error while creating query object", zap.Error(err)) + logger.Fatal("error while creating query object", zap.Error(err)) } spans, err := query.QueryTrace(options.TraceID) if err != nil { - logger.Error("error while querying for trace", zap.Error(err)) + logger.Fatal("error while querying for trace", zap.Error(err)) } for _, span := range spans { writer.WriteSpan(&span) } + writer.Close() }, } From 94b19498fb4b05d0c7fa8b7cf3f0901ad0f82330 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Thu, 12 Nov 2020 00:42:13 +0800 Subject: [PATCH 16/17] Feedbacks fixes Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/query/query.go | 20 ++++++++++++-------- cmd/anonymizer/app/writer/writer.go | 10 ++++------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/cmd/anonymizer/app/query/query.go b/cmd/anonymizer/app/query/query.go index a4f12959bb3..c6b261bde2e 100644 --- a/cmd/anonymizer/app/query/query.go +++ b/cmd/anonymizer/app/query/query.go @@ -50,6 +50,16 @@ func New(addr string) (*Query, error) { }, nil } +// isNotFoundErr is a helper function called when the trace isn't found +func isNotFoundErr(err error) ([]model.Span, error) { + if s, _ := status.FromError(err); s != nil { + if s.Message() == spanstore.ErrTraceNotFound.Error() { + return nil, spanstore.ErrTraceNotFound + } + } + return nil, fmt.Errorf("failed to fetch the provided trace id: %w", err) +} + // QueryTrace queries for a trace and returns all spans inside it func (q *Query) QueryTrace(traceID string) ([]model.Span, error) { mTraceID, err := model.TraceIDFromString(traceID) @@ -61,20 +71,14 @@ func (q *Query) QueryTrace(traceID string) ([]model.Span, error) { TraceID: mTraceID, }) if err != nil { - return nil, fmt.Errorf("failed to fetch the provided trace id: %w", err) + isNotFoundErr(err) } var spans []model.Span for received, err := stream.Recv(); err != io.EOF; received, err = stream.Recv() { if err != nil { - if s, _ := status.FromError(err); s != nil { - if s.Message() == spanstore.ErrTraceNotFound.Error() { - return nil, spanstore.ErrTraceNotFound - } - } - return nil, fmt.Errorf("grpc stream error: %w", err) + isNotFoundErr(err) } - for i := range received.Spans { spans = append(spans, received.Spans[i]) } diff --git a/cmd/anonymizer/app/writer/writer.go b/cmd/anonymizer/app/writer/writer.go index 327e6214d53..b8f1ee338d5 100644 --- a/cmd/anonymizer/app/writer/writer.go +++ b/cmd/anonymizer/app/writer/writer.go @@ -129,12 +129,6 @@ func (w *Writer) WriteSpan(msg *model.Span) error { if w.spanCount >= w.config.MaxSpansCount { w.logger.Info("Saved enough spans, exiting...") - w.capturedFile.WriteString("\n]\n") - w.capturedFile.Close() - w.anonymizedFile.WriteString("\n]\n") - w.anonymizedFile.Close() - w.anonymizer.SaveMapping() - os.Exit(0) } return nil @@ -142,6 +136,10 @@ func (w *Writer) WriteSpan(msg *model.Span) error { // Close closes the captured and anonymized files. func (w *Writer) Close() { + w.capturedFile.WriteString("\n]\n") w.capturedFile.Close() + w.anonymizedFile.WriteString("\n]\n") w.anonymizedFile.Close() + w.anonymizer.SaveMapping() + os.Exit(0) } From 854ad35335d2fa1b6e10fe21cfc558370faf79fd Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Thu, 12 Nov 2020 01:11:26 +0800 Subject: [PATCH 17/17] Feedbacks fixes Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/query/query.go | 12 ++++++------ cmd/anonymizer/app/writer/writer.go | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/anonymizer/app/query/query.go b/cmd/anonymizer/app/query/query.go index c6b261bde2e..ffbf2126d9e 100644 --- a/cmd/anonymizer/app/query/query.go +++ b/cmd/anonymizer/app/query/query.go @@ -50,14 +50,14 @@ func New(addr string) (*Query, error) { }, nil } -// isNotFoundErr is a helper function called when the trace isn't found -func isNotFoundErr(err error) ([]model.Span, error) { +// unwrapNotFoundErr is a conversion function +func unwrapNotFoundErr(err error) error { if s, _ := status.FromError(err); s != nil { if s.Message() == spanstore.ErrTraceNotFound.Error() { - return nil, spanstore.ErrTraceNotFound + return spanstore.ErrTraceNotFound } } - return nil, fmt.Errorf("failed to fetch the provided trace id: %w", err) + return err } // QueryTrace queries for a trace and returns all spans inside it @@ -71,13 +71,13 @@ func (q *Query) QueryTrace(traceID string) ([]model.Span, error) { TraceID: mTraceID, }) if err != nil { - isNotFoundErr(err) + return nil, unwrapNotFoundErr(err) } var spans []model.Span for received, err := stream.Recv(); err != io.EOF; received, err = stream.Recv() { if err != nil { - isNotFoundErr(err) + return nil, unwrapNotFoundErr(err) } for i := range received.Spans { spans = append(spans, received.Spans[i]) diff --git a/cmd/anonymizer/app/writer/writer.go b/cmd/anonymizer/app/writer/writer.go index b8f1ee338d5..5d04e61df53 100644 --- a/cmd/anonymizer/app/writer/writer.go +++ b/cmd/anonymizer/app/writer/writer.go @@ -34,7 +34,7 @@ type Config struct { CapturedFile string `yaml:"captured_file" name:"captured_file"` AnonymizedFile string `yaml:"anonymized_file" name:"anonymized_file"` MappingFile string `yaml:"mapping_file" name:"mapping_file"` - AnonymizerOpts anonymizer.Options `yaml:"anonymizer_opts" name:"anonymizer_opts"` + AnonymizerOpts anonymizer.Options `yaml:"anonymizer" name:"anonymizer"` } // Writer is a span Writer that obfuscates the span and writes it to a JSON file. @@ -129,6 +129,8 @@ func (w *Writer) WriteSpan(msg *model.Span) error { if w.spanCount >= w.config.MaxSpansCount { w.logger.Info("Saved enough spans, exiting...") + w.Close() + os.Exit(0) } return nil @@ -141,5 +143,4 @@ func (w *Writer) Close() { w.anonymizedFile.WriteString("\n]\n") w.anonymizedFile.Close() w.anonymizer.SaveMapping() - os.Exit(0) }