Implement anonymizer's main program

.gitignore

@@ -18,6 +18,8 @@ examples/memstore-plugin/memstore-plugin
 cmd/all-in-one/all-in-one-*
 cmd/agent/agent
 cmd/agent/agent-*
+cmd/anonymizer/anonymizer
+cmd/anonymizer/anonymizer-*
 cmd/collector/collector
 cmd/collector/collector-*
 cmd/ingester/ingester

Makefile

@@ -250,6 +250,10 @@ build-all-in-one build-all-in-one-debug: build-ui elasticsearch-mappings
 build-agent build-agent-debug:
 	$(GOBUILD) $(DISABLE_OPTIMIZATIONS) -o ./cmd/agent/agent$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/agent/main.go
 
+.PHONY: build-anonymizer
+build-anonymizer:
+	$(GOBUILD) $(DISABLE_OPTIMIZATIONS) -o ./cmd/anonymizer/anonymizer$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/anonymizer/main.go
+
 .PHONY: build-query build-query-debug
 build-query build-query-debug: build-ui
 	$(GOBUILD) $(DISABLE_OPTIMIZATIONS) -tags ui -o ./cmd/query/query$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/query/main.go

cmd/anonymizer/app/anonymizer/anonymizer.go

@@ -53,26 +53,32 @@ type mapping struct {
 //
 // The mapping from original to obfuscated strings is stored in a file and can be reused between runs.
 type Anonymizer struct {
-	mappingFile      string
-	logger           *zap.Logger
-	lock             sync.Mutex
-	mapping          mapping
-	hashStandardTags bool
-	hashCustomTags   bool
-	hashLogs         bool
-	hashProcess      bool
+	mappingFile string
+	logger      *zap.Logger
+	lock        sync.Mutex
+	mapping     mapping
+	options     Options
+}
+
+// Options represents the various options with which the anonymizer can be configured.
+type Options struct {
+	HashStandardTags bool `yaml:"hash_standard_tags" name:"hash_standard_tags"`
+	HashCustomTags   bool `yaml:"hash_custom_tags" name:"hash_custom_tags"`
+	HashLogs         bool `yaml:"hash_logs" name:"hash_logs"`
+	HashProcess      bool `yaml:"hash_process" name:"hash_process"`
 }
 
 // New creates new Anonymizer. The mappingFile stores the mapping from original to
 // obfuscated strings, in case later investigations require looking at the original traces.
-func New(mappingFile string, logger *zap.Logger) *Anonymizer {
+func New(mappingFile string, options Options, logger *zap.Logger) *Anonymizer {
 	a := &Anonymizer{
 		mappingFile: mappingFile,
 		logger:      logger,
 		mapping: mapping{
 			Services:   make(map[string]string),
 			Operations: make(map[string]string),
 		},
+		options: options,
 	}
 	if _, err := os.Stat(filepath.Clean(mappingFile)); err == nil {
 		dat, err := ioutil.ReadFile(filepath.Clean(mappingFile))
@@ -142,18 +148,18 @@ func (a *Anonymizer) AnonymizeSpan(span *model.Span) *uimodel.Span {
 
 	outputTags := filterStandardTags(span.Tags)
 	// when true, the allowedTags are hashed and when false they are preserved as it is
-	if a.hashStandardTags {
+	if a.options.HashStandardTags {
 		outputTags = hashTags(outputTags)
 	}
 	// when true, all tags other than allowedTags are hashed, when false they are dropped
-	if a.hashCustomTags {
+	if a.options.HashCustomTags {
 		customTags := hashTags(filterCustomTags(span.Tags))
 		outputTags = append(outputTags, customTags...)
 	}
 	span.Tags = outputTags
 
 	// when true, logs are hashed, when false, they are dropped
-	if a.hashLogs {
+	if a.options.HashLogs {
 		for _, log := range span.Logs {
 			log.Fields = hashTags(log.Fields)
 		}
@@ -164,7 +170,7 @@ func (a *Anonymizer) AnonymizeSpan(span *model.Span) *uimodel.Span {
 	span.Process.ServiceName = a.mapServiceName(service)
 
 	// when true, process tags are hashed, when false they are dropped
-	if a.hashProcess {
+	if a.options.HashProcess {
 		span.Process.Tags = hashTags(span.Process.Tags)
 	} else {
 		span.Process.Tags = nil

cmd/anonymizer/app/anonymizer/anonymizer_test.go

@@ -103,10 +103,12 @@ func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) {
 			Services:   make(map[string]string),
 			Operations: make(map[string]string),
 		},
-		hashStandardTags: true,
-		hashCustomTags:   true,
-		hashProcess:      true,
-		hashLogs:         true,
+		options: Options{
+			HashStandardTags: true,
+			HashCustomTags:   true,
+			HashProcess:      true,
+			HashLogs:         true,
+		},
 	}
 	_ = anonymizer.AnonymizeSpan(span1)
 	assert.Equal(t, 3, len(span1.Tags))
@@ -120,10 +122,12 @@ func TestAnonymizer_AnonymizeSpan_AllFalse(t *testing.T) {
 			Services:   make(map[string]string),
 			Operations: make(map[string]string),
 		},
-		hashStandardTags: false,
-		hashCustomTags:   false,
-		hashProcess:      false,
-		hashLogs:         false,
+		options: Options{
+			HashStandardTags: false,
+			HashCustomTags:   false,
+			HashProcess:      false,
+			HashLogs:         false,
+		},
 	}
 	_ = anonymizer.AnonymizeSpan(span2)
 	assert.Equal(t, 2, len(span2.Tags))

cmd/anonymizer/app/flags.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2020 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package app
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// Options represent configurable parameters for jaeger-anonymizer
+type Options struct {
+	QueryGRPCHostPort string
+	MaxSpansCount     int
+	TraceID           string
+	OutputDir         string
+	HashStandardTags  bool
+	HashCustomTags    bool
+	HashLogs          bool
+	HashProcess       bool
+}
+
+const (
+	queryGRPCHostPortFlag = "query-host-port"
+	outputDirFlag         = "output-dir"
+	traceIDFlag           = "trace-id"
+	hashStandardTagsFlag  = "hash-standard-tags"
+	hashCustomTagsFlag    = "hash-custom-tags"
+	hashLogsFlag          = "hash-logs"
+	hashProcessFlag       = "hash-process"
+	maxSpansCount         = "max-spans-count"
+)
+
+// AddFlags adds flags for anonymizer main program
+func (o *Options) AddFlags(command *cobra.Command) {
+	command.Flags().StringVar(
+		&o.QueryGRPCHostPort,
+		queryGRPCHostPortFlag,
+		"localhost:16686",
+		"The host:port of the jaeger-query endpoint")
+	command.Flags().StringVar(
+		&o.OutputDir,
+		outputDirFlag,
+		"/tmp",
+		"The directory to store the anonymized trace")
+	command.Flags().StringVar(
+		&o.TraceID,
+		traceIDFlag,
+		"",
+		"The trace-id of trace to anonymize")
+	command.Flags().BoolVar(
+		&o.HashStandardTags,
+		hashStandardTagsFlag,
+		false,
+		"Whether to hash standard tags")
+	command.Flags().BoolVar(
+		&o.HashCustomTags,
+		hashCustomTagsFlag,
+		false,
+		"Whether to hash custom tags")
+	command.Flags().BoolVar(
+		&o.HashLogs,
+		hashLogsFlag,
+		false,
+		"Whether to hash logs")
+	command.Flags().BoolVar(
+		&o.HashProcess,
+		hashProcessFlag,
+		false,
+		"Whether to hash process")
+	command.Flags().IntVar(
+		&o.MaxSpansCount,
+		maxSpansCount,
+		-1,
+		"The maximum number of spans to anonymize")
+
+	// mark traceid flag as mandatory
+	command.MarkFlagRequired(traceIDFlag)
+}

cmd/anonymizer/app/flags_test.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2020 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package app
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOptionsWithDefaultFlags(t *testing.T) {
+	o := Options{}
+	c := cobra.Command{}
+	o.AddFlags(&c)
+
+	assert.Equal(t, "localhost:16686", o.QueryGRPCHostPort)
+	assert.Equal(t, "/tmp", o.OutputDir)
+	assert.Equal(t, false, o.HashStandardTags)
+	assert.Equal(t, false, o.HashCustomTags)
+	assert.Equal(t, false, o.HashLogs)
+	assert.Equal(t, false, o.HashProcess)
+	assert.Equal(t, -1, o.MaxSpansCount)
+}
+
+func TestOptionsWithFlags(t *testing.T) {
+	o := Options{}
+	c := cobra.Command{}
+
+	o.AddFlags(&c)
+	c.ParseFlags([]string{
+		"--query-host-port=192.168.1.10:16686",
+		"--output-dir=/data",
+		"--trace-id=6ef2debb698f2f7c",
+		"--hash-standard-tags",
+		"--hash-custom-tags",
+		"--hash-logs",
+		"--hash-process",
+		"--max-spans-count=100",
+	})
+
+	assert.Equal(t, "192.168.1.10:16686", o.QueryGRPCHostPort)
+	assert.Equal(t, "/data", o.OutputDir)
+	assert.Equal(t, "6ef2debb698f2f7c", o.TraceID)
+	assert.Equal(t, true, o.HashStandardTags)
+	assert.Equal(t, true, o.HashCustomTags)
+	assert.Equal(t, true, o.HashLogs)
+	assert.Equal(t, true, o.HashProcess)
+	assert.Equal(t, 100, o.MaxSpansCount)
+}

cmd/anonymizer/app/query/.nocover

@@ -0,0 +1 @@
+non-critical test utility

cmd/anonymizer/app/query/query.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2020 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package query
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/status"
+
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/jaegertracing/jaeger/proto-gen/api_v2"
+	"github.com/jaegertracing/jaeger/storage/spanstore"
+)
+
+// Query represents a jaeger-query's query for trace-id
+type Query struct {
+	client api_v2.QueryServiceClient
+	conn   *grpc.ClientConn
+}
+
+// New creates a Query object
+func New(addr string) (*Query, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+
+	conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure())
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect with the jaeger-query service: %w", err)
+	}
+
+	return &Query{
+		client: api_v2.NewQueryServiceClient(conn),
+		conn:   conn,
+	}, nil
+}
+
+// unwrapNotFoundErr is a conversion function
+func unwrapNotFoundErr(err error) error {
+	if s, _ := status.FromError(err); s != nil {
+		if s.Message() == spanstore.ErrTraceNotFound.Error() {
+			return spanstore.ErrTraceNotFound
+		}
+	}
+	return err
+}
+
+// QueryTrace queries for a trace and returns all spans inside it
+func (q *Query) QueryTrace(traceID string) ([]model.Span, error) {
+	mTraceID, err := model.TraceIDFromString(traceID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert the provided trace id: %w", err)
+	}
+
+	stream, err := q.client.GetTrace(context.Background(), &api_v2.GetTraceRequest{
+		TraceID: mTraceID,
+	})
+	if err != nil {
+		return nil, unwrapNotFoundErr(err)
+	}
+
+	var spans []model.Span
+	for received, err := stream.Recv(); err != io.EOF; received, err = stream.Recv() {
+		if err != nil {
+			return nil, unwrapNotFoundErr(err)
+		}
+		for i := range received.Spans {
+			spans = append(spans, received.Spans[i])
+		}
+	}
+
+	return spans, nil
+}