Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement anonymizer's main program #2621

Merged
merged 22 commits into from
Nov 12, 2020
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ examples/memstore-plugin/memstore-plugin
cmd/all-in-one/all-in-one-*
cmd/agent/agent
cmd/agent/agent-*
cmd/anonymizer/anonymizer
cmd/anonymizer/anonymizer-*
cmd/collector/collector
cmd/collector/collector-*
cmd/ingester/ingester
Expand Down
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,10 @@ build-all-in-one build-all-in-one-debug: build-ui elasticsearch-mappings
build-agent build-agent-debug:
$(GOBUILD) $(DISABLE_OPTIMIZATIONS) -o ./cmd/agent/agent$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/agent/main.go

.PHONY: build-anonymizer
build-anonymizer:
$(GOBUILD) $(DISABLE_OPTIMIZATIONS) -o ./cmd/anonymizer/anonymizer$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/anonymizer/main.go

.PHONY: build-query build-query-debug
build-query build-query-debug: build-ui
$(GOBUILD) $(DISABLE_OPTIMIZATIONS) -tags ui -o ./cmd/query/query$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/query/main.go
Expand Down
32 changes: 19 additions & 13 deletions cmd/anonymizer/app/anonymizer/anonymizer.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,26 +53,32 @@ type mapping struct {
//
// The mapping from original to obfuscated strings is stored in a file and can be reused between runs.
type Anonymizer struct {
mappingFile string
logger *zap.Logger
lock sync.Mutex
mapping mapping
hashStandardTags bool
hashCustomTags bool
hashLogs bool
hashProcess bool
mappingFile string
logger *zap.Logger
lock sync.Mutex
mapping mapping
options *Options
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved
}

// Options represents the various options with which the anonymizer can be configured.
type Options struct {
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved
HashStandardTags bool
HashCustomTags bool
HashLogs bool
HashProcess bool
}

// New creates new Anonymizer. The mappingFile stores the mapping from original to
// obfuscated strings, in case later investigations require looking at the original traces.
func New(mappingFile string, logger *zap.Logger) *Anonymizer {
func New(mappingFile string, options *Options, logger *zap.Logger) *Anonymizer {
a := &Anonymizer{
mappingFile: mappingFile,
logger: logger,
mapping: mapping{
Services: make(map[string]string),
Operations: make(map[string]string),
},
options: options,
}
if _, err := os.Stat(filepath.Clean(mappingFile)); err == nil {
dat, err := ioutil.ReadFile(filepath.Clean(mappingFile))
Expand Down Expand Up @@ -142,18 +148,18 @@ func (a *Anonymizer) AnonymizeSpan(span *model.Span) *uimodel.Span {

outputTags := filterStandardTags(span.Tags)
// when true, the allowedTags are hashed and when false they are preserved as it is
if a.hashStandardTags {
if a.options.HashStandardTags {
outputTags = hashTags(outputTags)
}
// when true, all tags other than allowedTags are hashed, when false they are dropped
if a.hashCustomTags {
if a.options.HashCustomTags {
customTags := hashTags(filterCustomTags(span.Tags))
outputTags = append(outputTags, customTags...)
}
span.Tags = outputTags

// when true, logs are hashed, when false, they are dropped
if a.hashLogs {
if a.options.HashLogs {
for _, log := range span.Logs {
log.Fields = hashTags(log.Fields)
}
Expand All @@ -164,7 +170,7 @@ func (a *Anonymizer) AnonymizeSpan(span *model.Span) *uimodel.Span {
span.Process.ServiceName = a.mapServiceName(service)

// when true, process tags are hashed, when false they are dropped
if a.hashProcess {
if a.options.HashProcess {
span.Process.Tags = hashTags(span.Process.Tags)
} else {
span.Process.Tags = nil
Expand Down
22 changes: 14 additions & 8 deletions cmd/anonymizer/app/anonymizer/anonymizer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,15 +98,18 @@ func TestAnonymizer_Hash(t *testing.T) {
}

func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) {
options := &Options{
HashStandardTags: true,
HashCustomTags: true,
HashProcess: true,
HashLogs: true,
}
anonymizer := &Anonymizer{
mapping: mapping{
Services: make(map[string]string),
Operations: make(map[string]string),
},
hashStandardTags: true,
hashCustomTags: true,
hashProcess: true,
hashLogs: true,
options: options,
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved
}
_ = anonymizer.AnonymizeSpan(span1)
assert.Equal(t, 3, len(span1.Tags))
Expand All @@ -115,15 +118,18 @@ func TestAnonymizer_AnonymizeSpan_AllTrue(t *testing.T) {
}

func TestAnonymizer_AnonymizeSpan_AllFalse(t *testing.T) {
options := &Options{
HashStandardTags: false,
HashCustomTags: false,
HashProcess: false,
HashLogs: false,
}
anonymizer := &Anonymizer{
mapping: mapping{
Services: make(map[string]string),
Operations: make(map[string]string),
},
hashStandardTags: false,
hashCustomTags: false,
hashProcess: false,
hashLogs: false,
options: options,
}
_ = anonymizer.AnonymizeSpan(span2)
assert.Equal(t, 2, len(span2.Tags))
Expand Down
96 changes: 96 additions & 0 deletions cmd/anonymizer/app/flags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
// Copyright (c) 2020 The Jaeger Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package app

import (
"github.com/spf13/cobra"
)

// Options represent configurable parameters for jaeger-anonymizer
type Options struct {
QueryGRPCPort int
QueryGRPCHost string
MaxSpansCount int
TraceID string
OutputDir string
HashStandardTags bool
HashCustomTags bool
HashLogs bool
HashProcess bool
}

const (
queryGRPCHostFlag = "query-host"
queryGRPCPortFlag = "query-port"
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved
outputDirFlag = "output-dir"
traceIDFlag = "trace-id"
hashStandardTagsFlag = "hash-standard-tags"
hashCustomTagsFlag = "hash-custom-tags"
hashLogsFlag = "hash-logs"
hashProcessFlag = "hash-process"
maxSpansCount = "max-spans-count"
)

// AddFlags adds flags for anonymizer main program
func (o *Options) AddFlags(command *cobra.Command) {
command.Flags().StringVar(
&o.QueryGRPCHost,
queryGRPCHostFlag,
DefaultQueryGRPCHost,
"hostname of the jaeger-query endpoint")
command.Flags().IntVar(
&o.QueryGRPCPort,
queryGRPCPortFlag,
DefaultQueryGRPCPort,
"port of the jaeger-query endpoint")
command.Flags().StringVar(
&o.OutputDir,
outputDirFlag,
DefaultOutputDir,
"directory to store the anonymized trace")
command.Flags().StringVar(
&o.TraceID,
traceIDFlag,
"",
"trace-id of trace to anonymize")
command.Flags().BoolVar(
&o.HashStandardTags,
hashStandardTagsFlag,
DefaultHashStandardTags,
"whether to hash standard tags")
command.Flags().BoolVar(
&o.HashCustomTags,
hashCustomTagsFlag,
DefaultHashCustomTags,
"whether to hash custom tags")
command.Flags().BoolVar(
&o.HashLogs,
hashLogsFlag,
DefaultHashLogs,
"whether to hash logs")
command.Flags().BoolVar(
&o.HashProcess,
hashProcessFlag,
DefaultHashProcess,
"whether to hash process")
command.Flags().IntVar(
&o.MaxSpansCount,
maxSpansCount,
DefaultMaxSpansCount,
"maximum number of spans to anonymize")

// mark traceid flag as mandatory
command.MarkFlagRequired(traceIDFlag)
}
36 changes: 36 additions & 0 deletions cmd/anonymizer/app/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright (c) 2020 The Jaeger Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package app

import "github.com/jaegertracing/jaeger/ports"

const (
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved
// DefaultQueryGRPCHost is the default host for jaeger-query endpoint
DefaultQueryGRPCHost = "localhost"
// DefaultQueryGRPCPort is the default port for jaeger-query endpoint
DefaultQueryGRPCPort = ports.QueryHTTP
// DefaultOutputDir is the default output directory for spans
DefaultOutputDir = "/tmp"
// DefaultHashStandardTags is the default flag for whether to hash standard tags
DefaultHashStandardTags = true
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved
// DefaultHashCustomTags is the default flag for whether to hash custom tags
DefaultHashCustomTags = false
// DefaultHashLogs is the default flag for whether to hash logs
DefaultHashLogs = false
// DefaultHashProcess is the default flag for whether to hash process
DefaultHashProcess = false
// DefaultMaxSpansCount is the default value of maximum number of spans
DefaultMaxSpansCount = -1
)
91 changes: 91 additions & 0 deletions cmd/anonymizer/app/query/query.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Copyright (c) 2020 The Jaeger Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package query
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved

import (
"context"
"time"

"go.uber.org/zap"
"google.golang.org/grpc"

"github.com/jaegertracing/jaeger/model"
"github.com/jaegertracing/jaeger/proto-gen/api_v2"
"github.com/jaegertracing/jaeger/storage/spanstore"
)

// GrpcClient represents a grpc-client to jaeger-query
type GrpcClient struct {
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved
api_v2.QueryServiceClient
conn *grpc.ClientConn
}

// Query represents a jaeger-query's query for trace-id
type Query struct {
grpcClient *GrpcClient
logger *zap.Logger
}

// newGRPCClient returns a new GrpcClient
func newGRPCClient(addr string, logger *zap.Logger) *GrpcClient {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()

conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure())
if err != nil {
logger.Fatal("failed to connect with the jaeger-query service", zap.Error(err))
}

return &GrpcClient{
QueryServiceClient: api_v2.NewQueryServiceClient(conn),
conn: conn,
}
}

// New creates a Query object
func New(addr string, logger *zap.Logger) (*Query, error) {
client := newGRPCClient(addr, logger)

return &Query{
grpcClient: client,
logger: logger,
}, nil
}

// QueryTrace queries for a trace and returns all spans inside it
func (q *Query) QueryTrace(traceID string) []model.Span {
mTraceID, err := model.TraceIDFromString(traceID)
if err != nil {
q.logger.Fatal("failed to convert the provided trace id", zap.Error(err))
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved
}

response, err := q.grpcClient.GetTrace(context.Background(), &api_v2.GetTraceRequest{
TraceID: mTraceID,
})
if err != nil {
q.logger.Fatal("failed to fetch the provided trace id", zap.Error(err))
}

spanResponseChunk, err := response.Recv()
Ashmita152 marked this conversation as resolved.
Show resolved Hide resolved
if err == spanstore.ErrTraceNotFound {
q.logger.Fatal("failed to find the provided trace id", zap.Error(err))
}
if err != nil {
q.logger.Fatal("failed to fetch spans of provided trace id", zap.Error(err))
}

spans := spanResponseChunk.GetSpans()
return spans
}
Loading