From 71aa60e716a26db7baab38e4f96fff5d350e3c63 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 12 Nov 2020 19:00:07 -0500 Subject: [PATCH 01/13] [anonymizer]: save trace in UI format Signed-off-by: Yuri Shkuro --- cmd/anonymizer/app/uiconv/extractor.go | 90 ++++++++++++++++++++++ cmd/anonymizer/app/uiconv/module.go | 40 ++++++++++ cmd/anonymizer/app/uiconv/reader.go | 96 ++++++++++++++++++++++++ cmd/anonymizer/app/uiconv/reader_test.go | 56 ++++++++++++++ cmd/anonymizer/app/writer/writer.go | 2 +- cmd/anonymizer/main.go | 24 ++++-- 6 files changed, 301 insertions(+), 7 deletions(-) create mode 100644 cmd/anonymizer/app/uiconv/extractor.go create mode 100644 cmd/anonymizer/app/uiconv/module.go create mode 100644 cmd/anonymizer/app/uiconv/reader.go create mode 100644 cmd/anonymizer/app/uiconv/reader_test.go diff --git a/cmd/anonymizer/app/uiconv/extractor.go b/cmd/anonymizer/app/uiconv/extractor.go new file mode 100644 index 00000000000..6650531e6c9 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/extractor.go @@ -0,0 +1,90 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "go.uber.org/zap" + + uimodel "github.com/jaegertracing/jaeger/model/json" +) + +// Extractor reads the spans from reader, filters by traceID, and stores as JSON into uiFile. +type Extractor struct { + uiFile *os.File + traceID string + reader *Reader + logger *zap.Logger +} + +// NewExtractor creates Extractor. +func NewExtractor(uiFile string, traceID string, reader *Reader, logger *zap.Logger) (*Extractor, error) { + f, err := os.OpenFile(uiFile, os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("cannot create output file: %w", err) + } + logger.Sugar().Infof("Writing spans to UI file %s", uiFile) + + return &Extractor{ + uiFile: f, + traceID: traceID, + reader: reader, + logger: logger, + }, nil +} + +// Run executes the extraction. +func (e *Extractor) Run() error { + e.logger.Info("Parsing captured file for trace", zap.String("trace_id", e.traceID)) + var ( + spans []uimodel.Span + span *uimodel.Span + err error + ) + for span, err = e.reader.NextSpan(); err == nil; span, err = e.reader.NextSpan() { + if string(span.TraceID) == e.traceID { + spans = append(spans, *span) + } + } + if err != io.EOF { + e.logger.Fatal("Failed when scanning the file", zap.Error(err)) + } + trace := uimodel.Trace{ + TraceID: uimodel.TraceID(e.traceID), + Spans: spans, + Processes: make(map[uimodel.ProcessID]uimodel.Process), + } + for i := range spans { + span := &spans[i] + pid := uimodel.ProcessID(fmt.Sprintf("p%d", i)) + trace.Processes[pid] = *span.Process + span.Process = nil + span.ProcessID = pid + } + jsonBytes, err := json.Marshal(trace) + if err != nil { + return fmt.Errorf("failed to marshal UI trace: %w", err) + } + e.uiFile.Write([]byte(`{"data": [`)) + e.uiFile.Write(jsonBytes) + e.uiFile.Write([]byte(`]}`)) + e.logger.Sugar().Infof("Wrote spans to UI file %s", e.uiFile.Name()) + os.Exit(0) + return nil +} diff --git a/cmd/anonymizer/app/uiconv/module.go b/cmd/anonymizer/app/uiconv/module.go new file mode 100644 index 00000000000..bd19620dd0c --- /dev/null +++ b/cmd/anonymizer/app/uiconv/module.go @@ -0,0 +1,40 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "go.uber.org/zap" +) + +// Config for the extractor. +type Config struct { + CapturedFile string `yaml:"captured_file"` + UIFile string `yaml:"ui_file"` + TraceID string `yaml:"trace_id"` +} + +// Extract reads anonymized file, finds spans for a given trace, +// and writes out that trace in the UI format. +func Extract(config Config, logger *zap.Logger) error { + reader, err := NewReader(config.CapturedFile, logger) + if err != nil { + return err + } + ext, err := NewExtractor(config.UIFile, config.TraceID, reader, logger) + if err != nil { + return err + } + return ext.Run() +} diff --git a/cmd/anonymizer/app/uiconv/reader.go b/cmd/anonymizer/app/uiconv/reader.go new file mode 100644 index 00000000000..2ce0f0e0481 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/reader.go @@ -0,0 +1,96 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + + "go.uber.org/zap" + + uimodel "github.com/jaegertracing/jaeger/model/json" +) + +// Reader loads previously captured spans from a file. +type Reader struct { + logger *zap.Logger + capturedFile *os.File + reader *bufio.Reader + spansRead int + eofReached bool +} + +// NewReader creates a Reader. +func NewReader(capturedFile string, logger *zap.Logger) (*Reader, error) { + wd, err := os.Getwd() + if err != nil { + return nil, err + } + logger.Sugar().Infof("Current working dir is %s", wd) + + cf, err := os.OpenFile(capturedFile, os.O_RDONLY, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("cannot open captured file: %w", err) + } + logger.Sugar().Infof("Reading captured spans from file %s", capturedFile) + + return &Reader{ + logger: logger, + capturedFile: cf, + reader: bufio.NewReader(cf), + }, nil +} + +// NextSpan reads the next span from the capture file, or returns io.EOF. +func (r *Reader) NextSpan() (*uimodel.Span, error) { + if r.eofReached { + return nil, io.EOF + } + if r.spansRead == 0 { + b, err := r.reader.ReadByte() + if err != nil { + r.eofReached = true + return nil, fmt.Errorf("cannot read file: %w", err) + } + if b != '[' { + r.eofReached = true + return nil, fmt.Errorf("file must begin with '['") + } + } + s, err := r.reader.ReadString('\n') + if err != nil { + r.eofReached = true + return nil, fmt.Errorf("cannot read file: %w", err) + } + if s[len(s)-2] == ',' { // all but last span lines end with ,\n + s = s[0 : len(s)-2] + } else { + r.eofReached = true + } + var span uimodel.Span + err = json.Unmarshal([]byte(s), &span) + if err != nil { + r.eofReached = true + return nil, fmt.Errorf("cannot unmarshal span: %w; %s", err, s) + } + r.spansRead++ + if r.spansRead%1000 == 0 { + r.logger.Info("Scan progress", zap.Int("span_count", r.spansRead)) + } + return &span, nil +} diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go new file mode 100644 index 00000000000..5c98c3dbd36 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -0,0 +1,56 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "io" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +const spanData = `[{"traceId":"AAAAAAAAAABbgcpUtfnZBA==","spanId":"W4HKVLX52QQ=","operationName":"Meta::health","flags":1,"startTime":"2020-04-23T22:42:03.427158Z","duration":"0.000077s","process":{"serviceName":"acme"}}, +{"traceId":"AAAAAAAAAABkQX7HRbUymw==","spanId":"E+85jzZSdoY=","operationName":"foobar","references":[{"traceId":"AAAAAAAAAABkQX7HRbUymw==","spanId":"Pgd+mTq/Zh4="}],"flags":1,"startTime":"2020-04-23T22:42:01.289306Z","duration":"0.024253s","process":{"serviceName":"xyz"}} +]` + +func TestReader(t *testing.T) { + f, err := ioutil.TempFile("", "captured-spans.json") + require.NoError(t, err) + defer os.Remove(f.Name()) + + _, err = f.Write([]byte(spanData)) + require.NoError(t, err) + + r, err := NewReader( + f.Name(), + zap.NewNop(), + ) + require.NoError(t, err) + + s1, err := r.NextSpan() + require.NoError(t, err) + assert.Equal(t, "Meta::health", s1.OperationName) + + s2, err := r.NextSpan() + require.NoError(t, err) + assert.Equal(t, "foobar", s2.OperationName) + + _, err = r.NextSpan() + require.Equal(t, io.EOF, err) +} diff --git a/cmd/anonymizer/app/writer/writer.go b/cmd/anonymizer/app/writer/writer.go index 5d04e61df53..dee49e1735d 100644 --- a/cmd/anonymizer/app/writer/writer.go +++ b/cmd/anonymizer/app/writer/writer.go @@ -127,7 +127,7 @@ func (w *Writer) WriteSpan(msg *model.Span) error { w.logger.Info("progress", zap.Int("numSpans", w.spanCount)) } - if w.spanCount >= w.config.MaxSpansCount { + if w.config.MaxSpansCount > 0 && w.spanCount >= w.config.MaxSpansCount { w.logger.Info("Saved enough spans, exiting...") w.Close() os.Exit(0) diff --git a/cmd/anonymizer/main.go b/cmd/anonymizer/main.go index a4dd00e4a9f..10ee3390393 100644 --- a/cmd/anonymizer/main.go +++ b/cmd/anonymizer/main.go @@ -21,10 +21,11 @@ import ( "github.com/spf13/cobra" "go.uber.org/zap" - app "github.com/jaegertracing/jaeger/cmd/anonymizer/app" + "github.com/jaegertracing/jaeger/cmd/anonymizer/app" "github.com/jaegertracing/jaeger/cmd/anonymizer/app/anonymizer" - query "github.com/jaegertracing/jaeger/cmd/anonymizer/app/query" - writer "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" + "github.com/jaegertracing/jaeger/cmd/anonymizer/app/query" + "github.com/jaegertracing/jaeger/cmd/anonymizer/app/uiconv" + "github.com/jaegertracing/jaeger/cmd/anonymizer/app/writer" "github.com/jaegertracing/jaeger/pkg/version" ) @@ -41,9 +42,9 @@ func main() { prefix := options.OutputDir + "/" + options.TraceID conf := writer.Config{ MaxSpansCount: options.MaxSpansCount, - CapturedFile: prefix + ".original", - AnonymizedFile: prefix + ".anonymized", - MappingFile: prefix + ".mapping", + CapturedFile: prefix + ".original.json", + AnonymizedFile: prefix + ".anonymized.json", + MappingFile: prefix + ".mapping.json", AnonymizerOpts: anonymizer.Options{ HashStandardTags: options.HashStandardTags, HashCustomTags: options.HashCustomTags, @@ -70,7 +71,18 @@ func main() { for _, span := range spans { writer.WriteSpan(&span) } + logger.Info("finished writing spans") writer.Close() + + uiCfg := uiconv.Config{ + CapturedFile: conf.AnonymizedFile, + UIFile: prefix + ".anonymized-ui-trace.json", + TraceID: options.TraceID, + } + if err := uiconv.Extract(uiCfg, logger); err != nil { + logger.Fatal("error while extracing UI trace", zap.Error(err)) + } + logger.Sugar().Infof("Wrote UI-compatible anonymized file to %s", uiCfg.UIFile) }, } From dd5444fbda83b5f2df3ba45ee9e662093f6aef16 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 12 Nov 2020 19:01:15 -0500 Subject: [PATCH 02/13] remove log Signed-off-by: Yuri Shkuro --- cmd/anonymizer/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/anonymizer/main.go b/cmd/anonymizer/main.go index 10ee3390393..d80d609adaa 100644 --- a/cmd/anonymizer/main.go +++ b/cmd/anonymizer/main.go @@ -71,7 +71,6 @@ func main() { for _, span := range spans { writer.WriteSpan(&span) } - logger.Info("finished writing spans") writer.Close() uiCfg := uiconv.Config{ From c333eeee32a039335b823abefc3c653704aa8f4f Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 12 Nov 2020 20:00:04 -0500 Subject: [PATCH 03/13] Fix test Signed-off-by: Yuri Shkuro --- cmd/anonymizer/app/uiconv/extractor.go | 2 ++ cmd/anonymizer/app/uiconv/reader_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/anonymizer/app/uiconv/extractor.go b/cmd/anonymizer/app/uiconv/extractor.go index 6650531e6c9..297959bfa94 100644 --- a/cmd/anonymizer/app/uiconv/extractor.go +++ b/cmd/anonymizer/app/uiconv/extractor.go @@ -70,6 +70,8 @@ func (e *Extractor) Run() error { Spans: spans, Processes: make(map[uimodel.ProcessID]uimodel.Process), } + // (ys) The following is not exactly correct because it does not dedupe the processes, + // but I don't think it affects the UI. for i := range spans { span := &spans[i] pid := uimodel.ProcessID(fmt.Sprintf("p%d", i)) diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go index 5c98c3dbd36..08c3be603d0 100644 --- a/cmd/anonymizer/app/uiconv/reader_test.go +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -25,8 +25,8 @@ import ( "go.uber.org/zap" ) -const spanData = `[{"traceId":"AAAAAAAAAABbgcpUtfnZBA==","spanId":"W4HKVLX52QQ=","operationName":"Meta::health","flags":1,"startTime":"2020-04-23T22:42:03.427158Z","duration":"0.000077s","process":{"serviceName":"acme"}}, -{"traceId":"AAAAAAAAAABkQX7HRbUymw==","spanId":"E+85jzZSdoY=","operationName":"foobar","references":[{"traceId":"AAAAAAAAAABkQX7HRbUymw==","spanId":"Pgd+mTq/Zh4="}],"flags":1,"startTime":"2020-04-23T22:42:01.289306Z","duration":"0.024253s","process":{"serviceName":"xyz"}} +const spanData = `[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, +{"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34"}],"startTime":1605223981965074,"duration":32782,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"error","type":"bool","value":"true"}],"logs":[],"process":{"serviceName":"3c220036602f839e","tags":[]},"warnings":null} ]` func TestReader(t *testing.T) { @@ -45,11 +45,11 @@ func TestReader(t *testing.T) { s1, err := r.NextSpan() require.NoError(t, err) - assert.Equal(t, "Meta::health", s1.OperationName) + assert.Equal(t, "a071653098f9250d", s1.OperationName) s2, err := r.NextSpan() require.NoError(t, err) - assert.Equal(t, "foobar", s2.OperationName) + assert.Equal(t, "471418097747d04a", s2.OperationName) _, err = r.NextSpan() require.Equal(t, io.EOF, err) From 1c43b4970a97009578c75a0de1180d1db7f82933 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Sun, 15 Nov 2020 15:53:27 +0800 Subject: [PATCH 04/13] Add more tests for reader.go Signed-off-by: Ashmita Bohara --- .../app/uiconv/fixtures/trace_empty.json | 0 .../uiconv/fixtures/trace_invalid_json.json | 3 + .../app/uiconv/fixtures/trace_success.json | 3 + .../uiconv/fixtures/trace_wrong_format.json | 2 + cmd/anonymizer/app/uiconv/reader_test.go | 81 +++++++++++++++---- 5 files changed, 74 insertions(+), 15 deletions(-) create mode 100644 cmd/anonymizer/app/uiconv/fixtures/trace_empty.json create mode 100644 cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json create mode 100644 cmd/anonymizer/app/uiconv/fixtures/trace_success.json create mode 100644 cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_empty.json b/cmd/anonymizer/app/uiconv/fixtures/trace_empty.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json b/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json new file mode 100644 index 00000000000..94c754d3231 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json @@ -0,0 +1,3 @@ +[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, + {"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","startTime":1605223981965074,"duration": "invalid","tags":[{"key":"span.kind","type":"string","value":"client"}]} +] \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_success.json b/cmd/anonymizer/app/uiconv/fixtures/trace_success.json new file mode 100644 index 00000000000..7caed81fe70 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_success.json @@ -0,0 +1,3 @@ +[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, + {"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34"}],"startTime":1605223981965074,"duration":32782,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"error","type":"bool","value":"true"}],"logs":[],"process":{"serviceName":"3c220036602f839e","tags":[]},"warnings":null} +] \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json b/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json new file mode 100644 index 00000000000..cf30d39dcec --- /dev/null +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json @@ -0,0 +1,2 @@ +{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, +{"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34"}],"startTime":1605223981965074,"duration":32782,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"error","type":"bool","value":"true"}],"logs":[],"process":{"serviceName":"3c220036602f839e","tags":[]},"warnings":null} \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go index 08c3be603d0..144fe9265ad 100644 --- a/cmd/anonymizer/app/uiconv/reader_test.go +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -16,8 +16,6 @@ package uiconv import ( "io" - "io/ioutil" - "os" "testing" "github.com/stretchr/testify/assert" @@ -25,20 +23,10 @@ import ( "go.uber.org/zap" ) -const spanData = `[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, -{"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34"}],"startTime":1605223981965074,"duration":32782,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"error","type":"bool","value":"true"}],"logs":[],"process":{"serviceName":"3c220036602f839e","tags":[]},"warnings":null} -]` - -func TestReader(t *testing.T) { - f, err := ioutil.TempFile("", "captured-spans.json") - require.NoError(t, err) - defer os.Remove(f.Name()) - - _, err = f.Write([]byte(spanData)) - require.NoError(t, err) - +func TestReader_Trace_Success(t *testing.T) { + inputFile := "fixtures/trace_success.json" r, err := NewReader( - f.Name(), + inputFile, zap.NewNop(), ) require.NoError(t, err) @@ -46,11 +34,74 @@ func TestReader(t *testing.T) { s1, err := r.NextSpan() require.NoError(t, err) assert.Equal(t, "a071653098f9250d", s1.OperationName) + assert.Equal(t, 1, r.spansRead) + assert.Equal(t, false, r.eofReached) s2, err := r.NextSpan() require.NoError(t, err) assert.Equal(t, "471418097747d04a", s2.OperationName) + assert.Equal(t, 2, r.spansRead) + assert.Equal(t, true, r.eofReached) _, err = r.NextSpan() require.Equal(t, io.EOF, err) + assert.Equal(t, 2, r.spansRead) + assert.Equal(t, true, r.eofReached) +} + +func TestReader_Trace_NonExistent(t *testing.T) { + inputFile := "fixtures/trace_non_existent.json" + _, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.Equal(t, "cannot open captured file: open fixtures/trace_non_existent.json: no such file or directory", err.Error()) +} + +func TestReader_Trace_Empty(t *testing.T) { + inputFile := "fixtures/trace_empty.json" + r, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + _, err = r.NextSpan() + require.Equal(t, "cannot read file: EOF", err.Error()) + assert.Equal(t, 0, r.spansRead) + assert.Equal(t, true, r.eofReached) +} + +func TestReader_Trace_Wrong_Format(t *testing.T) { + inputFile := "fixtures/trace_wrong_format.json" + r, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + _, err = r.NextSpan() + require.Equal(t, "file must begin with '['", err.Error()) + assert.Equal(t, 0, r.spansRead) + assert.Equal(t, true, r.eofReached) } + +func TestReader_Trace_Invalid_Json(t *testing.T) { + inputFile := "fixtures/trace_invalid_json.json" + r, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + s1, err := r.NextSpan() + require.NoError(t, err) + assert.Equal(t, "a071653098f9250d", s1.OperationName) + assert.Equal(t, 1, r.spansRead) + assert.Equal(t, false, r.eofReached) + + _, err = r.NextSpan() + require.Equal(t, "cannot unmarshal span: json: cannot unmarshal string into Go struct field Span.duration of type uint64; {\"traceID\":\"2be38093ead7a083\",\"spanID\":\"7bd66f09ba90ea3d\",\"flags\":1,\"operationName\":\"471418097747d04a\",\"startTime\":1605223981965074,\"duration\": \"invalid\",\"tags\":[{\"key\":\"span.kind\",\"type\":\"string\",\"value\":\"client\"}]}\n", err.Error()) + assert.Equal(t, 1, r.spansRead) + assert.Equal(t, true, r.eofReached) +} \ No newline at end of file From b739de38099a59fa970860322a0f1b136923260b Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Mon, 16 Nov 2020 18:32:42 +0800 Subject: [PATCH 05/13] Add unit tests for extractor.go and module.go Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/uiconv/extractor.go | 6 +- cmd/anonymizer/app/uiconv/extractor_test.go | 118 ++++++++++++++++++ .../app/uiconv/fixtures/trace_scan_error.json | 3 + cmd/anonymizer/app/uiconv/module_test.go | 60 +++++++++ cmd/anonymizer/app/uiconv/reader_test.go | 12 +- 5 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 cmd/anonymizer/app/uiconv/extractor_test.go create mode 100644 cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json create mode 100644 cmd/anonymizer/app/uiconv/module_test.go diff --git a/cmd/anonymizer/app/uiconv/extractor.go b/cmd/anonymizer/app/uiconv/extractor.go index 297959bfa94..8679766318f 100644 --- a/cmd/anonymizer/app/uiconv/extractor.go +++ b/cmd/anonymizer/app/uiconv/extractor.go @@ -52,6 +52,7 @@ func NewExtractor(uiFile string, traceID string, reader *Reader, logger *zap.Log // Run executes the extraction. func (e *Extractor) Run() error { e.logger.Info("Parsing captured file for trace", zap.String("trace_id", e.traceID)) + var ( spans []uimodel.Span span *uimodel.Span @@ -63,7 +64,7 @@ func (e *Extractor) Run() error { } } if err != io.EOF { - e.logger.Fatal("Failed when scanning the file", zap.Error(err)) + return fmt.Errorf("failed when scanning the file: %w", err) } trace := uimodel.Trace{ TraceID: uimodel.TraceID(e.traceID), @@ -86,7 +87,8 @@ func (e *Extractor) Run() error { e.uiFile.Write([]byte(`{"data": [`)) e.uiFile.Write(jsonBytes) e.uiFile.Write([]byte(`]}`)) + e.uiFile.Sync() + e.uiFile.Close() e.logger.Sugar().Infof("Wrote spans to UI file %s", e.uiFile.Name()) - os.Exit(0) return nil } diff --git a/cmd/anonymizer/app/uiconv/extractor_test.go b/cmd/anonymizer/app/uiconv/extractor_test.go new file mode 100644 index 00000000000..fbcf6cef740 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/extractor_test.go @@ -0,0 +1,118 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/go-openapi/swag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/jaegertracing/jaeger/model" +) + +type UITrace struct { + Data []model.Trace +} + +func TestExtractor_TraceSuccess(t *testing.T) { + inputFile := "fixtures/trace_success.json" + outputFile := "fixtures/trace_success_ui_anonymized.json" + defer os.Remove(outputFile) + + reader, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + extractor, err := NewExtractor( + outputFile, + "2be38093ead7a083", + reader, + zap.NewNop(), + ) + require.NoError(t, err) + + err = extractor.Run() + require.NoError(t, err) + + var trace UITrace + loadJSON(t, outputFile, &trace) + + for i := range trace.Data { + for j := range trace.Data[i].Spans { + assert.Equal(t, "span.kind", trace.Data[i].Spans[j].Tags[0].Key) + } + } +} + +func TestExtractor_TraceOutputFileError(t *testing.T) { + inputFile := "fixtures/trace_success.json" + outputFile := "fixtures/trace_success_ui_anonymized.json" + defer os.Remove(outputFile) + + reader, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + err = os.Chmod("fixtures", 0000) + require.NoError(t, err) + defer os.Chmod("fixtures", 0755) + + _, err = NewExtractor( + outputFile, + "2be38093ead7a083", + reader, + zap.NewNop(), + ) + require.Equal(t, "cannot create output file: open fixtures/trace_success_ui_anonymized.json: permission denied", err.Error()) +} + +func TestExtractor_TraceScanError(t *testing.T) { + inputFile := "fixtures/trace_scan_error.json" + outputFile := "fixtures/trace_scan_error_ui_anonymized.json" + defer os.Remove(outputFile) + + reader, err := NewReader( + inputFile, + zap.NewNop(), + ) + require.NoError(t, err) + + extractor, err := NewExtractor( + outputFile, + "2be38093ead7a083", + reader, + zap.NewNop(), + ) + require.NoError(t, err) + + err = extractor.Run() + require.Equal(t, "failed when scanning the file: cannot read file: EOF", err.Error()) +} + +func loadJSON(t *testing.T, fileName string, i interface{}) { + b, err := ioutil.ReadFile(fileName) + require.NoError(t, err) + err = swag.ReadJSON(b, i) + require.NoError(t, err, "Failed to parse json fixture file %s", fileName) +} diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json b/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json new file mode 100644 index 00000000000..c3a905d095e --- /dev/null +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json @@ -0,0 +1,3 @@ +[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, + {"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34"}],"startTime":1605223981965074,"duration":32782,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"error","type":"bool","value":"true"}],"logs":[],"process":{"serviceName":"3c220036602f839e","tags":[]},"warnings":null}, +] \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/module_test.go b/cmd/anonymizer/app/uiconv/module_test.go new file mode 100644 index 00000000000..cabd339ffe2 --- /dev/null +++ b/cmd/anonymizer/app/uiconv/module_test.go @@ -0,0 +1,60 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uiconv + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "os" + "testing" +) + +func TestModule_TraceSuccess(t *testing.T) { + inputFile := "fixtures/trace_success.json" + outputFile := "fixtures/trace_success_ui_anonymized.json" + defer os.Remove(outputFile) + + config := Config{ + CapturedFile: inputFile, + UIFile: outputFile, + TraceID: "2be38093ead7a083", + } + err := Extract(config, zap.NewNop()) + require.NoError(t, err) + + var trace UITrace + loadJSON(t, outputFile, &trace) + + for i := range trace.Data { + for j := range trace.Data[i].Spans { + assert.Equal(t, "span.kind", trace.Data[i].Spans[j].Tags[0].Key) + } + } +} + +func TestModule_TraceNonExistent(t *testing.T) { + inputFile := "fixtures/trace_non_existent.json" + outputFile := "fixtures/trace_non_existent_ui_anonymized.json" + defer os.Remove(outputFile) + + config := Config{ + CapturedFile: inputFile, + UIFile: outputFile, + TraceID: "2be38093ead7a083", + } + err := Extract(config, zap.NewNop()) + require.Equal(t, "cannot open captured file: open fixtures/trace_non_existent.json: no such file or directory", err.Error()) +} diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go index 144fe9265ad..5d5b0e1e3da 100644 --- a/cmd/anonymizer/app/uiconv/reader_test.go +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -23,7 +23,7 @@ import ( "go.uber.org/zap" ) -func TestReader_Trace_Success(t *testing.T) { +func TestReader_TraceSuccess(t *testing.T) { inputFile := "fixtures/trace_success.json" r, err := NewReader( inputFile, @@ -49,7 +49,7 @@ func TestReader_Trace_Success(t *testing.T) { assert.Equal(t, true, r.eofReached) } -func TestReader_Trace_NonExistent(t *testing.T) { +func TestReader_TraceNonExistent(t *testing.T) { inputFile := "fixtures/trace_non_existent.json" _, err := NewReader( inputFile, @@ -58,7 +58,7 @@ func TestReader_Trace_NonExistent(t *testing.T) { require.Equal(t, "cannot open captured file: open fixtures/trace_non_existent.json: no such file or directory", err.Error()) } -func TestReader_Trace_Empty(t *testing.T) { +func TestReader_TraceEmpty(t *testing.T) { inputFile := "fixtures/trace_empty.json" r, err := NewReader( inputFile, @@ -72,7 +72,7 @@ func TestReader_Trace_Empty(t *testing.T) { assert.Equal(t, true, r.eofReached) } -func TestReader_Trace_Wrong_Format(t *testing.T) { +func TestReader_TraceWrongFormat(t *testing.T) { inputFile := "fixtures/trace_wrong_format.json" r, err := NewReader( inputFile, @@ -86,7 +86,7 @@ func TestReader_Trace_Wrong_Format(t *testing.T) { assert.Equal(t, true, r.eofReached) } -func TestReader_Trace_Invalid_Json(t *testing.T) { +func TestReader_TraceInvalidJson(t *testing.T) { inputFile := "fixtures/trace_invalid_json.json" r, err := NewReader( inputFile, @@ -104,4 +104,4 @@ func TestReader_Trace_Invalid_Json(t *testing.T) { require.Equal(t, "cannot unmarshal span: json: cannot unmarshal string into Go struct field Span.duration of type uint64; {\"traceID\":\"2be38093ead7a083\",\"spanID\":\"7bd66f09ba90ea3d\",\"flags\":1,\"operationName\":\"471418097747d04a\",\"startTime\":1605223981965074,\"duration\": \"invalid\",\"tags\":[{\"key\":\"span.kind\",\"type\":\"string\",\"value\":\"client\"}]}\n", err.Error()) assert.Equal(t, 1, r.spansRead) assert.Equal(t, true, r.eofReached) -} \ No newline at end of file +} From 23daf3675a0088cfa19dbb93c773739844357f38 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Mon, 16 Nov 2020 22:20:49 +0800 Subject: [PATCH 06/13] Add unit tests to module.go to increase overall uiconv package coverage above 95% Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/uiconv/module_test.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/cmd/anonymizer/app/uiconv/module_test.go b/cmd/anonymizer/app/uiconv/module_test.go index cabd339ffe2..df91fe97a10 100644 --- a/cmd/anonymizer/app/uiconv/module_test.go +++ b/cmd/anonymizer/app/uiconv/module_test.go @@ -15,11 +15,12 @@ package uiconv import ( + "os" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" - "os" - "testing" ) func TestModule_TraceSuccess(t *testing.T) { @@ -58,3 +59,22 @@ func TestModule_TraceNonExistent(t *testing.T) { err := Extract(config, zap.NewNop()) require.Equal(t, "cannot open captured file: open fixtures/trace_non_existent.json: no such file or directory", err.Error()) } + +func TestModule_TraceOutputFileError(t *testing.T) { + inputFile := "fixtures/trace_success.json" + outputFile := "fixtures/trace_success_ui_anonymized.json" + defer os.Remove(outputFile) + + config := Config{ + CapturedFile: inputFile, + UIFile: outputFile, + TraceID: "2be38093ead7a083", + } + + err := os.Chmod("fixtures", 0550) + require.NoError(t, err) + defer os.Chmod("fixtures", 0755) + + err = Extract(config, zap.NewNop()) + require.Equal(t, "cannot create output file: open fixtures/trace_success_ui_anonymized.json: permission denied", err.Error()) +} From d75c23c3ad7d37b2a2d078993da7b9ef49adfd72 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 17 Nov 2020 12:39:45 +0800 Subject: [PATCH 07/13] Simplify the wrong json tests Signed-off-by: Ashmita Bohara --- .../app/uiconv/fixtures/trace_invalid_json.json | 3 +-- .../app/uiconv/fixtures/trace_scan_error.json | 3 +-- .../app/uiconv/fixtures/trace_wrong_format.json | 3 +-- cmd/anonymizer/app/uiconv/reader_test.go | 10 ++-------- 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json b/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json index 94c754d3231..cc2337d97a1 100644 --- a/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_invalid_json.json @@ -1,3 +1,2 @@ -[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, - {"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","startTime":1605223981965074,"duration": "invalid","tags":[{"key":"span.kind","type":"string","value":"client"}]} +[{"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","duration": "invalid"} ] \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json b/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json index c3a905d095e..cb6d6e55c71 100644 --- a/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json @@ -1,3 +1,2 @@ -[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, - {"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34"}],"startTime":1605223981965074,"duration":32782,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"error","type":"bool","value":"true"}],"logs":[],"process":{"serviceName":"3c220036602f839e","tags":[]},"warnings":null}, +[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, ] \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json b/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json index cf30d39dcec..c81a85e3ede 100644 --- a/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json @@ -1,2 +1 @@ -{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"492770a15935810f"}],"startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, -{"traceID":"2be38093ead7a083","spanID":"7bd66f09ba90ea3d","flags":1,"operationName":"471418097747d04a","references":[{"refType":"CHILD_OF","traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34"}],"startTime":1605223981965074,"duration":32782,"tags":[{"key":"span.kind","type":"string","value":"client"},{"key":"error","type":"bool","value":"true"}],"logs":[],"process":{"serviceName":"3c220036602f839e","tags":[]},"warnings":null} \ No newline at end of file +{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null} diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go index 5d5b0e1e3da..dc873372b80 100644 --- a/cmd/anonymizer/app/uiconv/reader_test.go +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -94,14 +94,8 @@ func TestReader_TraceInvalidJson(t *testing.T) { ) require.NoError(t, err) - s1, err := r.NextSpan() - require.NoError(t, err) - assert.Equal(t, "a071653098f9250d", s1.OperationName) - assert.Equal(t, 1, r.spansRead) - assert.Equal(t, false, r.eofReached) - _, err = r.NextSpan() - require.Equal(t, "cannot unmarshal span: json: cannot unmarshal string into Go struct field Span.duration of type uint64; {\"traceID\":\"2be38093ead7a083\",\"spanID\":\"7bd66f09ba90ea3d\",\"flags\":1,\"operationName\":\"471418097747d04a\",\"startTime\":1605223981965074,\"duration\": \"invalid\",\"tags\":[{\"key\":\"span.kind\",\"type\":\"string\",\"value\":\"client\"}]}\n", err.Error()) - assert.Equal(t, 1, r.spansRead) + require.Equal(t, "cannot unmarshal span: json: cannot unmarshal string into Go struct field Span.duration of type uint64; {\"traceID\":\"2be38093ead7a083\",\"spanID\":\"7bd66f09ba90ea3d\",\"duration\": \"invalid\"}\n", err.Error()) + assert.Equal(t, 0, r.spansRead) assert.Equal(t, true, r.eofReached) } From b4db0aa56a3b4a526cb2ff04f64fb0aa48d4fad7 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 17 Nov 2020 12:59:31 +0800 Subject: [PATCH 08/13] Using require.Contains instead of require.Equal for some tests Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/uiconv/extractor_test.go | 2 +- cmd/anonymizer/app/uiconv/module_test.go | 6 ++++-- cmd/anonymizer/app/uiconv/reader_test.go | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/anonymizer/app/uiconv/extractor_test.go b/cmd/anonymizer/app/uiconv/extractor_test.go index fbcf6cef740..9a1426d7e09 100644 --- a/cmd/anonymizer/app/uiconv/extractor_test.go +++ b/cmd/anonymizer/app/uiconv/extractor_test.go @@ -84,7 +84,7 @@ func TestExtractor_TraceOutputFileError(t *testing.T) { reader, zap.NewNop(), ) - require.Equal(t, "cannot create output file: open fixtures/trace_success_ui_anonymized.json: permission denied", err.Error()) + require.Contains(t, err.Error(), "cannot create output file") } func TestExtractor_TraceScanError(t *testing.T) { diff --git a/cmd/anonymizer/app/uiconv/module_test.go b/cmd/anonymizer/app/uiconv/module_test.go index df91fe97a10..f0af7e9fbfc 100644 --- a/cmd/anonymizer/app/uiconv/module_test.go +++ b/cmd/anonymizer/app/uiconv/module_test.go @@ -57,7 +57,8 @@ func TestModule_TraceNonExistent(t *testing.T) { TraceID: "2be38093ead7a083", } err := Extract(config, zap.NewNop()) - require.Equal(t, "cannot open captured file: open fixtures/trace_non_existent.json: no such file or directory", err.Error()) + require.Contains(t, err.Error(), "cannot open captured file") + require.Contains(t, err.Error(), "no such file or directory") } func TestModule_TraceOutputFileError(t *testing.T) { @@ -76,5 +77,6 @@ func TestModule_TraceOutputFileError(t *testing.T) { defer os.Chmod("fixtures", 0755) err = Extract(config, zap.NewNop()) - require.Equal(t, "cannot create output file: open fixtures/trace_success_ui_anonymized.json: permission denied", err.Error()) + require.Contains(t, err.Error(), "cannot create output file") + require.Contains(t, err.Error(), "permission denied") } diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go index dc873372b80..0603bc72839 100644 --- a/cmd/anonymizer/app/uiconv/reader_test.go +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -55,7 +55,8 @@ func TestReader_TraceNonExistent(t *testing.T) { inputFile, zap.NewNop(), ) - require.Equal(t, "cannot open captured file: open fixtures/trace_non_existent.json: no such file or directory", err.Error()) + require.Contains(t, err.Error(), "cannot open captured file") + require.Contains(t, err.Error(), "no such file or directory") } func TestReader_TraceEmpty(t *testing.T) { @@ -95,7 +96,7 @@ func TestReader_TraceInvalidJson(t *testing.T) { require.NoError(t, err) _, err = r.NextSpan() - require.Equal(t, "cannot unmarshal span: json: cannot unmarshal string into Go struct field Span.duration of type uint64; {\"traceID\":\"2be38093ead7a083\",\"spanID\":\"7bd66f09ba90ea3d\",\"duration\": \"invalid\"}\n", err.Error()) + require.Contains(t, err.Error(), "cannot unmarshal span: json: cannot unmarshal string into Go struct field Span.duration of type uint64") assert.Equal(t, 0, r.spansRead) assert.Equal(t, true, r.eofReached) } From 360e29aa4d2a2d5b45654158e640f2a93313c60c Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 17 Nov 2020 13:08:08 +0800 Subject: [PATCH 09/13] Updated uiconv tests: remove loglines related to stdlib Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/uiconv/module_test.go | 2 -- cmd/anonymizer/app/uiconv/reader_test.go | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/anonymizer/app/uiconv/module_test.go b/cmd/anonymizer/app/uiconv/module_test.go index f0af7e9fbfc..cdaee6c9fb1 100644 --- a/cmd/anonymizer/app/uiconv/module_test.go +++ b/cmd/anonymizer/app/uiconv/module_test.go @@ -58,7 +58,6 @@ func TestModule_TraceNonExistent(t *testing.T) { } err := Extract(config, zap.NewNop()) require.Contains(t, err.Error(), "cannot open captured file") - require.Contains(t, err.Error(), "no such file or directory") } func TestModule_TraceOutputFileError(t *testing.T) { @@ -78,5 +77,4 @@ func TestModule_TraceOutputFileError(t *testing.T) { err = Extract(config, zap.NewNop()) require.Contains(t, err.Error(), "cannot create output file") - require.Contains(t, err.Error(), "permission denied") } diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go index 0603bc72839..784a7ea9b1b 100644 --- a/cmd/anonymizer/app/uiconv/reader_test.go +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -56,7 +56,6 @@ func TestReader_TraceNonExistent(t *testing.T) { zap.NewNop(), ) require.Contains(t, err.Error(), "cannot open captured file") - require.Contains(t, err.Error(), "no such file or directory") } func TestReader_TraceEmpty(t *testing.T) { @@ -96,7 +95,7 @@ func TestReader_TraceInvalidJson(t *testing.T) { require.NoError(t, err) _, err = r.NextSpan() - require.Contains(t, err.Error(), "cannot unmarshal span: json: cannot unmarshal string into Go struct field Span.duration of type uint64") + require.Contains(t, err.Error(), "cannot unmarshal span") assert.Equal(t, 0, r.spansRead) assert.Equal(t, true, r.eofReached) } From 6704a92170bfe34b0af1feed45efa1cf5b8d68ac Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 17 Nov 2020 13:11:31 +0800 Subject: [PATCH 10/13] Updated uiconv tests: remove loglines related to stdlib Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/uiconv/extractor_test.go | 2 +- cmd/anonymizer/app/uiconv/reader_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/anonymizer/app/uiconv/extractor_test.go b/cmd/anonymizer/app/uiconv/extractor_test.go index 9a1426d7e09..369ac11caba 100644 --- a/cmd/anonymizer/app/uiconv/extractor_test.go +++ b/cmd/anonymizer/app/uiconv/extractor_test.go @@ -107,7 +107,7 @@ func TestExtractor_TraceScanError(t *testing.T) { require.NoError(t, err) err = extractor.Run() - require.Equal(t, "failed when scanning the file: cannot read file: EOF", err.Error()) + require.Contains(t, err.Error(), "failed when scanning the file") } func loadJSON(t *testing.T, fileName string, i interface{}) { diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go index 784a7ea9b1b..897c398ab9a 100644 --- a/cmd/anonymizer/app/uiconv/reader_test.go +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -67,7 +67,7 @@ func TestReader_TraceEmpty(t *testing.T) { require.NoError(t, err) _, err = r.NextSpan() - require.Equal(t, "cannot read file: EOF", err.Error()) + require.Contains(t, err.Error(), "cannot read file") assert.Equal(t, 0, r.spansRead) assert.Equal(t, true, r.eofReached) } From 784b9d028308c2bc908dafbbe6cebac645dcbf81 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 17 Nov 2020 13:17:23 +0800 Subject: [PATCH 11/13] Simplify the wrong json tests Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json | 2 +- cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json b/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json index cb6d6e55c71..963a379ab25 100644 --- a/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_scan_error.json @@ -1,2 +1,2 @@ -[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null}, +[{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","duration":267037}, ] \ No newline at end of file diff --git a/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json b/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json index c81a85e3ede..672ae56f024 100644 --- a/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json +++ b/cmd/anonymizer/app/uiconv/fixtures/trace_wrong_format.json @@ -1 +1 @@ -{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","flags":1,"operationName":"a071653098f9250d","startTime":1605223981761425,"duration":267037,"tags":[{"key":"span.kind","type":"string","value":"server"}],"logs":[],"process":{"serviceName":"16af988c443cff37","tags":[]},"warnings":null} +{"traceID":"2be38093ead7a083","spanID":"7606ddfe69932d34","duration":267037} From a980b3a05b0700cde345b61fa687451fcf5e42d0 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Tue, 17 Nov 2020 17:00:25 +0800 Subject: [PATCH 12/13] Improve code coverage for uiconv package Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/uiconv/reader.go | 2 +- cmd/anonymizer/app/uiconv/reader_test.go | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/cmd/anonymizer/app/uiconv/reader.go b/cmd/anonymizer/app/uiconv/reader.go index 2ce0f0e0481..edc11853f78 100644 --- a/cmd/anonymizer/app/uiconv/reader.go +++ b/cmd/anonymizer/app/uiconv/reader.go @@ -39,7 +39,7 @@ type Reader struct { func NewReader(capturedFile string, logger *zap.Logger) (*Reader, error) { wd, err := os.Getwd() if err != nil { - return nil, err + return nil, fmt.Errorf("cannot get working directory: %w", err) } logger.Sugar().Infof("Current working dir is %s", wd) diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go index 897c398ab9a..f6fd543eb15 100644 --- a/cmd/anonymizer/app/uiconv/reader_test.go +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -16,6 +16,7 @@ package uiconv import ( "io" + "os" "testing" "github.com/stretchr/testify/assert" @@ -37,15 +38,17 @@ func TestReader_TraceSuccess(t *testing.T) { assert.Equal(t, 1, r.spansRead) assert.Equal(t, false, r.eofReached) + r.spansRead = 999 + s2, err := r.NextSpan() require.NoError(t, err) assert.Equal(t, "471418097747d04a", s2.OperationName) - assert.Equal(t, 2, r.spansRead) + assert.Equal(t, 1000, r.spansRead) assert.Equal(t, true, r.eofReached) _, err = r.NextSpan() require.Equal(t, io.EOF, err) - assert.Equal(t, 2, r.spansRead) + assert.Equal(t, 1000, r.spansRead) assert.Equal(t, true, r.eofReached) } @@ -99,3 +102,19 @@ func TestReader_TraceInvalidJson(t *testing.T) { assert.Equal(t, 0, r.spansRead) assert.Equal(t, true, r.eofReached) } + +func TestReader_GetWDError(t *testing.T) { + inputFile := "fixtures/trace_non_existent.json" + + wd, err := os.Getwd() + require.NoError(t, err) + err = os.Chmod(wd, 0000) + require.NoError(t, err) + defer os.Chmod(wd, 0755) + + _, err = NewReader( + inputFile, + zap.NewNop(), + ) + require.Contains(t, err.Error(), "cannot get working directory") +} From 4628dbb95384db4fce00a89052c98b28b56f3805 Mon Sep 17 00:00:00 2001 From: Ashmita Bohara Date: Wed, 18 Nov 2020 00:34:56 +0800 Subject: [PATCH 13/13] Remove unused code and test-case Signed-off-by: Ashmita Bohara --- cmd/anonymizer/app/uiconv/reader.go | 6 ------ cmd/anonymizer/app/uiconv/reader_test.go | 17 ----------------- 2 files changed, 23 deletions(-) diff --git a/cmd/anonymizer/app/uiconv/reader.go b/cmd/anonymizer/app/uiconv/reader.go index edc11853f78..ee26edb5093 100644 --- a/cmd/anonymizer/app/uiconv/reader.go +++ b/cmd/anonymizer/app/uiconv/reader.go @@ -37,12 +37,6 @@ type Reader struct { // NewReader creates a Reader. func NewReader(capturedFile string, logger *zap.Logger) (*Reader, error) { - wd, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("cannot get working directory: %w", err) - } - logger.Sugar().Infof("Current working dir is %s", wd) - cf, err := os.OpenFile(capturedFile, os.O_RDONLY, os.ModePerm) if err != nil { return nil, fmt.Errorf("cannot open captured file: %w", err) diff --git a/cmd/anonymizer/app/uiconv/reader_test.go b/cmd/anonymizer/app/uiconv/reader_test.go index f6fd543eb15..c9e230c876c 100644 --- a/cmd/anonymizer/app/uiconv/reader_test.go +++ b/cmd/anonymizer/app/uiconv/reader_test.go @@ -16,7 +16,6 @@ package uiconv import ( "io" - "os" "testing" "github.com/stretchr/testify/assert" @@ -102,19 +101,3 @@ func TestReader_TraceInvalidJson(t *testing.T) { assert.Equal(t, 0, r.spansRead) assert.Equal(t, true, r.eofReached) } - -func TestReader_GetWDError(t *testing.T) { - inputFile := "fixtures/trace_non_existent.json" - - wd, err := os.Getwd() - require.NoError(t, err) - err = os.Chmod(wd, 0000) - require.NoError(t, err) - defer os.Chmod(wd, 0755) - - _, err = NewReader( - inputFile, - zap.NewNop(), - ) - require.Contains(t, err.Error(), "cannot get working directory") -}