WIP - gRPC Plugin framework

Gopkg.toml

@@ -116,3 +116,17 @@
 [prune]
   go-tests = true
   unused-packages = true
+  non-go = true
+
+  [[prune.project]]
+    name = "github.com/gogo/protobuf"
+    non-go = false
+    unused-packages = false
+  [[prune.project]]
+    name = "github.com/grpc-ecosystem/grpc-gateway"
+    non-go = false
+    unused-packages = false
+  [[prune.project]]
+      name = "github.com/gogo/googleapis"
+      non-go = false
+      unused-packages = false

Makefile

@@ -323,9 +323,11 @@ PROTO_INCLUDES := \
 	-I model/proto \
 	-I vendor/github.com/grpc-ecosystem/grpc-gateway \
 	-I vendor/github.com/gogo/googleapis \
+	-I vendor/github.com/gogo/protobuf/protobuf \
 	-I vendor/github.com/gogo/protobuf
 # Remapping of std types to gogo types (must not contain spaces)
 PROTO_GOGO_MAPPINGS := $(shell echo \
+		Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/types, \
 		Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types, \
 		Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types, \
 		Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types, \
@@ -386,3 +388,15 @@ proto-install:
 		./vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
 		# ./vendor/github.com/mwitkow/go-proto-validators/protoc-gen-govalidators \
 		# ./vendor/github.com/rakyll/statik
+
+.PHONY: plugin-proto
+plugin-proto:
+	protoc \
+		-I model/proto \
+		-I vendor/github.com/grpc-ecosystem/grpc-gateway \
+		-I vendor/github.com/gogo/googleapis \
+		-I vendor/github.com/gogo/protobuf/protobuf \
+		-I vendor/github.com/gogo/protobuf \
+    	-I plugin/storage/grpc/proto \
+		--gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/plugin/storage/grpc/proto \
+		plugin/storage/grpc/proto/storage.proto

cmd/collector/app/span_processor.go

@@ -103,7 +103,7 @@ func (sp *spanProcessor) saveSpan(span *model.Span) {
 		sp.metrics.SavedErrBySvc.ReportServiceNameForSpan(span)
 	} else {
 		sp.logger.Debug("Span written to the storage by the collector",
-			zap.Stringer("trace-id", span.TraceID), zap.Stringer("span-id", span.SpanID))
+			zap.Stringer("trace-id", &span.TraceID), zap.Stringer("span-id", span.SpanID))
 		sp.metrics.SavedOkBySvc.ReportServiceNameForSpan(span)
 	}
 	sp.metrics.SaveLatency.Record(time.Since(startTime))

cmd/noop-grpc-plugin/load_test.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/jaegertracing/jaeger/plugin/storage/grpc"
+	"github.com/spf13/viper"
+	"github.com/uber/jaeger-lib/metrics/prometheus"
+	"go.uber.org/zap"
+	"testing"
+	"time"
+)
+
+var logger = zap.NewNop()
+
+func BenchmarkNoopSpanWriter(b *testing.B) {
+	s := &noopStore{}
+
+	for n := 0; n < b.N; n++ {
+		s.WriteSpan(&model.Span{
+			TraceID:model.NewTraceID(1, 2),
+			SpanID:model.NewSpanID(1),
+			OperationName: "test",
+			StartTime: time.Now(),
+			Duration: 1 * time.Second,
+			Process: model.NewProcess("process", []model.KeyValue{
+
+			}),
+			ProcessID: "process_id",
+			Tags: []model.KeyValue{
+				{
+					Key:"test",
+					VStr: "",
+				},
+			},
+		})
+	}
+}
+
+func BenchmarkGRPCNoopSpanWriter(b *testing.B) {
+	v := viper.New()
+
+	v.Set("grpc-storage-plugin.binary", "noop-grpc-plugin")
+
+	f := grpc.NewFactory()
+	f.InitFromViper(v)
+
+	metricsFactory := prometheus.New()
+
+	f.Initialize( metricsFactory, logger)
+
+	sw, err := f.CreateSpanWriter()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	for n := 0; n < b.N; n++ {
+		sw.WriteSpan(&model.Span{
+			TraceID:model.NewTraceID(1, 2),
+			SpanID:model.NewSpanID(1),
+			OperationName: "test",
+			StartTime: time.Now(),
+			Duration: 1 * time.Second,
+			Process: model.NewProcess("process", []model.KeyValue{
+
+			}),
+			ProcessID: "process_id",
+			Tags: []model.KeyValue{
+				{
+					Key:"test",
+					VStr: "",
+				},
+			},
+		})
+	}
+}

cmd/noop-grpc-plugin/main.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2018 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"github.com/hashicorp/go-plugin"
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
+	"github.com/jaegertracing/jaeger/plugin/storage/memory"
+	"github.com/jaegertracing/jaeger/storage/spanstore"
+	"github.com/spf13/viper"
+	"path"
+	"strings"
+)
+
+var configPath string
+
+func main() {
+	flag.StringVar(&configPath, "config", "", "A path to the plugin's configuration file")
+	flag.Parse()
+
+	if configPath != "" {
+		viper.SetConfigFile(path.Base(configPath))
+		viper.AddConfigPath(path.Dir(configPath))
+	}
+
+	v := viper.New()
+	v.AutomaticEnv()
+	v.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
+
+	opts := memory.Options{}
+	opts.InitFromViper(v)
+
+	plugin.Serve(&plugin.ServeConfig{
+		HandshakeConfig: shared.Handshake,
+		VersionedPlugins: map[int]plugin.PluginSet{
+			1: map[string]plugin.Plugin{
+				shared.StoragePluginIdentifier: &shared.StorageGRPCPlugin{
+					Impl: &noopStore{},
+				},
+			},
+		},
+		GRPCServer: plugin.DefaultGRPCServer,
+	})
+}
+
+type noopStore struct {
+
+}
+
+func (*noopStore) GetTrace(ctx context.Context, traceID model.TraceID) (*model.Trace, error) {
+	return nil, nil
+}
+
+func (*noopStore) GetServices(ctx context.Context) ([]string, error) {
+	return nil, nil
+}
+
+func (*noopStore) GetOperations(ctx context.Context, service string) ([]string, error) {
+	return nil, nil
+}
+
+func (*noopStore) FindTraces(ctx context.Context, query *spanstore.TraceQueryParameters) ([]*model.Trace, error) {
+	return nil,nil
+}
+
+func (*noopStore) FindTraceIDs(ctx context.Context, query *spanstore.TraceQueryParameters) ([]model.TraceID, error) {
+	panic("implement me")
+}
+
+func (*noopStore) WriteSpan(span *model.Span) error {
+	return nil
+}

model/ids.go

@@ -23,12 +23,6 @@ import (
 	"github.com/gogo/protobuf/jsonpb"
 )
 
-// TraceID is a random 128bit identifier for a trace
-type TraceID struct {
-	Low  uint64 `json:"lo"`
-	High uint64 `json:"hi"`
-}
-
 // SpanID is a random 64bit identifier for a span
 type SpanID uint64
 
@@ -39,13 +33,6 @@ func NewTraceID(high, low uint64) TraceID {
 	return TraceID{High: high, Low: low}
 }
 
-func (t TraceID) String() string {
-	if t.High == 0 {
-		return fmt.Sprintf("%x", t.Low)
-	}
-	return fmt.Sprintf("%x%016x", t.High, t.Low)
-}
-
 // TraceIDFromString creates a TraceID from a hexadecimal string
 func TraceIDFromString(s string) (TraceID, error) {
 	var hi, lo uint64
@@ -78,29 +65,6 @@ func (t *TraceID) UnmarshalText(text []byte) error {
 	return fmt.Errorf("unsupported method TraceID.UnmarshalText; please use github.com/gogo/protobuf/jsonpb for marshalling")
 }
 
-// Size returns the size of this datum in protobuf. It is always 16 bytes.
-func (t *TraceID) Size() int {
-	return 16
-}
-
-// MarshalTo converts trace ID into a binary representation. Called by protobuf serialization.
-func (t *TraceID) MarshalTo(data []byte) (n int, err error) {
-	var b [16]byte
-	binary.BigEndian.PutUint64(b[:8], uint64(t.High))
-	binary.BigEndian.PutUint64(b[8:], uint64(t.Low))
-	return marshalBytes(data, b[:])
-}
-
-// Unmarshal inflates this trace ID from binary representation. Called by protobuf serialization.
-func (t *TraceID) Unmarshal(data []byte) error {
-	if len(data) < 16 {
-		return fmt.Errorf("buffer is too short")
-	}
-	t.High = binary.BigEndian.Uint64(data[:8])
-	t.Low = binary.BigEndian.Uint64(data[8:])
-	return nil
-}
-
 func marshalBytes(dst []byte, src []byte) (n int, err error) {
 	if len(dst) < len(src) {
 		return 0, fmt.Errorf("buffer is too short")