Merge branch 'master' into add-agent-timeout

.gitignore

@@ -12,11 +12,13 @@ cover.html
 .mkdocs-virtual-env/
 vendor/
 examples/hotrod/hotrod
-cmd/standalone/standalone-linux
+cmd/all-in-one/all-in-one-*
 cmd/agent/agent
 cmd/agent/agent-linux
 cmd/collector/collector
 cmd/collector/collector-linux
+cmd/ingester/ingester
+cmd/ingester/ingester-linux
 cmd/query/query
 cmd/query/query-linux
 crossdock/crossdock-linux

.travis.yml

@@ -59,7 +59,6 @@ script:
   - if [ "$ES_INTEGRATION_TEST" == true ]; then bash ./scripts/travis/es-integration-test.sh ; else echo 'skipping elastic search integration test'; fi
   - if [ "$KAFKA_INTEGRATION_TEST" == true ]; then bash ./scripts/travis/kafka-integration-test.sh ; else echo 'skipping kafka integration test'; fi
   - if [ "$HOTROD" == true ]; then bash ./scripts/travis/hotrod-integration-test.sh ; else echo 'skipping hotrod example'; fi
-  - if [[ "$DEPLOY" == true && "$BRANCH" == "master" ]]; then make build-all-platforms ; else echo 'skipping linux'; fi
 
 after_success:
   - if [ "$COVERAGE" == true ]; then mv cover.out coverage.txt ; else echo 'skipping coverage'; fi
@@ -69,7 +68,7 @@ after_failure:
   - if [ "$CROSSDOCK" == true ]; then make crossdock-logs ; else echo 'skipping crossdock'; fi
 
 before_deploy:
-  - bash ./scripts/travis/package-deploy.sh
+  - make build-all-platforms && bash ./scripts/travis/package-deploy.sh
 
 deploy:
   provider: releases

CHANGELOG.md

@@ -1,15 +1,43 @@
 Changes by Version
 ==================
 
-TBD (pending)
+1.8.0 (unreleased)
+------------------
+
+#### Backend Changes
+
+##### Breaking Changes
+
+- Consolidate query metrics and include result tag ([#1075](https://github.com/jaegertracing/jaeger/pull/1075), [@objectiser](https://github.com/objectiser)
+- Make the metrics produced by jaeger query scoped to the query component, and generated for all span readers (not just ES) ([#1074](https://github.com/jaegertracing/jaeger/pull/1074), [@objectiser](https://github.com/objectiser)
+
+
+1.7.0 (2018-09-19)
 ------------------
 
 #### UI Changes
 
+- Compare two traces ([#228](https://github.com/jaegertracing/jaeger-ui/pull/228), [@tiffon](https://github.com/tiffon))
+- Make tags clickable ([#223](https://github.com/jaegertracing/jaeger-ui/pull/223), [@divdavem](https://github.com/divdavem))
+- Directed graph as React component ([#224](https://github.com/jaegertracing/jaeger-ui/pull/224), [@tiffon](https://github.com/tiffon))
+- Timeline Expand and Collapse Features ([#221](https://github.com/jaegertracing/jaeger-ui/issues/221), [@davit-y](https://github.com/davit-y))
+- Integrate Google Analytics into Search Page ([#220](https://github.com/jaegertracing/jaeger-ui/issues/220), [@davit-y](https://github.com/davit-y))
+
+#### Backend Changes
+
+##### Breaking changes
+
+- `jaeger-standalone` binary has been renamed to `jaeger-all-in-one`. This change also includes package rename from `standalone` to `all-in-one` ([#1062](https://github.com/jaegertracing/jaeger/pull/1062), [@pavolloffay](https://github.com/pavolloffay))
+
 ##### New Features
 
-- Timeline Expand and Collapse Features ([#221](https://github.com/jaegertracing/jaeger-ui/issues/221))
-- Integrate Google Analytics into Search Page ([#220](https://github.com/jaegertracing/jaeger-ui/issues/220))
+- (Experimental) Allow storing tags as object fields in Elasticsearch for better Kibana support(([#1018](https://github.com/jaegertracing/jaeger/pull/1018), [@pavolloffay](https://github.com/pavolloffay))
+- Enable tracing of Cassandra queries ([#1038](https://github.com/jaegertracing/jaeger/pull/1038), [@yurishkuro](https://github.com/yurishkuro))
+- Make Elasticsearch index configurable ([#1009](https://github.com/jaegertracing/jaeger/pull/1009), [@pavolloffay](https://github.com/pavoloffay))
+- Add flags to allow changing ports for HotROD services ([#951](https://github.com/jaegertracing/jaeger/pull/951), [@cboornaz17](https://github.com/cboornaz17))
+- (Experimental) Kafka ingester ([#952](https://github.com/jaegertracing/jaeger/pull/952), [#942](https://github.com/jaegertracing/jaeger/pull/942), [#944](https://github.com/jaegertracing/jaeger/pull/944), [#940](https://github.com/jaegertracing/jaeger/pull/940), [@davit-y](https://github.com/davit-y) and [@vprithvi](https://github.com/vprithvi)))
+- Use tags in agent metrics ([#950](https://github.com/jaegertracing/jaeger/pull/950), [@eundoosong](https://github.com/eundoosong))
+- Add support for Cassandra reconnect interval ([#934](https://github.com/jaegertracing/jaeger/pull/934), [@nyanshak](https://github.com/nyanshak))
 
 1.6.0 (2018-07-10)
 ------------------

CONTRIBUTING.md

@@ -43,13 +43,13 @@ make test
 The `jaeger-ui` submodule contains the source code for the UI assets (requires Node.js 6+). The assets must be compiled first with `make build_ui`, which runs Node.js build and then packages the assets into a Go file that is `.gitignore`-ed. The packaged assets can be enabled by providing a build tag `ui`, e.g.:
 
 ```
-$ go run -tags ui ./cmd/standalone/main.go
+$ go run -tags ui ./cmd/all-in-one/main.go
 ```
 
 Alternatively, the path to the built UI assets can be provided via `--query.static-files` flag:
 
 ```
-$ go run ./cmd/standalone/main.go --query.static-files jaeger-ui/build
+$ go run ./cmd/all-in-one/main.go --query.static-files jaeger-ui/build
 ```
 
 ## Project Structure

Makefile

@@ -34,7 +34,8 @@ GOTEST=go test -v $(RACE)
 GOLINT=golint
 GOVET=go vet
 GOFMT=gofmt
-GOSEC=gosec -quiet -exclude=G104
+GOSEC=gosec -quiet -exclude=G104,G107
+GOSIMPLE=gosimple
 FMT_LOG=fmt.log
 LINT_LOG=lint.log
 IMPORT_LOG=import.log
@@ -93,7 +94,7 @@ test: go-gen
 
 .PHONY: integration-test
 integration-test: go-gen
-	$(GOTEST) -tags=integration ./cmd/standalone/...
+	$(GOTEST) -tags=integration ./cmd/all-in-one/...
 
 .PHONY: storage-integration-test
 storage-integration-test: go-gen
@@ -132,6 +133,7 @@ lint-gosec:
 .PHONY: lint
 lint: lint-gosec
 	$(GOVET) $(TOP_PKGS)
+	$(GOSIMPLE) $(TOP_PKGS)
 	@cat /dev/null > $(LINT_LOG)
 	$(GOLINT) $(TOP_PKGS) | \
 		grep -v \
@@ -182,7 +184,7 @@ build-all-in-one-linux: build_ui
 
 .PHONY: build-all-in-one
 build-all-in-one:
-	CGO_ENABLED=0 installsuffix=cgo go build -tags ui -o ./cmd/standalone/standalone-$(GOOS) $(BUILD_INFO) ./cmd/standalone/main.go
+	CGO_ENABLED=0 installsuffix=cgo go build -tags ui -o ./cmd/all-in-one/all-in-one-$(GOOS) $(BUILD_INFO) ./cmd/all-in-one/main.go
 
 .PHONY: build-agent
 build-agent:
@@ -196,6 +198,10 @@ build-query:
 build-collector:
 	CGO_ENABLED=0 installsuffix=cgo go build -o ./cmd/collector/collector-$(GOOS) $(BUILD_INFO) ./cmd/collector/main.go
 
+.PHONY: build-ingester
+build-ingester:
+	CGO_ENABLED=0 installsuffix=cgo go build -o ./cmd/ingester/ingester-$(GOOS) $(BUILD_INFO) ./cmd/ingester/main.go
+
 .PHONY: docker-no-ui
 docker-no-ui: build-binaries-linux build-crossdock-linux
 	make docker-images-only
@@ -216,7 +222,7 @@ build-binaries-darwin:
 	GOOS=darwin $(MAKE) build-platform-binaries
 
 .PHONY: build-platform-binaries
-build-platform-binaries: build-agent build-collector build-query build-all-in-one build-examples
+build-platform-binaries: build-agent build-collector build-query build-ingester build-all-in-one build-examples
 
 .PHONY: build-all-platforms
 build-all-platforms: build-binaries-linux build-binaries-windows build-binaries-darwin
@@ -227,7 +233,7 @@ docker-images-only:
 	@echo "Finished building jaeger-cassandra-schema =============="
 	docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es
 	@echo "Finished building jaeger-es-indices-clean =============="
-	for component in agent collector query ; do \
+	for component in agent collector query ingester ; do \
 		docker build -t $(DOCKER_NAMESPACE)/jaeger-$$component:${DOCKER_TAG} cmd/$$component ; \
 		echo "Finished building $$component ==============" ; \
 	done
@@ -242,7 +248,7 @@ docker-push:
 	if [ $$CONFIRM != "y" ] && [ $$CONFIRM != "Y" ]; then \
 		echo "Exiting." ; exit 1 ; \
 	fi
-	for component in agent cassandra-schema es-index-cleaner collector query example-hotrod; do \
+	for component in agent cassandra-schema es-index-cleaner collector query ingester example-hotrod; do \
 		docker push $(DOCKER_NAMESPACE)/jaeger-$$component ; \
 	done
 
@@ -268,11 +274,12 @@ build-crossdock-fresh: build-crossdock-linux
 
 .PHONY: install-tools
 install-tools:
-	go get github.com/wadey/gocovmerge
-	go get golang.org/x/tools/cmd/cover
-	go get github.com/golang/lint/golint
-	go get github.com/sectioneight/md-to-godoc
-	go get github.com/securego/gosec/cmd/gosec/...
+	go get -u github.com/wadey/gocovmerge
+	go get -u golang.org/x/tools/cmd/cover
+	go get -u github.com/golang/lint/golint
+	go get -u github.com/sectioneight/md-to-godoc
+	go get -u github.com/securego/gosec/cmd/gosec/...
+	go get -u honnef.co/go/tools/cmd/gosimple
 
 .PHONY: install-ci
 install-ci: install install-tools
@@ -386,4 +393,4 @@ proto-install:
 		./vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway \
 		./vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
 		# ./vendor/github.com/mwitkow/go-proto-validators/protoc-gen-govalidators \
-		# ./vendor/github.com/rakyll/statik
+		# ./vendor/github.com/rakyll/statik

cmd/agent/app/httpserver/thrift-0.9.2/constants.go

@@ -20,6 +20,7 @@ package sampling
 import (
 	"bytes"
 	"fmt"
+
 	"github.com/apache/thrift/lib/go/thrift"
 )
 

cmd/agent/app/httpserver/thrift-0.9.2/ttypes.go

@@ -20,6 +20,7 @@ package sampling
 import (
 	"bytes"
 	"fmt"
+
 	"github.com/apache/thrift/lib/go/thrift"
 )
 

cmd/agent/app/servers/server.go

@@ -14,7 +14,9 @@
 
 package servers
 
-import "io"
+import (
+	"io"
+)
 
 // Server is the interface for servers that receive inbound span submissions from client.
 type Server interface {

cmd/standalone/Dockerfile → cmd/all-in-one/Dockerfile

@@ -18,8 +18,8 @@ EXPOSE 14268
 # Web HTTP
 EXPOSE 16686
 
-COPY ./cmd/standalone/standalone-linux /go/bin/
-COPY ./cmd/standalone/sampling_strategies.json /etc/jaeger/
+COPY ./cmd/all-in-one/all-in-one-linux /go/bin/
+COPY ./cmd/all-in-one/sampling_strategies.json /etc/jaeger/
 
-ENTRYPOINT ["/go/bin/standalone-linux"]
+ENTRYPOINT ["/go/bin/all-in-one-linux"]
 CMD ["--sampling.strategies-file=/etc/jaeger/sampling_strategies.json"]

cmd/standalone/standalone_test.go → cmd/all-in-one/all_in_one_test.go

@@ -51,7 +51,7 @@ var (
 	}
 )
 
-func TestStandalone(t *testing.T) {
+func TestAllInOne(t *testing.T) {
 	// Check if the query service is available
 	if err := healthCheck(); err != nil {
 		t.Fatal(err)

cmd/standalone/main.go → cmd/all-in-one/main.go

@@ -26,10 +26,12 @@ import (
 	"syscall"
 
 	"github.com/gorilla/mux"
+	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	jaegerClientConfig "github.com/uber/jaeger-client-go/config"
+	jaegerClientZapLog "github.com/uber/jaeger-client-go/log/zap"
 	"github.com/uber/jaeger-lib/metrics"
 	"github.com/uber/tchannel-go"
 	"github.com/uber/tchannel-go/thrift"
@@ -53,14 +55,15 @@ import (
 	"github.com/jaegertracing/jaeger/plugin/storage"
 	"github.com/jaegertracing/jaeger/storage/dependencystore"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
+	storageMetrics "github.com/jaegertracing/jaeger/storage/spanstore/metrics"
 	jc "github.com/jaegertracing/jaeger/thrift-gen/jaeger"
 	sc "github.com/jaegertracing/jaeger/thrift-gen/sampling"
 	zc "github.com/jaegertracing/jaeger/thrift-gen/zipkincore"
 )
 
-// standalone/main is a standalone full-stack jaeger backend, backed by a memory store
+// all-in-one/main is a standalone full-stack jaeger backend, backed by a memory store
 func main() {
-	var signalsChannel = make(chan os.Signal, 0)
+	var signalsChannel = make(chan os.Signal)
 	signal.Notify(signalsChannel, os.Interrupt, syscall.SIGTERM)
 
 	if os.Getenv(storage.SpanStorageTypeEnvVar) == "" {
@@ -76,7 +79,7 @@ func main() {
 	}
 	v := viper.New()
 	command := &cobra.Command{
-		Use:   "jaeger-standalone",
+		Use:   "jaeger-all-in-one",
 		Short: "Jaeger all-in-one distribution with agent, collector and query in one process.",
 		Long: `Jaeger all-in-one distribution with agent, collector and query. Use with caution this version
 		 uses only in-memory database.`,
@@ -128,18 +131,15 @@ func main() {
 			startCollector(cOpts, spanWriter, logger, metricsFactory, samplingHandler, hc)
 			startQuery(qOpts, spanReader, dependencyReader, logger, metricsFactory, mBldr, hc)
 			hc.Ready()
-
-			select {
-			case <-signalsChannel:
-				if closer, ok := spanWriter.(io.Closer); ok {
-					err := closer.Close()
-					if err != nil {
-						logger.Error("Failed to close span writer", zap.Error(err))
-					}
+			<-signalsChannel
+			logger.Info("Shutting down")
+			if closer, ok := spanWriter.(io.Closer); ok {
+				err := closer.Close()
+				if err != nil {
+					logger.Error("Failed to close span writer", zap.Error(err))
 				}
-
-				logger.Info("Jaeger Standalone is finishing")
 			}
+			logger.Info("Shutdown complete")
 			return nil
 		},
 	}
@@ -277,10 +277,18 @@ func startQuery(
 			Param: 1.0,
 		},
 		RPCMetrics: true,
-	}.New("jaeger-query", jaegerClientConfig.Metrics(baseFactory.Namespace("client", nil)))
+	}.New(
+		"jaeger-query",
+		jaegerClientConfig.Metrics(baseFactory.Namespace("client", nil)),
+		jaegerClientConfig.Logger(jaegerClientZapLog.NewLogger(logger)),
+	)
 	if err != nil {
 		logger.Fatal("Failed to initialize tracer", zap.Error(err))
 	}
+	opentracing.SetGlobalTracer(tracer)
+
+	spanReader = storageMetrics.NewReadMetricsDecorator(spanReader, baseFactory.Namespace("query", nil))
+
 	apiHandler := queryApp.NewAPIHandler(
 		spanReader,
 		depReader,

cmd/collector/app/metrics.go

@@ -28,7 +28,8 @@ const (
 )
 
 // SpanProcessorMetrics contains all the necessary metrics for the SpanProcessor
-type SpanProcessorMetrics struct { //TODO - initialize metrics in the traditional factory way. Initialize map afterward.
+type SpanProcessorMetrics struct {
+	//TODO - initialize metrics in the traditional factory way. Initialize map afterward.
 	// SaveLatency measures how long the actual save to storage takes
 	SaveLatency metrics.Timer
 	// InQueueLatency measures how long the span spends in the queue
@@ -41,10 +42,11 @@ type SpanProcessorMetrics struct { //TODO - initialize metrics in the traditiona
 	QueueLength metrics.Gauge
 	// ErrorBusy counts number of return ErrServerBusy
 	ErrorBusy metrics.Counter
-	// SavedBySvc contains span and trace counts by service
-	SavedBySvc   metricsBySvc  // spans actually saved
-	serviceNames metrics.Gauge // total number of unique service name metrics reported by this collector
-	spanCounts   map[string]CountsBySpanType
+	// SavedOkBySvc contains span and trace counts by service
+	SavedOkBySvc  metricsBySvc  // spans actually saved
+	SavedErrBySvc metricsBySvc  // spans failed to save
+	serviceNames  metrics.Gauge // total number of unique service name metrics reported by this collector
+	spanCounts    map[string]CountsBySpanType
 }
 
 type countsBySvc struct {
@@ -86,7 +88,8 @@ func NewSpanProcessorMetrics(serviceMetrics metrics.Factory, hostMetrics metrics
 		BatchSize:      hostMetrics.Gauge("batch-size", nil),
 		QueueLength:    hostMetrics.Gauge("queue-length", nil),
 		ErrorBusy:      hostMetrics.Counter("error.busy", nil),
-		SavedBySvc:     newMetricsBySvc(serviceMetrics, "saved-by-svc"),
+		SavedOkBySvc:   newMetricsBySvc(serviceMetrics.Namespace("", map[string]string{"result": "ok"}), "saved-by-svc"),
+		SavedErrBySvc:  newMetricsBySvc(serviceMetrics.Namespace("", map[string]string{"result": "err"}), "saved-by-svc"),
 		spanCounts:     spanCounts,
 		serviceNames:   hostMetrics.Gauge("spans.serviceNames", nil),
 	}

cmd/collector/app/model_consumer.go

@@ -14,7 +14,9 @@
 
 package app
 
-import "github.com/jaegertracing/jaeger/model"
+import (
+	"github.com/jaegertracing/jaeger/model"
+)
 
 // ProcessSpan processes a Domain Model Span
 type ProcessSpan func(span *model.Span)

cmd/collector/app/sanitizer/cache/auto_refresh_cache.go

@@ -1,3 +1,4 @@
+// Copyright (c) 2018 The Jaeger Authors.
 // Copyright (c) 2017 Uber Technologies, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -45,7 +46,7 @@ func NewAutoRefreshCache(
 	readRefreshInterval, saveRefreshInterval time.Duration,
 ) Cache {
 	return &autoRefreshCache{
-		cache:               make(map[string]string, 0),
+		cache:               make(map[string]string),
 		extSource:           extSource,
 		storage:             storage,
 		logger:              logger,