diff --git a/.github/workflows/ci-protogen-tests.yml b/.github/workflows/ci-protogen-tests.yml index f87061e5292..237375dc811 100644 --- a/.github/workflows/ci-protogen-tests.yml +++ b/.github/workflows/ci-protogen-tests.yml @@ -26,11 +26,14 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - submodules: true + submodules: recursive - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: 1.21.x - - name: Run protogen validation - run: make init-submodules && make proto && git diff --name-status --exit-code + - name: Verify Protobuf types are up to date + run: make proto && git diff --name-status --exit-code + + # - name: Verify Thrift types are up to date + # run: make thrift && git diff --name-status --exit-code diff --git a/Makefile b/Makefile index 3858c2ce101..5a56d4233df 100644 --- a/Makefile +++ b/Makefile @@ -4,13 +4,12 @@ SHELL := /bin/bash JAEGER_IMPORT_PATH = github.com/jaegertracing/jaeger STORAGE_PKGS = ./plugin/storage/integration/... -GO = go -include docker/Makefile -include crossdock/rules.mk +# These DOCKER_xxx vars are used when building Docker images. +DOCKER_NAMESPACE?=jaegertracing +DOCKER_TAG?=latest # TODO we can compartmentalize this Makefile better, by separting: -# - thrift and proto builds # - integration tests # - all the binary building targets @@ -51,6 +50,10 @@ ifeq ($(UNAME), s390x) else RACE=-race endif +# sed on Mac does not support the same syntax for in-place updates as sed on Linux +# When running on MacOS it's best to install gsed and run Makefile with SED=gsed +SED=sed +GO=go GOOS ?= $(shell $(GO) env GOOS) GOARCH ?= $(shell $(GO) env GOARCH) GOBUILD=CGO_ENABLED=0 installsuffix=cgo $(GO) build -trimpath @@ -60,41 +63,35 @@ GOFMT=gofmt GOFUMPT=gofumpt FMT_LOG=.fmt.log IMPORT_LOG=.import.log -COLORIZE ?= | $(SED) 's/PASS/✅ PASS/g' | $(SED) 's/FAIL/❌ FAIL/g' | $(SED) 's/SKIP/☠️ SKIP/g' +COLORIZE ?= | $(SED) 's/PASS/✅ PASS/g' | $(SED) 's/FAIL/❌ FAIL/g' | $(SED) 's/SKIP/🔕 SKIP/g' GIT_SHA=$(shell git rev-parse HEAD) GIT_CLOSEST_TAG=$(shell git describe --abbrev=0 --tags) ifneq ($(GIT_CLOSEST_TAG),$(shell echo ${GIT_CLOSEST_TAG} | grep -E "$(semver_regex)")) $(warning GIT_CLOSEST_TAG=$(GIT_CLOSEST_TAG) is not in the semver format $(semver_regex)) endif -GIT_CLOSEST_TAG_MAJOR := $(shell echo $(GIT_CLOSEST_TAG) | sed -n 's/v\([0-9]*\)\.[0-9]*\.[0-9]/\1/p') -GIT_CLOSEST_TAG_MINOR := $(shell echo $(GIT_CLOSEST_TAG) | sed -n 's/v[0-9]*\.\([0-9]*\)\.[0-9]/\1/p') -GIT_CLOSEST_TAG_PATCH := $(shell echo $(GIT_CLOSEST_TAG) | sed -n 's/v[0-9]*\.[0-9]*\.\([0-9]\)/\1/p') +GIT_CLOSEST_TAG_MAJOR := $(shell echo $(GIT_CLOSEST_TAG) | $(SED) -n 's/v\([0-9]*\)\.[0-9]*\.[0-9]/\1/p') +GIT_CLOSEST_TAG_MINOR := $(shell echo $(GIT_CLOSEST_TAG) | $(SED) -n 's/v[0-9]*\.\([0-9]*\)\.[0-9]/\1/p') +GIT_CLOSEST_TAG_PATCH := $(shell echo $(GIT_CLOSEST_TAG) | $(SED) -n 's/v[0-9]*\.[0-9]*\.\([0-9]\)/\1/p') DATE=$(shell TZ=UTC0 git show --quiet --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd") BUILD_INFO_IMPORT_PATH=$(JAEGER_IMPORT_PATH)/pkg/version BUILD_INFO=-ldflags "-X $(BUILD_INFO_IMPORT_PATH).commitSHA=$(GIT_SHA) -X $(BUILD_INFO_IMPORT_PATH).latestVersion=$(GIT_CLOSEST_TAG) -X $(BUILD_INFO_IMPORT_PATH).date=$(DATE)" -SED=sed -THRIFT_VER=0.14 -THRIFT_IMG=jaegertracing/thrift:$(THRIFT_VER) -THRIFT=docker run --rm -u ${shell id -u} -v "${PWD}:/data" $(THRIFT_IMG) thrift -THRIFT_GO_ARGS=thrift_import="github.com/apache/thrift/lib/go/thrift" -THRIFT_GEN_DIR=thrift-gen - SWAGGER_VER=0.27.0 SWAGGER_IMAGE=quay.io/goswagger/swagger:v$(SWAGGER_VER) SWAGGER=docker run --rm -it -u ${shell id -u} -v "${PWD}:/go/src/" -w /go/src/ $(SWAGGER_IMAGE) SWAGGER_GEN_DIR=swagger-gen -JAEGER_DOCKER_PROTOBUF=jaegertracing/protobuf:0.4.0 - -DOCKER_NAMESPACE?=jaegertracing -DOCKER_TAG?=latest - MOCKERY=mockery GOVERSIONINFO=goversioninfo SYSOFILE=resource.syso +# import other Makefiles after the variables are defined +include docker/Makefile +include Makefile.Protobuf.mk +include Makefile.Thrift.mk +include Makefile.Crossdock.mk + .DEFAULT_GOAL := test-and-lint .PHONY: test-and-lint @@ -425,25 +422,6 @@ docker-images-elastic: create-baseimg docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE) --build-arg TARGETARCH=$(GOARCH) cmd/es-rollover @echo "Finished building jaeger-es-indices-clean ==============" -# TODO does this target need to exist? It's only called from crossdock, apparently. -.PHONY: docker-images-jaeger-backend -docker-images-jaeger-backend: create-baseimg create-debugimg - for component in "jaeger-agent" "jaeger-collector" "jaeger-query" "jaeger-ingester" "all-in-one" ; do \ - regex="jaeger-(.*)"; \ - component_suffix=$$component; \ - if [[ $$component =~ $$regex ]]; then \ - component_suffix="$${BASH_REMATCH[1]}"; \ - fi; \ - docker buildx build --target $(TARGET) \ - --tag $(DOCKER_NAMESPACE)/$$component$(SUFFIX):${DOCKER_TAG} \ - --build-arg base_image=$(BASE_IMAGE) \ - --build-arg debug_image=$(DEBUG_IMAGE) \ - --build-arg TARGETARCH=$(GOARCH) \ - --load \ - cmd/$$component_suffix; \ - echo "Finished building $$component ==============" ; \ - done; - .PHONY: docker-images-tracegen docker-images-tracegen: docker build -t $(DOCKER_NAMESPACE)/jaeger-tracegen:${DOCKER_TAG} cmd/tracegen/ --build-arg TARGETARCH=$(GOARCH) @@ -454,34 +432,6 @@ docker-images-anonymizer: docker build -t $(DOCKER_NAMESPACE)/jaeger-anonymizer:${DOCKER_TAG} cmd/anonymizer/ --build-arg TARGETARCH=$(GOARCH) @echo "Finished building jaeger-anonymizer ==============" -.PHONY: build-crossdock-binary -build-crossdock-binary: - $(GOBUILD) -o ./crossdock/crossdock-$(GOOS)-$(GOARCH) ./crossdock/main.go - -.PHONY: build-crossdock-linux -build-crossdock-linux: - GOOS=linux $(MAKE) build-crossdock-binary - -# Crossdock tests do not require fully functioning UI, so we skip it to speed up the build. -.PHONY: build-crossdock-ui-placeholder -build-crossdock-ui-placeholder: - mkdir -p jaeger-ui/packages/jaeger-ui/build/ - cp cmd/query/app/ui/placeholder/index.html jaeger-ui/packages/jaeger-ui/build/index.html - $(MAKE) build-ui - -.PHONY: build-crossdock -build-crossdock: build-crossdock-ui-placeholder build-binaries-linux build-crossdock-linux docker-images-cassandra docker-images-jaeger-backend - docker build -t $(DOCKER_NAMESPACE)/test-driver:${DOCKER_TAG} --build-arg TARGETARCH=$(GOARCH) crossdock/ - @echo "Finished building test-driver ==============" ; \ - -.PHONY: build-and-run-crossdock -build-and-run-crossdock: build-crossdock - make crossdock - -.PHONY: build-crossdock-fresh -build-crossdock-fresh: build-crossdock-linux - make crossdock-fresh - .PHONY: changelog changelog: ./scripts/release-notes.py --exclude-dependabot @@ -515,30 +465,10 @@ test-ci: install-test-tools build-examples cover test-report test-report: cat test-results.json | go-junit-report -parser gojson > junit-report.xml -.PHONY: thrift -thrift: idl/thrift/jaeger.thrift thrift-image - [ -d $(THRIFT_GEN_DIR) ] || mkdir $(THRIFT_GEN_DIR) - $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/agent.thrift -# TODO sed is GNU and BSD compatible - sed -i.bak 's|"zipkincore"|"$(JAEGER_IMPORT_PATH)/thrift-gen/zipkincore"|g' $(THRIFT_GEN_DIR)/agent/*.go - sed -i.bak 's|"jaeger"|"$(JAEGER_IMPORT_PATH)/thrift-gen/jaeger"|g' $(THRIFT_GEN_DIR)/agent/*.go - $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/jaeger.thrift - $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/sampling.thrift - $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/baggage.thrift - $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/zipkincore.thrift - rm -rf thrift-gen/*/*-remote thrift-gen/*/*.bak - -idl/thrift/jaeger.thrift: - $(MAKE) init-submodules - .PHONY: init-submodules init-submodules: git submodule update --init --recursive -.PHONY: thrift-image -thrift-image: - $(THRIFT) -version - .PHONY: generate-zipkin-swagger generate-zipkin-swagger: init-submodules $(SWAGGER) generate server -f ./idl/swagger/zipkin2-api.yaml -t $(SWAGGER_GEN_DIR) -O PostSpans --exclude-main @@ -554,160 +484,6 @@ generate-mocks: install-tools echo-version: @echo $(GIT_CLOSEST_TAG) -PROTO_INTERMEDIATE_DIR = proto-gen/.patched-otel-proto -PROTOC := docker run --rm -u ${shell id -u} -v${PWD}:${PWD} -w${PWD} ${JAEGER_DOCKER_PROTOBUF} --proto_path=${PWD} -PROTO_INCLUDES := \ - -Iidl/proto/api_v2 \ - -Iidl/proto/api_v3 \ - -Imodel/proto/metrics \ - -I$(PROTO_INTERMEDIATE_DIR) \ - -I/usr/include/github.com/gogo/protobuf -# Remapping of std types to gogo types (must not contain spaces) -PROTO_GOGO_MAPPINGS := $(shell echo \ - Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/types, \ - Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types, \ - Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types, \ - Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types, \ - Mgoogle/api/annotations.proto=github.com/gogo/googleapis/google/api, \ - Mmodel.proto=github.com/jaegertracing/jaeger/model \ - | sed 's/ //g') - -.PHONY: proto -proto: proto-prepare-otel - # Generate gogo, swagger, go-validators, gRPC-storage-plugin output. - # - # -I declares import folders, in order of importance - # This is how proto resolves the protofile imports. - # It will check for the protofile relative to each of these - # folders and use the first one it finds. - # - # --gogo_out generates GoGo Protobuf output with gRPC plugin enabled. - # --govalidators_out generates Go validation files for our messages types, if specified. - # - # The lines starting with Mgoogle/... are proto import replacements, - # which cause the generated file to import the specified packages - # instead of the go_package's declared by the imported protof files. - # - # $$GOPATH/src is the output directory. It is relative to the GOPATH/src directory - # since we've specified a go_package option relative to that directory. - # - # model/proto/jaeger.proto is the location of the protofile we use. - # - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/model/ \ - idl/proto/api_v2/model.proto - - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v2 \ - idl/proto/api_v2/query.proto - ### --swagger_out=allow_merge=true:$(PWD)/proto-gen/openapi/ \ - - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v2/metrics \ - model/proto/metrics/otelspankind.proto - - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v2/metrics \ - model/proto/metrics/openmetrics.proto - - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v2/metrics \ - model/proto/metrics/metricsquery.proto - - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v2 \ - idl/proto/api_v2/collector.proto - - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v2 \ - idl/proto/api_v2/sampling.proto - - $(PROTOC) \ - $(PROTO_INCLUDES) \ - -Iplugin/storage/grpc/proto \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/storage_v1 \ - plugin/storage/grpc/proto/storage.proto - - $(PROTOC) \ - -Imodel/proto \ - --go_out=$(PWD)/model/ \ - model/proto/model_test.proto - - $(PROTOC) \ - -Iplugin/storage/grpc/proto \ - --go_out=$(PWD)/plugin/storage/grpc/proto/ \ - plugin/storage/grpc/proto/storage_test.proto - - $(PROTOC) \ - -Iidl/proto \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/zipkin \ - idl/proto/zipkin.proto - - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,paths=source_relative,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/otel \ - $(PROTO_INTERMEDIATE_DIR)/common/v1/common.proto - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,paths=source_relative,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/otel \ - $(PROTO_INTERMEDIATE_DIR)/resource/v1/resource.proto - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,paths=source_relative,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/otel \ - $(PROTO_INTERMEDIATE_DIR)/trace/v1/trace.proto - - # Target proto-prepare-otel modifies OTEL proto to use import path jaeger.proto.* - # The modification is needed because OTEL collector already uses opentelemetry.proto.* - # and two complied protobuf types cannot have the same import path. The root cause is that the compiled OTLP - # in the collector is in private package, hence it cannot be used in Jaeger. - # The following statements revert changes in OTEL proto and only modify go package. - # This way the service will use opentelemetry.proto.trace.v1.ResourceSpans but in reality at runtime - # it uses jaeger.proto.trace.v1.ResourceSpans which is the same type in a different package which - # prevents panic of two equal proto types. - rm -rf $(PROTO_INTERMEDIATE_DIR)/* - cp -R idl/opentelemetry-proto/* $(PROTO_INTERMEDIATE_DIR) - find $(PROTO_INTERMEDIATE_DIR) -name "*.proto" | xargs -L 1 sed -i 's+go.opentelemetry.io/proto/otlp+github.com/jaegertracing/jaeger/proto-gen/otel+g' - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v3 \ - idl/proto/api_v3/query_service.proto - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --grpc-gateway_out=logtostderr=true,grpc_api_configuration=idl/proto/api_v3/query_service_http.yaml,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v3 \ - idl/proto/api_v3/query_service.proto - rm -rf $(PROTO_INTERMEDIATE_DIR) - -.PHONY: proto-prepare-otel -proto-prepare-otel: - @echo -- - @echo -- Copying to $(PROTO_INTERMEDIATE_DIR) - @echo -- - mkdir -p $(PROTO_INTERMEDIATE_DIR) - cp -R idl/opentelemetry-proto/opentelemetry/proto/* $(PROTO_INTERMEDIATE_DIR) - - @echo -- - @echo -- Editing proto - @echo -- - @# Change: - @# go import from github.com/open-telemetry/opentelemetry-proto/gen/go/* to github.com/jaegertracing/jaeger/proto-gen/otel/* - @# proto package from opentelemetry.proto.* to jaeger.proto.* - @# remove import opentelemetry/proto - find $(PROTO_INTERMEDIATE_DIR) -name "*.proto" | xargs -L 1 sed -i -f otel_proto_patch.sed - -.PHONY: proto-hotrod -proto-hotrod: - $(PROTOC) \ - $(PROTO_INCLUDES) \ - --gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/ \ - examples/hotrod/services/driver/driver.proto - .PHONY: certs certs: cd pkg/config/tlscfg/testdata && ./gen-certs.sh diff --git a/Makefile.Crossdock.mk b/Makefile.Crossdock.mk new file mode 100644 index 00000000000..96cbecaecdd --- /dev/null +++ b/Makefile.Crossdock.mk @@ -0,0 +1,50 @@ +# Copyright (c) 2023 The Jaeger Authors. +# SPDX-License-Identifier: Apache-2.0 + +include crossdock/rules.mk + +.PHONY: build-crossdock-binary +build-crossdock-binary: + $(GOBUILD) -o ./crossdock/crossdock-$(GOOS)-$(GOARCH) ./crossdock/main.go + +.PHONY: build-crossdock-linux +build-crossdock-linux: + GOOS=linux $(MAKE) build-crossdock-binary + +# Crossdock tests do not require fully functioning UI, so we skip it to speed up the build. +.PHONY: build-crossdock-ui-placeholder +build-crossdock-ui-placeholder: + mkdir -p jaeger-ui/packages/jaeger-ui/build/ + cp cmd/query/app/ui/placeholder/index.html jaeger-ui/packages/jaeger-ui/build/index.html + $(MAKE) build-ui + +.PHONY: build-crossdock +build-crossdock: build-crossdock-ui-placeholder build-binaries-linux build-crossdock-linux docker-images-cassandra crossdock-docker-images-jaeger-backend + docker build -t $(DOCKER_NAMESPACE)/test-driver:${DOCKER_TAG} --build-arg TARGETARCH=$(GOARCH) crossdock/ + @echo "Finished building test-driver ==============" ; \ + +.PHONY: build-and-run-crossdock +build-and-run-crossdock: build-crossdock + make crossdock + +.PHONY: build-crossdock-fresh +build-crossdock-fresh: build-crossdock-linux + make crossdock-fresh + +.PHONY: crossdock-docker-images-jaeger-backend +crossdock-docker-images-jaeger-backend: create-baseimg create-debugimg + for component in "jaeger-agent" "jaeger-collector" "jaeger-query" "jaeger-ingester" "all-in-one" ; do \ + regex="jaeger-(.*)"; \ + component_suffix=$$component; \ + if [[ $$component =~ $$regex ]]; then \ + component_suffix="$${BASH_REMATCH[1]}"; \ + fi; \ + docker buildx build --target $(TARGET) \ + --tag $(DOCKER_NAMESPACE)/$$component$(SUFFIX):${DOCKER_TAG} \ + --build-arg base_image=$(BASE_IMAGE) \ + --build-arg debug_image=$(DEBUG_IMAGE) \ + --build-arg TARGETARCH=$(GOARCH) \ + --load \ + cmd/$$component_suffix; \ + echo "Finished building $$component ==============" ; \ + done; diff --git a/Makefile.Protobuf.mk b/Makefile.Protobuf.mk new file mode 100644 index 00000000000..b1a492f2098 --- /dev/null +++ b/Makefile.Protobuf.mk @@ -0,0 +1,160 @@ +# Copyright (c) 2023 The Jaeger Authors. +# SPDX-License-Identifier: Apache-2.0 + +# Generate gogo, swagger, go-validators, gRPC-storage-plugin output. +# +# -I declares import folders, in order of importance. This is how proto resolves the protofile imports. +# It will check for the protofile relative to each of thesefolders and use the first one it finds. +# +# --gogo_out generates GoGo Protobuf output with gRPC plugin enabled. +# --govalidators_out generates Go validation files for our messages types, if specified. +# +# The lines starting with Mgoogle/... are proto import replacements, +# which cause the generated file to import the specified packages +# instead of the go_package's declared by the imported protof files. +# + +DOCKER_PROTOBUF_VERSION=0.5.0 +DOCKER_PROTOBUF=jaegertracing/protobuf:$(DOCKER_PROTOBUF_VERSION) +PROTOC := docker run --rm -u ${shell id -u} -v${PWD}:${PWD} -w${PWD} ${DOCKER_PROTOBUF} --proto_path=${PWD} + +PATCHED_OTEL_PROTO_DIR = proto-gen/.patched-otel-proto + +PROTO_INCLUDES := \ + -Iidl/proto/api_v2 \ + -Iidl/proto/api_v3 \ + -Imodel/proto/metrics \ + -I/usr/include/github.com/gogo/protobuf + +# Remapping of std types to gogo types (must not contain spaces) +PROTO_GOGO_MAPPINGS := $(shell echo \ + Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/types \ + Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types \ + Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types \ + Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types \ + Mgoogle/api/annotations.proto=github.com/gogo/googleapis/google/api \ + Mmodel.proto=github.com/jaegertracing/jaeger/model \ + | $(SED) 's/ */,/g') + +OPENMETRICS_PROTO_FILES=$(wildcard model/proto/metrics/*.proto) + +# The source directory for OTLP Protobufs from the sub-sub-module. +OTEL_PROTO_SRC_DIR=idl/opentelemetry-proto/opentelemetry/proto + +# Find all OTEL .proto files, remove leading path (only keep relevant namespace dirs). +OTEL_PROTO_FILES=$(subst $(OTEL_PROTO_SRC_DIR)/,,\ + $(shell ls $(OTEL_PROTO_SRC_DIR)/{common,resource,trace}/v1/*.proto)) + +# Macro to execute a command passed as argument. +# DO NOT DELETE EMPTY LINE at the end of the macro, it's required to separate commands. +define exec-command +$(1) + +endef + +# DO NOT DELETE EMPTY LINE at the end of the macro, it's required to separate commands. +define print_caption + @echo "🏗️ " + @echo "🏗️ " $1 + @echo "🏗️ " + +endef + +# Macro to compile Protobuf $(2) into directory $(1). $(3) can provide additional flags. +# DO NOT DELETE EMPTY LINE at the end of the macro, it's required to separate commands. +# Arguments: +# $(1) - output directory +# $(2) - path to the .proto file +# $(3) - additional flags to pass to protoc, e.g. extra -Ixxx +# $(4) - additional options to pass to gogo plugin +define proto_compile + $(call print_caption, "Processing $(2) --> $(1)") + + $(PROTOC) \ + $(PROTO_INCLUDES) \ + --gogo_out=plugins=grpc,$(strip $(4)),$(PROTO_GOGO_MAPPINGS):$(PWD)/$(strip $(1)) \ + $(3) $(2) + +endef + +# TODO add proto-hotrod to the list after regenerating its file (may need linter tweaking) +.PHONY: proto +proto: proto-model proto-api-v2 proto-storage-v1 proto-zipkin proto-openmetrics proto-otel proto-api-v3 + +.PHONY: proto-model +proto-model: + $(call proto_compile, model, idl/proto/api_v2/model.proto) + $(PROTOC) -Imodel/proto --go_out=$(PWD)/model/ model/proto/model_test.proto + +.PHONY: proto-api-v2 +proto-api-v2: + $(call proto_compile, proto-gen/api_v2, idl/proto/api_v2/query.proto) + $(call proto_compile, proto-gen/api_v2, idl/proto/api_v2/collector.proto) + $(call proto_compile, proto-gen/api_v2, idl/proto/api_v2/sampling.proto) + +.PHONY: proto-openmetrics +proto-openmetrics: + $(call print_caption, Processing OpenMetrics Protos) + $(foreach file,$(OPENMETRICS_PROTO_FILES),$(call proto_compile, proto-gen/api_v2/metrics, $(file))) + @# TODO why is this file included in model/proto/metrics/ in the first place? + rm proto-gen/api_v2/metrics/otelmetric.pb.go + +.PHONY: proto-storage-v1 +proto-storage-v1: + $(call proto_compile, proto-gen/storage_v1, plugin/storage/grpc/proto/storage.proto, -Iplugin/storage/grpc/proto) + $(PROTOC) \ + -Iplugin/storage/grpc/proto \ + --go_out=$(PWD)/plugin/storage/grpc/proto/ \ + plugin/storage/grpc/proto/storage_test.proto + +.PHONY: proto-hotrod +proto-hotrod: + $(call proto_compile, , examples/hotrod/services/driver/driver.proto) + +.PHONY: proto-zipkin +proto-zipkin: + $(call proto_compile, proto-gen/zipkin, idl/proto/zipkin.proto, -Iidl/proto) + +# Target 'proto-prepare-otel' modifies OTEL proto to use proto-import path jaeger.proto.* +# The modification is needed because OTEL collector already uses opentelemetry.proto.* +# and two complied protobuf types cannot have the same import path. The root cause is that the compiled OTLP +# in the collector is in private package, hence it cannot be used in Jaeger. +.PHONY: proto-prepare-otel +proto-prepare-otel: + $(call print_caption, Enriching OpenTelemetry Protos into $(PATCHED_OTEL_PROTO_DIR)) + + rm -rf $(PATCHED_OTEL_PROTO_DIR) + mkdir -p $(PATCHED_OTEL_PROTO_DIR) + + @# TODO replace otel_proto_patch.sed below with otel/collector/proto_patch.sed to include gogo annotations. + @$(foreach file,$(OTEL_PROTO_FILES), \ + $(call exec-command,\ + echo $(file); \ + mkdir -p $(shell dirname $(PATCHED_OTEL_PROTO_DIR)/$(file)); \ + $(SED) -f otel_proto_patch.sed $(OTEL_PROTO_SRC_DIR)/$(file) > $(PATCHED_OTEL_PROTO_DIR)/$(file))) + +# Target 'proto-otel' generates classes for OpenTelemetry OTLP format. We cannot reuse similar classes +# already generated by OTel Collector because those are private / internal. +.PHONY: proto-otel +proto-otel: proto-prepare-otel + $(foreach file,$(OTEL_PROTO_FILES), \ + $(call proto_compile, proto-gen/otel, $(file), -I$(PATCHED_OTEL_PROTO_DIR), paths=source_relative)) + +# Similar to 'proto-prepare-otel', this target modifies OTEL Protos by changing their Go package. +# This way the API v3 service uses official OTEL types like opentelemetry.proto.trace.v1.ResourceSpans +# which at runtime are mapped to our internal classes generated in proto-gen/otel by 'proto-otel' target. +.PHONY: proto-api-v3 +proto-api-v3: + $(call print_caption, Enriching OpenTelemetry Protos into $(PATCHED_OTEL_PROTO_DIR)) + rm -rf $(PATCHED_OTEL_PROTO_DIR)/* + cp -R idl/opentelemetry-proto/* $(PATCHED_OTEL_PROTO_DIR) + find $(PATCHED_OTEL_PROTO_DIR) -name "*.proto" | xargs -L 1 $(SED) -i 's+go.opentelemetry.io/proto/otlp+github.com/jaegertracing/jaeger/proto-gen/otel+g' + + $(call proto_compile, proto-gen/api_v3, idl/proto/api_v3/query_service.proto, -I$(PATCHED_OTEL_PROTO_DIR)) + + $(call print_caption, Generate API v3 gRPC Gateway) + $(PROTOC) \ + $(PROTO_INCLUDES) \ + -I$(PATCHED_OTEL_PROTO_DIR) \ + --grpc-gateway_out=logtostderr=true,grpc_api_configuration=idl/proto/api_v3/query_service_http.yaml,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v3 \ + idl/proto/api_v3/query_service.proto diff --git a/Makefile.Thrift.mk b/Makefile.Thrift.mk new file mode 100644 index 00000000000..9bef42a02be --- /dev/null +++ b/Makefile.Thrift.mk @@ -0,0 +1,27 @@ +# Copyright (c) 2023 The Jaeger Authors. +# SPDX-License-Identifier: Apache-2.0 + +THRIFT_VER=0.14 +THRIFT_IMG=jaegertracing/thrift:$(THRIFT_VER) +THRIFT=docker run --rm -u ${shell id -u} -v "${PWD}:/data" $(THRIFT_IMG) thrift +THRIFT_GO_ARGS=thrift_import="github.com/apache/thrift/lib/go/thrift" +THRIFT_GEN_DIR=thrift-gen + +.PHONY: thrift-image +thrift-image: + $(THRIFT) -version + +.PHONY: thrift +thrift: idl/thrift/jaeger.thrift thrift-image + [ -d $(THRIFT_GEN_DIR) ] || mkdir $(THRIFT_GEN_DIR) + $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/agent.thrift + $(SED) -i.bak 's|"zipkincore"|"$(JAEGER_IMPORT_PATH)/thrift-gen/zipkincore"|g' $(THRIFT_GEN_DIR)/agent/*.go + $(SED) -i.bak 's|"jaeger"|"$(JAEGER_IMPORT_PATH)/thrift-gen/jaeger"|g' $(THRIFT_GEN_DIR)/agent/*.go + $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/jaeger.thrift + $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/sampling.thrift + $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/baggage.thrift + $(THRIFT) -o /data --gen go:$(THRIFT_GO_ARGS) --out /data/$(THRIFT_GEN_DIR) /data/idl/thrift/zipkincore.thrift + rm -rf thrift-gen/*/*-remote thrift-gen/*/*.bak + +idl/thrift/jaeger.thrift: + $(MAKE) init-submodules diff --git a/cmd/agent/app/reporter/client_metrics_test.go b/cmd/agent/app/reporter/client_metrics_test.go index 004c84e17f2..64c1edb3d80 100644 --- a/cmd/agent/app/reporter/client_metrics_test.go +++ b/cmd/agent/app/reporter/client_metrics_test.go @@ -56,6 +56,7 @@ func testClientMetricsWithParams(params ClientMetricsReporterParams, fn func(tr r1 := testutils.NewInMemoryReporter() zapCore, logs := observer.New(zap.DebugLevel) mb := metricstest.NewFactory(time.Hour) + defer mb.Stop() params.Reporter = r1 params.Logger = zap.New(zapCore) diff --git a/cmd/agent/app/reporter/connect_metrics_test.go b/cmd/agent/app/reporter/connect_metrics_test.go index 524752c84c2..d57a445284e 100644 --- a/cmd/agent/app/reporter/connect_metrics_test.go +++ b/cmd/agent/app/reporter/connect_metrics_test.go @@ -26,6 +26,7 @@ import ( func TestConnectMetrics(t *testing.T) { mf := metricstest.NewFactory(time.Hour) + defer mf.Stop() cm := NewConnectMetrics(mf) getGauge := func() map[string]int64 { diff --git a/cmd/agent/app/reporter/metrics_test.go b/cmd/agent/app/reporter/metrics_test.go index 59e5d890662..6795781b28f 100644 --- a/cmd/agent/app/reporter/metrics_test.go +++ b/cmd/agent/app/reporter/metrics_test.go @@ -102,6 +102,7 @@ func TestMetricsReporter(t *testing.T) { for _, test := range tests { metricsFactory := metricstest.NewFactory(time.Microsecond) + defer metricsFactory.Stop() r := WrapWithMetrics(test.rep, metricsFactory) test.action(r) metricsFactory.AssertCounterMetrics(t, test.expectedCounters...) diff --git a/cmd/agent/app/reporter/package_test.go b/cmd/agent/app/reporter/package_test.go new file mode 100644 index 00000000000..e0c68918bf5 --- /dev/null +++ b/cmd/agent/app/reporter/package_test.go @@ -0,0 +1,25 @@ +// Copyright (c) 2023 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reporter + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/otel_proto_patch.sed b/otel_proto_patch.sed index ade5733c92a..9821ab8d072 100644 --- a/otel_proto_patch.sed +++ b/otel_proto_patch.sed @@ -1,4 +1,8 @@ -s+go.opentelemetry.io/proto/otlp+github.com/jaegertracing/jaeger/proto-gen/otel+g +# Substitute Go package name. +s|go.opentelemetry.io/proto/otlp|github.com/jaegertracing/jaeger/proto-gen/otel|g -s+ opentelemetry.proto+ jaeger+g -s+import "opentelemetry/proto/+import "+g +# Substitute Proto package name. +s| opentelemetry.proto| jaeger|g + +# Remove opentelemetry/proto prefix from imports. +s|import "opentelemetry/proto/|import "|g diff --git a/pkg/httpmetrics/metrics_test.go b/pkg/httpmetrics/metrics_test.go index 03f62d30f0b..25371b360a9 100644 --- a/pkg/httpmetrics/metrics_test.go +++ b/pkg/httpmetrics/metrics_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/goleak" "go.uber.org/zap" "github.com/jaegertracing/jaeger/internal/metrics/prometheus" @@ -37,6 +38,7 @@ func TestNewMetricsHandler(t *testing.T) { }) mb := metricstest.NewFactory(time.Hour) + defer mb.Stop() handler := Wrap(dummyHandlerFunc, mb, zap.NewNop()) req, err := http.NewRequest(http.MethodGet, "/subdir/qwerty", nil) @@ -56,6 +58,7 @@ func TestNewMetricsHandler(t *testing.T) { func TestMaxEntries(t *testing.T) { mf := metricstest.NewFactory(time.Hour) + defer mf.Stop() r := newRequestDurations(mf, zap.NewNop()) r.maxEntries = 1 r.record(recordedRequest{ @@ -85,3 +88,7 @@ func TestIllegalPrometheusLabel(t *testing.T) { require.NoError(t, err) handler.ServeHTTP(httptest.NewRecorder(), req) } + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/pkg/recoveryhandler/zap_test.go b/pkg/recoveryhandler/zap_test.go index dc22bb97b2d..e39c1dff71f 100644 --- a/pkg/recoveryhandler/zap_test.go +++ b/pkg/recoveryhandler/zap_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/goleak" "github.com/jaegertracing/jaeger/pkg/testutils" ) @@ -45,3 +46,7 @@ func TestNewRecoveryHandler(t *testing.T) { "msg": "Unexpected error!", }, log.JSONLine(0)) } + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +}