Merge branch 'pm_params_refactor_service' of https://github.com/pmaty…

…jasek-sumo/opentelemetry-collector into pm_params_refactor_service

.circleci/config.yml

@@ -1,3 +1,26 @@
+# Using Contexts:
+#   some jobs depend on secrets like API tokens to work correctly such as publishing to docker hub
+#   or reporting issues to GitHub. All such tokens are stored in CircleCI contexts (https://circleci.com/docs/2.0/contexts).
+#   
+#   All tokens stored in a contexts are injected into a job as environment variables IF the pipeline that runs the job
+#   explicitly enables the context for the job. 
+#
+#   Contexts are protected with security groups. Jobs that use contexts will not run for commits from people who are not
+#   part of the approved security groups for the given context. This means that contributors who are not part of the
+#   OpenTelemetry GitHub organisation will not be able to run jobs that depend on contexts. As a result, PR pipelines
+#   should never depend on any contexts and never use any tokens/secrets.
+#
+#   This CI pipeline uses two contexts:
+#      - github-release-and-issues-api-token
+#        This context makes GITHUB_TOKEN available to jobs. Jobs can use the token to authenticate with the GitHub API.
+#        We use this to report failures as issues back to the GitHub project.
+#        Any member of the OpenTelemetry GitHub organisation can run jobs that require this context e.g, loadtest-with-github-reports.
+#        
+#      - dockerhub-token
+#        This contexts makes DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD environment variables available to the jobs.
+#        This is used to publish docker images to Docker Hub.
+#        Only project approvers and maintainers can run jobs that depend on this context such e.g, publish-stable.
+
 version: 2.1
 
 orbs:
@@ -91,6 +114,32 @@ commands:
                 command: go run cmd/issuegenerator/main.go ${TEST_RESULTS}
                 when: on_fail
 
+  run_loadtest:
+    steps:
+      - attach_to_workspace
+      - run:
+          name: Loadtest
+          command: TEST_ARGS="-test.run=$(make -s testbed-list-loadtest | circleci tests split|xargs echo|sed 's/ /|/g')" make testbed-loadtest
+      - store_artifacts:
+          path: testbed/tests/results
+      - store_test_results:
+          path: testbed/tests/results/junit
+
+  run_tests:
+    steps:
+      - attach_to_workspace
+      - run:
+          name: Unit tests
+          command: |
+            mkdir -p unit-test-results/junit
+            trap "go-junit-report  -set-exit-code < unit-test-results/go-unit-tests.out > unit-test-results/junit/results.xml" EXIT
+            make gotest | tee unit-test-results/go-unit-tests.out
+      - store_artifacts:
+          path: unit-test-results
+      - store_test_results:
+          path: unit-test-results/junit
+      - save_module_cache
+
 workflows:
   version: 2
   build-and-test:
@@ -105,18 +154,38 @@ workflows:
           filters:
             tags:
               only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
-      - loadtest:
+      - loadtest-with-github-reports:
+          context:
+            - github-release-and-issues-api-token
           requires:
             - cross-compile
           filters:
+            branches:
+              only: main
             tags:
               only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
-      - test:
+      - loadtest:
+          requires:
+            - cross-compile
+          filters:
+            branches:
+              ignore: main
+      - test-with-github-reports:
+          context:
+            - github-release-and-issues-api-token
           requires:
             - setup-environment
           filters:
+            branches:
+              only: main
             tags:
               only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
+      - test:
+          requires:
+            - setup-environment
+          filters:
+            branches:
+              ignore: main
       - coverage:
           requires:
             - setup-environment
@@ -129,6 +198,18 @@ workflows:
           filters:
             tags:
               only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
+      - publish-check:
+          requires:
+            - cross-compile
+            - loadtest-with-github-reports
+            - test-with-github-reports
+            - coverage
+            - windows-msi
+            - deb-package
+            - rpm-package
+          filters:
+            branches:
+              only: main
       - publish-check:
           requires:
             - cross-compile
@@ -138,7 +219,13 @@ workflows:
             - windows-msi
             - deb-package
             - rpm-package
+          filters:
+            branches:
+              ignore: main
       - publish-stable:
+          context:
+            - github-release-and-issues-api-token
+            - dockerhub-token
           requires:
             - cross-compile
             - loadtest
@@ -153,6 +240,8 @@ workflows:
             tags:
               only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
       - publish-dev:
+          context:
+            - dockerhub-token
           requires:
             - cross-compile
             - loadtest
@@ -217,41 +306,39 @@ jobs:
           root: ~/
           paths: project/bin
 
+  loadtest-with-github-reports:
+    executor: golang
+    environment:
+      TEST_RESULTS: testbed/tests/results/junit/results.xml
+    parallelism: 6
+    resource_class: medium+
+    steps:
+      - run_loadtest
+      - github_issue_generator
+
   loadtest:
     executor: golang
     environment:
       TEST_RESULTS: testbed/tests/results/junit/results.xml
     parallelism: 6
     resource_class: medium+
     steps:
-      - attach_to_workspace
-      - run:
-          name: Loadtest
-          command: TEST_ARGS="-test.run=$(make -s testbed-list-loadtest | circleci tests split|xargs echo|sed 's/ /|/g')" make testbed-loadtest
-      - store_artifacts:
-          path: testbed/tests/results
-      - store_test_results:
-          path: testbed/tests/results/junit
+      - run_loadtest
+
+  test-with-github-reports:
+    executor: golang
+    environment:
+      TEST_RESULTS: unit-test-results/junit/results.xml
+    steps:
+      - run_loadtest
       - github_issue_generator
 
   test:
     executor: golang
     environment:
       TEST_RESULTS: unit-test-results/junit/results.xml
     steps:
-      - attach_to_workspace
-      - run:
-          name: Unit tests
-          command: |
-            mkdir -p unit-test-results/junit
-            trap "go-junit-report  -set-exit-code < unit-test-results/go-unit-tests.out > unit-test-results/junit/results.xml" EXIT
-            make gotest | tee unit-test-results/go-unit-tests.out
-      - store_artifacts:
-          path: unit-test-results
-      - store_test_results:
-          path: unit-test-results/junit
-      - save_module_cache
-      - github_issue_generator
+      - run_loadtest
 
   coverage:
     executor: golang

.github/dependabot.yml

@@ -2,10 +2,18 @@
 
 version: 2
 updates:
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "gomod"
+    directory: "/cmd/checkdoc"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "gomod"
     directory: "/cmd/issuegenerator"
     schedule:
@@ -22,7 +30,3 @@ updates:
     directory: "/internal/tools"
     schedule:
       interval: "weekly"
-  - package-ecosystem: "gomod"
-    directory: "/cmd/checkdoc"
-    schedule:
-      interval: "weekly"

.github/workflows/docs-update.yml

@@ -0,0 +1,36 @@
+name: Update OpenTelemetry Website Docs
+
+on:
+  # triggers only on a manual dispatch
+  workflow_dispatch:
+
+jobs:
+  update-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2.3.4
+      - name: make-pr
+        env:
+          API_TOKEN_GITHUB: ${{secrets.DOC_UPDATE_TOKEN}}
+          # Destination repo should always be 'open-telemetry/opentelemetry.io'
+          DESTINATION_REPO: open-telemetry/opentelemetry.io
+          # Destination path should be the absolute path to your language's friendly name in the docs tree (i.e, 'content/en/docs/java')
+          DESTINATION_PATH: content/en/docs/collector
+          # Source path should be 'website_docs', all files and folders are copied from here to dest
+          SOURCE_PATH: website_docs
+        run: |
+          TARGET_DIR=$(mktemp -d)
+          export GITHUB_TOKEN=$API_TOKEN_GITHUB
+          git config --global user.name austinlparker
+          git config --global user.email austin@lightstep.com
+          git clone "https://$API_TOKEN_GITHUB@github.com/$DESTINATION_REPO.git" "$TARGET_DIR"
+          rsync -av --delete "$SOURCE_PATH/" "$TARGET_DIR/$DESTINATION_PATH/"
+          cd "$TARGET_DIR"
+          git checkout -b docs-$GITHUB_REPOSITORY-$GITHUB_SHA
+          git add .
+          git commit -m "Docs update from $GITHUB_REPOSITORY"
+          git push -u origin HEAD:docs-$GITHUB_REPOSITORY-$GITHUB_SHA
+          gh pr create -t "Docs Update from $GITHUB_REPOSITORY" -b "This is an automated pull request." -B main -H docs-$GITHUB_REPOSITORY-$GITHUB_SHA
+          echo "done"
+          

.github/workflows/scripts/setup_load_tests.sh

@@ -14,5 +14,5 @@ else
     curr="${TESTS[$i]}"
 fi
 done
-MATRIX+="]}"
+MATRIX+=",{\"test\":\"$curr\"}]}"
 echo "::set-output name=matrix::$MATRIX"

.golangci.yml

@@ -69,7 +69,7 @@ linters-settings:
     disable:
       - fieldalignment
 
-  golint:
+  revive:
     # minimal confidence for issues, default is 0.8
     min-confidence: 0.8
 
@@ -103,10 +103,10 @@ linters:
     - gocritic
     - gofmt
     - goimports
-    - golint
     - gosec
     - govet
     - misspell
+    - revive
     - staticcheck
     - unconvert
     - unparam
@@ -133,11 +133,11 @@ issues:
     - path: ".*internal.*|.*testbed.*"
       text: "should have comment|should be of the form"
       linters:
-        - golint
+        - revive
     # Exclude documenting constant blocks.
     - text: "or a comment on this block"
       linters:
-        - golint
+        - revive
 
   # The list of ids of default excludes to include or disable. By default it's empty.
   # See the list of default excludes here https://golangci-lint.run/usage/configuration.

CHANGELOG.md

@@ -4,13 +4,53 @@
 
 ## 🛑 Breaking changes 🛑
 
+- Remove unused logstest package (#3222)
+- Rename `service.Parameters` to `service.AppSettings` (#3163)
+
+## v0.27.0 Beta
+
+## 🛑 Breaking changes 🛑
+
+- Change `Marshal` signatures in kafkaexporter's Marshalers to directly convert pdata to `sarama.ProducerMessage` (#3162)
 - Remove `tracetranslator.DetermineValueType`, only used internally by Zipkin (#3114)
 - Remove OpenCensus conventions, should not be used (#3113)
 - Remove Zipkin specific translation constants, move to internal (#3112)
 - Remove `tracetranslator.TagHTTPStatusCode`, use `conventions.AttributeHTTPStatusCode` (#3111)
 - Remove OpenCensus status constants and transformation (#3110)
 - Remove `tracetranslator.AttributeArrayToSlice`, not used in core or contrib (#3109)
-- Introduce `SvcSettings` and `AppSettings` instead of `Parameters` (#3163)
+- Remove `internaldata.MetricsData`, same APIs as for traces (#3156)
+- Rename `config.IDFromString` to `NewIDFromString`, remove `MustIDFromString` (#3177)
+- Move consumerfanout package to internal (#3207)
+- Canonicalize enum names in pdata. Fix usage of uppercase names (#3208)
+
+## 💡 Enhancements 💡
+
+- Use `config.ComponentID` for obsreport receiver/scraper (#3098)
+- Add initial implementation of the consumerhelper (#3146)
+- Add Collector version to Prometheus Remote Write Exporter user-agent header (#3094)
+- Refactor processorhelper to use consumerhelper, split by signal type (#3180)
+- Use consumerhelper for exporterhelper, add WithCapabilities (#3186)
+- Set capabilities for all core exporters, remove unnecessary funcs (#3190)
+- Add an internal sharedcomponent to be shared by receivers with shared resources (#3198)
+- Allow users to configure the Prometheus remote write queue (#3046)
+- Mark internaldata traces translation as deprecated for external usage (#3176)
+- Change receiver obsreport helpers pattern to match the Processor/Exporter (#3227)
+
+## 🧰 Bug fixes 🧰
+
+- Fix Prometheus receiver metric start time and reset determination logic. (#3047)
+  - The receiver will no longer drop the first sample for `counter`, `summary`, and `histogram` metrics.
+- The Prometheus remote write exporter will no longer force `counter` metrics to have a `_total` suffix. (#2993)
+- Remove locking from jaeger receiver start and stop processes (#3070)
+- Fix batch processor metrics reorder, improve performance (#3034)
+- Fix batch processor traces reorder, improve performance (#3107)
+- Fix batch processor logs reorder, improve performance (#3125)
+- Avoid one unnecessary allocation in grpc OTLP exporter (#3122)
+- `batch` processor: Validate that batch config max size is greater than send size (#3126)
+- Add capabilities to consumer, remove from processor (#2770)
+- Remove internal protos usage in Prometheusremotewrite exporter (#3184)
+- `prometheus` receiver: Honor Prometheus external labels (#3127)
+- Validate that remote write queue settings are not negative (#3213)
 
 ## v0.26.0 Beta
 

Makefile

@@ -263,18 +263,26 @@ build-binary-internal:
 genmdata:
 	$(MAKE) for-all CMD="go generate ./..."
 
-DEPENDABOT_PATH=./.github/dependabot.yml
+DEPENDABOT_PATH=".github/dependabot.yml"
+.PHONY: internal-gendependabot
+internal-gendependabot:
+	@echo "Add rule for \"${PACKAGE}\" in \"${DIR}\"";
+	@echo "  - package-ecosystem: \"${PACKAGE}\"" >> ${DEPENDABOT_PATH};
+	@echo "    directory: \"${DIR}\"" >> ${DEPENDABOT_PATH};
+	@echo "    schedule:" >> ${DEPENDABOT_PATH};
+	@echo "      interval: \"weekly\"" >> ${DEPENDABOT_PATH};
+
 .PHONY: gendependabot
 gendependabot:
-	@echo "Recreate dependabot.yml file"
-	@echo "# File generated by \"make gendependabot\"; DO NOT EDIT.\n" > ${DEPENDABOT_PATH}
+	@echo "Recreating ${DEPENDABOT_PATH} file"
+	@echo "# File generated by \"make gendependabot\"; DO NOT EDIT." > ${DEPENDABOT_PATH}
+	@echo "" >> ${DEPENDABOT_PATH}
 	@echo "version: 2" >> ${DEPENDABOT_PATH}
 	@echo "updates:" >> ${DEPENDABOT_PATH}
-	@echo "Add entry for \"/\""
-	@echo "  - package-ecosystem: \"gomod\"\n    directory: \"/\"\n    schedule:\n      interval: \"weekly\"" >> ${DEPENDABOT_PATH}
+	$(MAKE) internal-gendependabot DIR="/" PACKAGE="docker"
+	$(MAKE) internal-gendependabot DIR="/" PACKAGE="gomod"
 	@set -e; for dir in $(ALL_MODULES); do \
-		(echo "Add entry for \"$${dir:1}\"" && \
-		  echo "  - package-ecosystem: \"gomod\"\n    directory: \"$${dir:1}\"\n    schedule:\n      interval: \"weekly\"" >> ${DEPENDABOT_PATH} ); \
+		$(MAKE) internal-gendependabot DIR=$${dir:1} PACKAGE="gomod"; \
 	done
 
 # Definitions for ProtoBuf generation.