Merge branch 'master' into feat/span-normalization-allow-host-config

.env

@@ -11,7 +11,6 @@ SNUBA_IMAGE=getsentry/snuba:nightly
 RELAY_IMAGE=getsentry/relay:nightly
 SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly
 VROOM_IMAGE=getsentry/vroom:nightly
-WAL2JSON_VERSION=latest
 HEALTHCHECK_INTERVAL=30s
 HEALTHCHECK_TIMEOUT=1m30s
 HEALTHCHECK_RETRIES=10

.github/ISSUE_TEMPLATE/release.yml

@@ -10,10 +10,8 @@ body:
         [previous YY.M.N](https://github.com/getsentry/self-hosted/issues) | ***YY.M.N*** | [next YY.M.N](https://github.com/getsentry/self-hosted/issues)
 
         - [ ] Release all components (_replace items with [publish repo issue links](https://github.com/getsentry/publish/issues)_).
-          - [ ] [`develop`](https://github.com/getsentry/develop/actions/workflows/prepare-release.yml)
           - [ ] [`relay`](https://github.com/getsentry/relay/actions/workflows/release_binary.yml)
           - [ ] [`sentry`](https://github.com/getsentry/sentry/actions/workflows/release.yml)
-          - [ ] [`sentry-docs`](https://github.com/getsentry/sentry-docs/actions/workflows/prepare-release.yml)
           - [ ] [`snuba`](https://github.com/getsentry/snuba/actions/workflows/release.yml)
           - [ ] [`symbolicator`](https://github.com/getsentry/symbolicator/actions/workflows/release.yml)
           - [ ] [`vroom`](https://github.com/getsentry/vroom/actions/workflows/release.yaml)
@@ -25,7 +23,6 @@ body:
         - [ ] Follow up.
           - [ ] [Create the next release issue](https://github.com/getsentry/self-hosted/issues/new?assignees=&labels=&projects=&template=release.yml) and link it from this one.
             - _replace with link_
-          - [ ] Update the [quarterly ticket](https://github.com/getsentry/team-ospo/issues).
           - [ ] Update the [release issue template](https://github.com/getsentry/self-hosted/blob/master/.github/ISSUE_TEMPLATE/release.yml).
           - [ ] Create a PR to update relocation release tests to add the new version.
             - _replace with link_

.gitignore

@@ -98,9 +98,6 @@ geoip/GeoIP.conf
 geoip/*.mmdb
 geoip/.geoipupdate.lock
 
-# wal2json download
-postgres/wal2json
-
 # integration testing
 _integration-test/custom-ca-roots/nginx/*
 sentry/test-custom-ca-roots.py

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 24.8.0
+
+### Various fixes & improvements
+
+- Migrate to zookeeper-less kafka  (#3263) by @hubertdeng123
+- Revert "ref(feedback): cleanup topic rollout option" (#3262) by @aliu39
+- ref(feedback): cleanup topic rollout option (#3256) by @aliu39
+- Remove cdc and wal2json and use the default postgres entrypoint (#3260) by @beezz
+- add `-euo pipefail` to enhance-image.example.sh (#3246) by @asottile-sentry
+- remove python-dev (#3242) by @asottile-sentry
+- feat: enable user feedback feature (#3193) by @aldy505
+- Use CDN by default for JS SDK Loader (#3213) by @stayallive
+
 ## 24.7.1
 
 ### Various fixes & improvements

_unit-test/create-docker-volumes-test.sh

@@ -14,8 +14,7 @@ sentry-data
 sentry-kafka
 sentry-postgres
 sentry-redis
-sentry-symbolicator
-sentry-zookeeper"
+sentry-symbolicator"
 
 before=$(get_volumes)
 

docker-compose.yml

@@ -126,59 +126,33 @@ services:
       [
         "postgres",
         "-c",
-        "wal_level=logical",
-        "-c",
-        "max_replication_slots=1",
-        "-c",
-        "max_wal_senders=1",
-        "-c",
         "max_connections=${POSTGRES_MAX_CONNECTIONS:-100}",
       ]
     environment:
       POSTGRES_HOST_AUTH_METHOD: "trust"
-    entrypoint: /opt/sentry/postgres-entrypoint.sh
     volumes:
       - "sentry-postgres:/var/lib/postgresql/data"
-      - type: bind
-        read_only: true
-        source: ./postgres/
-        target: /opt/sentry/
-  zookeeper:
-    <<: *restart_policy
-    image: "confluentinc/cp-zookeeper:7.6.1"
-    environment:
-      ZOOKEEPER_CLIENT_PORT: "2181"
-      CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
-      ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN"
-      ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN"
-      KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok"
-    ulimits:
-      nofile:
-        soft: 4096
-        hard: 4096
-    volumes:
-      - "sentry-zookeeper:/var/lib/zookeeper/data"
-      - "sentry-zookeeper-log:/var/lib/zookeeper/log"
-      - "sentry-secrets:/etc/zookeeper/secrets"
-    healthcheck:
-      <<: *healthcheck_defaults
-      test: ["CMD-SHELL", 'echo "ruok" | nc -w 2 localhost 2181 | grep imok']
   kafka:
     <<: *restart_policy
-    depends_on:
-      zookeeper:
-        <<: *depends_on-healthy
     image: "confluentinc/cp-kafka:7.6.1"
     environment:
-      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092"
+      # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example
+      KAFKA_PROCESS_ROLES: "broker,controller"
+      KAFKA_CONTROLLER_QUORUM_VOTERS: "1001@127.0.0.1:29093"
+      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
+      KAFKA_NODE_ID: "1001"
+      CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk"
+      KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:29092,INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093"
+      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:29092,INTERNAL://kafka:9093,EXTERNAL://kafka:9092"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"
+      KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
       KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1"
       KAFKA_LOG_RETENTION_HOURS: "24"
       KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust
       KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too
       CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
-      KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN"
+      KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,state.change.logger=WARN"
       KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN"
       KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN"
     ulimits:
@@ -549,8 +523,6 @@ volumes:
     external: true
   sentry-redis:
     external: true
-  sentry-zookeeper:
-    external: true
   sentry-kafka:
     external: true
   sentry-clickhouse:
@@ -566,7 +538,6 @@ volumes:
   sentry-secrets:
   sentry-smtp:
   sentry-nginx-cache:
-  sentry-zookeeper-log:
   sentry-kafka-log:
   sentry-smtp-log:
   sentry-clickhouse-log:

install.sh

@@ -32,7 +32,6 @@ source install/ensure-relay-credentials.sh
 source install/generate-secret-key.sh
 source install/update-docker-images.sh
 source install/build-docker-images.sh
-source install/install-wal2json.sh
 source install/bootstrap-snuba.sh
 source install/upgrade-postgres.sh
 source install/set-up-and-migrate-database.sh

install/_min-requirements.sh

@@ -1,7 +1,9 @@
-# Don't forget to update the README and othes docs when you change these!
+# Don't forget to update the README and other docs when you change these!
 MIN_DOCKER_VERSION='19.03.6'
 MIN_COMPOSE_VERSION='2.19.0'
-MIN_RAM_HARD=3800 # MB
-MIN_RAM_SOFT=7800 # MB
-MIN_CPU_HARD=2
-MIN_CPU_SOFT=4
+
+# 16 GB minimum host RAM, but there'll be some overhead outside of what
+# can be allotted to docker
+MIN_RAM_HARD=14000 # MB
+
+MIN_CPU_HARD=4

install/check-minimum-requirements.sh

@@ -36,16 +36,12 @@ CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all)
 if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then
   echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER"
   exit 1
-elif [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then
-  echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT, found $CPU_AVAILABLE_IN_DOCKER"
 fi
 
 RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}')
 if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then
   echo "FAIL: Required minimum RAM available to Docker is $MIN_RAM_HARD MB, found $RAM_AVAILABLE_IN_DOCKER MB"
   exit 1
-elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_SOFT" ]]; then
-  echo "WARN: Recommended minimum RAM available to Docker is $MIN_RAM_SOFT MB, found $RAM_AVAILABLE_IN_DOCKER MB"
 fi
 
 #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/)

install/create-docker-volumes.sh

@@ -6,6 +6,5 @@ echo "Created $(docker volume create --name=sentry-kafka)."
 echo "Created $(docker volume create --name=sentry-postgres)."
 echo "Created $(docker volume create --name=sentry-redis)."
 echo "Created $(docker volume create --name=sentry-symbolicator)."
-echo "Created $(docker volume create --name=sentry-zookeeper)."
 
 echo "${_endgroup}"

install/wrap-up.sh

@@ -28,3 +28,8 @@ else
   echo "-----------------------------------------------------------------"
   echo ""
 fi
+
+# TODO(getsentry/self-hosted#2489)
+if docker volume ls | grep -qw sentry-zookeeper; then
+  docker volume rm sentry-zookeeper
+fi

scripts/bump-version.sh

@@ -4,11 +4,7 @@ set -eu
 OLD_VERSION="$1"
 NEW_VERSION="$2"
 
-WAL2JSON_VERSION=${WAL2JSON_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/wal2json/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')}
-
-sed -i -e "s/^WAL2JSON_VERSION=\([^:]\+\):.\+\$/WAL2JSON_VERSION=\1:$WAL2JSON_VERSION/" .env
 sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\|SYMBOLICATOR\|VROOM\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env
 sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md
 
 echo "New version: $NEW_VERSION"
-echo "New wal2json version: $WAL2JSON_VERSION"

sentry/enhance-image.example.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -euo pipefail
 
 # Enhance the base $SENTRY_IMAGE with additional dependencies, plugins - see https://github.com/getsentry/self-hosted#enhance-sentry-image
 # For example:

sentry/sentry.conf.example.py

@@ -325,10 +325,6 @@ def get_internal_network():
     }
 )
 
-# Temporary flag to mark User Feedback to be produced to the dedicated feedback topic by relay.
-# This will be removed at a later time after it's considered stable and fully rolled out.
-SENTRY_OPTIONS["feedback.ingest-topic.rollout-rate"] = 1.0
-
 #######################
 # MaxMind Integration #
 #######################