Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Kafka 3.1.0 to CI matrix, migrate to bitnami kafka image #2124

Merged
merged 4 commits into from
Feb 1, 2022
Merged

Add Kafka 3.1.0 to CI matrix, migrate to bitnami kafka image #2124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2d3a2f4
Add Kafka 3.1.0 to CI matrix, migrate to bitnami kafka image
bai Jan 31, 2022
c51d007
Run functional tests on latest Go only
bai Feb 1, 2022
8228604
Fail early if lint fails
bai Feb 1, 2022
c195179
Set GOFLAGS for lint step
bai Feb 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on: [push, pull_request]

jobs:
test:
name: Go ${{ matrix.go-version }} with Kafka ${{ matrix.kafka-version }} on Ubuntu
name: Go ${{ matrix.go-version }} with Kafka ${{ matrix.kafka-version }}
runs-on: ubuntu-latest
# We want to run on external PRs, but not on our own internal PRs as they'll be run
# by the push to the branch.
Expand All @@ -14,7 +14,7 @@ jobs:
fail-fast: false
matrix:
go-version: [1.16.x, 1.17.x]
kafka-version: [2.7.1, 2.8.1, 3.0.0]
kafka-version: [2.8.1, 3.0.0, 3.1.0]
platform: [ubuntu-latest]

env:
Expand Down Expand Up @@ -46,13 +46,18 @@ jobs:
restore-keys: |
${{ runner.os }}-go-

- name: Install dependencies
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1
export REPOSITORY_ROOT=${GITHUB_WORKSPACE}
- name: Lint
uses: golangci/golangci-lint-action@v2
if: ${{ matrix.go-version == '1.17.x' }}
env:
GOFLAGS: -tags=functional
with:
version: v1.44.0

- name: Run test suite
run: make test_functional
- name: Test (Unit)
run: make test
if: ${{ matrix.go-version != '1.17.x' }}

- name: Run linter
run: make lint
- name: Test (Functional)
run: make test_functional
if: ${{ matrix.go-version == '1.17.x' }}
151 changes: 78 additions & 73 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: '3.7'
services:
zookeeper-1:
image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'confluentinc/cp-zookeeper:7.0.1'
Copy link
Contributor Author

@bai bai Feb 1, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't expect Zookeeper version requirement to change as often as Kafka versions change, so oped to hardcode Zookeeper version instead of adding an extra configuration parameter.

We might explore moving to non-proprietary zookeeper docker image in future.

restart: always
environment:
ZOOKEEPER_SERVER_ID: '1'
Expand All @@ -13,7 +13,7 @@ services:
ZOOKEEPER_SYNC_LIMIT: '5'
ZOOKEEPER_MAX_CLIENT_CONNS: '0'
zookeeper-2:
image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'confluentinc/cp-zookeeper:7.0.1'
restart: always
environment:
ZOOKEEPER_SERVER_ID: '2'
Expand All @@ -25,7 +25,7 @@ services:
ZOOKEEPER_SYNC_LIMIT: '5'
ZOOKEEPER_MAX_CLIENT_CONNS: '0'
zookeeper-3:
image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'confluentinc/cp-zookeeper:7.0.1'
restart: always
environment:
ZOOKEEPER_SERVER_ID: '3'
Expand All @@ -37,90 +37,95 @@ services:
ZOOKEEPER_SYNC_LIMIT: '5'
ZOOKEEPER_MAX_CLIENT_CONNS: '0'
kafka-1:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '1'
KAFKA_BROKER_RACK: '1'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '1'
KAFKA_CFG_BROKER_RACK: '1'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
kafka-2:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '2'
KAFKA_BROKER_RACK: '2'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '2'
KAFKA_CFG_BROKER_RACK: '2'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
kafka-3:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '3'
KAFKA_BROKER_RACK: '3'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '3'
KAFKA_CFG_BROKER_RACK: '3'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
kafka-4:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '4'
KAFKA_BROKER_RACK: '4'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '4'
KAFKA_CFG_BROKER_RACK: '4'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
kafka-5:
image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
restart: always
environment:
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_BROKER_ID: '5'
KAFKA_BROKER_RACK: '5'
KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
KAFKA_CFG_BROKER_ID: '5'
KAFKA_CFG_BROKER_RACK: '5'
KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
ALLOW_PLAINTEXT_LISTENER: 'true'
toxiproxy:
image: 'ghcr.io/shopify/toxiproxy:2.3.0'
ports:
Expand Down
21 changes: 3 additions & 18 deletions functional_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,28 +136,13 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err
if version, ok := os.LookupEnv("KAFKA_VERSION"); ok {
env.KafkaVersion = version
} else {
// We have cp-7.0.0 as the default in the docker-compose file, so that's kafka 3.0.0.
env.KafkaVersion = "3.0.0"
}

// the mapping of confluent platform docker image versions -> kafka versions can be
// found here: https://docs.confluent.io/current/installation/versions-interoperability.html
var confluentPlatformVersion string
switch env.KafkaVersion {
case "3.0.0":
confluentPlatformVersion = "7.0.1"
case "2.8.1":
confluentPlatformVersion = "6.2.2"
case "2.7.1":
confluentPlatformVersion = "6.1.4"
default:
return fmt.Errorf("don't know what confluent platform version to use for kafka %s", env.KafkaVersion)
env.KafkaVersion = "3.1.0"
}

c := exec.Command("docker-compose", "up", "-d")
c.Stdout = os.Stdout
c.Stderr = os.Stderr
c.Env = append(os.Environ(), fmt.Sprintf("CONFLUENT_PLATFORM_VERSION=%s", confluentPlatformVersion))
c.Env = append(os.Environ(), fmt.Sprintf("KAFKA_VERSION=%s", env.KafkaVersion))
err := c.Run()
if err != nil {
return fmt.Errorf("failed to run docker-compose to start test environment: %w", err)
Expand All @@ -169,7 +154,7 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err

// Wait for the kafka broker to come up
allBrokersUp := false
for i := 0; i < 45 && !allBrokersUp; i++ {
for i := 0; i < 90 && !allBrokersUp; i++ {
Logger.Println("waiting for kafka brokers to come up")
time.Sleep(1 * time.Second)
config := NewTestConfig()
Expand Down
1 change: 1 addition & 0 deletions utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func TestVersionParsing(t *testing.T) {
"2.8.0",
"2.8.1",
"3.0.0",
"3.1.0",
}
for _, s := range validVersions {
v, err := ParseKafkaVersion(s)
Expand Down