Add Kafka 3.1.0 to CI matrix, migrate to bitnami kafka image

IBM · Feb 1, 2022 · 2d3a2f4 · 2d3a2f4
1 parent f214fdd
commit 2d3a2f4
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 93 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         go-version: [1.16.x, 1.17.x]
-        kafka-version: [2.7.1, 2.8.1, 3.0.0]
+        kafka-version: [3.0.0, 3.1.0]
         platform: [ubuntu-latest]
 
     env:
@@ -48,7 +48,7 @@ jobs:
 
     - name: Install dependencies
       run: |
-        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.44.0
         export REPOSITORY_ROOT=${GITHUB_WORKSPACE}
 
     - name: Run test suite

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '3.7'
 services:
   zookeeper-1:
-    image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
+    image: 'confluentinc/cp-zookeeper:7.0.1'
     restart: always
     environment:
       ZOOKEEPER_SERVER_ID: '1'
@@ -13,7 +13,7 @@ services:
       ZOOKEEPER_SYNC_LIMIT: '5'
       ZOOKEEPER_MAX_CLIENT_CONNS: '0'
   zookeeper-2:
-    image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
+    image: 'confluentinc/cp-zookeeper:7.0.1'
     restart: always
     environment:
       ZOOKEEPER_SERVER_ID: '2'
@@ -25,7 +25,7 @@ services:
       ZOOKEEPER_SYNC_LIMIT: '5'
       ZOOKEEPER_MAX_CLIENT_CONNS: '0'
   zookeeper-3:
-    image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
+    image: 'confluentinc/cp-zookeeper:7.0.1'
     restart: always
     environment:
       ZOOKEEPER_SERVER_ID: '3'
@@ -37,90 +37,95 @@ services:
       ZOOKEEPER_SYNC_LIMIT: '5'
       ZOOKEEPER_MAX_CLIENT_CONNS: '0'
   kafka-1:
-    image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
+    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
     restart: always
     environment:
-      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
-      KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
-      KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
-      KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
-      KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
-      KAFKA_BROKER_ID: '1'
-      KAFKA_BROKER_RACK: '1'
-      KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
-      KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
-      KAFKA_DELETE_TOPIC_ENABLE: 'true'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
+      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
+      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
+      KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
+      KAFKA_CFG_BROKER_ID: '1'
+      KAFKA_CFG_BROKER_RACK: '1'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
+      KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
+      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      ALLOW_PLAINTEXT_LISTENER: 'true'
   kafka-2:
-    image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
+    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
     restart: always
     environment:
-      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
-      KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
-      KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
-      KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
-      KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
-      KAFKA_BROKER_ID: '2'
-      KAFKA_BROKER_RACK: '2'
-      KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
-      KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
-      KAFKA_DELETE_TOPIC_ENABLE: 'true'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
+      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
+      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
+      KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
+      KAFKA_CFG_BROKER_ID: '2'
+      KAFKA_CFG_BROKER_RACK: '2'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
+      KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
+      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      ALLOW_PLAINTEXT_LISTENER: 'true'
   kafka-3:
-    image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
+    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
     restart: always
     environment:
-      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
-      KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
-      KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
-      KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
-      KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
-      KAFKA_BROKER_ID: '3'
-      KAFKA_BROKER_RACK: '3'
-      KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
-      KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
-      KAFKA_DELETE_TOPIC_ENABLE: 'true'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
+      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
+      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
+      KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
+      KAFKA_CFG_BROKER_ID: '3'
+      KAFKA_CFG_BROKER_RACK: '3'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
+      KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
+      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      ALLOW_PLAINTEXT_LISTENER: 'true'
   kafka-4:
-    image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
+    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
     restart: always
     environment:
-      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
-      KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
-      KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
-      KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
-      KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
-      KAFKA_BROKER_ID: '4'
-      KAFKA_BROKER_RACK: '4'
-      KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
-      KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
-      KAFKA_DELETE_TOPIC_ENABLE: 'true'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
+      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
+      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
+      KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
+      KAFKA_CFG_BROKER_ID: '4'
+      KAFKA_CFG_BROKER_RACK: '4'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
+      KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
+      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      ALLOW_PLAINTEXT_LISTENER: 'true'
   kafka-5:
-    image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-7.0.1}'
+    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
     restart: always
     environment:
-      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
-      KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
-      KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
-      KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
-      KAFKA_DEFAULT_REPLICATION_FACTOR: '2'
-      KAFKA_BROKER_ID: '5'
-      KAFKA_BROKER_RACK: '5'
-      KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
-      KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
-      KAFKA_DELETE_TOPIC_ENABLE: 'true'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
+      KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
+      KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
+      KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
+      KAFKA_CFG_BROKER_ID: '5'
+      KAFKA_CFG_BROKER_RACK: '5'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
+      KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
+      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      ALLOW_PLAINTEXT_LISTENER: 'true'
   toxiproxy:
     image: 'ghcr.io/shopify/toxiproxy:2.3.0'
     ports:

diff --git a/functional_test.go b/functional_test.go
@@ -136,28 +136,13 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err
 	if version, ok := os.LookupEnv("KAFKA_VERSION"); ok {
 		env.KafkaVersion = version
 	} else {
-		// We have cp-7.0.0 as the default in the docker-compose file, so that's kafka 3.0.0.
-		env.KafkaVersion = "3.0.0"
-	}
-
-	// the mapping of confluent platform docker image versions -> kafka versions can be
-	// found here: https://docs.confluent.io/current/installation/versions-interoperability.html
-	var confluentPlatformVersion string
-	switch env.KafkaVersion {
-	case "3.0.0":
-		confluentPlatformVersion = "7.0.1"
-	case "2.8.1":
-		confluentPlatformVersion = "6.2.2"
-	case "2.7.1":
-		confluentPlatformVersion = "6.1.4"
-	default:
-		return fmt.Errorf("don't know what confluent platform version to use for kafka %s", env.KafkaVersion)
+		env.KafkaVersion = "3.1.0"
 	}
 
 	c := exec.Command("docker-compose", "up", "-d")
 	c.Stdout = os.Stdout
 	c.Stderr = os.Stderr
-	c.Env = append(os.Environ(), fmt.Sprintf("CONFLUENT_PLATFORM_VERSION=%s", confluentPlatformVersion))
+	c.Env = append(os.Environ(), fmt.Sprintf("KAFKA_VERSION=%s", env.KafkaVersion))
 	err := c.Run()
 	if err != nil {
 		return fmt.Errorf("failed to run docker-compose to start test environment: %w", err)
@@ -169,7 +154,7 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err
 
 	// Wait for the kafka broker to come up
 	allBrokersUp := false
-	for i := 0; i < 45 && !allBrokersUp; i++ {
+	for i := 0; i < 90 && !allBrokersUp; i++ {
 		Logger.Println("waiting for kafka brokers to come up")
 		time.Sleep(1 * time.Second)
 		config := NewTestConfig()

diff --git a/utils_test.go b/utils_test.go
@@ -69,6 +69,7 @@ func TestVersionParsing(t *testing.T) {
 		"2.8.0",
 		"2.8.1",
 		"3.0.0",
+		"3.1.0",
 	}
 	for _, s := range validVersions {
 		v, err := ParseKafkaVersion(s)