From 8d29b9d32b3dc9f3946c9f61c8507cc59ffc2179 Mon Sep 17 00:00:00 2001 From: Alexey Biryukov Date: Fri, 8 Jul 2022 17:23:25 -0400 Subject: [PATCH 1/2] config: increased default partitions # for __consumer_offsets When a consumer tries to locate a consumer group coordinator of a cluster for the first time, the __consumer_offsets topic is created with the number of partitions as per the group_topic_partitions property. The default value for that property was 1 which means that unless a different value was explicitly specified by the customer at a very early stage of cluster's life, all OffsetCommit requests from all consumers will be going to a single broker. This change increases the default value to 16 as a reasonable trade-off between OffsetCommit parallelism for the clusters that will use consumer groups later in their life, and the overhead for the clusters that won't use consumer groups. #5222 --- src/v/config/configuration.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v/config/configuration.cc b/src/v/config/configuration.cc index 0d0f72c39cd6..b4716730d019 100644 --- a/src/v/config/configuration.cc +++ b/src/v/config/configuration.cc @@ -474,7 +474,7 @@ configuration::configuration() "group_topic_partitions", "Number of partitions in the internal group membership topic", {.needs_restart = needs_restart::no, .visibility = visibility::tunable}, - 1) + 16) , default_topic_replication( *this, "default_topic_replications", From 20c701a4653ca7a9c6dd481f81faef0123b9f09f Mon Sep 17 00:00:00 2001 From: Alexey Biryukov Date: Tue, 12 Jul 2022 23:50:55 -0400 Subject: [PATCH 2/2] ducktape: run GroupMetricsTest with group_topic_partitions=1 test_leadership_transfer relies on kafka_group_offset metric source as an indication of the node being a group coordinator. However in admin API there is no way to know which __consumer_groups partition a specific consumer group uses, and the testcase always works with partition 0. To make that work, number of __consumer_groups partition is set to 1 explicitly. --- tests/rptest/tests/group_membership_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/rptest/tests/group_membership_test.py b/tests/rptest/tests/group_membership_test.py index 389bc1cf3a6b..d57fce734e79 100644 --- a/tests/rptest/tests/group_membership_test.py +++ b/tests/rptest/tests/group_membership_test.py @@ -127,7 +127,8 @@ def __init__(self, ctx, *args, **kwargs): # Require internal_kafka topic to have an increased replication factor extra_rp_conf = dict(default_topic_replications=3, - enable_leader_balancer=False) + enable_leader_balancer=False, + group_topic_partitions=1) super(GroupMetricsTest, self).__init__(test_context=ctx, num_brokers=3, extra_rp_conf=extra_rp_conf) @@ -374,6 +375,9 @@ def select_next_leader(): timeout_sec=30, backoff_sec=5) + self.logger.debug( + f"Waiting for metrics from the single node: {new_leader.account.hostname}" + ) wait_until(lambda: metrics_from_single_node(new_leader), timeout_sec=30, backoff_sec=5)