From 6cb5a71e009803a818400db6ad6336b01b24171b Mon Sep 17 00:00:00 2001 From: Travis Downs Date: Thu, 14 Jul 2022 13:38:51 -0700 Subject: [PATCH] Include broker list in metadata estimation Prior to this change, we used only the topic and partition data to estimate the size of the metadata response. Now, we also include the approximate size of the broker metadata portion of the response, which may be important if the list of brokers is very large or they have very long hostnames. --- src/v/kafka/server/handlers/metadata.cc | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/v/kafka/server/handlers/metadata.cc b/src/v/kafka/server/handlers/metadata.cc index 92e034fb4af8..2667b11176f5 100644 --- a/src/v/kafka/server/handlers/metadata.cc +++ b/src/v/kafka/server/handlers/metadata.cc @@ -14,6 +14,7 @@ #include "cluster/types.h" #include "config/configuration.h" #include "config/node_config.h" +#include "kafka/protocol/schemata/metadata_response.h" #include "kafka/server/errors.h" #include "kafka/server/fwd.h" #include "kafka/server/handlers/details/leader_epoch.h" @@ -446,9 +447,21 @@ metadata_memory_estimator(size_t request_size, connection_context& conn_ctx) { // fit in this 10000k slush fund. size_t size_estimate = 10000; - auto& md = conn_ctx.server().metadata_cache().all_topics_metadata(); + auto& md_cache = conn_ctx.server().metadata_cache(); - for (auto& [tp_ns, topic_metadata] : md) { + // The size will vary with the number of brokers, though this effect is + // probably small if there are large numbers of partitions + + // This covers the variable part of the broker response, i.e., the broker + // hostname + rack We just hope these are less than this amount, because we + // don't want to execute the relatively complex logic to guess the listener + // just for the size estimate. + constexpr size_t extra_bytes_per_broker = 200; + size_estimate + += md_cache.all_brokers().size() + * (sizeof(metadata_response_broker) + extra_bytes_per_broker); + + for (auto& [tp_ns, topic_metadata] : md_cache.all_topics_metadata()) { // metadata_response_topic size_estimate += sizeof(kafka::metadata_response_topic); size_estimate += tp_ns.tp().size();