UAF during shutdown in AutomaticLeadershipBalancingTest.test_automatic_rebalance

[mmaslankaprv]

FAIL test: AutomaticLeadershipBalancingTest.test_automatic_rebalance (1/60 runs)
  failure at 2022-07-30T01:43:27.563Z: <BadLogLines nodes=docker-rp-11(1) example="==9310==ERROR: AddressSanitizer: heap-use-after-free on address 0x61d000006190 at pc 0x55b4c7361296 bp 0x7ffe49915800 sp 0x7ffe499157f8">

https://buildkite.com/redpanda/redpanda/builds/13352#01824c53-bbf9-47bc-93df-4dc00b8fe0bb

[mmaslankaprv]

This is crash during redpanda shutdown sequence. Accessing already freed consensus pointer:

addr2line: DWARF error: invalid or unhandled FORM value: 0x23
READ of size 8 at 0x61d000006190 thread T0
[Backtrace #0]
0x32f02295: absl::lts_20210324::container_internal::btree<absl::lts_20210324::container_internal::map_params<detail::base_named_type<long, model::model_offset_type, std::__1::integral_constant<bool, true> >, raft::configuration_manager::indexed_configuration, std::__1::less<detail::base_named_type<long, model::model_offset_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<long, model::model_offset_type, std::__1::integral_constant<bool, true> > const, raft::configuration_manager::indexed_configuration> >, 256, false> >::empty() const at ??:?
0x35a84238: absl::lts_20210324::container_internal::btree_container<absl::lts_20210324::container_internal::btree<absl::lts_20210324::container_internal::map_params<detail::base_named_type<long, model::model_offset_type, std::__1::integral_constant<bool, true> >, raft::configuration_manager::indexed_configuration, std::__1::less<detail::base_named_type<long, model::model_offset_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<long, model::model_offset_type, std::__1::integral_constant<bool, true> > const, raft::configuration_manager::indexed_configuration> >, 256, false> > >::empty() const at ??:?
0x35a3d434: raft::configuration_manager::get_latest() const at ??:?
0x34de2104: raft::consensus::process_append_entries_reply(detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >, detail::base_named_type<unsigned long, raft::follower_req_seq_tag, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::model_offset_type, std::__1::integral_constant<bool, true> >) at ??:?
0x34ed8daa: raft::consensus::linearizable_barrier()::$_4::operator()(raft::vnode) const::{lambda(boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >)#1}::operator()(boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >) const at consensus.cc:?
0x34ed888c: seastar::future<void> seastar::futurize<void>::invoke<raft::consensus::linearizable_barrier()::$_4::operator()(raft::vnode) const::{lambda(boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >)#1}&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >(raft::consensus::linearizable_barrier()::$_4::operator()(raft::vnode) const::{lambda(boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >)#1}&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&) at consensus.cc:?
0x34ed8687: auto seastar::futurize_invoke<raft::consensus::linearizable_barrier()::$_4::operator()(raft::vnode) const::{lambda(boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >)#1}&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >(raft::consensus::linearizable_barrier()::$_4::operator()(raft::vnode) const::{lambda(boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >)#1}&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&) at consensus.cc:?
0x34ed8565: _ZZN7seastar6futureIN5boost10outcome_v212basic_resultIN4raft20append_entries_replyENSt3__110error_codeENS2_6policy32error_code_throw_as_system_errorIS5_S7_vEEEEE4thenIZZNS4_9consensus20linearizable_barrierEvENK3$_4clENS4_5vnodeEEUlSB_E_NS0_IvEEEET0_OT_ENUlDpOT_E_clIJSB_EEEDaSO_ at consensus.cc:?
0x34ed8463: _ZN7seastar20noncopyable_functionIFNS_6futureIvEEON5boost10outcome_v212basic_resultIN4raft20append_entries_replyENSt3__110error_codeENS4_6policy32error_code_throw_as_system_errorIS7_S9_vEEEEEE17direct_vtable_forIZNS1_ISD_E4thenIZZNS6_9consensus20linearizable_barrierEvENK3$_4clENS6_5vnodeEEUlSD_E_S2_EET0_OT_EUlDpOT_E_E4callEPKSG_SE_ at consensus.cc:?
0x35149d53: seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>::operator()(boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&) const at ??:?
0x35149b5f: decltype ((static_cast<seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&>({parm#1}))(static_cast<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >>({parm#2}))) std::__1::__invoke<seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >(seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&) at consensus.cc:?
0x35149a47: std::__1::invoke_result<seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >::type std::__1::invoke<seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >(seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&) at ??:?
0x35149787: auto seastar::internal::future_invoke<seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >(seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&) at ??:?
0x35149661: seastar::future<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, seastar::future_state<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, seastar::future_state<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >&&) const::{lambda()#1}::operator()() const at ??:?
0x351493dc: void seastar::futurize<seastar::future<void> >::satisfy_with_result_of<seastar::future<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, seastar::future_state<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, seastar::future_state<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >&&) const::{lambda()#1}>(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&&) at ??:?
0x351491d6: seastar::future<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, seastar::future_state<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, seastar::future_state<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >&&) const at ??:?
0x35148770: seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>, seastar::future<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> >&&)>&, seastar::future_state<boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >&&)#1}, boost::outcome_v2::basic_result<raft::append_entries_reply, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft::append_entries_reply, std::__1::error_code, void> > >::run_and_dispose() at ??:?
0x439c47cd: seastar::reactor::run_tasks(seastar::reactor::task_queue&) at reactor.cc:?
0x439ca7e6: seastar::reactor::run_some_tasks() at reactor.cc:?
0x439cf33f: seastar::reactor::do_run() at reactor.cc:?
addr2line: '(/var/lib/buildkite-agent/builds/buildkite-amd64-xfs-builders-i-0626f4c16d90628dd-1/redpanda/redpanda/vbuild/debug/clang/dist/local/redpanda/lib/libc.so.6': No such file
0x439ccec8: seastar::reactor::run() at reactor.cc:?
0x436ba80c: seastar::app_template::run_deprecated(int, char**, std::__1::function<void ()>&&) at app-template.cc:?
0x436b7ec5: seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&) at app-template.cc:?
0x32a2ee2e: application::run(int, char**) at ??:?
0x329edd1f: main at ??:?
{(/var/lib/buildkite-agent/builds/buildkite-amd64-xfs-builders-i-0626f4c16d90628dd-1/redpanda/redpanda/vbuild/debug/clang/dist/local/redpanda/lib/libc.so.6} 0x27b74: (/var/lib/buildkite-agent/builds/buildkite-amd64-xfs-builders-i-0626f4c16d90628dd-1/redpanda/redpanda/vbuild/debug/clang/dist/local/redpanda/lib/libc.so.6 0x27b74 
0x3292d84d: _start at ??:?

freed by thread T0 here:
[Backtrace #1]
0x329ebdbd: operator delete(void*) at ??:?
0x32ef9fac: seastar::internal::lw_shared_ptr_accessors_no_esft<raft::consensus>::dispose(seastar::lw_shared_ptr_counter_base*) at ??:?
0x32f18b73: cluster::leader_balancer::~leader_balancer() at ??:?
0x32f188ac: std::__1::default_delete<cluster::leader_balancer>::operator()(cluster::leader_balancer*) const at application.cc:?
0x32f18684: std::__1::unique_ptr<cluster::leader_balancer, std::__1::default_delete<cluster::leader_balancer> >::reset(cluster::leader_balancer*) at application.cc:?
0x32ef837c: std::__1::unique_ptr<cluster::leader_balancer, std::__1::default_delete<cluster::leader_balancer> >::~unique_ptr() at application.cc:?
0x32ef804f: cluster::controller::~controller() at ??:?
0x32ef7d5c: std::__1::default_delete<cluster::controller>::operator()(cluster::controller*) const at application.cc:?
0x32ef7b34: std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >::reset(cluster::controller*) at application.cc:?
0x33488e71: application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}::operator()() const at ??:?
0x33488a50: decltype ((static_cast<application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}&>({parm#1}))()) std::__1::__invoke<application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}&>(application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}&) at application.cc:?
0x3348890c: void std::__1::__invoke_void_return_wrapper<void, true>::__call<application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}&>(application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}&) at ??:?
0x33488874: std::__1::__function::__alloc_func<application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}, std::__1::allocator<application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}>, void ()>::operator()() at application.cc:?
0x33483bd0: std::__1::__function::__func<application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}, std::__1::allocator<application::construct_single_service<cluster::controller, cluster::config_manager::preload_result, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> > >(std::__1::unique_ptr<cluster::controller, std::__1::default_delete<cluster::controller> >&, cluster::config_manager::preload_result&&, seastar::sharded<rpc::connection_cache>&, seastar::sharded<cluster::partition_manager>&, seastar::sharded<cluster::shard_table>&, seastar::sharded<storage::api>&, seastar::sharded<storage::node_api>&, std::__1::reference_wrapper<seastar::sharded<raft::group_manager> >&&, seastar::sharded<v8_engine::data_policy_table>&, std::__1::reference_wrapper<seastar::sharded<cluster::feature_table> >&&, std::__1::reference_wrapper<seastar::sharded<cloud_storage::remote> >&&)::{lambda()#1}>, void ()>::operator()() at ??:?
0x32d3dc55: std::__1::__function::__value_func<void ()>::operator()() const at application.cc:?
0x32c4f758: std::__1::function<void ()>::operator()() const at application.cc:?
0x32f510d8: seastar::deferred_action<std::__1::function<void ()> >::~deferred_action() at ??:?
0x32f50fe0: void std::__1::__destroy_at<seastar::deferred_action<std::__1::function<void ()> >, 0>(seastar::deferred_action<std::__1::function<void ()> >*) at application.cc:?
0x32f50f7c: void std::__1::destroy_at<seastar::deferred_action<std::__1::function<void ()> >, 0>(seastar::deferred_action<std::__1::function<void ()> >*) at application.cc:?
0x32f50f50: void std::__1::allocator_traits<std::__1::allocator<seastar::deferred_action<std::__1::function<void ()> > > >::destroy<seastar::deferred_action<std::__1::function<void ()> >, void, void>(std::__1::allocator<seastar::deferred_action<std::__1::function<void ()> > >&, seastar::deferred_action<std::__1::function<void ()> >*) at ??:?
0x32f50ebf: std::__1::vector<seastar::deferred_action<std::__1::function<void ()> >, std::__1::allocator<seastar::deferred_action<std::__1::function<void ()> > > >::__base_destruct_at_end(seastar::deferred_action<std::__1::function<void ()> >*) at application.cc:?
0x32f8d5bf: std::__1::vector<seastar::deferred_action<std::__1::function<void ()> >, std::__1::allocator<seastar::deferred_action<std::__1::function<void ()> > > >::__destruct_at_end(seastar::deferred_action<std::__1::function<void ()> >*) at application.cc:?
0x32f8d0b5: std::__1::vector<seastar::deferred_action<std::__1::function<void ()> >, std::__1::allocator<seastar::deferred_action<std::__1::function<void ()> > > >::pop_back() at application.cc:?
0x32a79331: application::run(int, char**)::$_1::operator()() const::{lambda()#1}::operator()() const::{lambda()#1}::operator()() const at application.cc:?
0x32a78f78: seastar::deferred_action<application::run(int, char**)::$_1::operator()() const::{lambda()#1}::operator()() const::{lambda()#1}>::~deferred_action() at application.cc:?
0x32a78841: application::run(int, char**)::$_1::operator()() const::{lambda()#1}::operator()() const at application.cc:?
0x32a77da0: decltype ((static_cast<application::run(int, char**)::$_1::operator()() const::{lambda()#1}>({parm#1}))()) std::__1::__invoke_constexpr<application::run(int, char**)::$_1::operator()() const::{lambda()#1}>(application::run(int, char**)::$_1::operator()() const::{lambda()#1}&&) at application.cc:?
0x32a77d40: decltype(auto) std::__1::__apply_tuple_impl<application::run(int, char**)::$_1::operator()() const::{lambda()#1}, std::__1::tuple<>>(application::run(int, char**)::$_1::operator()() const::{lambda()#1}&&, std::__1::tuple<>&&, std::__1::__tuple_indices<>) at application.cc:?
0x32a77c9d: decltype(auto) std::__1::apply<application::run(int, char**)::$_1::operator()() const::{lambda()#1}, std::__1::tuple<> >(application::run(int, char**)::$_1::operator()() const::{lambda()#1}&&, std::__1::tuple<>&&) at application.cc:?
0x32a77a4d: seastar::future<int> seastar::futurize<int>::apply<application::run(int, char**)::$_1::operator()() const::{lambda()#1}>(application::run(int, char**)::$_1::operator()() const::{lambda()#1}&&, std::__1::tuple<>&&) at application.cc:?

previously allocated by thread T0 here:
[Backtrace #2]
0x329eb55d: operator new(unsigned long) at ??:?
0x3566ae2e: seastar::lw_shared_ptr<raft::consensus> seastar::lw_shared_ptr<raft::consensus>::make<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >&, detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >&, raft::group_configuration, simple_time_jitter<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> > >, storage::log&, raft::scheduling_config, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> >&, raft::consensus_client_protocol&, raft::group_manager::create_group(detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >, std::__1::vector<model::broker, std::__1::allocator<model::broker> >, storage::log)::$_5, storage::api&, raft::recovery_throttle&, raft::recovery_memory_quota&, raft::raft_feature_table&>(detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >&, detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >&, raft::group_configuration&&, simple_time_jitter<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> > >&&, storage::log&, raft::scheduling_config&&, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> >&, raft::consensus_client_protocol&, raft::group_manager::create_group(detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >, std::__1::vector<model::broker, std::__1::allocator<model::broker> >, storage::log)::$_5&&, storage::api&, raft::recovery_throttle&, raft::recovery_memory_quota&, raft::raft_feature_table&) at group_manager.cc:?
0x3566166a: seastar::lw_shared_ptr<raft::consensus> seastar::make_lw_shared<raft::consensus, detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >&, detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >&, raft::group_configuration, simple_time_jitter<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> > >, storage::log&, raft::scheduling_config, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> >&, raft::consensus_client_protocol&, raft::group_manager::create_group(detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >, std::__1::vector<model::broker, std::__1::allocator<model::broker> >, storage::log)::$_5, storage::api&, raft::recovery_throttle&, raft::recovery_memory_quota&, raft::raft_feature_table&>(detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >&, detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >&, raft::group_configuration&&, simple_time_jitter<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> > >&&, storage::log&, raft::scheduling_config&&, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> >&, raft::consensus_client_protocol&, raft::group_manager::create_group(detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >, std::__1::vector<model::broker, std::__1::allocator<model::broker> >, storage::log)::$_5&&, storage::api&, raft::recovery_throttle&, raft::recovery_memory_quota&, raft::raft_feature_table&) at group_manager.cc:?
0x35660859: raft::group_manager::create_group(detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >, std::__1::vector<model::broker, std::__1::allocator<model::broker> >, storage::log) at ??:?
0x3ad37547: cluster::partition_manager::manage(storage::ntp_config, detail::base_named_type<long, raft::raft_group_id_type, std::__1::integral_constant<bool, true> >, std::__1::vector<model::broker, std::__1::allocator<model::broker> >, std::__1::optional<cluster::remote_topic_properties>) [clone .resume] at partition_manager.cc:?
0x3ad5de30: std::__1::coroutine_handle<seastar::internal::coroutine_traits_base<seastar::lw_shared_ptr<raft::consensus> >::promise_type>::resume() const at partition_manager.cc:?
0x3ad5d9e5: seastar::internal::coroutine_traits_base<seastar::lw_shared_ptr<raft::consensus> >::promise_type::run_and_dispose() at ??:?
0x439c47cd: seastar::reactor::run_tasks(seastar::reactor::task_queue&) at reactor.cc:?
0x439ca7e6: seastar::reactor::run_some_tasks() at reactor.cc:?
0x439cf33f: seastar::reactor::do_run() at reactor.cc:?
0x439ccec8: seastar::reactor::run() at reactor.cc:?
0x436ba80c: seastar::app_template::run_deprecated(int, char**, std::__1::function<void ()>&&) at app-template.cc:?
0x436b7ec5: seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&) at app-template.cc:?
0x32a2ee2e: application::run(int, char**) at ??:?
0x329edd1f: main at ??:?