Exceptional future ignored: seastar::gate_closed_exception in handle_leadership_notification

[VadimPlh]

I got this error with si-verifier in stress ducktape test #3459

WARN  2022-01-13 15:19:20,230 [shard 0] seastar - Exceptional future ignored: seastar::gate_closed_exception (gate closed), backtrace: 0x3bd7384 0x38c3872 0x2ba98ac 0x3952d14 0x39561d7 0x39535b5 0x38a53b9 0x38a3450 0x11ebb94 0x3c9942c /home/vadim/redpanda/vbuild/release/clang/dist/local/redpanda/lib/libc.so.6+0x270b2 0x11e908d
   --------
   seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&)::'lambda'(), false>, seastar::futurize<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>::type seastar::future<void>::then_wrapped_nrvo<seastar::future<void>, seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&)::'lambda'(), false> >(seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&)::'lambda'(), false>&&)::'lambda'(seastar::internal::promise_base_with_type<void>&&, seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&)::'lambda'(), false>&, seastar::future_state<seastar::internal::monostate>&&), void>

[dotnwat]

@VadimPlh can you decode and print the back trace backtrace: 0x3bd7384 0x38c3872 0x2ba98ac 0x3952d14 0x39561d7 0x39535b5 0x38a53b9 0x38a3450 0x11ebb94 0x3c9942c? it's not clear if what you posted below the ------- is the back trace or not; it's all on a single line...

[dotnwat]

@VadimPlh can you post some of the context from the log? for example, what was happening in the moments before the exception was thrown?

[VadimPlh]

@dotnwat I posted in description lines from redpanda log

[VadimPlh]

More info:
It was on shutdown.
Log: link
back trace:

[Backtrace #0]
void seastar::backtrace<seastar::current_backtrace_tasklocal()::$_3>(seastar::current_backtrace_tasklocal()::$_3&&) at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/include/seastar/util/backtrace.hh:59
 (inlined by) seastar::current_backtrace_tasklocal() at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/util/backtrace.cc:86
 (inlined by) seastar::current_tasktrace() at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/util/backtrace.cc:137
 (inlined by) seastar::current_backtrace() at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/util/backtrace.cc:170
seastar::report_failed_future(std::exception_ptr const&) at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/core/future.cc:210
 (inlined by) seastar::report_failed_future(seastar::future_state_base::any&&) at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/core/future.cc:218
seastar::future_state_base::any::check_failure() at /home/vadim/redpanda/vbuild/release/clang/rp_deps_install/include/seastar/core/future.hh:567
 (inlined by) seastar::future_state<seastar::internal::monostate>::clear() at /home/vadim/redpanda/vbuild/release/clang/rp_deps_install/include/seastar/core/future.hh:609
 (inlined by) ~future_state at /home/vadim/redpanda/vbuild/release/clang/rp_deps_install/include/seastar/core/future.hh:614
 (inlined by) ~future at /home/vadim/redpanda/vbuild/release/clang/rp_deps_install/include/seastar/core/scheduling.hh:43
 (inlined by) void seastar::futurize<seastar::future<void> >::satisfy_with_result_of<seastar::future<void>::then_wrapped_nrvo<seastar::future<void>, seastar::future<void>::finally_body<seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&)::{lambda()#1}, false> >(seastar::future<void>::finally_body<seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&)::{lambda()#1}, false>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, auto:1&&)::{lambda()#1}&, seastar::future_state<seastar::internal::monostate>&&)#1}::operator()(seastar::internal::promise_base_with_type<void>, seastar::internal::promise_base_with_type<void>&&, seastar::future_state<seastar::internal::monostate>) const::{lambda()#1}>(seastar::internal::promise_base_with_type<void>, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&) at /home/vadim/redpanda/vbuild/release/clang/rp_deps_install/include/seastar/core/future.hh:2120
 (inlined by) operator() at /home/vadim/redpanda/vbuild/release/clang/rp_deps_install/include/seastar/core/future.hh:1667
 (inlined by) seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::future<void>::finally_body<seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&)::{lambda()#1}, false>, seastar::future<void>::then_wrapped_nrvo<seastar::future<void>, seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&)::{lambda()#1}>(seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2&&)::{lambda()#1}&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::internal::invoke_func_with_gate<cluster::metadata_dissemination_service::handle_leadership_notification(model::ntp, detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true> >, std::__1::optional<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >)::$_2>(seastar::gate&, auto:1&&)::{lambda()#1}&, seastar::future_state<seastar::internal::monostate>&&)#1}, void>::run_and_dispose() at /home/vadim/redpanda/vbuild/release/clang/rp_deps_install/include/seastar/core/future.hh:767
seastar::reactor::run_tasks(seastar::reactor::task_queue&) at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/core/reactor.cc:2230
 (inlined by) seastar::reactor::run_some_tasks() at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/core/reactor.cc:2639
seastar::reactor::do_run() at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/core/reactor.cc:2808
seastar::reactor::run() at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/core/reactor.cc:2691
seastar::app_template::run_deprecated(int, char**, std::__1::function<void ()>&&) at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/core/app-template.cc:217
seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&) at /home/vadim/redpanda/vbuild/release/clang/v_deps_build/seastar-prefix/src/seastar/src/core/app-template.cc:125
application::run(int, char**) at /home/vadim/redpanda/vbuild/release/clang/../../../src/v/redpanda/application.cc:175
main at /home/vadim/redpanda/vbuild/release/clang/../../../src/v/redpanda/main.cc:38