Hi, all rgw crash on thread notif-worker0 when using rgw persistent notification function in ceph version: v16.2.13. I haven't find root cause. Backtrace: 0> 2025-05-13T13:36:45.435+0000 7f0813c64700 -1 *** Caught signal (Aborted) ** in thread 7f0813c64700 thread_name:notif-worker0 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f084607acf0] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f084506809b] 5: /lib64/libstdc++.so.6(+0x9654c) [0x7f084506e54c] 6: /lib64/libstdc++.so.6(+0x965a7) [0x7f084506e5a7] 7: /lib64/libstdc++.so.6(+0x96808) [0x7f084506e808] 8: (rgw::notify::Manager::Manager(ceph::common::CephContext*, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, rgw::sal::RGWRadosStore*)::{lambda()#2}::operator()() const+0x240) [0x7f0852c176c0] 9: /lib64/libstdc++.so.6(+0xc2b23) [0x7f084509ab23] 10: /lib64/libpthread.so.0(+0x81ca) [0x7f08460701ca] 11: clone() NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this. =================== Above coredump info had no symbol info , so I compile same version with debuginfo. However, there are differen coredump info: gdb) bt #0 0x00007fd4edc6dacf in raise () from /lib64/libc.so.6 #1 0x00007fd4edc40ea5 in abort () from /lib64/libc.so.6 #2 0x00007fd4edc40d79 in __assert_fail_base.cold.0 () from /lib64/libc.so.6 #3 0x00007fd4edc66426 in __assert_fail () from /lib64/libc.so.6 #4 0x00007fd4fb81d7cd in boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>::allocate() () from /lib64/libradosgw.so.2 #5 0x00007fd4fbe8491b in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> > boost::context::callcc<boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> >::operator()()::{lambda(boost::context::continuation&&)#1}>(std::allocator_arg_t, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>&&, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> >::operator()()::{lambda(boost::context::continuation&&)#1}&&) () from /lib64/libradosgw.so.2 #6 0x00007fd4fbe8554f in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >::operator()() () from /lib64/libradosgw.so.2 #7 0x00007fd4fbe85999 in void boost::asio::detail::strand_executor_service::dispatch<boost::asio::io_context::executor_type const, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >&, std::allocator<void> >(std::shared_ptr<boost::asio::detail::strand_executor_service::strand_impl> const&, boost::asio::io_context::executor_type const&, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >&, std::allocator<void> const&) () from /lib64/libradosgw.so.2 #8 0x00007fd4fbe85c55 in std::enable_if<spawn::detail::is_stack_allocator<std::decay<boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >::type, void>::value, void>::type spawn::spawn<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}&&, std::decay&&) () from /lib64/libradosgw.so.2 #9 0x00007fd4fbe888fe in rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) () from /lib64/libradosgw.so.2 #10 0x00007fd4fbe8c711 in rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}::operator()(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) const () from /lib64/libradosgw.so.2 #11 0x00007fd4fbe8cab4 in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}::stack_traits> >::operator()()::{lambda(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}::continuation&&)#1}::operator()(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}::continuation) const () from /lib64/libradosgw.so.2 #12 0x00007fd4fbe8cc90 in boost::context::detail::record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> >::operator()()::{lambda(boost::context::continuation&&)#1}>::run(void*) () from /lib64/libradosgw.so.2 #13 0x00007fd4fbe8cd58 in void boost::context::detail::context_entry<boost::context::detail::record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_c--Type <RET> for more, q to quit, c to continue without paging-- ontext<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> >::operator()()::{lambda(boost::context::continuation&&)#1}> >(boost::context::detail::transfer_t) () from /lib64/libradosgw.so.2 #14 0x00007fd4fbf5e94f in make_fcontext () from /lib64/libradosgw.so.2 #15 0x0000000000000000 in ?? () I have found a similar issue in https://github.com/ceph/ceph/pull/39521, but it haven't reslove it.