rgw crash on thread notif-worker0
Hi, all rgw crash on thread notif-worker0 when using rgw persistent notification function in ceph version: v16.2.13. I haven't find root cause. Backtrace: 0> 2025-05-13T13:36:45.435+0000 7f0813c64700 -1 *** Caught signal (Aborted) ** in thread 7f0813c64700 thread_name:notif-worker0 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f084607acf0] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f084506809b] 5: /lib64/libstdc++.so.6(+0x9654c) [0x7f084506e54c] 6: /lib64/libstdc++.so.6(+0x965a7) [0x7f084506e5a7] 7: /lib64/libstdc++.so.6(+0x96808) [0x7f084506e808] 8: (rgw::notify::Manager::Manager(ceph::common::CephContext*, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, rgw::sal::RGWRadosStore*)::{lambda()#2}::operator()() const+0x240) [0x7f0852c176c0] 9: /lib64/libstdc++.so.6(+0xc2b23) [0x7f084509ab23] 10: /lib64/libpthread.so.0(+0x81ca) [0x7f08460701ca] 11: clone() NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this. =================== Above coredump info had no symbol info , so I compile same version with debuginfo. However, there are differen coredump info: gdb) bt #0 0x00007fd4edc6dacf in raise () from /lib64/libc.so.6 #1 0x00007fd4edc40ea5 in abort () from /lib64/libc.so.6 #2 0x00007fd4edc40d79 in __assert_fail_base.cold.0 () from /lib64/libc.so.6 #3 0x00007fd4edc66426 in __assert_fail () from /lib64/libc.so.6 #4 0x00007fd4fb81d7cd in boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>::allocate() () from /lib64/libradosgw.so.2 #5 0x00007fd4fbe8491b in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> > boost::context::callcc<boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> >::operator()()::{lambda(boost::context::continuation&&)#1}>(std::allocator_arg_t, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>&&, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> >::operator()()::{lambda(boost::context::continuation&&)#1}&&) () from /lib64/libradosgw.so.2 #6 0x00007fd4fbe8554f in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >::operator()() () from /lib64/libradosgw.so.2 #7 0x00007fd4fbe85999 in void boost::asio::detail::strand_executor_service::dispatch<boost::asio::io_context::executor_type const, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >&, std::allocator<void> >(std::shared_ptr<boost::asio::detail::strand_executor_service::strand_impl> const&, boost::asio::io_context::executor_type const&, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >&, std::allocator<void> const&) () from /lib64/libradosgw.so.2 #8 0x00007fd4fbe85c55 in std::enable_if<spawn::detail::is_stack_allocator<std::decay<boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >::type, void>::value, void>::type spawn::spawn<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}::stack_traits> >(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}&&, std::decay&&) () from /lib64/libradosgw.so.2 #9 0x00007fd4fbe888fe in rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) () from /lib64/libradosgw.so.2 #10 0x00007fd4fbe8c711 in rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}::operator()(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) const () from /lib64/libradosgw.so.2 #11 0x00007fd4fbe8cab4 in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}::stack_traits> >::operator()()::{lambda(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}::continuation&&)#1}::operator()(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}::continuation) const () from /lib64/libradosgw.so.2 #12 0x00007fd4fbe8cc90 in boost::context::detail::record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> >::operator()()::{lambda(boost::context::continuation&&)#1}>::run(void*) () from /lib64/libradosgw.so.2 #13 0x00007fd4fbe8cd58 in void boost::context::detail::context_entry<boost::context::detail::record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)::{lambda(spawn::basic_yield_c--Type <RET> for more, q to quit, c to continue without paging-- ontext<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> >::operator()()::{lambda(boost::context::continuation&&)#1}> >(boost::context::detail::transfer_t) () from /lib64/libradosgw.so.2 #14 0x00007fd4fbf5e94f in make_fcontext () from /lib64/libradosgw.so.2 #15 0x0000000000000000 in ?? () I have found a similar issue in https://github.com/ceph/ceph/pull/39521, but it haven't reslove it.
xu chenhui wrote:
Hi, all
rgw crash on thread notif-worker0 when using rgw persistent notification function in ceph version: v16.2.13. I haven't find root cause.
Backtrace:
0> 2025-05-13T13:36:45.435+0000 7f0813c64700 -1 *** Caught signal (Aborted) ** in thread 7f0813c64700 thread_name:notif-worker0
1: /lib64/libpthread.so.0(+0x12cf0) [0x7f084607acf0] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f084506809b] 5: /lib64/libstdc++.so.6(+0x9654c) [0x7f084506e54c] 6: /lib64/libstdc++.so.6(+0x965a7) [0x7f084506e5a7] 7: /lib64/libstdc++.so.6(+0x96808) [0x7f084506e808] 8: (rgw::notify::Manager::Manager(ceph::common::CephContext*, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, rgw::sal::RGWRadosStore*)::{lambda()#2}::operator()() const+0x240) [0x7f0852c176c0] 9: /lib64/libstdc++.so.6(+0xc2b23) [0x7f084509ab23] 10: /lib64/libpthread.so.0(+0x81ca) [0x7f08460701ca] 11: clone() NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
=================== Above coredump info had no symbol info , so I compile same version with debuginfo. However, there are differen coredump info:
gdb) bt #0 0x00007fd4edc6dacf in raise () from /lib64/libc.so.6 #1 0x00007fd4edc40ea5 in abort () from /lib64/libc.so.6 #2 0x00007fd4edc40d79 in __assert_fail_base.cold.0 () from /lib64/libc.so.6 #3 0x00007fd4edc66426 in __assert_fail () from /lib64/libc.so.6 #4 0x00007fd4fb81d7cd in boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>::allocate() () from /lib64/libradosgw.so.2 #5 0x00007fd4fbe8491b in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> > boost::context::callcc<boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> ::operator()()::{lambda(boost::context::continuation&&)#1}>(std::allocator_arg_t, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>&&, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> ::operator()()::{lambda(boost::context::continuation&&)#1}&&) () from /lib64/libradosgw.so.2 #6 0x00007fd4fbe8554f in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >::operator()() () from /lib64/libradosgw.so.2 #7 0x00007fd4fbe85999 in void boost::asio::detail::strand_executor_service::dispatch<boost::asio::io_context::executor_type const, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >&, std::allocator<void> (std::shared_ptr<boost::asio::detail::strand_executor_service::strand_impl> const&, boost::asio::io_context::executor_type const&, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >&, std::allocator<void> const&) () from /lib64/libradosgw.so.2 #8 0x00007fd4fbe85c55 in std::enable_if<spawn::detail::is_stack_allocator<std::decay<boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >::type, void>::value, void>::type spawn::spawn<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> (spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}&&, std::decay&&) () from /lib64/libradosgw.so.2 #9 0x00007fd4fbe888fe in rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) () from /lib64/libradosgw.so.2 #10 0x00007fd4fbe8c711 in rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::operator()(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) const () from /lib64/libradosgw.so.2 #11 0x00007fd4fbe8cab4 in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::stack_traits> ::operator()()::{lambda(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::continuation&&)#1}::operator()(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::continuation) const () from /lib64/libradosgw.so.2 #12 0x00007fd4fbe8cc90 in boost::context::detail::record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> ::operator()()::{lambda(boost::context::continuation&&)#1}>::run(void*) () from /lib64/libradosgw.so.2 #13 0x00007fd4fbe8cd58 in void boost::context::detail::context_entry<boost::context::detail::record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_c--Type <RET> for more, q to quit, c to continue without paging-- ontext<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> ::operator()()::{lambda(boost::context::continuation&&)#1}> (boost::context::detail::transfer_t) () from /lib64/libradosgw.so.2 #14 0x00007fd4fbf5e94f in make_fcontext () from /lib64/libradosgw.so.2 #15 0x0000000000000000 in ?? ()
I have found a similar issue in https://github.com/ceph/ceph/pull/39521, but it haven't reslove it.
Rgw crash was disappeared when set vm.max_map_count = 2000000. The default value 65536 of max map count of per process is limite allocation for coroutine. However, a new problem has arisen. We are using the method of pushing to an HTTP server. If the HTTP server is abnormal or the HTTP server configuration of the RGW topic is incorrect, a memory leak will occur. I haven't reslove it. Any help is very gratefully. Thanks
Hi Xu, I opened a tracker to address the issue: https://tracker.ceph.com/issues/71402 feel free to comment there. To workaround the problem, you can try using: time_to_live, max_retries and retry_sleep_duration topic parameters to limit the number of retries we make to the faulty HTTP server. see: https://docs.ceph.com/en/quincy/radosgw/notifications/ Please let me know if it helped? Yuval On Sun, May 18, 2025 at 6:04 PM xu chenhui <xuchenhuig@gmail.com> wrote:
xu chenhui wrote:
Hi, all
rgw crash on thread notif-worker0 when using rgw persistent notification function in ceph version: v16.2.13. I haven't find root cause.
Backtrace:
0> 2025-05-13T13:36:45.435+0000 7f0813c64700 -1 *** Caught signal (Aborted) ** in thread 7f0813c64700 thread_name:notif-worker0
1: /lib64/libpthread.so.0(+0x12cf0) [0x7f084607acf0] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f084506809b] 5: /lib64/libstdc++.so.6(+0x9654c) [0x7f084506e54c] 6: /lib64/libstdc++.so.6(+0x965a7) [0x7f084506e5a7] 7: /lib64/libstdc++.so.6(+0x96808) [0x7f084506e808] 8: (rgw::notify::Manager::Manager(ceph::common::CephContext*, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, rgw::sal::RGWRadosStore*)::{lambda()#2}::operator()() const+0x240) [0x7f0852c176c0] 9: /lib64/libstdc++.so.6(+0xc2b23) [0x7f084509ab23] 10: /lib64/libpthread.so.0(+0x81ca) [0x7f08460701ca] 11: clone() NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
=================== Above coredump info had no symbol info , so I compile same version with debuginfo. However, there are differen coredump info:
gdb) bt #0 0x00007fd4edc6dacf in raise () from /lib64/libc.so.6 #1 0x00007fd4edc40ea5 in abort () from /lib64/libc.so.6 #2 0x00007fd4edc40d79 in __assert_fail_base.cold.0 () from /lib64/libc.so.6 #3 0x00007fd4edc66426 in __assert_fail () from /lib64/libc.so.6 #4 0x00007fd4fb81d7cd in
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>::allocate()
() from /lib64/libradosgw.so.2 #5 0x00007fd4fbe8491b in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>
boost::context::callcc<boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>,
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>
::operator()()::{lambda(boost::context::continuation&&)#1}>(std::allocator_arg_t,
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>&&,
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>
::operator()()::{lambda(boost::context::continuation&&)#1}&&) () from /lib64/libradosgw.so.2 #6 0x00007fd4fbe8554f in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >::operator()() () from /lib64/libradosgw.so.2 #7 0x00007fd4fbe85999 in void
boost::asio::detail::strand_executor_service::dispatch<boost::asio::io_context::executor_type
const, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >&, std::allocator<void>
(std::shared_ptr<boost::asio::detail::strand_executor_service::strand_impl> const&, boost::asio::io_context::executor_type const&, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >&, std::allocator<void> const&) () from /lib64/libradosgw.so.2 #8 0x00007fd4fbe85c55 in
std::enable_if<spawn::detail::is_stack_allocator<std::decay<boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >::type, void>::value, void>::type spawn::spawn<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> (spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}&&, std::decay&&) () from /lib64/libradosgw.so.2 #9 0x00007fd4fbe888fe in rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) () from /lib64/libradosgw.so.2 #10 0x00007fd4fbe8c711 in
rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)#6}::operator()(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) const () from /lib64/libradosgw.so.2 #11 0x00007fd4fbe8cab4 in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::stack_traits>
::operator()()::{lambda(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)#6}::continuation&&)#1}::operator()(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::continuation) const () from /lib64/libradosgw.so.2 #12 0x00007fd4fbe8cc90 in boost::context::detail::record<boost::context::continuation,
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>,
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>
::operator()()::{lambda(boost::context::continuation&&)#1}>::run(void*) () from /lib64/libradosgw.so.2 #13 0x00007fd4fbe8cd58 in void
boost::context::detail::context_entry<boost::context::detail::record<boost::context::continuation,
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>,
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_c--Type <RET> for more, q to quit, c to continue without paging-- ontext<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>
::operator()()::{lambda(boost::context::continuation&&)#1}> (boost::context::detail::transfer_t) () from /lib64/libradosgw.so.2 #14 0x00007fd4fbf5e94f in make_fcontext () from /lib64/libradosgw.so.2 #15 0x0000000000000000 in ?? ()
I have found a similar issue in https://github.com/ceph/ceph/pull/39521, but it haven't reslove it.
Rgw crash was disappeared when set vm.max_map_count = 2000000. The default value 65536 of max map count of per process is limite allocation for coroutine. However, a new problem has arisen. We are using the method of pushing to an HTTP server. If the HTTP server is abnormal or the HTTP server configuration of the RGW topic is incorrect, a memory leak will occur. I haven't reslove it. Any help is very gratefully.
Thanks _______________________________________________ ceph-users mailing list -- ceph-users@ceph.io To unsubscribe send an email to ceph-users-leave@ceph.io
participants (2)
-
xu chenhui
-
Yuval Lifshitz