xu chenhui wrote:
Hi, all
rgw crash on thread notif-worker0 when using rgw persistent notification function in ceph version: v16.2.13. I haven't find root cause.
Backtrace:
0> 2025-05-13T13:36:45.435+0000 7f0813c64700 -1 *** Caught signal (Aborted) ** in thread 7f0813c64700 thread_name:notif-worker0
1: /lib64/libpthread.so.0(+0x12cf0) [0x7f084607acf0] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f084506809b] 5: /lib64/libstdc++.so.6(+0x9654c) [0x7f084506e54c] 6: /lib64/libstdc++.so.6(+0x965a7) [0x7f084506e5a7] 7: /lib64/libstdc++.so.6(+0x96808) [0x7f084506e808] 8: (rgw::notify::Manager::Manager(ceph::common::CephContext*, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, rgw::sal::RGWRadosStore*)::{lambda()#2}::operator()() const+0x240) [0x7f0852c176c0] 9: /lib64/libstdc++.so.6(+0xc2b23) [0x7f084509ab23] 10: /lib64/libpthread.so.0(+0x81ca) [0x7f08460701ca] 11: clone() NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
=================== Above coredump info had no symbol info , so I compile same version with debuginfo. However, there are differen coredump info:
gdb) bt #0 0x00007fd4edc6dacf in raise () from /lib64/libc.so.6 #1 0x00007fd4edc40ea5 in abort () from /lib64/libc.so.6 #2 0x00007fd4edc40d79 in __assert_fail_base.cold.0 () from /lib64/libc.so.6 #3 0x00007fd4edc66426 in __assert_fail () from /lib64/libc.so.6 #4 0x00007fd4fb81d7cd in boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>::allocate() () from /lib64/libradosgw.so.2 #5 0x00007fd4fbe8491b in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> > boost::context::callcc<boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> ::operator()()::{lambda(boost::context::continuation&&)#1}>(std::allocator_arg_t, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>&&, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> ::operator()()::{lambda(boost::context::continuation&&)#1}&&) () from /lib64/libradosgw.so.2 #6 0x00007fd4fbe8554f in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >::operator()() () from /lib64/libradosgw.so.2 #7 0x00007fd4fbe85999 in void boost::asio::detail::strand_executor_service::dispatch<boost::asio::io_context::executor_type const, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >&, std::allocator<void> (std::shared_ptr<boost::asio::detail::strand_executor_service::strand_impl> const&, boost::asio::io_context::executor_type const&, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >&, std::allocator<void> const&) () from /lib64/libradosgw.so.2 #8 0x00007fd4fbe85c55 in std::enable_if<spawn::detail::is_stack_allocator<std::decay<boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> >::type, void>::value, void>::type spawn::spawn<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#7}::stack_traits> (spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >, rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}&&, std::decay&&) () from /lib64/libradosgw.so.2 #9 0x00007fd4fbe888fe in rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) () from /lib64/libradosgw.so.2 #10 0x00007fd4fbe8c711 in rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::operator()(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) const () from /lib64/libradosgw.so.2 #11 0x00007fd4fbe8cab4 in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::stack_traits> ::operator()()::{lambda(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::continuation&&)#1}::operator()(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )#6}::continuation) const () from /lib64/libradosgw.so.2 #12 0x00007fd4fbe8cc90 in boost::context::detail::record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> ::operator()()::{lambda(boost::context::continuation&&)#1}>::run(void*) () from /lib64/libradosgw.so.2 #13 0x00007fd4fbe8cd58 in void boost::context::detail::context_entry<boost::context::detail::record<boost::context::continuation, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > )::{lambda(spawn::basic_yield_c--Type <RET> for more, q to quit, c to continue without paging-- ontext<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >)#6}, boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> ::operator()()::{lambda(boost::context::continuation&&)#1}> (boost::context::detail::transfer_t) () from /lib64/libradosgw.so.2 #14 0x00007fd4fbf5e94f in make_fcontext () from /lib64/libradosgw.so.2 #15 0x0000000000000000 in ?? ()
I have found a similar issue in https://github.com/ceph/ceph/pull/39521, but it haven't reslove it.
Rgw crash was disappeared when set vm.max_map_count = 2000000. The default value 65536 of max map count of per process is limite allocation for coroutine. However, a new problem has arisen. We are using the method of pushing to an HTTP server. If the HTTP server is abnormal or the HTTP server configuration of the RGW topic is incorrect, a memory leak will occur. I haven't reslove it. Any help is very gratefully. Thanks