Hi, RGW enable bucket notification for http push_endpoint, it will crash when exec delete multi objects command. ceph version: v16.2.13 Thread 483 "radosgw" received signal SIGABRT, Aborted. [Switching to Thread 0x7f94c0ac1700 (LWP 500)] 0x00007f95b86f2acf in raise () from /lib64/libc.so.6 (gdb) bt #0 0x00007f95b86f2acf in raise () from /lib64/libc.so.6 #1 0x00007f95b86c5ea5 in abort () from /lib64/libc.so.6 #2 0x00007f95b86c5d79 in __assert_fail_base.cold.0 () from /lib64/libc.so.6 #3 0x00007f95b86eb426 in __assert_fail () from /lib64/libc.so.6 #4 0x00007f95c7480332 in boost::context::continuation::resume() && (this=0x55a1b3705158) at /opt/ceph-16.2.13/build/boost/include/boost/context/continuation_fcontext.hpp:263 #5 0x00007f95c74802cd in boost::context::continuation::resume() & (this=0x55a1b3705158) at /opt/ceph-16.2.13/build/boost/include/boost/context/continuation_fcontext.hpp:259 #6 0x00007f95c74803db in spawn::detail::continuation_context::resume (this=0x55a1b3705158) at /opt/ceph-16.2.13/src/spawn/include/spawn/impl/spawn.hpp:37 #7 0x00007f95c7499de0 in spawn::detail::coro_async_result<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, void>::get (this=0x55a1b4b0b248) at /opt/ceph-16.2.13/src/spawn/include/spawn/impl/spawn.hpp:256 #8 0x00007f95c7d28659 in librados::async_operate<boost::asio::io_context, spawn::basic_yield_context<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> > > > (ctx=..., io=..., oid="ifind-project-other-tv", write_op=0x55a1b4b0b460, flags=1024, token=...) at /opt/ceph-16.2.13/src/librados/librados_asio.h:185 #9 0x00007f95c7d24d9d in rgw_rados_operate (dpp=0x55a1b463c000, ioctx=..., oid="ifind-project-other-tv", op=0x55a1b4b0b460, y=..., flags=1024) at /opt/ceph-16.2.13/src/rgw/rgw_tools.cc:269 #10 0x00007f95c7f4caff in rgw::notify::publish_reserve (dpp=0x55a1b463c000, event_type=rgw::notify::ObjectRemovedDelete, res=..., req_tags=0x0) at /opt/ceph-16.2.13/src/rgw/rgw_notify.cc:796 #11 0x00007f95c7b182e0 in RGWDeleteMultiObj::handle_individual_object (this=0x55a1b463c000, o=..., y=..., formatter_flush_cond=0x7f9485248130) at /opt/ceph-16.2.13/src/rgw/rgw_op.cc:7164 #12 0x00007f95c7b18c45 in RGWDeleteMultiObj::<lambda(yield_context)>::operator()(yield_context) const (__closure=0x55a1b0160fa8, yield=...) at /opt/ceph-16.2.13/src/rgw/rgw_op.cc:7267 #13 0x00007f95c7b2f43f in spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, RGWDeleteMultiObj::execute(optional_yield)::<lambda(yield_context)>, boost::context::basic_fixedsize_stack<boost::context::stack_traits> >::<lambda(boost::context::continuation&&)>::operator()(boost::context::continuation &&) const ( this=0x7f9485247f60, c=...) at /opt/ceph-16.2.13/src/spawn/include/spawn/impl/spawn.hpp:390 #14 0x00007f95c7b30610 in std::__invoke_impl<boost::context::continuation, spawn::detail::spawn_helper<Handler, Function, StackAllocator>::operator()() [with Handler = boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >; Function = RGWDeleteMultiObj::execute(optional_yield)::<lambda(yield_context)>; StackAllocator = boost::context::basic_fixedsize_stack<boost::context::stack_traits>]::<lambda(boost::context::continuation&&)>&, boost::context::continuation>(std::__invoke_other, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, RGWDeleteMultiObj::execute(optional_yield)::<lambda(yield_context)>, boost::context::basic_fixedsize_stack<boost::context::stack_traits> >::<lambda(boost::context::continuation&&)> &, boost::context::continuation &&) (__f=..., __args#0=...) at /usr/include/c++/8/bits/invoke.h:60 #15 0x00007f95c7b30531 in std::__invoke<spawn::detail::spawn_helper<Handler, Function, StackAllocator>::operator()() [with Handler = boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >; Function = RGWDeleteMultiObj::execute(optional_yield)::<lambda(yield_context)>; StackAllocator = boost::context::basic_fixedsize_stack<boost::context::stack_traits>]::<lambda(boost::context::continuation&&)>&, boost::context::continuation>(spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, RGWDeleteMultiObj::execute(optional_yield)::<lambda(yield_context)>, boost::context::basic_fixedsize_stack<boost::context::stack_traits> >::<lambda(boost::context::continuation&&)> &, boost::context::continuation &&) (__fn=..., __args#0=...) at /usr/include/c++/8/bits/invoke.h:96 #16 0x00007f95c7b304a2 in std::invoke<spawn::detail::spawn_helper<Handler, Function, StackAllocator>::operator()() [with Handler = boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >; Function = RGWDeleteMultiObj::execute(optional_yield)::<lambda(yield_context)>; StackAllocator = boost::context::basic_fixedsize_stack<boost::context::stack_traits>]::<lambda(boost::context::continuation&&)>&, boost::context::continuation>(spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >, RGWDeleteMultiObj::execute(optional_yield)::<lambda(yield_context)>, boost::context::basic_fixedsize_stack<boost::context::stack_traits> >::<lambda(boost::context::continuation&&)> &, boost::context::continuation &&) (__fn=..., __args#0=...) at /usr/include/c++/8/functional:82 #17 0x00007f95c7b3036f in boost::context::detail::record<boost::context::continuation, boost::context::basic_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<Handler, Function, StackAllocator>::operator()() [with Handler = boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >; Function = RGWDeleteMultiObj::execute(optional_yield)::<lambda(yield_context)>; StackAllocator = boost::context::basic_fixedsize_stack<boost::context::stack_traits>]::<lambda(boost::context::continuation&&)> >::run(boost::context::detail::fcontext_t) ( this=0x55a1b4b0bf00, fctx=0x7f9485247d10) at /opt/ceph-16.2.13/build/boost/include/boost/context/continuation_fcontext.hpp:146 #18 0x00007f95c7b3015a in boost::context::detail::context_entry<boost::context::detail::record<boost::context::continuation, boost::context::basic_fixedsize_stack<boost::context::stack_traits>, spawn::detail::spawn_helper<Handler, Function, StackAllocator>::operator()() [with Handler = boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::executor_type> >; Function = RGWDeleteMultiObj::execute(optional_yield)::<lambda(yield_context)>; StackAllocator = boost::context::basic_fixedsize_stack<boost::context::stack_traits>]::<lambda(boost::context::continuation&&)> > >(boost::context::detail::transfer_t) (t=...) at /opt/ceph-16.2.13/build/boost/include/boost/context/continuation_fcontext.hpp:80 #19 0x00007f95c809175f in make_fcontext () from /lib64/libradosgw.so.2 #20 0x9435c9c14cf40130 in ?? () #21 0xacf2d40e6f64164e in ?? () #22 0x00000000d9ff03a8 in ?? () #23 0x0000000000000025 in ?? () #24 0x5f7265767265535f in ?? () #25 0x000055a1b4b0bef8 in ?? () #26 0x0000000000000007 in ?? () #27 0x00322d7466697773 in ?? () #28 0x0000000000020000 in ?? () #29 0x000055a1b4b0c000 in ?? () #30 0x0000000000020000 in ?? () #31 0x00007f9485247f60 in ?? () #32 0x3234346330623365 in ?? () Reproduce： 1. create topic rgw_topic='{"push-endpoint": "'$httpserver'","verify-ssl": "false","persistent":"true"}' aws --region=default --endpoint-url $srcEndpoint sns create-topic --name=$userId --attributes="$rgw_topic" 2. create bucket notification bucket_notify='{"TopicConfigurations": [{"Id": "'$bk'", "TopicArn": "'$topic'", "Events": ["s3:ObjectCreated:*", "s3:ObjectRemoved:*"]}]}' aws --region=default --endpoint-url $srcEndpoint s3api put-bucket-notification-configuration --bucket=$bucketName --notification-configuration="$bucket_notify" 3. delete multi objects aws s3api --endpoint-url $srcEndpoint delete-objects --bucket $bucketName --delete '{"Objects":[{"Key":"obj1"}]}' I had report this problem in https://tracker.ceph.com/issues/71427 rgw_multi_object_del_aio set to 1 didn't help, anyone can have a workaround to resolve this problem ? thanks.