Hi, Since the recent update to 16.2.14-1~bpo11+1 on Debian Bullseye I've started seeing OSD crashes being registered almost daily across all six physical machines (6xOSD disks per machine). There's a --block-db for each osd on a LV from an NVMe. If anyone has any idea what might be causing these I'd appreciate some insight. Happy to provide any other info which might be useful. Thanks, Luke { "assert_condition": "cur2 >= p.length", "assert_file": "./src/os/bluestore/BlueStore.h", "assert_func": "virtual void RocksDBBlueFSVolumeSelector::sub_usage(void*, const bluefs_fnode_t&)", "assert_line": 3875, "assert_msg": "./src/os/bluestore/BlueStore.h: In function 'virtual void RocksDBBlueFSVolumeSelector::sub_usage(void*, const bluefs_fnode_t&)' thread 7f7f54f25700 time 2023-09-20T14:24:00.455721+0100\n./src/os/bluestore/BlueStore.h: 3875: FAILED ceph_assert(cur2 >= p.length)\n", "assert_thread_name": "bstore_kv_sync", "backtrace": [ "/lib/x86_64-linux-gnu/libpthread.so.0(+0x13140) [0x7f7f68632140]", "gsignal()", "abort()", "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x16e) [0x55b22a49b5fa]", "/usr/bin/ceph-osd(+0xac673b) [0x55b22a49b73b]", "(RocksDBBlueFSVolumeSelector::sub_usage(void*, bluefs_fnode_t const&)+0x11e) [0x55b22ab0077e]", "(BlueFS::_flush_range_F(BlueFS::FileWriter*, unsigned long, unsigned long)+0x5bd) [0x55b22ab9b8ed]", "(BlueFS::_flush_F(BlueFS::FileWriter*, bool, bool*)+0x9a) [0x55b22ab9bd7a]", "(BlueFS::fsync(BlueFS::FileWriter*)+0x79) [0x55b22aba97a9]", "(BlueRocksWritableFile::Sync()+0x15) [0x55b22abbf405]", "(rocksdb::LegacyWritableFileWrapper::Sync(rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x3f) [0x55b22b0914d1]", "(rocksdb::WritableFileWriter::SyncInternal(bool)+0x1f4) [0x55b22b26b7c6]", "(rocksdb::WritableFileWriter::Sync(bool)+0x18c) [0x55b22b26b1f8]", "(rocksdb::DBImpl::WriteToWAL(rocksdb::WriteThread::WriteGroup const&, rocksdb::log::Writer*, unsigned long*, bool, bool, unsigned long)+0x366) [0x55b22b0e4a98]", "(rocksdb::DBImpl::WriteImpl(rocksdb::WriteOptions const&, rocksdb::WriteBatch*, rocksdb::WriteCallback*, unsigned long*, unsigned long, bool, unsigned long*, unsigned long, rocksdb::PreReleaseCallback*)+0x12cc) [0x55b22b0e0c5a]", "(rocksdb::DBImpl::Write(rocksdb::WriteOptions const&, rocksdb::WriteBatch*)+0x4a) [0x55b22b0df92a]", "(RocksDBStore::submit_common(rocksdb::WriteOptions&, std::shared_ptr<KeyValueDB::TransactionImpl>)+0x82) [0x55b22b036c42]", "(RocksDBStore::submit_transaction_sync(std::shared_ptr<KeyValueDB::TransactionImpl>)+0x96) [0x55b22b037cc6]", "(BlueStore::_kv_sync_thread()+0x1201) [0x55b22aafc891]", "(BlueStore::KVSyncThread::entry()+0xd) [0x55b22ab2792d]", "/lib/x86_64-linux-gnu/libpthread.so.0(+0x7ea7) [0x7f7f68626ea7]", "clone()" ], "ceph_version": "16.2.14", "crash_id": "2023-09-20T13:24:00.562318Z_beb5c664-9ffb-4a4e-8c61-166865fd4e0b", "entity_name": "osd.8", "os_id": "11", "os_name": "Debian GNU/Linux 11 (bullseye)", "os_version": "11 (bullseye)", "os_version_id": "11", "process_name": "ceph-osd", "stack_sig": "90d1fb6954f0f5b1e98659a93a1b9ce5a5a42cd5e0b2990a65dc336567adcb26", "timestamp": "2023-09-20T13:24:00.562318Z", "utsname_hostname": "cphosd02", "utsname_machine": "x86_64", "utsname_release": "5.10.0-23-amd64", "utsname_sysname": "Linux", "utsname_version": "#1 SMP Debian 5.10.179-1 (2023-05-12)" } -- All postal correspondence to: The Positive Internet Company, 24 Ganton Street, London. W1F 7QY *Follow us on Twitter* @posipeople The Positive Internet Company Limited is registered in England and Wales. Registered company number: 3673639. VAT no: 726 7072 28. Registered office: Northside House, Mount Pleasant, Barnet, Herts, EN4 9EE.