Unfortunately there are problems even with the removal of the object: [root@ceph-osd-19 ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-250 --pgid 43.1b0s0 '{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":4210196912,"max":0,"pool":43,"namespace":"","generation":44630682,"shard_id":0}' remove remove 0#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2a9029a /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.2.3/rpm/el9/BUILD/ceph-19.2.3/src/os/bluestore/bluestore_types.cc: In function 'bool bluestore_blob_use_tracker_t::put(uint32_t, uint32_t, PExtentVector*)' thread 7f7bfa1f76c0 time 2026-09-17T16:42:25.345681+0200 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.2.3/rpm/el9/BUILD/ceph-19.2.3/src/os/bluestore/bluestore_types.cc: 511: FAILED ceph_assert(diff <= bytes_per_au[pos]) ceph version 19.2.3 (c92aebb279828e9c3c1f5d24613efca272649e62) squid (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x11e) [0x7f7bfb783e02] 2: /usr/lib64/ceph/libceph-common.so.2(+0x183fc1) [0x7f7bfb783fc1] 3: (bluestore_blob_use_tracker_t::put(unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t>
*)+0x1a7) [0x556997ebb877] 4: (BlueStore::Blob::put_ref(BlueStore::Collection*, unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> *)+0xae) [0x556997df3bfe] 5: (BlueStore::OldExtent::create(boost::intrusive_ptr<BlueStore::Collection>, unsigned int, unsigned int, unsigned int, boost::intrusive_ptr<BlueStore::Blob>&)+0x11d) [0x556997dfa85d] 6: (BlueStore::ExtentMap::punch_hole(boost::intrusive_ptr<BlueStore::Collection>&, unsigned long, unsigned long, boost::intrusive::list<BlueStore::OldExtent, boost::intrusive::member_hook<BlueStore::OldExtent, boost::intrusive::list_member_hook<>, &BlueStore::OldExtent::old_extent_item> >*)+0x491) [0x556997e038f1] 7: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x365) [0x556997e79e85] 8: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0xc3) [0x556997e8bfa3] 9: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x80) [0x556997e8da70] 10: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x12d6) [0x556997e7fd96] 11: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> &, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2ef) [0x556997e80e2f] 12: ceph-objectstore-tool(+0x386fd8) [0x556997936fd8] 13: (do_remove_object(ObjectStore*, coll_t, ghobject_t&, bool, bool, rmtype)+0xc58) [0x556997952f08] 14: main() 15: /lib64/libc.so.6(+0x2a610) [0x7f7bfa82a610] 16: __libc_start_main() 17: _start() *** Caught signal (Aborted) ** in thread 7f7bfa1f76c0 thread_name:ceph-objectstor ceph version 19.2.3 (c92aebb279828e9c3c1f5d24613efca272649e62) squid (stable) 1: /lib64/libc.so.6(+0x3fc30) [0x7f7bfa83fc30] 2: /lib64/libc.so.6(+0x8d02c) [0x7f7bfa88d02c] 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x178) [0x7f7bfb783e5c] 6: /usr/lib64/ceph/libceph-common.so.2(+0x183fc1) [0x7f7bfb783fc1] 7: (bluestore_blob_use_tracker_t::put(unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> *)+0x1a7) [0x556997ebb877] 8: (BlueStore::Blob::put_ref(BlueStore::Collection*, unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> *)+0xae) [0x556997df3bfe] 9: (BlueStore::OldExtent::create(boost::intrusive_ptr<BlueStore::Collection>, unsigned int, unsigned int, unsigned int, boost::intrusive_ptr<BlueStore::Blob>&)+0x11d) [0x556997dfa85d] 10: (BlueStore::ExtentMap::punch_hole(boost::intrusive_ptr<BlueStore::Collection>&, unsigned long, unsigned long, boost::intrusive::list<BlueStore::OldExtent, boost::intrusive::member_hook<BlueStore::OldExtent, boost::intrusive::list_member_hook<>, &BlueStore::OldExtent::old_extent_item> >*)+0x491) [0x556997e038f1] 11: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x365) [0x556997e79e85] 12: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0xc3) [0x556997e8bfa3] 13: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x80) [0x556997e8da70] 14: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x12d6) [0x556997e7fd96] 15: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> &, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2ef) [0x556997e80e2f] 16: ceph-objectstore-tool(+0x386fd8) [0x556997936fd8] 17: (do_remove_object(ObjectStore*, coll_t, ghobject_t&, bool, bool, rmtype)+0xc58) [0x556997952f08] 18: main() 19: /lib64/libc.so.6(+0x2a610) [0x7f7bfa82a610] 20: __libc_start_main() 21: _start() Aborted (core dumped) [root@ceph-osd-19 ~]#
On Thu, Sep 17, 2026 at 12:24 PM Enrico Bocchi <enrico.bocchi@cern.ch> wrote:
Hi Massimo,
Have you attempted to export the PG manually with the OSD stopped? Something like: `ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-206 --pgid 43.1b0 --op export --file ~/pg-43.1b0`
I cannot tell whether the export will work, but it is probably worth the shot before deleting data. If it does, you may want to re-import it (again with objectsotre-tool) on another OSD. I am usure if there is any risk of crashing the OSD you are importing the PG to, though...
The signature of the crash matches the known issue about elastic blobs in Squid (https://tracker.ceph.com/issues/70390#note-7). Beware that the issue is remediated in v19.2.4, but this does NOT fix OSDs created with previous Squid versions. You will have to drain + zap + re-create those. Also, you should set `ceph config set osd bluestore_elastic_shared_blobs 0` if you plan to create new OSDs while on v19.2.3 but, again, this does NOT fix previously-created OSDs on buggy Squids.
Cheers, Enrico
On 9/17/26 10:23, Andrew wrote:
Hi Massimo,
Sorry to say that this isn't my strong area, and I wouldn't want to give the wrong advice. Hopefully someone else in this group will be able to give some specifics.
In our situation, we were fortunate enough that one of the crashing OSD's was able to be restarted (fsck still failed when we tried). It didn't crash straight away, and the data replicated out. In the end all OSD's had to be rebuilt (one by one, with the correct settings to work around the bug) - and then an upgrade at the end.
Best Regards,
Andrew
On 17/9/26 17:03, Massimo Sgaravatto wrote:
So the question what is the best way to recover what is recoverable Do you think that it makes sense to do what I wrote in the first mail, i.e. deleting the objects reported by fsck on the OSD which crashed, doing e.g.:
ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-206 --pgid 43.1b0
'{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":7151\
31231,"max":0,"pool":43,"namespace":"","shardid":3}' remove
And then trying to restart the OSD ?
I would do this operation only on 3 OSDs needed to restore the PG in down state
This will clearly corrupt the openstack cinder volumes where these objects are used ... not an ideal scenario but if there aren't better options ...
And then I will update ceph and recreate all the OSDs which were created with 19.2.3 ceph release
Thanks again
Cheers, Massimo
On Thu, Sep 17, 2026 at 6:20 AM Massimo Sgaravatto < massimo.sgaravatto@gmail.com> wrote:
Uhmm, and this also could explain why I see these problems only on some OSDs (i.e. the ones created after the update to v. 19)
I will update, but first I would like to recover (at least what it is possible to recover) the down pg ...
Thanks, Massimo
On Thu, Sep 17, 2026 at 6:13 AM Andrew <andrew@donehue.net> wrote:
Hi Massimo,
There are some critical crashing issues with 19.2.3 (and earlier)
https://docs.clyso.com/docs/kb/known-bugs/squid/
Best Regards,
Andrew.
On 17/9/26 13:20, Massimo Sgaravatto wrote:
Hi Anthony, all
I am running ceph squid 19.2.3
I copied the relevant part of the log (from the start till the crash) for a OSD (osd.206) in https://cernbox.cern.ch/s/56sdGTQsotbN2XE
So as far as I can understand it crashes because of:
2026-09-16T21:16:06.859+0200 7f5431ee1640 4 rocksdb: [db/compaction/compaction_job.cc:1588] [O-2] [JOB 9] Generated table #148643: 151504 keys, 67943642 bytes, temperature: kUnknown 2026-09-16T21:16:06.859+0200 7f5431ee1640 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1789586166860678, "cf_name": "O-2", "job": 9, "event": "table_file_creation", "file_number": 148643, "file_size": 67943642, "file_checksum": "", "f\ ile_checksum_func_name": "Unknown", "smallest_seqno": 0, "largest_seqno": 0, "table_properties": {"data_size": 67111852, "index_size": 1538071, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_v\ alue_is_delta_encoded": 1, "filter_size": 378821, "raw_key_size": 12956410, "raw_average_key_size": 85, "raw_value_size": 67640956, "raw_average_value_size": 446, "num_data_blocks": 18954, "num_entries": 151504, "num_filter_entrie\ s": 151504, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "O-2", "column_family_id": 9, "comparator": "leveldb\ .BytewiseComparator", "merge_operator": "nullptr", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "LZ4", "compression_options": "window_bits=-14; level=32767; strategy=0; \ max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1778598719, "oldest_key_time": 0, "file_creation_time": 1789586166, "slow_compression_estimated_data_size":\ 0, "fast_compression_estimated_data_size": 0, "db_id": "1103608a-90ce-4939-82c0-de96ff245167", "db_session_id": "FO9J42Q67M9AK232DQXE", "orig_file_number": 148643, "seqno_to_time_mapping": "N/A"}} 2026-09-16T21:16:07.421+0200 7f541fcaf640 -1
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.2.3/rpm/el9/BUILD/ceph-19.2.3/src/o\
s/bluestore/bluestore_types.cc: In function 'bool bluestore_blob_use_tracker_t::put(uint32_t, uint32_t, PExtentVector*)' thread 7f541fcaf640 time 2026-09-16T21:16:07.418902+0200
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.2.3/rpm/el9/BUILD/ceph-19.2.3/src/os/bluestore/bluestore_types.cc:
511: FAILED c\ eph_assert(diff <= bytes_per_au[pos])
ceph version 19.2.3 (c92aebb279828e9c3c1f5d24613efca272649e62) squid (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x113) [0x560cc2d4c85d] 2: /usr/bin/ceph-osd(+0x401a14) [0x560cc2d4ca14] 3: /usr/bin/ceph-osd(+0x3f2930) [0x560cc2d3d930] 4: (BlueStore::Blob::put_ref(BlueStore::Collection*, unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> > *)+0xaa) [0x560cc32bb2fa] 5:
(BlueStore::OldExtent::create(boost::intrusive_ptr<BlueStore::Collection>,
unsigned int, unsigned int, unsigned int, boost::intrusive_ptr<BlueStore::Blob>&)+0x11d) [0x560cc32bf47d] 6:
(BlueStore::ExtentMap::punch_hole(boost::intrusive_ptr<BlueStore::Collection>&,
unsigned long, unsigned long, boost::intrusive::list<BlueStore::OldExtent, boost::intrusive::member_hook<BlueStore::OldExtent, boost::intrusive::l\ ist_member_hook<>, &BlueStore::OldExtent::old_extent_item> >*)+0x451) [0x560cc32cbe21] 7: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::\ allocator<BlueStore::SharedBlob*> >*)+0x205) [0x560cc33483f5] 8: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0xc1) [0x560cc334d781] 9: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x7f) [0x560cc334f0af] 10: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x12a6) [0x560cc3334206] 11:
(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&,
std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> > &, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2f3) > [0\ x560cc3335123] 12: /usr/bin/ceph-osd(+0x5239a8) [0x560cc2e6e9a8] 13: (OSD::dispatch_context(PeeringCtx&, PG*, std::shared_ptr<OSDMap const>, ThreadPool::TPHandle*)+0x115) [0x560cc2edf495] 14: (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x2bc) [0x560cc2eea72c] 15: (ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x51) [0x560cc31359b1] 16: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xcd0) [0x560cc2f04b70] 17: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x2aa) [0x560cc34238aa] 18: /usr/bin/ceph-osd(+0xad8e64) [0x560cc3423e64] 19: /lib64/libc.so.6(+0x8b2ea) [0x7f543ce8b2ea] 20: /lib64/libc.so.6(+0x1103d0) [0x7f543cf103d0]
2026-09-16T21:16:07.426+0200 7f541fcaf640 -1 *** Caught signal (Aborted) ** in thread 7f541fcaf640 thread_name:tp_osd_tp
ceph version 19.2.3 (c92aebb279828e9c3c1f5d24613efca272649e62) squid (stable) 1: /lib64/libc.so.6(+0x3fc30) [0x7f543ce3fc30] 2: /lib64/libc.so.6(+0x8d02c) [0x7f543ce8d02c] 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x169) [0x560cc2d4c8b3] 6: /usr/bin/ceph-osd(+0x401a14) [0x560cc2d4ca14] 7: /usr/bin/ceph-osd(+0x3f2930) [0x560cc2d3d930] 8: (BlueStore::Blob::put_ref(BlueStore::Collection*, unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> > *)+0xaa) [0x560cc32bb2fa] 9:
(BlueStore::OldExtent::create(boost::intrusive_ptr<BlueStore::Collection>,
unsigned int, unsigned int, unsigned int, boost::intrusive_ptr<BlueStore::Blob>&)+0x11d) [0x560cc32bf47d] 10:
(BlueStore::ExtentMap::punch_hole(boost::intrusive_ptr<BlueStore::Collection>&,
unsigned long, unsigned long, boost::intrusive::list<BlueStore::OldExtent, boost::intrusive::member_hook<BlueStore::OldExtent, boost::intrusive::\ list_member_hook<>, &BlueStore::OldExtent::old_extent_item> >*)+0x451) [0x560cc32cbe21] 11: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std:\ :allocator<BlueStore::SharedBlob*> >*)+0x205) [0x560cc33483f5] 12: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0xc1) [0x560cc334d781] 13: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x7f) [0x560cc334f0af] 14: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x12a6) [0x560cc3334206] 15:
(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&,
std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> > &, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2f3) > [0\ x560cc3335123] 16: /usr/bin/ceph-osd(+0x5239a8) [0x560cc2e6e9a8] 17: (OSD::dispatch_context(PeeringCtx&, PG*, std::shared_ptr<OSDMap const>, ThreadPool::TPHandle*)+0x115) [0x560cc2edf495] 18: (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x2bc) [0x560cc2eea72c] 19: (ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x51) [0x560cc31359b1] 20: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xcd0) [0x560cc2f04b70] 21: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x2aa) [0x560cc34238aa] 22: /usr/bin/ceph-osd(+0xad8e64) [0x560cc3423e64] 23: /lib64/libc.so.6(+0x8b2ea) [0x7f543ce8b2ea] 24: /lib64/libc.so.6(+0x1103d0) [0x7f543cf103d0]
Thanks, Massimo
On Thu, Sep 17, 2026 at 4:44 AM Anthony D'Atri <anthony.datri@gmail.com
wrote:
> Start with what they log when they try to start. > > And what release you’re running. > >> On Sep 16, 2026, at 4:47 PM, Massimo Sgaravatto >> <ceph-users@ceph.io> > wrote: >> Dear all >> Today we had a problem in our ceph cluster: basically for some >> (still >> unknown) reasons, 5 OSDs (206, 233, 250, 257, 277) crashed and now they >> refuse to start. >> >> >> The main problem is that 1 pg is in down+remapped state because >> it is >> waiting for 3 of these OSDs [*] >> >> >> Doing a ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-xxx --op fsck, > I >> see [**] that the problem is with 2 objects >> (rbd_data.44.f397cc4c21bbfb.000000000000319b >> and rbd_data.44.f397cc4c21bbfb.0000000000000e66 >> A "--op repair" is not able to fix the problem >> >> >> I am not sure how to proceed now >> >> Is the only option removing the problematic object from each OSD, doing >> e.g.: >> >> >> ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-206 --pgid > 43.1b0 >
'{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":7151\
>> 31231,"max":0,"pool":43,"namespace":"","shardid":3}' remove >> >> ? >> >> Then the OSDs should hopefully restart, right ? >> >> Or are there better options ? >> >> Your help will be really appreciated ! >> >> >> Thanks, Massimo >> >> [*] >> ceph pg 43.1b0 query >> ... >> ... >> >> "blocked": "peering is blocked due to down osds", >> "down_osds_we_would_probe": [ >> 206, >> 233, >> 250 >> ], >> "peering_blocked_by": [ >> { >> "osd": 206, >> "current_lost_at": 0, >> "comment": "starting or marking this osd >> lost may let >> us proceed" >> }, >> { >> "osd": 233, >> "current_lost_at": 0, >> "comment": "starting or marking this osd >> lost may let >> us proceed" >> }, >> { >> "osd": 250, >> "current_lost_at": 0, >> "comment": "starting or marking this osd >> lost may let >> us proceed" >> } >> ] >> >> [**] >> >> [root@ceph-osd-17 ~]# ceph-bluestore-tool --path > /var/lib/ceph/osd/ceph-206 >> --op fsck >> 2026-09-16T21:26:23.609+0200 7fb5aa3ddac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-206) fsck error: >> 3#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ >> a9029a lextent at 0x6c000~4000 spans a shard boundary >> 2026-09-16T21:26:23.609+0200 7fb5aa3ddac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-206) fsck error: >> 3#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ >> a9029a lextent at 0x6d000 overlaps with the previous, which ends at > 0x70000 >> 2026-09-16T21:26:23.609+0200 7fb5aa3ddac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-206) fsck error: >> 3#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ >> a9029a blob Blob(0x5605e5760b60 spanning 1 >>
blob([!~4000,0xa211d395000~1000,!~3000,0x23b97906000~1000,0x23b97226000~3000]
>> llen=0xc000 csum+shared crc32c/0x1000/48\ >> ) use_tracker(0xc*0x1000 >> 0x[0,0,0,0,1000,0,0,0,1000,1000,1000,1000]) >> SharedBlob(0x5606c99ebfc0 sbid 0x37ae23b)) doesn't match expected ref_map >> use_tracker(0xc*0x\ >> 1000 0x[0,0,0,0,1000,0,0,0,1000,2000,2000,2000]) >> fsck status: remaining 3 error(s) and warning(s) >> >> root@ceph-osd-17 ~]# ceph-objectstore-tool --data-path >> /var/lib/ceph/osd/ceph-206 --op list | grep >> "rbd_data.44.f397cc4c21bbfb.000000000000319b" >>
["43.1b0s3",{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":4210196912,"max":0,"pool":43,"namespace":"","generation":44630682,"s\
>> hard_id":3,"max":0}] >> >> >> ---------------------------------------- >> ---------------------------------------- >> >> [root@ceph-osd-18 ~]# ceph-bluestore-tool --path > /var/lib/ceph/osd/ceph-233 >> --op fsck >> 2026-09-16T21:37:09.598+0200 7f8ee4c49ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-233) fsck error: >> 4#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ >> a9029a lextent at 0x6c000~4000 spans a shard boundary >> 2026-09-16T21:37:09.598+0200 7f8ee4c49ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-233) fsck error: >> 4#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ >> a9029a lextent at 0x6d000 overlaps with the previous, which ends at > 0x70000 >> 2026-09-16T21:37:09.598+0200 7f8ee4c49ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-233) fsck error: >> 4#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ >> a9029a blob Blob(0x5557c8bef2b0 spanning 1 >>
blob([!~4000,0x2ce88f4c000~1000,!~3000,0x2ce89560000~1000,0x2ce88a80000~3000]
>> llen=0xc000 csum+shared crc32c/0x1000/48\ >> ) use_tracker(0xc*0x1000 >> 0x[0,0,0,0,1000,0,0,0,1000,1000,1000,1000]) >> SharedBlob(0x5557fff8b1a0 sbid 0x323170b)) doesn't match expected ref_map >> use_tracker(0xc*0x\ >> 1000 0x[0,0,0,0,1000,0,0,0,1000,2000,2000,2000]) >> fsck status: remaining 3 error(s) and warning(s) >> >> [root@ceph-osd-18 ~]# ceph-objectstore-tool --data-path >> /var/lib/ceph/osd/ceph-233 --op list | grep >> "rbd_data.44.f397cc4c21bbfb.000000000000319b" >>
["43.1b0s4",{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":4210196912,"max":0,"pool":43,"namespace":"","generation":44630682,"s\
>> hard_id":4,"max":0}] >> [root@ceph-osd-18 ~]# >> >> >> ---------------------------------------- >> >> >> [root@ceph-osd-19 ~]# ceph-bluestore-tool --path > /var/lib/ceph/osd/ceph-250 >> --op fsck >> 2026-09-16T21:59:02.401+0200 7fa2e8795ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-250) fsck error: >> 0#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ >> a9029a lextent at 0x6c000~4000 spans a shard boundary >> 2026-09-16T21:59:02.401+0200 7fa2e8795ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-250) fsck error: >> 0#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ >> a9029a lextent at 0x6d000 overlaps with the previous, which ends at > 0x70000 >> 2026-09-16T21:59:02.401+0200 7fa2e8795ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-250) fsck error: >> 0#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ >> a9029a blob Blob(0x556a29cc4820 spanning 1 >>
blob([!~4000,0x464866d9000~1000,!~3000,0x4648731f000~1000,0x46485e09000~3000]
>> llen=0xc000 csum+shared crc32c/0x1000/48\ >> ) use_tracker(0xc*0x1000 >> 0x[0,0,0,0,1000,0,0,0,1000,1000,1000,1000]) >> SharedBlob(0x556a1fb84e40 sbid 0x98f249)) doesn't match expected ref_map >> use_tracker(0xc*0x1\ >> 000 0x[0,0,0,0,1000,0,0,0,1000,2000,2000,2000]) >> fsck status: remaining 3 error(s) and warning(s) >> >> [root@ceph-osd-19 ~]# ceph-objectstore-tool --data-path >> /var/lib/ceph/osd/ceph-250 --op list | grep >> "rbd_data.44.f397cc4c21bbfb.000000000000319b" >>
["43.1b0s0",{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":4210196912,"max":0,"pool":43,"namespace":"","generation":44630682,"s\
>> hard_id":0,"max":0}] >> ---------------------------------------- >> >> [root@ceph-osd-19 ~]# ceph-bluestore-tool --path > /var/lib/ceph/osd/ceph-257 >> --op fsck >> 2026-09-16T22:06:09.627+0200 7fded8db6ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-257) fsck error: >> 0#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ >> dbe3c7 lextent at 0x6a000~6000 spans a shard boundary >> 2026-09-16T22:06:09.627+0200 7fded8db6ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-257) fsck error: >> 0#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ >> dbe3c7 lextent at 0x6e000 overlaps with the previous, which ends at > 0x70000 >> 2026-09-16T22:06:09.627+0200 7fded8db6ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-257) fsck error: >> 0#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ >> dbe3c7 blob Blob(0x561e442541a0 spanning 2 >>
blob([!~4000,0x19c87d9b000~1000,0x46e6508e000~3000,0x46e64693000~2000]
>> >> llen=0xa000 csum+shared crc32c/0x1000/40) use_t\ >> racker(0xa*0x1000 0x[0,0,0,0,1000,1000,1000,1000,1000,1000]) >> SharedBlob(0x561e440f7620 sbid 0x3e6b92e)) doesn't match expected ref_map >> use_tracker(0xa*0x1000 0x[\ >> 0,0,0,0,1000,1000,1000,1000,2000,2000]) >> fsck status: remaining 3 error(s) and warning(s) >> [root@ceph-osd-19 ~]# >> >> [root@ceph-osd-19 ~]# ceph-objectstore-tool --data-path >> /var/lib/ceph/osd/ceph-257 --op list | grep >> "rbd_data.44.f397cc4c21bbfb.0000000000000e66" >>
["43.70s0",{"oid":"rbd_data.44.f397cc4c21bbfb.0000000000000e66","key":"","snapid":-2,"hash":2802327152,"max":0,"pool":43,"namespace":"","generation":47965127,"sh\
>> ard_id":0,"max":0}] >> >> >> >> ---------------------------------------- >> >> ---------------------------------------- >> >> [root@ceph-osd-20 ~]# ceph-bluestore-tool --path > /var/lib/ceph/osd/ceph-277 >> --op fsck >> 2026-09-16T22:08:04.530+0200 7f35041c8ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-277) fsck error: >> 3#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ >> dbe3c7 lextent at 0x6a000~6000 spans a shard boundary >> 2026-09-16T22:08:04.530+0200 7f35041c8ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-277) fsck error: >> 3#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ >> dbe3c7 lextent at 0x6e000 overlaps with the previous, which ends at > 0x70000 >> 2026-09-16T22:08:04.530+0200 7f35041c8ac0 -1 >> bluestore(/var/lib/ceph/osd/ceph-277) fsck error: >> 3#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ >> dbe3c7 blob Blob(0x55ead1a808f0 spanning 2 >>
blob([!~3000,0x1754e9e3000~1000,0x1754eb48000~3000,0x1754e809000~2000]
>> >> llen=0x9000 csum crc32c/0x1000/36) use_tracker(\ >> 0x9*0x1000 0x[0,0,0,1000,1000,1000,1000,1000,1000]) (shared_blob=NULL)) >> doesn't match expected ref_map use_tracker(0x9*0x1000 >> 0x[0,0,0,1000,1000,1000,1000,2000,2\ >> 000]) >> fsck status: remaining 3 error(s) and warning(s) >> [root@ceph-osd-20 ~]# >> >> >> root@ceph-osd-20 ~]# ceph-objectstore-tool --data-path >> /var/lib/ceph/osd/ceph-277 --op list | grep >> "rbd_data.44.f397cc4c21bbfb.0000000000000e66" >>
["43.70s3",{"oid":"rbd_data.44.f397cc4c21bbfb.0000000000000e66","key":"","snapid":-2,"hash":2802327152,"max":0,"pool":43,"namespace":"","generation":47965127,"sh\
>> ard_id":3,"max":0}] >> _______________________________________________ >> ceph-users mailing list -- ceph-users@ceph.io >> To unsubscribe send an email to ceph-users-leave@ceph.io _______________________________________________ ceph-users mailing list -- ceph-users@ceph.io To unsubscribe send an email to ceph-users-leave@ceph.io
ceph-users mailing list -- ceph-users@ceph.io To unsubscribe send an email to ceph-users-leave@ceph.io
ceph-users mailing list -- ceph-users@ceph.io To unsubscribe send an email to ceph-users-leave@ceph.io
-- Enrico Bocchi CERN European Laboratory for Particle Physics IT - Storage & Data Management - General Storage Services Mailbox: G20500 - Office: 31-2-010 1211 Genève 23 Switzerland