Dear all Today we had a problem in our ceph cluster: basically for some (still unknown) reasons, 5 OSDs (206, 233, 250, 257, 277) crashed and now they refuse to start. The main problem is that 1 pg is in down+remapped state because it is waiting for 3 of these OSDs [*] Doing a ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-xxx --op fsck, I see [**] that the problem is with 2 objects (rbd_data.44.f397cc4c21bbfb.000000000000319b and rbd_data.44.f397cc4c21bbfb.0000000000000e66 A "--op repair" is not able to fix the problem I am not sure how to proceed now Is the only option removing the problematic object from each OSD, doing e.g.: ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-206 --pgid 43.1b0 '{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":7151\ 31231,"max":0,"pool":43,"namespace":"","shardid":3}' remove ? Then the OSDs should hopefully restart, right ? Or are there better options ? Your help will be really appreciated ! Thanks, Massimo [*] ceph pg 43.1b0 query ... ... "blocked": "peering is blocked due to down osds", "down_osds_we_would_probe": [ 206, 233, 250 ], "peering_blocked_by": [ { "osd": 206, "current_lost_at": 0, "comment": "starting or marking this osd lost may let us proceed" }, { "osd": 233, "current_lost_at": 0, "comment": "starting or marking this osd lost may let us proceed" }, { "osd": 250, "current_lost_at": 0, "comment": "starting or marking this osd lost may let us proceed" } ] [**] [root@ceph-osd-17 ~]# ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-206 --op fsck 2026-09-16T21:26:23.609+0200 7fb5aa3ddac0 -1 bluestore(/var/lib/ceph/osd/ceph-206) fsck error: 3#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ a9029a lextent at 0x6c000~4000 spans a shard boundary 2026-09-16T21:26:23.609+0200 7fb5aa3ddac0 -1 bluestore(/var/lib/ceph/osd/ceph-206) fsck error: 3#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ a9029a lextent at 0x6d000 overlaps with the previous, which ends at 0x70000 2026-09-16T21:26:23.609+0200 7fb5aa3ddac0 -1 bluestore(/var/lib/ceph/osd/ceph-206) fsck error: 3#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ a9029a blob Blob(0x5605e5760b60 spanning 1 blob([!~4000,0xa211d395000~1000,!~3000,0x23b97906000~1000,0x23b97226000~3000] llen=0xc000 csum+shared crc32c/0x1000/48\ ) use_tracker(0xc*0x1000 0x[0,0,0,0,1000,0,0,0,1000,1000,1000,1000]) SharedBlob(0x5606c99ebfc0 sbid 0x37ae23b)) doesn't match expected ref_map use_tracker(0xc*0x\ 1000 0x[0,0,0,0,1000,0,0,0,1000,2000,2000,2000]) fsck status: remaining 3 error(s) and warning(s) root@ceph-osd-17 ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-206 --op list | grep "rbd_data.44.f397cc4c21bbfb.000000000000319b" ["43.1b0s3",{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":4210196912,"max":0,"pool":43,"namespace":"","generation":44630682,"s\ hard_id":3,"max":0}] ---------------------------------------- ---------------------------------------- [root@ceph-osd-18 ~]# ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-233 --op fsck 2026-09-16T21:37:09.598+0200 7f8ee4c49ac0 -1 bluestore(/var/lib/ceph/osd/ceph-233) fsck error: 4#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ a9029a lextent at 0x6c000~4000 spans a shard boundary 2026-09-16T21:37:09.598+0200 7f8ee4c49ac0 -1 bluestore(/var/lib/ceph/osd/ceph-233) fsck error: 4#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ a9029a lextent at 0x6d000 overlaps with the previous, which ends at 0x70000 2026-09-16T21:37:09.598+0200 7f8ee4c49ac0 -1 bluestore(/var/lib/ceph/osd/ceph-233) fsck error: 4#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ a9029a blob Blob(0x5557c8bef2b0 spanning 1 blob([!~4000,0x2ce88f4c000~1000,!~3000,0x2ce89560000~1000,0x2ce88a80000~3000] llen=0xc000 csum+shared crc32c/0x1000/48\ ) use_tracker(0xc*0x1000 0x[0,0,0,0,1000,0,0,0,1000,1000,1000,1000]) SharedBlob(0x5557fff8b1a0 sbid 0x323170b)) doesn't match expected ref_map use_tracker(0xc*0x\ 1000 0x[0,0,0,0,1000,0,0,0,1000,2000,2000,2000]) fsck status: remaining 3 error(s) and warning(s) [root@ceph-osd-18 ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-233 --op list | grep "rbd_data.44.f397cc4c21bbfb.000000000000319b" ["43.1b0s4",{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":4210196912,"max":0,"pool":43,"namespace":"","generation":44630682,"s\ hard_id":4,"max":0}] [root@ceph-osd-18 ~]# ---------------------------------------- [root@ceph-osd-19 ~]# ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-250 --op fsck 2026-09-16T21:59:02.401+0200 7fa2e8795ac0 -1 bluestore(/var/lib/ceph/osd/ceph-250) fsck error: 0#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ a9029a lextent at 0x6c000~4000 spans a shard boundary 2026-09-16T21:59:02.401+0200 7fa2e8795ac0 -1 bluestore(/var/lib/ceph/osd/ceph-250) fsck error: 0#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ a9029a lextent at 0x6d000 overlaps with the previous, which ends at 0x70000 2026-09-16T21:59:02.401+0200 7fa2e8795ac0 -1 bluestore(/var/lib/ceph/osd/ceph-250) fsck error: 0#43:0d814f5f:::rbd_data.44.f397cc4c21bbfb.000000000000319b:head#2\ a9029a blob Blob(0x556a29cc4820 spanning 1 blob([!~4000,0x464866d9000~1000,!~3000,0x4648731f000~1000,0x46485e09000~3000] llen=0xc000 csum+shared crc32c/0x1000/48\ ) use_tracker(0xc*0x1000 0x[0,0,0,0,1000,0,0,0,1000,1000,1000,1000]) SharedBlob(0x556a1fb84e40 sbid 0x98f249)) doesn't match expected ref_map use_tracker(0xc*0x1\ 000 0x[0,0,0,0,1000,0,0,0,1000,2000,2000,2000]) fsck status: remaining 3 error(s) and warning(s) [root@ceph-osd-19 ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-250 --op list | grep "rbd_data.44.f397cc4c21bbfb.000000000000319b" ["43.1b0s0",{"oid":"rbd_data.44.f397cc4c21bbfb.000000000000319b","key":"","snapid":-2,"hash":4210196912,"max":0,"pool":43,"namespace":"","generation":44630682,"s\ hard_id":0,"max":0}] ---------------------------------------- [root@ceph-osd-19 ~]# ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-257 --op fsck 2026-09-16T22:06:09.627+0200 7fded8db6ac0 -1 bluestore(/var/lib/ceph/osd/ceph-257) fsck error: 0#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ dbe3c7 lextent at 0x6a000~6000 spans a shard boundary 2026-09-16T22:06:09.627+0200 7fded8db6ac0 -1 bluestore(/var/lib/ceph/osd/ceph-257) fsck error: 0#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ dbe3c7 lextent at 0x6e000 overlaps with the previous, which ends at 0x70000 2026-09-16T22:06:09.627+0200 7fded8db6ac0 -1 bluestore(/var/lib/ceph/osd/ceph-257) fsck error: 0#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ dbe3c7 blob Blob(0x561e442541a0 spanning 2 blob([!~4000,0x19c87d9b000~1000,0x46e6508e000~3000,0x46e64693000~2000] llen=0xa000 csum+shared crc32c/0x1000/40) use_t\ racker(0xa*0x1000 0x[0,0,0,0,1000,1000,1000,1000,1000,1000]) SharedBlob(0x561e440f7620 sbid 0x3e6b92e)) doesn't match expected ref_map use_tracker(0xa*0x1000 0x[\ 0,0,0,0,1000,1000,1000,1000,2000,2000]) fsck status: remaining 3 error(s) and warning(s) [root@ceph-osd-19 ~]# [root@ceph-osd-19 ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-257 --op list | grep "rbd_data.44.f397cc4c21bbfb.0000000000000e66" ["43.70s0",{"oid":"rbd_data.44.f397cc4c21bbfb.0000000000000e66","key":"","snapid":-2,"hash":2802327152,"max":0,"pool":43,"namespace":"","generation":47965127,"sh\ ard_id":0,"max":0}] ---------------------------------------- ---------------------------------------- [root@ceph-osd-20 ~]# ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-277 --op fsck 2026-09-16T22:08:04.530+0200 7f35041c8ac0 -1 bluestore(/var/lib/ceph/osd/ceph-277) fsck error: 3#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ dbe3c7 lextent at 0x6a000~6000 spans a shard boundary 2026-09-16T22:08:04.530+0200 7f35041c8ac0 -1 bluestore(/var/lib/ceph/osd/ceph-277) fsck error: 3#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ dbe3c7 lextent at 0x6e000 overlaps with the previous, which ends at 0x70000 2026-09-16T22:08:04.530+0200 7f35041c8ac0 -1 bluestore(/var/lib/ceph/osd/ceph-277) fsck error: 3#43:0e7810e5:::rbd_data.44.f397cc4c21bbfb.0000000000000e66:head#2\ dbe3c7 blob Blob(0x55ead1a808f0 spanning 2 blob([!~3000,0x1754e9e3000~1000,0x1754eb48000~3000,0x1754e809000~2000] llen=0x9000 csum crc32c/0x1000/36) use_tracker(\ 0x9*0x1000 0x[0,0,0,1000,1000,1000,1000,1000,1000]) (shared_blob=NULL)) doesn't match expected ref_map use_tracker(0x9*0x1000 0x[0,0,0,1000,1000,1000,1000,2000,2\ 000]) fsck status: remaining 3 error(s) and warning(s) [root@ceph-osd-20 ~]# root@ceph-osd-20 ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-277 --op list | grep "rbd_data.44.f397cc4c21bbfb.0000000000000e66" ["43.70s3",{"oid":"rbd_data.44.f397cc4c21bbfb.0000000000000e66","key":"","snapid":-2,"hash":2802327152,"max":0,"pool":43,"namespace":"","generation":47965127,"sh\ ard_id":3,"max":0}]