Hi community We have a new Ceph cluster deployment with 100 nodes. When we are draining an OSD host from the cluster, we see a small amount of PGs that cannot make any progress to the end. From the logs and metrics, it seems like the recovery progress is stuck (0 recovery ops for several days). Would like to get some ideas on this. Re-peering and OSD restart do resolve to mitigate the issue but we want to get to the root cause of it as draining and recovery happen frequently. I have put some debugging information below. Any help is appreciated, thanks! ceph -s pgs: 4210926/7380034104 objects misplaced (0.057%) 41198 active+clean 71 active+remapped+backfilling 12 active+recovering One of the stuck PG: 6.38f1 active+remapped+backfilling [313,643,727] 313 [313,643,717] 313 PG query result: ceph pg 6.38f1 query { "snap_trimq": "[]", "snap_trimq_len": 0, "state": "active+remapped+backfilling", "epoch": 246856, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "backfill_targets": [ "727" ], "acting_recovery_backfill": [ "313", "643", "717", "727" ], "info": { "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 38833, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 243987, "last_interval_started": 243986, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "212333'38916", "reported_seq": 413425, "reported_epoch": 246856, "state": "active+remapped+backfilling", "last_fresh": "2024-02-05T21:14:40.838785+0000", "last_change": "2024-02-03T22:33:43.052272+0000", "last_active": "2024-02-05T21:14:40.838785+0000", "last_peered": "2024-02-05T21:14:40.838785+0000", "last_clean": "2024-02-03T04:26:35.168232+0000", "last_became_active": "2024-02-03T22:31:16.037823+0000", "last_became_peered": "2024-02-03T22:31:16.037823+0000", "last_unstale": "2024-02-05T21:14:40.838785+0000", "last_undegraded": "2024-02-05T21:14:40.838785+0000", "last_fullsized": "2024-02-05T21:14:40.838785+0000", "mapping_epoch": 243986, "log_start": "80608'37589", "ondisk_log_start": "80608'37589", "created": 3726, "last_epoch_clean": 220174, "parent": "0.0", "parent_split_bits": 14, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "objects_scrubbed": 17743, "log_size": 1327, "log_dups_size": 3000, "ondisk_log_size": 1327, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 29, "scrub_schedule": "queued for deep scrub", "scrub_duration": 28.699844475999999, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 67167022618, "num_objects": 17743, "num_object_clones": 0, "num_object_copies": 53229, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 12401, "num_objects_unfound": 0, "num_objects_dirty": 17743, "num_whiteouts": 0, "num_read": 16640, "num_read_kb": 30923655, "num_write": 55094, "num_write_kb": 71824558, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 40746, "num_bytes_recovered": 154245233300, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [ "313", "643", "717" ], "object_location_counts": [ { "shards": "313,643,717", "objects": 17743 } ], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 243987, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "643", "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 38833, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 243987, "last_interval_started": 243986, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "212331'38915", "reported_seq": 362849, "reported_epoch": 212333, "state": "active+clean+scrubbing+deep", "last_fresh": "2024-01-28T07:43:40.515729+0000", "last_change": "2024-01-28T06:20:14.969313+0000", "last_active": "2024-01-28T07:43:40.515729+0000", "last_peered": "2024-01-28T07:43:40.515729+0000", "last_clean": "2024-01-28T07:43:40.515729+0000", "last_became_active": "2024-01-23T10:01:07.710478+0000", "last_became_peered": "2024-01-23T10:01:07.710478+0000", "last_unstale": "2024-01-28T07:43:40.515729+0000", "last_undegraded": "2024-01-28T07:43:40.515729+0000", "last_fullsized": "2024-01-28T07:43:40.515729+0000", "mapping_epoch": 243986, "log_start": "80608'37589", "ondisk_log_start": "80608'37589", "created": 3726, "last_epoch_clean": 123203, "parent": "0.0", "parent_split_bits": 14, "last_scrub": "80609'38697", "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", "last_deep_scrub": "80609'38697", "last_deep_scrub_stamp": "2024-01-07T08:20:59.246940+0000", "last_clean_scrub_stamp": "2024-01-09T20:27:40.795531+0000", "objects_scrubbed": 17724, "log_size": 1326, "log_dups_size": 3000, "ondisk_log_size": 1326, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 29, "scrub_schedule": "deep scrubbing for 5005s", "scrub_duration": 28.446739284, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 67167022618, "num_objects": 17743, "num_object_clones": 0, "num_object_copies": 53229, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 17743, "num_whiteouts": 0, "num_read": 16640, "num_read_kb": 30923655, "num_write": 55094, "num_write_kb": 71824558, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 35403, "num_bytes_recovered": 134021819371, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 243987, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "717", "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 243987, "last_interval_started": 243986, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "212331'38915", "reported_seq": 362849, "reported_epoch": 212333, "state": "active+clean+scrubbing+deep", "last_fresh": "2024-01-28T07:43:40.515729+0000", "last_change": "2024-01-28T06:20:14.969313+0000", "last_active": "2024-01-28T07:43:40.515729+0000", "last_peered": "2024-01-28T07:43:40.515729+0000", "last_clean": "2024-01-28T07:43:40.515729+0000", "last_became_active": "2024-01-23T10:01:07.710478+0000", "last_became_peered": "2024-01-23T10:01:07.710478+0000", "last_unstale": "2024-01-28T07:43:40.515729+0000", "last_undegraded": "2024-01-28T07:43:40.515729+0000", "last_fullsized": "2024-01-28T07:43:40.515729+0000", "mapping_epoch": 243986, "log_start": "80608'37589", "ondisk_log_start": "80608'37589", "created": 3726, "last_epoch_clean": 123203, "parent": "0.0", "parent_split_bits": 14, "last_scrub": "80609'38697", "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", "last_deep_scrub": "80609'38697", "last_deep_scrub_stamp": "2024-01-07T08:20:59.246940+0000", "last_clean_scrub_stamp": "2024-01-09T20:27:40.795531+0000", "objects_scrubbed": 17724, "log_size": 1326, "log_dups_size": 3000, "ondisk_log_size": 1326, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 29, "scrub_schedule": "deep scrubbing for 5005s", "scrub_duration": 28.446739284, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 67167022618, "num_objects": 17743, "num_object_clones": 0, "num_object_copies": 53229, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 17743, "num_whiteouts": 0, "num_read": 16640, "num_read_kb": 30923655, "num_write": 55094, "num_write_kb": 71824558, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 35403, "num_bytes_recovered": 134021819371, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 243987, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "726", "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 238348, "last_interval_started": 238347, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 11.834917336 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 243986, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 4194304, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 17743, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 238320, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "727", "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 0, "last_backfill": "6:8f1d3293:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.V-JlvDFjbQNgn8GtBLN2LePecv5fLKf_3:head", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 243987, "last_interval_started": 243986, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 243986, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 20227608233, "num_objects": 5342, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 12401, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 5342, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 243987, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started/Primary/Active", "enter_time": "2024-02-03T22:31:15.023323+0000", "might_have_unfound": [], "recovery_progress": { "backfill_targets": [ "727" ], "waiting_on_backfill": [], "last_backfill_started": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", "backfill_info": { "begin": "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", "end": "6:8f1d3549:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.PTxEYjlIXk2vkWpRgKUOu4ygweU-Jqy_5:head", "objects": [ { "object": "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", "version": "80588'26367" }, { "object": "6:8f1d32b0:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_18-71435:head", "version": "80588'26368" }, { "object": "6:8f1d32c2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.o_s0NGR7mrH-4JKpvYS6Ip14hbm4w5o_5:head", "version": "80588'26369" }, { "object": "6:8f1d32c3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WV1Ja5V21BKfmI5e1W90a7iQXdcTnSG_6:head", "version": "80588'26370" }, { "object": "6:8f1d32c4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.96tkpnCzKmtnPTLewpai2cXSr_z_QcR_1:head", "version": "80588'26371" }, { "object": "6:8f1d331d:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.InACTSlwV7irTe0g9cLnadT8EbQr4Cl_5:head", "version": "80588'26372" }, { "object": "6:8f1d3326:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_233-155987:head", "version": "80588'26373" }, { "object": "6:8f1d3332:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.5aipEOj0vV3h4jdSf6tkAUn64yJmvoH_1:head", "version": "80588'26374" }, { "object": "6:8f1d3343:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QTzZJhDV6JLC5wJ5BDxHvK4sfp3Hr6o_6:head", "version": "80588'26375" }, { "object": "6:8f1d334b:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QgMCmHWV87HlAjqkfvbO9X5OUZrgIPc_2:head", "version": "80588'26376" }, { "object": "6:8f1d3355:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.H4j6DWgQa0jsrl-xzgugsW22GHCOgGL_7:head", "version": "80588'26377" }, { "object": "6:8f1d3379:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.25g7c0w-VnGqX-z_m53YmnoWUO0R56N_5:head", "version": "80588'26378" }, { "object": "6:8f1d337c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WMmV4BoLMm4q6BztoZ1uPc4boiJ-zhW_1:head", "version": "80588'26379" }, { "object": "6:8f1d3385:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_184-108817:head", "version": "80588'26380" }, { "object": "6:8f1d3389:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sGNiGV9K-TGyTYRqWSOcSIXkBBOT93X_3:head", "version": "80588'26381" }, { "object": "6:8f1d339c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.NkMm_dhUxRyxXhBDYqGCuOaSvwNvSuJ_3:head", "version": "80588'26382" }, { "object": "6:8f1d33a3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.BV__z8jIWqcjMsTf40MvaARU1dVtCv2_4:head", "version": "80588'26383" }, { "object": "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.XJLSyRSnrDupnOjWzHKc4wykPJjqR5g_1:head", "version": "80588'26384" }, { "object": "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.oYSnMakOIpFmfLdym2oCx-fLBDCLDmr_4:head", "version": "80588'26385" }, { "object": "6:8f1d33b4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_135-2779:head", "version": "80588'26386" }, { "object": "6:8f1d33b6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sEHqthyEGg3FQP4-vdL9zkhSJmQEaIc_2:head", "version": "80588'26387" }, { "object": "6:8f1d33ec:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.wL36lNEfHGE6B4V-YT6fMuAI-n2uV30_1:head", "version": "80588'26388" }, { "object": "6:8f1d33f3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.j1UloV7DpRwG0-4AKULi3CXOTC0CrdU_4:head", "version": "80588'26389" }, { "object": "6:8f1d33f5:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.q3ExUB7Mp-LggBIsRBBKyIob574rNpG_3:head", "version": "80588'26390" }, { "object": "6:8f1d33f9:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_72-13403:head", "version": "80588'26391" }, { "object": "6:8f1d33fd:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_157-172390:head", "version": "80588'26392" }, { "object": "6:8f1d3403:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Wn4ed97PvQp_oDnk_Y4xlZMke0pQ4z1_3:head", "version": "80588'26393" }, { "object": "6:8f1d3405:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.uFg6eFSD3BCEqzJLCZlw546nDqqg80U_4:head", "version": "80588'26394" }, { "object": "6:8f1d3436:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xPwN1sJsNFAg2Nr_Vnu4ItFgHFyu_Nx_7:head", "version": "80588'26395" }, { "object": "6:8f1d3438:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Qgd9EXWAraBWnYcZ3OhFvepsT8r0Ksi_1:head", "version": "80588'26396" }, { "object": "6:8f1d3464:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.1KnQBzx5u-CkXROQVUzUPfJiuTq52zH_3:head", "version": "80588'26397" }, { "object": "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Aw2QRjM0ZXkf20MtQlhyq31hwVMEWwe_1:head", "version": "80588'26398" }, { "object": "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xprsKnjtYu1ITOsgDtkPup9Nk6xzGTN_6:head", "version": "80588'26399" }, { "object": "6:8f1d3475:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.jo4-jEQV1kB9kHewUYdbU1UUNX60EUm_1:head", "version": "80588'26400" }, { "object": "6:8f1d3488:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.f9EE8lnu2FzW1_vGo9J41KY-iFpI9-f_1:head", "version": "80588'26401" }, { "object": "6:8f1d3491:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_11-138415:head", "version": "80588'26402" }, { "object": "6:8f1d34ac:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_33-123891:head", "version": "80588'26403" }, { "object": "6:8f1d34ad:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.x3tIcVCAeRnOJmmHhJ2hxCjsUZ3GBsK_3:head", "version": "80588'26404" }, { "object": "6:8f1d34bc:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.9yQELiJhCNs5Z-foJHH3vEfQ_1TZdBF_1:head", "version": "80588'26405" }, { "object": "6:8f1d34d1:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Me9Xl2_j9z2_yE1laHWpHdJ2fXvwZDZ_1:head", "version": "80588'26406" }, { "object": "6:8f1d34fa:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Y5VD6lU4UqjXEh5Yb9GVwusKpsqGzuG_4:head", "version": "80588'26407" }, { "object": "6:8f1d3503:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_58-178651:head", "version": "80588'26408" }, { "object": "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.FFyoywHasB0i44Y9ZtCi_YgtYhm8cP6_6:head", "version": "80588'26409" }, { "object": "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.lHmY-Z-Q3Ewx7kHwCoLFoFU70Xtrgzu_1:head", "version": "80588'26410" }, { "object": "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.ETPvUyLCwz5Bfmh22s_odAwEIMCC_Vn_2:head", "version": "80588'26411" }, { "object": "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.EdYbzDvA-7HpfrM0DMfHRgalz8ZiLhJ_1:head", "version": "80588'26412" }, { "object": "6:8f1d3529:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1186045.1_164-1500436:head", "version": "80588'26413" } ] }, "peer_backfill_info": [ "727", { "begin": "MAX", "end": "MAX", "objects": [] } ], "backfills_in_flight": [ "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" ], "recovering": [ "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" ], "pg_backend": { "pull_from_peer": [], "pushing": [ { "pushing": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", "pushing_to": [ { "pushing_to": "727", "push_info": { "recovery_progress": { "first?": 0, "data_complete?": 1, "data_recovered_to": 4194304, "omap_complete?": 1, "omap_recovered_to": "" }, "recovery_info": { "object": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", "at_version": "80588'26366", "size": "4194304", "object_info": { "oid": { "oid": "d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605", "key": "", "snapid": -2, "hash": 4182554865, "max": 0, "pool": 6, "namespace": "" }, "version": "80588'26366", "prior_version": "6896'10987", "last_reqid": "osd.313.0:4380013", "user_version": 10987, "size": 4194304, "mtime": "2023-12-19T07:17:04.766167+0000", "local_mtime": "2023-12-19T07:17:04.799712+0000", "lost": 0, "flags": [ "dirty", "data_digest", "omap_digest" ], "truncate_seq": 0, "truncate_size": 0, "data_digest": "0x9cdfa22f", "omap_digest": "0xffffffff", "expected_object_size": 0, "expected_write_size": 0, "alloc_hint_flags": 0, "manifest": { "type": 0 }, "watchers": {} }, "snapset": { "seq": 0, "clones": [] }, "copy_subset": "[0~4194304]", "clone_subset": "{}", "object_exist": "0" } } } ] } ] } } }, { "name": "Started", "enter_time": "2024-02-03T22:31:13.882074+0000" } ], "scrubber": { "active": false, "must_scrub": false, "must_deep_scrub": false, "must_repair": false, "need_auto": false, "scrub_reg_stamp": "2024-01-30T15:39:19.044093+0000", "schedule": "queued for deep scrub" }, "agent_state": {} }