{ "snap_trimq": "[]", "snap_trimq_len": 0, "state": "active+remapped+backfill_wait", "epoch": 2261586, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "backfill_targets": [ "442(3)", "458(5)", "506(4)" ], "acting_recovery_backfill": [ "269(2)", "273(3)", "275(1)", "298(0)", "409(5)", "440(4)", "442(3)", "458(5)", "506(4)" ], "info": { "pgid": "11.fffs0", "last_update": "2261478'3114248", "last_complete": "2261478'3114248", "log_tail": "2257134'3113294", "last_user_version": 3114248, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2261584, "last_interval_started": 2261583, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2261478'3114248", "reported_seq": 27298090, "reported_epoch": 2261586, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T21:43:54.465643+0000", "last_change": "2024-03-23T21:41:58.944447+0000", "last_active": "2024-03-23T21:43:54.465643+0000", "last_peered": "2024-03-23T21:43:54.465643+0000", "last_clean": "2024-03-19T12:59:01.132410+0000", "last_became_active": "2024-03-23T21:41:58.933991+0000", "last_became_peered": "2024-03-23T21:41:58.933991+0000", "last_unstale": "2024-03-23T21:43:54.465643+0000", "last_undegraded": "2024-03-23T21:43:54.465643+0000", "last_fullsized": "2024-03-23T21:43:54.465643+0000", "mapping_epoch": 2261583, "log_start": "2257134'3113294", "ondisk_log_start": "2257134'3113294", "created": 1755101, "last_epoch_clean": 2248989, "parent": "0.0", "parent_split_bits": 12, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "objects_scrubbed": 45979, "log_size": 954, "log_dups_size": 3000, "ondisk_log_size": 954, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 6, "scrub_schedule": "queued for scrub", "scrub_duration": 5.3869062870000004, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 192684589056, "num_objects": 46279, "num_object_clones": 9, "num_object_copies": 277674, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 137058, "num_objects_unfound": 0, "num_objects_dirty": 46279, "num_whiteouts": 0, "num_read": 1766106, "num_read_kb": 177513600, "num_write": 187542, "num_write_kb": 124853392, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 931582, "num_bytes_recovered": 3891444797440, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [ "298(0)", "269(2)", "273(3)", "275(1)", "409(5)", "440(4)" ], "object_location_counts": [ { "shards": "269(2),273(3),275(1),298(0),409(5),440(4)", "objects": 46279 } ], "blocked_by": [], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261584, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "237(4)", "pgid": "11.fffs4", "last_update": "2250634'3112256", "last_complete": "2250634'3112256", "log_tail": "2244781'3111291", "last_user_version": 3112256, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2260405, "last_interval_started": 2260401, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261583, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 46279, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249821, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "269(2)", "pgid": "11.fffs2", "last_update": "2261478'3114248", "last_complete": "2261478'3114248", "log_tail": "2257134'3113294", "last_user_version": 3114248, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2261584, "last_interval_started": 2261583, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2261478'3114248", "reported_seq": 27298075, "reported_epoch": 2261583, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T19:39:12.241953+0000", "last_change": "2024-03-23T13:38:04.290356+0000", "last_active": "2024-03-23T19:39:12.241953+0000", "last_peered": "2024-03-23T19:39:12.241953+0000", "last_clean": "2024-03-19T12:59:01.132410+0000", "last_became_active": "2024-03-23T13:20:41.305412+0000", "last_became_peered": "2024-03-23T13:20:41.305412+0000", "last_unstale": "2024-03-23T19:39:12.241953+0000", "last_undegraded": "2024-03-23T19:39:12.241953+0000", "last_fullsized": "2024-03-23T19:39:12.241953+0000", "mapping_epoch": 2261583, "log_start": "2257134'3113294", "ondisk_log_start": "2257134'3113294", "created": 1755101, "last_epoch_clean": 2248989, "parent": "0.0", "parent_split_bits": 12, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "objects_scrubbed": 45979, "log_size": 954, "log_dups_size": 3000, "ondisk_log_size": 954, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 6, "scrub_schedule": "queued for scrub", "scrub_duration": 5.3869062870000004, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 192684589056, "num_objects": 46279, "num_object_clones": 9, "num_object_copies": 277674, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 138381, "num_objects_unfound": 0, "num_objects_dirty": 46279, "num_whiteouts": 0, "num_read": 1766106, "num_read_kb": 177513600, "num_write": 187542, "num_write_kb": 124853392, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 931582, "num_bytes_recovered": 3891444797440, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [ "298(0)", "269(2)", "273(3)", "275(1)", "409(5)", "440(4)" ], "object_location_counts": [ { "shards": "269(2),273(3),275(1),298(0),409(5),440(4)", "objects": 46279 } ], "blocked_by": [], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261584, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "273(3)", "pgid": "11.fffs3", "last_update": "2261478'3114248", "last_complete": "2261478'3114248", "log_tail": "2257134'3113294", "last_user_version": 3114248, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2261584, "last_interval_started": 2261583, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2261478'3114248", "reported_seq": 27296956, "reported_epoch": 2261478, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T19:39:12.241953+0000", "last_change": "2024-03-23T13:38:04.290356+0000", "last_active": "2024-03-23T19:39:12.241953+0000", "last_peered": "2024-03-23T19:39:12.241953+0000", "last_clean": "2024-03-19T12:59:01.132410+0000", "last_became_active": "2024-03-23T13:20:41.305412+0000", "last_became_peered": "2024-03-23T13:20:41.305412+0000", "last_unstale": "2024-03-23T19:39:12.241953+0000", "last_undegraded": "2024-03-23T19:39:12.241953+0000", "last_fullsized": "2024-03-23T19:39:12.241953+0000", "mapping_epoch": 2261583, "log_start": "2257134'3113294", "ondisk_log_start": "2257134'3113294", "created": 1755101, "last_epoch_clean": 2248989, "parent": "0.0", "parent_split_bits": 12, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "objects_scrubbed": 45979, "log_size": 954, "log_dups_size": 3000, "ondisk_log_size": 954, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 6, "scrub_schedule": "queued for scrub", "scrub_duration": 5.3869062870000004, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 192684589056, "num_objects": 46279, "num_object_clones": 9, "num_object_copies": 277674, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 138381, "num_objects_unfound": 0, "num_objects_dirty": 46279, "num_whiteouts": 0, "num_read": 1766106, "num_read_kb": 177513600, "num_write": 187542, "num_write_kb": 124853392, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 931582, "num_bytes_recovered": 3891444797440, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [ "298(0)", "269(2)", "273(3)", "275(1)", "409(5)", "440(4)" ], "object_location_counts": [ { "shards": "269(2),273(3),275(1),298(0),409(5),440(4)", "objects": 46279 } ], "blocked_by": [], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261584, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "275(1)", "pgid": "11.fffs1", "last_update": "2261478'3114248", "last_complete": "2261478'3114248", "log_tail": "2257134'3113294", "last_user_version": 3114248, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2261584, "last_interval_started": 2261583, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2261478'3114248", "reported_seq": 27298735, "reported_epoch": 2261582, "state": "remapped+peering", "last_fresh": "2024-03-23T21:41:55.558794+0000", "last_change": "2024-03-23T21:41:55.558794+0000", "last_active": "2024-03-23T21:41:55.558652+0000", "last_peered": "2024-03-23T21:41:55.519091+0000", "last_clean": "2024-03-19T12:59:01.132410+0000", "last_became_active": "2024-03-23T21:41:19.687245+0000", "last_became_peered": "2024-03-23T21:41:19.687245+0000", "last_unstale": "2024-03-23T21:41:55.558794+0000", "last_undegraded": "2024-03-23T21:41:55.558794+0000", "last_fullsized": "2024-03-23T21:41:55.558794+0000", "mapping_epoch": 2261583, "log_start": "2257134'3113294", "ondisk_log_start": "2257134'3113294", "created": 1755101, "last_epoch_clean": 2248989, "parent": "0.0", "parent_split_bits": 12, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "objects_scrubbed": 45979, "log_size": 954, "log_dups_size": 3000, "ondisk_log_size": 954, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 6, "scrub_schedule": "queued for scrub", "scrub_duration": 5.3869062870000004, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 192684589056, "num_objects": 46279, "num_object_clones": 9, "num_object_copies": 277674, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 46279, "num_whiteouts": 0, "num_read": 1766106, "num_read_kb": 177513600, "num_write": 187542, "num_write_kb": 124853392, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 932022, "num_bytes_recovered": 3893286993920, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [ 269, 273, 298, 409, 440, 442, 458, 506 ], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261584, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "409(5)", "pgid": "11.fffs5", "last_update": "2261478'3114248", "last_complete": "2261478'3114248", "log_tail": "2257134'3113294", "last_user_version": 3114248, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2261584, "last_interval_started": 2261583, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2261478'3114248", "reported_seq": 27296956, "reported_epoch": 2261478, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T19:39:12.241953+0000", "last_change": "2024-03-23T13:38:04.290356+0000", "last_active": "2024-03-23T19:39:12.241953+0000", "last_peered": "2024-03-23T19:39:12.241953+0000", "last_clean": "2024-03-19T12:59:01.132410+0000", "last_became_active": "2024-03-23T13:20:41.305412+0000", "last_became_peered": "2024-03-23T13:20:41.305412+0000", "last_unstale": "2024-03-23T19:39:12.241953+0000", "last_undegraded": "2024-03-23T19:39:12.241953+0000", "last_fullsized": "2024-03-23T19:39:12.241953+0000", "mapping_epoch": 2261583, "log_start": "2257134'3113294", "ondisk_log_start": "2257134'3113294", "created": 1755101, "last_epoch_clean": 2248989, "parent": "0.0", "parent_split_bits": 12, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "objects_scrubbed": 45979, "log_size": 954, "log_dups_size": 3000, "ondisk_log_size": 954, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 6, "scrub_schedule": "queued for scrub", "scrub_duration": 5.3869062870000004, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 192684589056, "num_objects": 46279, "num_object_clones": 9, "num_object_copies": 277674, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 138381, "num_objects_unfound": 0, "num_objects_dirty": 46279, "num_whiteouts": 0, "num_read": 1766106, "num_read_kb": 177513600, "num_write": 187542, "num_write_kb": 124853392, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 931582, "num_bytes_recovered": 3891444797440, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [ "298(0)", "269(2)", "273(3)", "275(1)", "409(5)", "440(4)" ], "object_location_counts": [ { "shards": "269(2),273(3),275(1),298(0),409(5),440(4)", "objects": 46279 } ], "blocked_by": [], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261584, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "440(4)", "pgid": "11.fffs4", "last_update": "2261478'3114248", "last_complete": "2261478'3114248", "log_tail": "2257134'3113294", "last_user_version": 3114248, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2261584, "last_interval_started": 2261583, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2261478'3114248", "reported_seq": 27296956, "reported_epoch": 2261478, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T19:39:12.241953+0000", "last_change": "2024-03-23T13:38:04.290356+0000", "last_active": "2024-03-23T19:39:12.241953+0000", "last_peered": "2024-03-23T19:39:12.241953+0000", "last_clean": "2024-03-19T12:59:01.132410+0000", "last_became_active": "2024-03-23T13:20:41.305412+0000", "last_became_peered": "2024-03-23T13:20:41.305412+0000", "last_unstale": "2024-03-23T19:39:12.241953+0000", "last_undegraded": "2024-03-23T19:39:12.241953+0000", "last_fullsized": "2024-03-23T19:39:12.241953+0000", "mapping_epoch": 2261583, "log_start": "2257134'3113294", "ondisk_log_start": "2257134'3113294", "created": 1755101, "last_epoch_clean": 2248989, "parent": "0.0", "parent_split_bits": 12, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "objects_scrubbed": 45979, "log_size": 954, "log_dups_size": 3000, "ondisk_log_size": 954, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 6, "scrub_schedule": "queued for scrub", "scrub_duration": 5.3869062870000004, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 192684589056, "num_objects": 46279, "num_object_clones": 9, "num_object_copies": 277674, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 138381, "num_objects_unfound": 0, "num_objects_dirty": 46279, "num_whiteouts": 0, "num_read": 1766106, "num_read_kb": 177513600, "num_write": 187542, "num_write_kb": 124853392, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 931582, "num_bytes_recovered": 3891444797440, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [ "298(0)", "269(2)", "273(3)", "275(1)", "409(5)", "440(4)" ], "object_location_counts": [ { "shards": "269(2),273(3),275(1),298(0),409(5),440(4)", "objects": 46279 } ], "blocked_by": [], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261584, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "442(3)", "pgid": "11.fffs3", "last_update": "2261478'3114248", "last_complete": "2261478'3114248", "log_tail": "2257134'3113294", "last_user_version": 3114248, "last_backfill": "11:fff0351f:::rbd_data.10.8425f92e18b601.0000000000f49360:head", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2261584, "last_interval_started": 2261583, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261583, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 2465595392, "num_objects": 593, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 45686, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 593, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261584, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "458(5)", "pgid": "11.fffs5", "last_update": "2261478'3114248", "last_complete": "2261478'3114248", "log_tail": "2257134'3113294", "last_user_version": 3114248, "last_backfill": "11:fff0351f:::rbd_data.10.8425f92e18b601.0000000000f49360:head", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2261584, "last_interval_started": 2261583, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261583, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 2465595392, "num_objects": 593, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 45686, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 593, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261584, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "506(4)", "pgid": "11.fffs4", "last_update": "2261478'3114248", "last_complete": "2261478'3114248", "log_tail": "2257134'3113294", "last_user_version": 3114248, "last_backfill": "11:fff0351f:::rbd_data.10.8425f92e18b601.0000000000f49360:head", "purged_snaps": [], "history": { "epoch_created": 1755101, "epoch_pool_created": 1183522, "last_epoch_started": 2261584, "last_interval_started": 2261583, "last_epoch_clean": 2248989, "last_interval_clean": 2248988, "last_epoch_split": 1755101, "last_epoch_marked_full": 0, "same_up_since": 2261582, "same_interval_since": 2261583, "same_primary_since": 2261583, "last_scrub": "2237175'3110177", "last_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "last_deep_scrub": "2230303'3107531", "last_deep_scrub_stamp": "2024-03-09T00:03:48.057602+0000", "last_clean_scrub_stamp": "2024-03-14T02:44:04.637166+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261583, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 2465595392, "num_objects": 593, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 45686, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 593, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 298, 275, 269, 442, 506, 458 ], "acting": [ 298, 275, 269, 273, 440, 409 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 298, "acting_primary": 298, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261584, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started/Primary/Active", "enter_time": "2024-03-23T21:41:58.766906+0000", "might_have_unfound": [], "recovery_progress": { "backfill_targets": [ "442(3)", "458(5)", "506(4)" ], "waiting_on_backfill": [], "last_backfill_started": "11:fff0351f:::rbd_data.10.8425f92e18b601.0000000000f49360:head", "backfill_info": { "begin": "MIN", "end": "MIN", "objects": [] }, "peer_backfill_info": [], "backfills_in_flight": [], "recovering": [], "pg_backend": { "recovery_ops": [], "read_ops": [] } } }, { "name": "Started", "enter_time": "2024-03-23T21:41:57.170485+0000" } ], "scrubber": { "active": false, "must_scrub": false, "must_deep_scrub": false, "must_repair": false, "need_auto": false, "scrub_reg_stamp": "2024-03-15T02:44:04.637166+0000", "schedule": "queued for scrub" }, "agent_state": {} }