{ "snap_trimq": "[]", "snap_trimq_len": 0, "state": "active+remapped+backfill_wait", "epoch": 2261510, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "backfill_targets": [ "56(5)", "78(4)", "183(6)", "256(2)", "265(8)", "293(7)", "463(0)", "491(1)" ], "acting_recovery_backfill": [ "30(6)", "33(7)", "56(5)", "56(8)", "78(4)", "183(0)", "183(6)", "256(2)", "256(5)", "265(2)", "265(8)", "293(1)", "293(7)", "424(4)", "436(3)", "463(0)", "491(1)" ], "info": { "pgid": "37.1s0", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257923'162308", "last_user_version": 163591, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2261507'163591", "reported_seq": 717548, "reported_epoch": 2261510, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T20:30:37.154379+0000", "last_change": "2024-03-23T13:21:35.209393+0000", "last_active": "2024-03-23T20:30:37.154379+0000", "last_peered": "2024-03-23T20:30:37.154379+0000", "last_clean": "2024-03-19T13:01:08.581589+0000", "last_became_active": "2024-03-23T13:21:35.209252+0000", "last_became_peered": "2024-03-23T13:21:35.209252+0000", "last_unstale": "2024-03-23T20:30:37.154379+0000", "last_undegraded": "2024-03-23T20:30:37.154379+0000", "last_fullsized": "2024-03-23T20:30:37.154379+0000", "mapping_epoch": 2261102, "log_start": "2257923'162308", "ondisk_log_start": "2257923'162308", "created": 2139449, "last_epoch_clean": 2249011, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "objects_scrubbed": 78567, "log_size": 1283, "log_dups_size": 3000, "ondisk_log_size": 1283, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 14, "scrub_schedule": "queued for deep scrub", "scrub_duration": 13.170252527000001, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 93359062318, "num_objects": 78813, "num_object_clones": 0, "num_object_copies": 709317, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 630504, "num_objects_unfound": 0, "num_objects_dirty": 78813, "num_whiteouts": 0, "num_read": 80686, "num_read_kb": 112206672, "num_write": 228767, "num_write_kb": 109403261, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 34565, "num_bytes_recovered": 65940324388, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [ "183(0)", "30(6)", "33(7)", "56(8)", "256(5)", "265(2)", "293(1)", "424(4)", "436(3)" ], "object_location_counts": [ { "shards": "30(6),33(7),56(8),183(0),256(5),265(2),293(1),424(4),436(3)", "objects": 78813 } ], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "30(3)", "pgid": "37.1s3", "last_update": "2249296'156942", "last_complete": "2249296'156942", "log_tail": "2242483'155558", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2259141, "last_interval_started": 2259140, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249712, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "30(6)", "pgid": "37.1s6", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'163465", "reported_seq": 713683, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:49.686570+0000", "last_change": "2024-03-23T08:03:18.391295+0000", "last_active": "2024-03-23T08:23:49.686570+0000", "last_peered": "2024-03-23T08:23:49.686570+0000", "last_clean": "2024-03-19T13:01:08.581589+0000", "last_became_active": "2024-03-23T08:03:18.390854+0000", "last_became_peered": "2024-03-23T08:03:18.390854+0000", "last_unstale": "2024-03-23T08:23:49.686570+0000", "last_undegraded": "2024-03-23T08:23:49.686570+0000", "last_fullsized": "2024-03-23T08:23:49.686570+0000", "mapping_epoch": 2261102, "log_start": "2257907'162208", "ondisk_log_start": "2257907'162208", "created": 2139449, "last_epoch_clean": 2249011, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "objects_scrubbed": 78567, "log_size": 1257, "log_dups_size": 3000, "ondisk_log_size": 1257, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 14, "scrub_schedule": "queued for deep scrub", "scrub_duration": 13.170252527000001, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92981574958, "num_objects": 78723, "num_object_clones": 0, "num_object_copies": 708507, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 629784, "num_objects_unfound": 0, "num_objects_dirty": 78723, "num_whiteouts": 0, "num_read": 78736, "num_read_kb": 112024442, "num_write": 228641, "num_write_kb": 109034621, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 34565, "num_bytes_recovered": 65940324388, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [ "183(0)", "30(6)", "33(7)", "56(8)", "256(5)", "265(2)", "293(1)", "424(4)", "436(3)" ], "object_location_counts": [ { "shards": "30(6),33(7),56(8),183(0),256(5),265(2),293(1),424(4),436(3)", "objects": 78723 } ], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "33(4)", "pgid": "37.1s4", "last_update": "2249296'156942", "last_complete": "2249296'156942", "log_tail": "2242483'155558", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2259141, "last_interval_started": 2259140, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249712, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "33(7)", "pgid": "37.1s7", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'163465", "reported_seq": 713683, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:49.686570+0000", "last_change": "2024-03-23T08:03:18.391295+0000", "last_active": "2024-03-23T08:23:49.686570+0000", "last_peered": "2024-03-23T08:23:49.686570+0000", "last_clean": "2024-03-19T13:01:08.581589+0000", "last_became_active": "2024-03-23T08:03:18.390854+0000", "last_became_peered": "2024-03-23T08:03:18.390854+0000", "last_unstale": "2024-03-23T08:23:49.686570+0000", "last_undegraded": "2024-03-23T08:23:49.686570+0000", "last_fullsized": "2024-03-23T08:23:49.686570+0000", "mapping_epoch": 2261102, "log_start": "2257907'162208", "ondisk_log_start": "2257907'162208", "created": 2139449, "last_epoch_clean": 2249011, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "objects_scrubbed": 78567, "log_size": 1257, "log_dups_size": 3000, "ondisk_log_size": 1257, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 14, "scrub_schedule": "queued for deep scrub", "scrub_duration": 13.170252527000001, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92981574958, "num_objects": 78723, "num_object_clones": 0, "num_object_copies": 708507, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 629784, "num_objects_unfound": 0, "num_objects_dirty": 78723, "num_whiteouts": 0, "num_read": 78736, "num_read_kb": 112024442, "num_write": 228641, "num_write_kb": 109034621, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 34565, "num_bytes_recovered": 65940324388, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [ "183(0)", "30(6)", "33(7)", "56(8)", "256(5)", "265(2)", "293(1)", "424(4)", "436(3)" ], "object_location_counts": [ { "shards": "30(6),33(7),56(8),183(0),256(5),265(2),293(1),424(4),436(3)", "objects": 78723 } ], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "56(5)", "pgid": "37.1s5", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "56(8)", "pgid": "37.1s8", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'163465", "reported_seq": 713683, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:49.686570+0000", "last_change": "2024-03-23T08:03:18.391295+0000", "last_active": "2024-03-23T08:23:49.686570+0000", "last_peered": "2024-03-23T08:23:49.686570+0000", "last_clean": "2024-03-19T13:01:08.581589+0000", "last_became_active": "2024-03-23T08:03:18.390854+0000", "last_became_peered": "2024-03-23T08:03:18.390854+0000", "last_unstale": "2024-03-23T08:23:49.686570+0000", "last_undegraded": "2024-03-23T08:23:49.686570+0000", "last_fullsized": "2024-03-23T08:23:49.686570+0000", "mapping_epoch": 2261102, "log_start": "2257907'162208", "ondisk_log_start": "2257907'162208", "created": 2139449, "last_epoch_clean": 2249011, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "objects_scrubbed": 78567, "log_size": 1257, "log_dups_size": 3000, "ondisk_log_size": 1257, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 14, "scrub_schedule": "queued for deep scrub", "scrub_duration": 13.170252527000001, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92981574958, "num_objects": 78723, "num_object_clones": 0, "num_object_copies": 708507, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 629784, "num_objects_unfound": 0, "num_objects_dirty": 78723, "num_whiteouts": 0, "num_read": 78736, "num_read_kb": 112024442, "num_write": 228641, "num_write_kb": 109034621, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 34565, "num_bytes_recovered": 65940324388, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [ "183(0)", "30(6)", "33(7)", "56(8)", "256(5)", "265(2)", "293(1)", "424(4)", "436(3)" ], "object_location_counts": [ { "shards": "30(6),33(7),56(8),183(0),256(5),265(2),293(1),424(4),436(3)", "objects": 78723 } ], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "78(4)", "pgid": "37.1s4", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "78(7)", "pgid": "37.1s7", "last_update": "2249296'156942", "last_complete": "2249296'156942", "log_tail": "2242483'155558", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2259141, "last_interval_started": 2259140, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249821, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "183(6)", "pgid": "37.1s6", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "256(2)", "pgid": "37.1s2", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "256(5)", "pgid": "37.1s5", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'163465", "reported_seq": 713683, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:49.686570+0000", "last_change": "2024-03-23T08:03:18.391295+0000", "last_active": "2024-03-23T08:23:49.686570+0000", "last_peered": "2024-03-23T08:23:49.686570+0000", "last_clean": "2024-03-19T13:01:08.581589+0000", "last_became_active": "2024-03-23T08:03:18.390854+0000", "last_became_peered": "2024-03-23T08:03:18.390854+0000", "last_unstale": "2024-03-23T08:23:49.686570+0000", "last_undegraded": "2024-03-23T08:23:49.686570+0000", "last_fullsized": "2024-03-23T08:23:49.686570+0000", "mapping_epoch": 2261102, "log_start": "2257907'162208", "ondisk_log_start": "2257907'162208", "created": 2139449, "last_epoch_clean": 2249011, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "objects_scrubbed": 78567, "log_size": 1257, "log_dups_size": 3000, "ondisk_log_size": 1257, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 14, "scrub_schedule": "queued for deep scrub", "scrub_duration": 13.170252527000001, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92981574958, "num_objects": 78723, "num_object_clones": 0, "num_object_copies": 708507, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 629784, "num_objects_unfound": 0, "num_objects_dirty": 78723, "num_whiteouts": 0, "num_read": 78736, "num_read_kb": 112024442, "num_write": 228641, "num_write_kb": 109034621, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 34565, "num_bytes_recovered": 65940324388, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [ "183(0)", "30(6)", "33(7)", "56(8)", "256(5)", "265(2)", "293(1)", "424(4)", "436(3)" ], "object_location_counts": [ { "shards": "30(6),33(7),56(8),183(0),256(5),265(2),293(1),424(4),436(3)", "objects": 78723 } ], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "265(2)", "pgid": "37.1s2", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'163465", "reported_seq": 713683, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:49.686570+0000", "last_change": "2024-03-23T08:03:18.391295+0000", "last_active": "2024-03-23T08:23:49.686570+0000", "last_peered": "2024-03-23T08:23:49.686570+0000", "last_clean": "2024-03-19T13:01:08.581589+0000", "last_became_active": "2024-03-23T08:03:18.390854+0000", "last_became_peered": "2024-03-23T08:03:18.390854+0000", "last_unstale": "2024-03-23T08:23:49.686570+0000", "last_undegraded": "2024-03-23T08:23:49.686570+0000", "last_fullsized": "2024-03-23T08:23:49.686570+0000", "mapping_epoch": 2261102, "log_start": "2257907'162208", "ondisk_log_start": "2257907'162208", "created": 2139449, "last_epoch_clean": 2249011, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "objects_scrubbed": 78567, "log_size": 1257, "log_dups_size": 3000, "ondisk_log_size": 1257, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 14, "scrub_schedule": "queued for deep scrub", "scrub_duration": 13.170252527000001, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92981574958, "num_objects": 78723, "num_object_clones": 0, "num_object_copies": 708507, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 629784, "num_objects_unfound": 0, "num_objects_dirty": 78723, "num_whiteouts": 0, "num_read": 78736, "num_read_kb": 112024442, "num_write": 228641, "num_write_kb": 109034621, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 34565, "num_bytes_recovered": 65940324388, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [ "183(0)", "30(6)", "33(7)", "56(8)", "256(5)", "265(2)", "293(1)", "424(4)", "436(3)" ], "object_location_counts": [ { "shards": "30(6),33(7),56(8),183(0),256(5),265(2),293(1),424(4),436(3)", "objects": 78723 } ], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "265(8)", "pgid": "37.1s8", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "293(1)", "pgid": "37.1s1", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'163465", "reported_seq": 713804, "reported_epoch": 2260315, "state": "remapped+peering", "last_fresh": "2024-03-23T09:00:41.322076+0000", "last_change": "2024-03-23T09:00:41.322076+0000", "last_active": "2024-03-23T09:00:41.322025+0000", "last_peered": "2024-03-23T09:00:39.236240+0000", "last_clean": "2024-03-19T13:01:08.581589+0000", "last_became_active": "2024-03-23T08:51:11.086602+0000", "last_became_peered": "2024-03-23T08:51:11.086602+0000", "last_unstale": "2024-03-23T09:00:41.322076+0000", "last_undegraded": "2024-03-23T09:00:41.322076+0000", "last_fullsized": "2024-03-23T09:00:41.322076+0000", "mapping_epoch": 2261102, "log_start": "2257907'162208", "ondisk_log_start": "2257907'162208", "created": 2139449, "last_epoch_clean": 2249011, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "objects_scrubbed": 78567, "log_size": 1257, "log_dups_size": 3000, "ondisk_log_size": 1257, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 14, "scrub_schedule": "queued for deep scrub", "scrub_duration": 13.170252527000001, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92981574958, "num_objects": 78723, "num_object_clones": 0, "num_object_copies": 708507, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 78723, "num_whiteouts": 0, "num_read": 78753, "num_read_kb": 112026786, "num_write": 228641, "num_write_kb": 109034621, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 34565, "num_bytes_recovered": 65940324388, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [ 30, 33, 56, 78, 183, 256, 265, 293, 424, 436, 463, 491 ], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "293(7)", "pgid": "37.1s7", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "424(4)", "pgid": "37.1s4", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'163465", "reported_seq": 713683, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:49.686570+0000", "last_change": "2024-03-23T08:03:18.391295+0000", "last_active": "2024-03-23T08:23:49.686570+0000", "last_peered": "2024-03-23T08:23:49.686570+0000", "last_clean": "2024-03-19T13:01:08.581589+0000", "last_became_active": "2024-03-23T08:03:18.390854+0000", "last_became_peered": "2024-03-23T08:03:18.390854+0000", "last_unstale": "2024-03-23T08:23:49.686570+0000", "last_undegraded": "2024-03-23T08:23:49.686570+0000", "last_fullsized": "2024-03-23T08:23:49.686570+0000", "mapping_epoch": 2261102, "log_start": "2257907'162208", "ondisk_log_start": "2257907'162208", "created": 2139449, "last_epoch_clean": 2249011, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "objects_scrubbed": 78567, "log_size": 1257, "log_dups_size": 3000, "ondisk_log_size": 1257, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 14, "scrub_schedule": "queued for deep scrub", "scrub_duration": 13.170252527000001, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92981574958, "num_objects": 78723, "num_object_clones": 0, "num_object_copies": 708507, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 629784, "num_objects_unfound": 0, "num_objects_dirty": 78723, "num_whiteouts": 0, "num_read": 78736, "num_read_kb": 112024442, "num_write": 228641, "num_write_kb": 109034621, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 34565, "num_bytes_recovered": 65940324388, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [ "183(0)", "30(6)", "33(7)", "56(8)", "256(5)", "265(2)", "293(1)", "424(4)", "436(3)" ], "object_location_counts": [ { "shards": "30(6),33(7),56(8),183(0),256(5),265(2),293(1),424(4),436(3)", "objects": 78723 } ], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "436(0)", "pgid": "37.1s0", "last_update": "2249296'156942", "last_complete": "2249296'156942", "log_tail": "2242483'155558", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2259141, "last_interval_started": 2259140, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249712, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "436(3)", "pgid": "37.1s3", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'163465", "reported_seq": 713683, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:49.686570+0000", "last_change": "2024-03-23T08:03:18.391295+0000", "last_active": "2024-03-23T08:23:49.686570+0000", "last_peered": "2024-03-23T08:23:49.686570+0000", "last_clean": "2024-03-19T13:01:08.581589+0000", "last_became_active": "2024-03-23T08:03:18.390854+0000", "last_became_peered": "2024-03-23T08:03:18.390854+0000", "last_unstale": "2024-03-23T08:23:49.686570+0000", "last_undegraded": "2024-03-23T08:23:49.686570+0000", "last_fullsized": "2024-03-23T08:23:49.686570+0000", "mapping_epoch": 2261102, "log_start": "2257907'162208", "ondisk_log_start": "2257907'162208", "created": 2139449, "last_epoch_clean": 2249011, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "objects_scrubbed": 78567, "log_size": 1257, "log_dups_size": 3000, "ondisk_log_size": 1257, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 14, "scrub_schedule": "queued for deep scrub", "scrub_duration": 13.170252527000001, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92981574958, "num_objects": 78723, "num_object_clones": 0, "num_object_copies": 708507, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 629784, "num_objects_unfound": 0, "num_objects_dirty": 78723, "num_whiteouts": 0, "num_read": 78736, "num_read_kb": 112024442, "num_write": 228641, "num_write_kb": 109034621, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 34565, "num_bytes_recovered": 65940324388, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [ "183(0)", "30(6)", "33(7)", "56(8)", "256(5)", "265(2)", "293(1)", "424(4)", "436(3)" ], "object_location_counts": [ { "shards": "30(6),33(7),56(8),183(0),256(5),265(2),293(1),424(4),436(3)", "objects": 78723 } ], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "436(6)", "pgid": "37.1s6", "last_update": "2249296'156942", "last_complete": "2249296'156942", "log_tail": "2242483'155558", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2259141, "last_interval_started": 2259140, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249821, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "463(0)", "pgid": "37.1s0", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "463(3)", "pgid": "37.1s3", "last_update": "2249296'156942", "last_complete": "2249296'156942", "log_tail": "2242483'155558", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2259141, "last_interval_started": 2259140, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249821, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "471(1)", "pgid": "37.1s1", "last_update": "2249296'156942", "last_complete": "2249296'156942", "log_tail": "2242483'155558", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2259141, "last_interval_started": 2259140, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249712, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "471(4)", "pgid": "37.1s4", "last_update": "2249296'156942", "last_complete": "2249296'156942", "log_tail": "2242483'155558", "last_user_version": 156942, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2259141, "last_interval_started": 2259140, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249054, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "491(1)", "pgid": "37.1s1", "last_update": "2261507'163591", "last_complete": "2261507'163591", "log_tail": "2257907'162208", "last_user_version": 163465, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2261103, "last_interval_started": 2261102, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261103, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "491(4)", "pgid": "37.1s4", "last_update": "2249296'156942", "last_complete": "2249296'156942", "log_tail": "2242483'155558", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139449, "epoch_pool_created": 2139449, "last_epoch_started": 2259141, "last_interval_started": 2259140, "last_epoch_clean": 2249011, "last_interval_clean": 2249010, "last_epoch_split": 2139452, "last_epoch_marked_full": 0, "same_up_since": 2260999, "same_interval_since": 2261102, "same_primary_since": 2260316, "last_scrub": "2235826'154521", "last_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "last_deep_scrub": "2198936'124827", "last_deep_scrub_stamp": "2024-02-16T04:20:58.564794+0000", "last_clean_scrub_stamp": "2024-03-12T18:26:58.560460+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261102, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78813, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 463, 491, 256, 436, 78, 56, 183, 293, 265 ], "acting": [ 183, 293, 265, 436, 424, 256, 30, 33, 56 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 463, "acting_primary": 183, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249821, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started/Primary/Active", "enter_time": "2024-03-23T13:21:35.135388+0000", "might_have_unfound": [], "recovery_progress": { "backfill_targets": [ "56(5)", "78(4)", "183(6)", "256(2)", "265(8)", "293(7)", "463(0)", "491(1)" ], "waiting_on_backfill": [], "last_backfill_started": "MIN", "backfill_info": { "begin": "MIN", "end": "MIN", "objects": [] }, "peer_backfill_info": [], "backfills_in_flight": [], "recovering": [], "pg_backend": { "recovery_ops": [], "read_ops": [] } } }, { "name": "Started", "enter_time": "2024-03-23T13:21:33.475653+0000" } ], "scrubber": { "active": false, "must_scrub": false, "must_deep_scrub": false, "must_repair": false, "need_auto": false, "scrub_reg_stamp": "2024-03-13T18:26:58.560460+0000", "schedule": "queued for deep scrub" }, "agent_state": {} }