{ "snap_trimq": "[]", "snap_trimq_len": 0, "state": "active+remapped+backfill_wait", "epoch": 2261510, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "backfill_targets": [ "81(3)", "149(1)", "203(7)", "221(8)", "288(6)", "441(4)", "474(0)", "524(5)" ], "acting_recovery_backfill": [ "9(8)", "51(2)", "54(7)", "81(3)", "81(6)", "149(1)", "203(4)", "203(7)", "221(5)", "221(8)", "288(3)", "288(6)", "408(0)", "441(1)", "441(4)", "474(0)", "524(5)" ], "info": { "pgid": "37.7dcs0", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258312'159756", "last_user_version": 160457, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2261510'160457", "reported_seq": 570807, "reported_epoch": 2261510, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T20:34:09.066718+0000", "last_change": "2024-03-23T13:33:55.254328+0000", "last_active": "2024-03-23T20:34:09.066718+0000", "last_peered": "2024-03-23T20:34:09.066718+0000", "last_clean": "2024-03-19T12:57:39.093899+0000", "last_became_active": "2024-03-23T13:33:55.254025+0000", "last_became_peered": "2024-03-23T13:33:55.254025+0000", "last_unstale": "2024-03-23T20:34:09.066718+0000", "last_undegraded": "2024-03-23T20:34:09.066718+0000", "last_fullsized": "2024-03-23T20:34:09.066718+0000", "mapping_epoch": 2261149, "log_start": "2258312'159756", "ondisk_log_start": "2258312'159756", "created": 2139486, "last_epoch_clean": 2238064, "parent": "0.0", "parent_split_bits": 11, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "objects_scrubbed": 77852, "log_size": 701, "log_dups_size": 3000, "ondisk_log_size": 701, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 13, "scrub_schedule": "queued for deep scrub", "scrub_duration": 12.784428346, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92853372351, "num_objects": 78103, "num_object_clones": 0, "num_object_copies": 702927, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 623322, "num_objects_unfound": 0, "num_objects_dirty": 78103, "num_whiteouts": 0, "num_read": 81761, "num_read_kb": 111785098, "num_write": 222131, "num_write_kb": 108212113, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 26566, "num_bytes_recovered": 52884763428, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [ "408(0)", "9(8)", "51(2)", "54(7)", "81(6)", "203(4)", "221(5)", "288(3)", "441(1)" ], "object_location_counts": [ { "shards": "9(8),51(2),54(7),81(6),203(4),221(5),288(3),408(0),441(1)", "objects": 78103 } ], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "9(8)", "pgid": "37.7dcs8", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'160305", "reported_seq": 567018, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:41.335971+0000", "last_change": "2024-03-23T08:03:34.887737+0000", "last_active": "2024-03-23T08:23:41.335971+0000", "last_peered": "2024-03-23T08:23:41.335971+0000", "last_clean": "2024-03-19T12:57:39.093899+0000", "last_became_active": "2024-03-23T08:03:34.887640+0000", "last_became_peered": "2024-03-23T08:03:34.887640+0000", "last_unstale": "2024-03-23T08:23:41.335971+0000", "last_undegraded": "2024-03-23T08:23:41.335971+0000", "last_fullsized": "2024-03-23T08:23:41.335971+0000", "mapping_epoch": 2261149, "log_start": "2258026'159556", "ondisk_log_start": "2258026'159556", "created": 2139486, "last_epoch_clean": 2238064, "parent": "0.0", "parent_split_bits": 11, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "objects_scrubbed": 77852, "log_size": 749, "log_dups_size": 3000, "ondisk_log_size": 749, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 13, "scrub_schedule": "queued for deep scrub", "scrub_duration": 12.784428346, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92400387519, "num_objects": 77995, "num_object_clones": 0, "num_object_copies": 701955, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 622466, "num_objects_unfound": 0, "num_objects_dirty": 77995, "num_whiteouts": 0, "num_read": 80501, "num_read_kb": 111657864, "num_write": 221979, "num_write_kb": 107769745, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 26566, "num_bytes_recovered": 52884763428, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [ "408(0)", "9(8)", "51(2)", "54(7)", "81(6)", "203(4)", "221(5)", "288(3)", "441(1)" ], "object_location_counts": [ { "shards": "9(8),51(2),54(7),81(6),203(4),221(5),288(3),408(0),441(1)", "objects": 77995 } ], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "51(2)", "pgid": "37.7dcs2", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'160305", "reported_seq": 567018, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:41.335971+0000", "last_change": "2024-03-23T08:03:34.887737+0000", "last_active": "2024-03-23T08:23:41.335971+0000", "last_peered": "2024-03-23T08:23:41.335971+0000", "last_clean": "2024-03-19T12:57:39.093899+0000", "last_became_active": "2024-03-23T08:03:34.887640+0000", "last_became_peered": "2024-03-23T08:03:34.887640+0000", "last_unstale": "2024-03-23T08:23:41.335971+0000", "last_undegraded": "2024-03-23T08:23:41.335971+0000", "last_fullsized": "2024-03-23T08:23:41.335971+0000", "mapping_epoch": 2261149, "log_start": "2258026'159556", "ondisk_log_start": "2258026'159556", "created": 2139486, "last_epoch_clean": 2238064, "parent": "0.0", "parent_split_bits": 11, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "objects_scrubbed": 77852, "log_size": 749, "log_dups_size": 3000, "ondisk_log_size": 749, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 13, "scrub_schedule": "queued for deep scrub", "scrub_duration": 12.784428346, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92400387519, "num_objects": 77995, "num_object_clones": 0, "num_object_copies": 701955, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 622466, "num_objects_unfound": 0, "num_objects_dirty": 77995, "num_whiteouts": 0, "num_read": 80501, "num_read_kb": 111657864, "num_write": 221979, "num_write_kb": 107769745, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 26566, "num_bytes_recovered": 52884763428, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [ "408(0)", "9(8)", "51(2)", "54(7)", "81(6)", "203(4)", "221(5)", "288(3)", "441(1)" ], "object_location_counts": [ { "shards": "9(8),51(2),54(7),81(6),203(4),221(5),288(3),408(0),441(1)", "objects": 77995 } ], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "54(7)", "pgid": "37.7dcs7", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'160305", "reported_seq": 567018, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:41.335971+0000", "last_change": "2024-03-23T08:03:34.887737+0000", "last_active": "2024-03-23T08:23:41.335971+0000", "last_peered": "2024-03-23T08:23:41.335971+0000", "last_clean": "2024-03-19T12:57:39.093899+0000", "last_became_active": "2024-03-23T08:03:34.887640+0000", "last_became_peered": "2024-03-23T08:03:34.887640+0000", "last_unstale": "2024-03-23T08:23:41.335971+0000", "last_undegraded": "2024-03-23T08:23:41.335971+0000", "last_fullsized": "2024-03-23T08:23:41.335971+0000", "mapping_epoch": 2261149, "log_start": "2258026'159556", "ondisk_log_start": "2258026'159556", "created": 2139486, "last_epoch_clean": 2238064, "parent": "0.0", "parent_split_bits": 11, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "objects_scrubbed": 77852, "log_size": 749, "log_dups_size": 3000, "ondisk_log_size": 749, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 13, "scrub_schedule": "queued for deep scrub", "scrub_duration": 12.784428346, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92400387519, "num_objects": 77995, "num_object_clones": 0, "num_object_copies": 701955, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 622466, "num_objects_unfound": 0, "num_objects_dirty": 77995, "num_whiteouts": 0, "num_read": 80501, "num_read_kb": 111657864, "num_write": 221979, "num_write_kb": 107769745, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 26566, "num_bytes_recovered": 52884763428, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [ "408(0)", "9(8)", "51(2)", "54(7)", "81(6)", "203(4)", "221(5)", "288(3)", "441(1)" ], "object_location_counts": [ { "shards": "9(8),51(2),54(7),81(6),203(4),221(5),288(3),408(0),441(1)", "objects": 77995 } ], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "58(8)", "pgid": "37.7dcs8", "last_update": "2252592'155929", "last_complete": "2252592'155929", "log_tail": "2252130'155256", "last_user_version": 155929, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2259550, "last_interval_started": 2259549, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78103, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249821, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "81(3)", "pgid": "37.7dcs3", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "37:3be0144d:::100002efcf4.00000000:head", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 234651648, "num_objects": 194, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 77909, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 194, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 4, "num_write_kb": 8192, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "81(6)", "pgid": "37.7dcs6", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'160305", "reported_seq": 567018, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:41.335971+0000", "last_change": "2024-03-23T08:03:34.887737+0000", "last_active": "2024-03-23T08:23:41.335971+0000", "last_peered": "2024-03-23T08:23:41.335971+0000", "last_clean": "2024-03-19T12:57:39.093899+0000", "last_became_active": "2024-03-23T08:03:34.887640+0000", "last_became_peered": "2024-03-23T08:03:34.887640+0000", "last_unstale": "2024-03-23T08:23:41.335971+0000", "last_undegraded": "2024-03-23T08:23:41.335971+0000", "last_fullsized": "2024-03-23T08:23:41.335971+0000", "mapping_epoch": 2261149, "log_start": "2258026'159556", "ondisk_log_start": "2258026'159556", "created": 2139486, "last_epoch_clean": 2238064, "parent": "0.0", "parent_split_bits": 11, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "objects_scrubbed": 77852, "log_size": 749, "log_dups_size": 3000, "ondisk_log_size": 749, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 13, "scrub_schedule": "queued for deep scrub", "scrub_duration": 12.784428346, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92400387519, "num_objects": 77995, "num_object_clones": 0, "num_object_copies": 701955, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 622466, "num_objects_unfound": 0, "num_objects_dirty": 77995, "num_whiteouts": 0, "num_read": 80501, "num_read_kb": 111657864, "num_write": 221979, "num_write_kb": 107769745, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 26566, "num_bytes_recovered": 52884763428, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [ "408(0)", "9(8)", "51(2)", "54(7)", "81(6)", "203(4)", "221(5)", "288(3)", "441(1)" ], "object_location_counts": [ { "shards": "9(8),51(2),54(7),81(6),203(4),221(5),288(3),408(0),441(1)", "objects": 77995 } ], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "149(1)", "pgid": "37.7dcs1", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "37:3be0144d:::100002efcf4.00000000:head", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 234651648, "num_objects": 194, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 77909, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 194, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 4, "num_write_kb": 8192, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "203(4)", "pgid": "37.7dcs4", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'160305", "reported_seq": 567018, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:41.335971+0000", "last_change": "2024-03-23T08:03:34.887737+0000", "last_active": "2024-03-23T08:23:41.335971+0000", "last_peered": "2024-03-23T08:23:41.335971+0000", "last_clean": "2024-03-19T12:57:39.093899+0000", "last_became_active": "2024-03-23T08:03:34.887640+0000", "last_became_peered": "2024-03-23T08:03:34.887640+0000", "last_unstale": "2024-03-23T08:23:41.335971+0000", "last_undegraded": "2024-03-23T08:23:41.335971+0000", "last_fullsized": "2024-03-23T08:23:41.335971+0000", "mapping_epoch": 2261149, "log_start": "2258026'159556", "ondisk_log_start": "2258026'159556", "created": 2139486, "last_epoch_clean": 2238064, "parent": "0.0", "parent_split_bits": 11, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "objects_scrubbed": 77852, "log_size": 749, "log_dups_size": 3000, "ondisk_log_size": 749, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 13, "scrub_schedule": "queued for deep scrub", "scrub_duration": 12.784428346, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92400387519, "num_objects": 77995, "num_object_clones": 0, "num_object_copies": 701955, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 622466, "num_objects_unfound": 0, "num_objects_dirty": 77995, "num_whiteouts": 0, "num_read": 80501, "num_read_kb": 111657864, "num_write": 221979, "num_write_kb": 107769745, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 26566, "num_bytes_recovered": 52884763428, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [ "408(0)", "9(8)", "51(2)", "54(7)", "81(6)", "203(4)", "221(5)", "288(3)", "441(1)" ], "object_location_counts": [ { "shards": "9(8),51(2),54(7),81(6),203(4),221(5),288(3),408(0),441(1)", "objects": 77995 } ], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "203(7)", "pgid": "37.7dcs7", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "37:3be0144d:::100002efcf4.00000000:head", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 234651648, "num_objects": 194, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 77909, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 194, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 4, "num_write_kb": 8192, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "221(5)", "pgid": "37.7dcs5", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'160305", "reported_seq": 567018, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:41.335971+0000", "last_change": "2024-03-23T08:03:34.887737+0000", "last_active": "2024-03-23T08:23:41.335971+0000", "last_peered": "2024-03-23T08:23:41.335971+0000", "last_clean": "2024-03-19T12:57:39.093899+0000", "last_became_active": "2024-03-23T08:03:34.887640+0000", "last_became_peered": "2024-03-23T08:03:34.887640+0000", "last_unstale": "2024-03-23T08:23:41.335971+0000", "last_undegraded": "2024-03-23T08:23:41.335971+0000", "last_fullsized": "2024-03-23T08:23:41.335971+0000", "mapping_epoch": 2261149, "log_start": "2258026'159556", "ondisk_log_start": "2258026'159556", "created": 2139486, "last_epoch_clean": 2238064, "parent": "0.0", "parent_split_bits": 11, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "objects_scrubbed": 77852, "log_size": 749, "log_dups_size": 3000, "ondisk_log_size": 749, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 13, "scrub_schedule": "queued for deep scrub", "scrub_duration": 12.784428346, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92400387519, "num_objects": 77995, "num_object_clones": 0, "num_object_copies": 701955, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 622466, "num_objects_unfound": 0, "num_objects_dirty": 77995, "num_whiteouts": 0, "num_read": 80501, "num_read_kb": 111657864, "num_write": 221979, "num_write_kb": 107769745, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 26566, "num_bytes_recovered": 52884763428, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [ "408(0)", "9(8)", "51(2)", "54(7)", "81(6)", "203(4)", "221(5)", "288(3)", "441(1)" ], "object_location_counts": [ { "shards": "9(8),51(2),54(7),81(6),203(4),221(5),288(3),408(0),441(1)", "objects": 77995 } ], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "221(8)", "pgid": "37.7dcs8", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "37:3be0144d:::100002efcf4.00000000:head", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 234651648, "num_objects": 194, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 77909, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 194, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 4, "num_write_kb": 8192, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "288(3)", "pgid": "37.7dcs3", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'160305", "reported_seq": 567018, "reported_epoch": 2260182, "state": "active+remapped+backfill_wait", "last_fresh": "2024-03-23T08:23:41.335971+0000", "last_change": "2024-03-23T08:03:34.887737+0000", "last_active": "2024-03-23T08:23:41.335971+0000", "last_peered": "2024-03-23T08:23:41.335971+0000", "last_clean": "2024-03-19T12:57:39.093899+0000", "last_became_active": "2024-03-23T08:03:34.887640+0000", "last_became_peered": "2024-03-23T08:03:34.887640+0000", "last_unstale": "2024-03-23T08:23:41.335971+0000", "last_undegraded": "2024-03-23T08:23:41.335971+0000", "last_fullsized": "2024-03-23T08:23:41.335971+0000", "mapping_epoch": 2261149, "log_start": "2258026'159556", "ondisk_log_start": "2258026'159556", "created": 2139486, "last_epoch_clean": 2238064, "parent": "0.0", "parent_split_bits": 11, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "objects_scrubbed": 77852, "log_size": 749, "log_dups_size": 3000, "ondisk_log_size": 749, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 13, "scrub_schedule": "queued for deep scrub", "scrub_duration": 12.784428346, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92400387519, "num_objects": 77995, "num_object_clones": 0, "num_object_copies": 701955, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 622466, "num_objects_unfound": 0, "num_objects_dirty": 77995, "num_whiteouts": 0, "num_read": 80501, "num_read_kb": 111657864, "num_write": 221979, "num_write_kb": 107769745, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 26566, "num_bytes_recovered": 52884763428, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [ "408(0)", "9(8)", "51(2)", "54(7)", "81(6)", "203(4)", "221(5)", "288(3)", "441(1)" ], "object_location_counts": [ { "shards": "9(8),51(2),54(7),81(6),203(4),221(5),288(3),408(0),441(1)", "objects": 77995 } ], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "288(6)", "pgid": "37.7dcs6", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "37:3be0144d:::100002efcf4.00000000:head", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 234651648, "num_objects": 194, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 77909, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 194, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 4, "num_write_kb": 8192, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "429(2)", "pgid": "37.7dcs2", "last_update": "2249473'154298", "last_complete": "2249473'154298", "log_tail": "2246259'153487", "last_user_version": 154298, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261112, "last_interval_started": 2261111, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 12.283888945999999 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78103, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2248955, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "441(1)", "pgid": "37.7dcs1", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "2260182'160305", "reported_seq": 567142, "reported_epoch": 2261072, "state": "active+undersized+degraded+remapped+backfill_wait", "last_fresh": "2024-03-23T13:20:16.597975+0000", "last_change": "2024-03-23T13:07:19.906293+0000", "last_active": "2024-03-23T13:20:16.597975+0000", "last_peered": "2024-03-23T13:20:16.597975+0000", "last_clean": "2024-03-19T12:57:39.093899+0000", "last_became_active": "2024-03-23T13:07:19.906163+0000", "last_became_peered": "2024-03-23T13:07:19.906163+0000", "last_unstale": "2024-03-23T13:20:16.597975+0000", "last_undegraded": "2024-03-23T13:07:19.698384+0000", "last_fullsized": "2024-03-23T13:07:19.698249+0000", "mapping_epoch": 2261149, "log_start": "2258026'159556", "ondisk_log_start": "2258026'159556", "created": 2139486, "last_epoch_clean": 2238064, "parent": "0.0", "parent_split_bits": 11, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "objects_scrubbed": 77852, "log_size": 749, "log_dups_size": 3000, "ondisk_log_size": 749, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 13, "scrub_schedule": "queued for deep scrub", "scrub_duration": 12.784428346, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 92400387519, "num_objects": 77995, "num_object_clones": 0, "num_object_copies": 701955, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 77802, "num_objects_misplaced": 544664, "num_objects_unfound": 0, "num_objects_dirty": 77995, "num_whiteouts": 0, "num_read": 80540, "num_read_kb": 111660510, "num_write": 221979, "num_write_kb": 107769745, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 26566, "num_bytes_recovered": 52884763428, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [ "441(1)", "9(8)", "51(2)", "54(7)", "81(6)", "203(4)", "221(5)", "288(3)" ], "object_location_counts": [ { "shards": "9(8),51(2),54(7),81(6),203(4),221(5),288(3),441(1)", "objects": 77995 } ], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "441(4)", "pgid": "37.7dcs4", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "37:3be00f4c:::10006d2cddd.00000000:head", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 158138368, "num_objects": 144, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 77959, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 144, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 3, "num_write_kb": 8192, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "441(7)", "pgid": "37.7dcs7", "last_update": "2252592'155929", "last_complete": "2252592'155929", "log_tail": "2252130'155256", "last_user_version": 155929, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2259550, "last_interval_started": 2259549, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 78103, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2249821, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "474(0)", "pgid": "37.7dcs0", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "37:3be0144d:::100002efcf4.00000000:head", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 234651648, "num_objects": 194, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 77909, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 194, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 4, "num_write_kb": 8192, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "524(5)", "pgid": "37.7dcs5", "last_update": "2261510'160457", "last_complete": "2261510'160457", "log_tail": "2258026'159556", "last_user_version": 160305, "last_backfill": "37:3be0144d:::100002efcf4.00000000:head", "purged_snaps": [], "history": { "epoch_created": 2139486, "epoch_pool_created": 2139449, "last_epoch_started": 2261150, "last_interval_started": 2261149, "last_epoch_clean": 2238064, "last_interval_clean": 2238063, "last_epoch_split": 2139486, "last_epoch_marked_full": 0, "same_up_since": 2261148, "same_interval_since": 2261149, "same_primary_since": 2261073, "last_scrub": "2236051'152185", "last_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "last_deep_scrub": "2192897'113065", "last_deep_scrub_stamp": "2024-02-12T05:54:48.898543+0000", "last_clean_scrub_stamp": "2024-03-13T02:41:17.440734+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 2261149, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 234651648, "num_objects": 194, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 77909, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 194, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 4, "num_write_kb": 8192, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 474, 149, 51, 81, 441, 524, 288, 203, 221 ], "acting": [ 408, 441, 51, 288, 203, 221, 81, 54, 9 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 474, "acting_primary": 408, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 2261150, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started/Primary/Active", "enter_time": "2024-03-23T13:33:55.142083+0000", "might_have_unfound": [], "recovery_progress": { "backfill_targets": [ "81(3)", "149(1)", "203(7)", "221(8)", "288(6)", "441(4)", "474(0)", "524(5)" ], "waiting_on_backfill": [], "last_backfill_started": "37:3be00f4c:::10006d2cddd.00000000:head", "backfill_info": { "begin": "MIN", "end": "MIN", "objects": [] }, "peer_backfill_info": [], "backfills_in_flight": [], "recovering": [], "pg_backend": { "recovery_ops": [], "read_ops": [] } } }, { "name": "Started", "enter_time": "2024-03-23T13:33:53.532161+0000" } ], "scrubber": { "active": false, "must_scrub": false, "must_deep_scrub": false, "must_repair": false, "need_auto": false, "scrub_reg_stamp": "2024-03-14T02:41:17.440734+0000", "schedule": "queued for deep scrub" }, "agent_state": {} }