Hi, I have 4 inactive pg in my cluster, the osds are dies where it was before. How can I make it work again? Maybe just threw away because last backfill=max? Based on the pg query it is totally up on other osds. It is an EC 3+1. This is an example inactive pg: ceph pg 44.1f0 query { "snap_trimq": "[]", "snap_trimq_len": 0, "state": "incomplete", "epoch": 5541839, "up": [ 46, 34, 62, 74 ], "acting": [ 46, 34, 62, 74 ], "info": { "pgid": "44.1f0s0", "last_update": "4863820'2109288", "last_complete": "4863820'2109288", "log_tail": "3881944'2103139", "last_user_version": 11189093, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 192123, "epoch_pool_created": 175799, "last_epoch_started": 4865266, "last_interval_started": 4865265, "last_epoch_clean": 4865007, "last_interval_clean": 4865006, "last_epoch_split": 192123, "last_epoch_marked_full": 0, "same_up_since": 5541572, "same_interval_since": 5541572, "same_primary_since": 5520915, "last_scrub": "4863820'2109288", "last_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "last_deep_scrub": "4808731'2109261", "last_deep_scrub_stamp": "2021-01-15T21:17:56.729962+0100", "last_clean_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "prior_readable_until_ub": 0 }, "stats": { "version": "4863820'2109288", "reported_seq": "12355046", "reported_epoch": "5541839", "state": "incomplete", "last_fresh": "2021-03-16T10:39:21.058569+0100", "last_change": "2021-03-16T10:39:21.058569+0100", "last_active": "2021-01-20T01:07:18.246158+0100", "last_peered": "2021-01-20T01:07:13.931842+0100", "last_clean": "2021-01-20T01:07:07.392736+0100", "last_became_active": "2021-01-20T01:07:09.187047+0100", "last_became_peered": "2021-01-20T01:07:09.187047+0100", "last_unstale": "2021-03-16T10:39:21.058569+0100", "last_undegraded": "2021-03-16T10:39:21.058569+0100", "last_fullsized": "2021-03-16T10:39:21.058569+0100", "mapping_epoch": 5541572, "log_start": "3881944'2103139", "ondisk_log_start": "3881944'2103139", "created": 192123, "last_epoch_clean": 4865007, "parent": "0.0", "parent_split_bits": 9, "last_scrub": "4863820'2109288", "last_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "last_deep_scrub": "4808731'2109261", "last_deep_scrub_stamp": "2021-01-15T21:17:56.729962+0100", "last_clean_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "log_size": 6149, "ondisk_log_size": 6149, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 356705195545, "num_objects": 98594, "num_object_clones": 758, "num_object_copies": 394376, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 98594, "num_whiteouts": 750, "num_read": 30767, "num_read_kb": 37109881, "num_write": 31023, "num_write_kb": 66655410, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 1000136, "num_bytes_recovered": 3631523354253, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 46, 34, 62, 74 ], "acting": [ 46, 34, 62, 74 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [ 33, 68 ], "up_primary": 46, "acting_primary": 46, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 4865266, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "34(1)", "pgid": "44.1f0s1", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 192123, "epoch_pool_created": 175799, "last_epoch_started": 4865266, "last_interval_started": 4865265, "last_epoch_clean": 4865007, "last_interval_clean": 4865006, "last_epoch_split": 192123, "last_epoch_marked_full": 0, "same_up_since": 5541572, "same_interval_since": 5541572, "same_primary_since": 5520915, "last_scrub": "4863820'2109288", "last_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "last_deep_scrub": "4808731'2109261", "last_deep_scrub_stamp": "2021-01-15T21:17:56.729962+0100", "last_clean_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": "119", "reported_epoch": "5520802", "state": "down", "last_fresh": "2021-03-14T07:03:14.383913+0100", "last_change": "2021-03-14T07:03:14.383913+0100", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "2021-03-14T07:03:14.383913+0100", "last_undegraded": "2021-03-14T07:03:14.383913+0100", "last_fullsized": "2021-03-14T07:03:14.383913+0100", "mapping_epoch": 5541572, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 192123, "last_epoch_clean": 4865007, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "4863820'2109288", "last_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "last_deep_scrub": "4808731'2109261", "last_deep_scrub_stamp": "2021-01-15T21:17:56.729962+0100", "last_clean_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 46, 34, 62, 74 ], "acting": [ 46, 34, 62, 74 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [ 33, 46, 68 ], "up_primary": 46, "acting_primary": 46, "purged_snaps": [] }, "empty": 1, "dne": 0, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "54(3)", "pgid": "44.1f0s3", "last_update": "4863820'2109288", "last_complete": "4863820'2109288", "log_tail": "3881944'2103139", "last_user_version": 11189093, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 192123, "epoch_pool_created": 175799, "last_epoch_started": 4865266, "last_interval_started": 4865265, "last_epoch_clean": 4865007, "last_interval_clean": 4865006, "last_epoch_split": 192123, "last_epoch_marked_full": 0, "same_up_since": 5541572, "same_interval_since": 5541572, "same_primary_since": 5520915, "last_scrub": "4863820'2109288", "last_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "last_deep_scrub": "4808731'2109261", "last_deep_scrub_stamp": "2021-01-15T21:17:56.729962+0100", "last_clean_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "prior_readable_until_ub": 0 }, "stats": { "version": "4863820'2109288", "reported_seq": "11820744", "reported_epoch": "5336744", "state": "down", "last_fresh": "2021-03-05T16:47:09.479395+0100", "last_change": "2021-03-05T16:45:14.475314+0100", "last_active": "2021-02-15T11:57:07.430181+0100", "last_peered": "2021-01-19T23:23:39.318842+0100", "last_clean": "2021-01-19T23:23:39.318842+0100", "last_became_active": "2021-01-18T23:18:46.888782+0100", "last_became_peered": "2021-01-18T23:18:46.888782+0100", "last_unstale": "2021-03-05T16:47:09.479395+0100", "last_undegraded": "2021-03-05T16:47:09.479395+0100", "last_fullsized": "2021-03-05T16:47:09.479395+0100", "mapping_epoch": 5541572, "log_start": "3881944'2103139", "ondisk_log_start": "3881944'2103139", "created": 192123, "last_epoch_clean": 4865007, "parent": "0.0", "parent_split_bits": 9, "last_scrub": "4863820'2109288", "last_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "last_deep_scrub": "4808731'2109261", "last_deep_scrub_stamp": "2021-01-15T21:17:56.729962+0100", "last_clean_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "log_size": 6149, "ondisk_log_size": 6149, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 356705195545, "num_objects": 98594, "num_object_clones": 758, "num_object_copies": 394376, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 98594, "num_whiteouts": 750, "num_read": 30767, "num_read_kb": 37109881, "num_write": 31023, "num_write_kb": 66655410, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 1000136, "num_bytes_recovered": 3631523354253, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 46, 34, 62, 74 ], "acting": [ 46, 34, 62, 74 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [ 33, 34, 46, 68, 74, 131 ], "up_primary": 46, "acting_primary": 46, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 4865007, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "62(2)", "pgid": "44.1f0s2", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 0, "epoch_pool_created": 0, "last_epoch_started": 0, "last_interval_started": 0, "last_epoch_clean": 0, "last_interval_clean": 0, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 0, "same_interval_since": 0, "same_primary_since": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": "0", "reported_epoch": "0", "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [], "acting": [], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1, "purged_snaps": [] }, "empty": 1, "dne": 1, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "72(2)", "pgid": "44.1f0s2", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 192123, "epoch_pool_created": 175799, "last_epoch_started": 4865266, "last_interval_started": 4865265, "last_epoch_clean": 4865007, "last_interval_clean": 4865006, "last_epoch_split": 192123, "last_epoch_marked_full": 0, "same_up_since": 5541572, "same_interval_since": 5541572, "same_primary_since": 5520915, "last_scrub": "4863820'2109288", "last_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "last_deep_scrub": "4808731'2109261", "last_deep_scrub_stamp": "2021-01-15T21:17:56.729962+0100", "last_clean_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": "130", "reported_epoch": "5520914", "state": "down", "last_fresh": "2021-03-14T07:09:19.787519+0100", "last_change": "2021-03-14T07:03:21.216689+0100", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "2021-03-14T07:09:19.787519+0100", "last_undegraded": "2021-03-14T07:09:19.787519+0100", "last_fullsized": "2021-03-14T07:09:19.787519+0100", "mapping_epoch": 5541572, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 192123, "last_epoch_clean": 4865007, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "4863820'2109288", "last_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "last_deep_scrub": "4808731'2109261", "last_deep_scrub_stamp": "2021-01-15T21:17:56.729962+0100", "last_clean_scrub_stamp": "2021-01-19T23:44:40.885414+0100", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 46, 34, 62, 74 ], "acting": [ 46, 34, 62, 74 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [ 33, 34, 46, 68 ], "up_primary": 46, "acting_primary": 46, "purged_snaps": [] }, "empty": 1, "dne": 0, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "74(2)", "pgid": "44.1f0s2", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 0, "epoch_pool_created": 0, "last_epoch_started": 0, "last_interval_started": 0, "last_epoch_clean": 0, "last_interval_clean": 0, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 0, "same_interval_since": 0, "same_primary_since": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": "0", "reported_epoch": "0", "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [], "acting": [], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1, "purged_snaps": [] }, "empty": 1, "dne": 1, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "74(3)", "pgid": "44.1f0s3", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 0, "epoch_pool_created": 0, "last_epoch_started": 0, "last_interval_started": 0, "last_epoch_clean": 0, "last_interval_clean": 0, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 0, "same_interval_since": 0, "same_primary_since": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": "0", "reported_epoch": "0", "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [], "acting": [], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1, "purged_snaps": [] }, "empty": 1, "dne": 1, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "131(2)", "pgid": "44.1f0s2", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 0, "epoch_pool_created": 0, "last_epoch_started": 0, "last_interval_started": 0, "last_epoch_clean": 0, "last_interval_clean": 0, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 0, "same_interval_since": 0, "same_primary_since": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": "0", "reported_epoch": "0", "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [], "acting": [], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1, "purged_snaps": [] }, "empty": 1, "dne": 1, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started/Primary/Peering/Incomplete", "enter_time": "2021-03-16T10:39:21.058568+0100", "comment": "not enough complete instances of this PG" }, { "name": "Started/Primary/Peering", "enter_time": "2021-03-16T10:39:21.058483+0100", "past_intervals": [ { "first": "4865006", "last": "5541571", "all_participants": [ { "osd": 33, "shard": 1 }, { "osd": 34, "shard": 1 }, { "osd": 46, "shard": 0 }, { "osd": 54, "shard": 3 }, { "osd": 68, "shard": 2 }, { "osd": 72, "shard": 2 }, { "osd": 74, "shard": 2 }, { "osd": 131, "shard": 2 } ], "intervals": [ { "first": "4865265", "last": "4865269", "acting": "33(1),46(0),68(2)" }, { "first": "4865496", "last": "4865498", "acting": "33(1),46(0),54(3)" }, { "first": "5085650", "last": "5085653", "acting": "34(1),54(3),68(2)" }, { "first": "5114555", "last": "5114581", "acting": "34(1),46(0),68(2)" }, { "first": "5132241", "last": "5132272", "acting": "34(1),46(0),54(3)" }, { "first": "5236601", "last": "5236617", "acting": "46(0),54(3),68(2)" }, { "first": "5520781", "last": "5520798", "acting": "34(1),54(3),72(2)" }, { "first": "5520915", "last": "5520919", "acting": "46(0),54(3),72(2)" }, { "first": "5520920", "last": "5541571", "acting": "34(1),46(0),54(3),72(2)" } ] } ], "probing_osds": [ "34(1)", "46(0)", "54(3)", "62(2)", "72(2)", "74(2)", "74(3)", "131(2)" ], "down_osds_we_would_probe": [ 33, 68 ], "peering_blocked_by": [], "peering_blocked_by_detail": [ { "detail": "peering_blocked_by_history_les_bound" } ] }, { "name": "Started", "enter_time": "2021-03-16T10:39:21.058461+0100" } ], "agent_state": {} } Istvan Szabo Senior Infrastructure Engineer --------------------------------------------------- Agoda Services Co., Ltd. e: istvan.szabo@agoda.com<mailto:istvan.szabo@agoda.com> --------------------------------------------------- ________________________________ This message is confidential and is for the sole use of the intended recipient(s). It may also be privileged or otherwise protected by copyright or other legal rules. If you have received it by mistake please let us know by reply email and delete it from your system. It is prohibited to copy this message or disclose its content to anyone. Any confidentiality or privilege is not waived or lost by any mistaken delivery or unauthorized disclosure of the message. All messages sent to and from Agoda may be monitored to ensure compliance with company policies, to protect the company's interests and to remove potential malware. Electronic messages may be intercepted, amended, lost or deleted, or contain viruses.