Hello, I am running into an "interesting" issue with a PG that is being flagged as inconsistent during scrub (causing the cluster to go to HEALTH_ERR), but doesn't actually appear to contain any inconsistent objects. $ ceph health detail HEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent OSD_SCRUB_ERRORS 1 scrub errors PG_DAMAGED Possible data damage: 1 pg inconsistent pg 10.10d is active+clean+inconsistent, acting [15,13] $ rados list-inconsistent-obj 10.10d {"epoch":12138,"inconsistents":[]} "ceph pg query" (see below) on that PG does report num_scrub_errors=1, num_shallow_scrub_errors=1, and num_objects_dirty=1. "osd scrub auto repair = true" is set on all OSDs, but the PG never auto-repairs. (This is a test cluster, the pool size is 2 — this may preclude auto repair from ever kicking in; I'm not sure on that one.) "ceph pg repair" does repair, but the issue reappears on the next scheduled scrub. This issue was first discovered while the cluster was on Jewel/Filestore. In an event like this I would normally suspect either a problem with an individual OSD, or a bug in the FileStore code. But the cluster has had *all* of it's OSDs replaced since, as part of a full Jewel→Luminous→Nautilus upgrade and a FileStore→BlueStore conversion. The issue still persists. A full "ceph pg 10.10d query" result is below. If anyone has ideas on how to permanently fix this issue, I'd be most grateful. Thanks! Cheers, Florian { "state": "active+clean+inconsistent", "snap_trimq": "[]", "snap_trimq_len": 0, "epoch": 12143, "up": [ 15, 13 ], "acting": [ 15, 13 ], "acting_recovery_backfill": [ "13", "15" ], "info": { "pgid": "10.10d", "last_update": "100'11", "last_complete": "100'11", "log_tail": "0'0", "last_user_version": 11, "last_backfill": "MAX", "last_backfill_bitwise": 0, "purged_snaps": [], "history": { "epoch_created": 45, "epoch_pool_created": 45, "last_epoch_started": 12139, "last_interval_started": 12138, "last_epoch_clean": 12139, "last_interval_clean": 12138, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 12138, "same_interval_since": 12138, "same_primary_since": 12114, "last_scrub": "100'11", "last_scrub_stamp": "2019-10-14 08:33:57.347097", "last_deep_scrub": "100'11", "last_deep_scrub_stamp": "2019-10-11 14:09:29.016946", "last_clean_scrub_stamp": "2019-10-11 14:09:29.016946" }, "stats": { "version": "100'11", "reported_seq": "4927", "reported_epoch": "12143", "state": "active+clean+inconsistent", "last_fresh": "2019-10-14 08:33:57.347147", "last_change": "2019-10-14 08:33:57.347147", "last_active": "2019-10-14 08:33:57.347147", "last_peered": "2019-10-14 08:33:57.347147", "last_clean": "2019-10-14 08:33:57.347147", "last_became_active": "2019-10-11 14:44:09.312226", "last_became_peered": "2019-10-11 14:44:09.312226", "last_unstale": "2019-10-14 08:33:57.347147", "last_undegraded": "2019-10-14 08:33:57.347147", "last_fullsized": "2019-10-14 08:33:57.347147", "mapping_epoch": 12138, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 45, "last_epoch_clean": 12139, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "100'11", "last_scrub_stamp": "2019-10-14 08:33:57.347097", "last_deep_scrub": "100'11", "last_deep_scrub_stamp": "2019-10-11 14:09:29.016946", "last_clean_scrub_stamp": "2019-10-11 14:09:29.016946", "log_size": 11, "ondisk_log_size": 11, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": true, "pin_stats_invalid": true, "manifest_stats_invalid": true, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 11, "num_objects": 1, "num_object_clones": 0, "num_object_copies": 2, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 1, "num_whiteouts": 0, "num_read": 33, "num_read_kb": 22, "num_write": 11, "num_write_kb": 6, "num_scrub_errors": 1, "num_shallow_scrub_errors": 1, "num_deep_scrub_errors": 0, "num_objects_recovered": 2, "num_bytes_recovered": 22, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 15, 13 ], "acting": [ 15, 13 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 15, "acting_primary": 15, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 12139, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "13", "pgid": "10.10d", "last_update": "100'11", "last_complete": "100'11", "log_tail": "0'0", "last_user_version": 11, "last_backfill": "MAX", "last_backfill_bitwise": 0, "purged_snaps": [], "history": { "epoch_created": 45, "epoch_pool_created": 45, "last_epoch_started": 12139, "last_interval_started": 12138, "last_epoch_clean": 12139, "last_interval_clean": 12138, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 12138, "same_interval_since": 12138, "same_primary_since": 12114, "last_scrub": "100'11", "last_scrub_stamp": "2019-10-14 08:33:57.347097", "last_deep_scrub": "100'11", "last_deep_scrub_stamp": "2019-10-11 14:09:29.016946", "last_clean_scrub_stamp": "2019-10-11 14:09:29.016946" }, "stats": { "version": "100'11", "reported_seq": "36", "reported_epoch": "12113", "state": "active+undersized+degraded", "last_fresh": "2019-10-11 14:39:58.946532", "last_change": "2019-10-11 14:39:58.924989", "last_active": "2019-10-11 14:39:58.946532", "last_peered": "2019-10-11 14:39:58.946532", "last_clean": "2014-11-05 15:48:35.131248", "last_became_active": "2019-10-11 14:39:58.924989", "last_became_peered": "2019-10-11 14:39:58.924989", "last_unstale": "2019-10-11 14:39:58.946532", "last_undegraded": "2019-10-11 14:39:58.892352", "last_fullsized": "2019-10-11 14:39:58.892331", "mapping_epoch": 12138, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 45, "last_epoch_clean": 12103, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "100'11", "last_scrub_stamp": "2019-10-11 14:09:29.016946", "last_deep_scrub": "100'11", "last_deep_scrub_stamp": "2019-10-11 14:09:29.016946", "last_clean_scrub_stamp": "2019-10-11 14:09:29.016946", "log_size": 11, "ondisk_log_size": 11, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": true, "pin_stats_invalid": true, "manifest_stats_invalid": true, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 11, "num_objects": 1, "num_object_clones": 0, "num_object_copies": 2, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 1, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 1, "num_whiteouts": 0, "num_read": 33, "num_read_kb": 22, "num_write": 11, "num_write_kb": 6, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 2, "num_bytes_recovered": 22, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 15, 13 ], "acting": [ 15, 13 ], "avail_no_missing": [ "13" ], "object_location_counts": [ { "shards": "13", "objects": 1 } ], "blocked_by": [], "up_primary": 15, "acting_primary": 15, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 12139, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started/Primary/Active", "enter_time": "2019-10-11 14:44:09.175574", "might_have_unfound": [], "recovery_progress": { "backfill_targets": [], "waiting_on_backfill": [], "last_backfill_started": "MIN", "backfill_info": { "begin": "MIN", "end": "MIN", "objects": [] }, "peer_backfill_info": [], "backfills_in_flight": [], "recovering": [], "pg_backend": { "pull_from_peer": [], "pushing": [] } }, "scrub": { "scrubber.epoch_start": "12138", "scrubber.active": false, "scrubber.state": "INACTIVE", "scrubber.start": "MIN", "scrubber.end": "MIN", "scrubber.max_end": "MIN", "scrubber.subset_last_update": "0'0", "scrubber.deep": false, "scrubber.waiting_on_whom": [] } }, { "name": "Started", "enter_time": "2019-10-11 14:44:08.833757" } ], "agent_state": {} }