February 2024 - ceph-users

by LeonGao

Hi community We have a new Ceph cluster deployment with 100 nodes. When we are draining an OSD host from the cluster, we see a small amount of PGs that cannot make any progress to the end. From the logs and metrics, it seems like the recovery progress is stuck (0 recovery ops for several days). Would like to get some ideas on this. Re-peering and OSD restart do resolve to mitigate the issue but we want to get to the root cause of it as draining and recovery happen frequently. I have put some debugging information below. Any help is appreciated, thanks! ceph -s pgs: 4210926/7380034104 objects misplaced (0.057%) 41198 active+clean 71 active+remapped+backfilling 12 active+recovering One of the stuck PG: 6.38f1 active+remapped+backfilling [313,643,727] 313 [313,643,717] 313 PG query result: ceph pg 6.38f1 query { "snap_trimq": "[]", "snap_trimq_len": 0, "state": "active+remapped+backfilling", "epoch": 246856, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "backfill_targets": [ "727" ], "acting_recovery_backfill": [ "313", "643", "717", "727" ], "info": { "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 38833, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 243987, "last_interval_started": 243986, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "212333'38916", "reported_seq": 413425, "reported_epoch": 246856, "state": "active+remapped+backfilling", "last_fresh": "2024-02-05T21:14:40.838785+0000", "last_change": "2024-02-03T22:33:43.052272+0000", "last_active": "2024-02-05T21:14:40.838785+0000", "last_peered": "2024-02-05T21:14:40.838785+0000", "last_clean": "2024-02-03T04:26:35.168232+0000", "last_became_active": "2024-02-03T22:31:16.037823+0000", "last_became_peered": "2024-02-03T22:31:16.037823+0000", "last_unstale": "2024-02-05T21:14:40.838785+0000", "last_undegraded": "2024-02-05T21:14:40.838785+0000", "last_fullsized": "2024-02-05T21:14:40.838785+0000", "mapping_epoch": 243986, "log_start": "80608'37589", "ondisk_log_start": "80608'37589", "created": 3726, "last_epoch_clean": 220174, "parent": "0.0", "parent_split_bits": 14, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "objects_scrubbed": 17743, "log_size": 1327, "log_dups_size": 3000, "ondisk_log_size": 1327, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 29, "scrub_schedule": "queued for deep scrub", "scrub_duration": 28.699844475999999, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 67167022618, "num_objects": 17743, "num_object_clones": 0, "num_object_copies": 53229, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 12401, "num_objects_unfound": 0, "num_objects_dirty": 17743, "num_whiteouts": 0, "num_read": 16640, "num_read_kb": 30923655, "num_write": 55094, "num_write_kb": 71824558, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 40746, "num_bytes_recovered": 154245233300, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [ "313", "643", "717" ], "object_location_counts": [ { "shards": "313,643,717", "objects": 17743 } ], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 243987, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "643", "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 38833, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 243987, "last_interval_started": 243986, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "212331'38915", "reported_seq": 362849, "reported_epoch": 212333, "state": "active+clean+scrubbing+deep", "last_fresh": "2024-01-28T07:43:40.515729+0000", "last_change": "2024-01-28T06:20:14.969313+0000", "last_active": "2024-01-28T07:43:40.515729+0000", "last_peered": "2024-01-28T07:43:40.515729+0000", "last_clean": "2024-01-28T07:43:40.515729+0000", "last_became_active": "2024-01-23T10:01:07.710478+0000", "last_became_peered": "2024-01-23T10:01:07.710478+0000", "last_unstale": "2024-01-28T07:43:40.515729+0000", "last_undegraded": "2024-01-28T07:43:40.515729+0000", "last_fullsized": "2024-01-28T07:43:40.515729+0000", "mapping_epoch": 243986, "log_start": "80608'37589", "ondisk_log_start": "80608'37589", "created": 3726, "last_epoch_clean": 123203, "parent": "0.0", "parent_split_bits": 14, "last_scrub": "80609'38697", "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", "last_deep_scrub": "80609'38697", "last_deep_scrub_stamp": "2024-01-07T08:20:59.246940+0000", "last_clean_scrub_stamp": "2024-01-09T20:27:40.795531+0000", "objects_scrubbed": 17724, "log_size": 1326, "log_dups_size": 3000, "ondisk_log_size": 1326, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 29, "scrub_schedule": "deep scrubbing for 5005s", "scrub_duration": 28.446739284, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 67167022618, "num_objects": 17743, "num_object_clones": 0, "num_object_copies": 53229, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 17743, "num_whiteouts": 0, "num_read": 16640, "num_read_kb": 30923655, "num_write": 55094, "num_write_kb": 71824558, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 35403, "num_bytes_recovered": 134021819371, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 243987, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "717", "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 243987, "last_interval_started": 243986, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "212331'38915", "reported_seq": 362849, "reported_epoch": 212333, "state": "active+clean+scrubbing+deep", "last_fresh": "2024-01-28T07:43:40.515729+0000", "last_change": "2024-01-28T06:20:14.969313+0000", "last_active": "2024-01-28T07:43:40.515729+0000", "last_peered": "2024-01-28T07:43:40.515729+0000", "last_clean": "2024-01-28T07:43:40.515729+0000", "last_became_active": "2024-01-23T10:01:07.710478+0000", "last_became_peered": "2024-01-23T10:01:07.710478+0000", "last_unstale": "2024-01-28T07:43:40.515729+0000", "last_undegraded": "2024-01-28T07:43:40.515729+0000", "last_fullsized": "2024-01-28T07:43:40.515729+0000", "mapping_epoch": 243986, "log_start": "80608'37589", "ondisk_log_start": "80608'37589", "created": 3726, "last_epoch_clean": 123203, "parent": "0.0", "parent_split_bits": 14, "last_scrub": "80609'38697", "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", "last_deep_scrub": "80609'38697", "last_deep_scrub_stamp": "2024-01-07T08:20:59.246940+0000", "last_clean_scrub_stamp": "2024-01-09T20:27:40.795531+0000", "objects_scrubbed": 17724, "log_size": 1326, "log_dups_size": 3000, "ondisk_log_size": 1326, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 29, "scrub_schedule": "deep scrubbing for 5005s", "scrub_duration": 28.446739284, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 67167022618, "num_objects": 17743, "num_object_clones": 0, "num_object_copies": 53229, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 17743, "num_whiteouts": 0, "num_read": 16640, "num_read_kb": 30923655, "num_write": 55094, "num_write_kb": 71824558, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 35403, "num_bytes_recovered": 134021819371, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 0, "last_epoch_started": 243987, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "726", "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 0, "last_backfill": "MIN", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 238348, "last_interval_started": 238347, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 11.834917336 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 243986, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 4194304, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 17743, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 238320, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "727", "pgid": "6.38f1", "last_update": "212333'38916", "last_complete": "212333'38916", "log_tail": "80608'37589", "last_user_version": 0, "last_backfill": "6:8f1d3293:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.V-JlvDFjbQNgn8GtBLN2LePecv5fLKf_3:head", "purged_snaps": [], "history": { "epoch_created": 3726, "epoch_pool_created": 3279, "last_epoch_started": 243987, "last_interval_started": 243986, "last_epoch_clean": 220174, "last_interval_clean": 220173, "last_epoch_split": 3726, "last_epoch_marked_full": 0, "same_up_since": 238347, "same_interval_since": 243986, "same_primary_since": 3728, "last_scrub": "212333'38916", "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "last_deep_scrub": "212333'38916", "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 243986, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "log_dups_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 20227608233, "num_objects": 5342, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 12401, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 5342, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 313, 643, 727 ], "acting": [ 313, 643, 717 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 313, "acting_primary": 313, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 243987, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started/Primary/Active", "enter_time": "2024-02-03T22:31:15.023323+0000", "might_have_unfound": [], "recovery_progress": { "backfill_targets": [ "727" ], "waiting_on_backfill": [], "last_backfill_started": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", "backfill_info": { "begin": "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", "end": "6:8f1d3549:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.PTxEYjlIXk2vkWpRgKUOu4ygweU-Jqy_5:head", "objects": [ { "object": "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", "version": "80588'26367" }, { "object": "6:8f1d32b0:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_18-71435:head", "version": "80588'26368" }, { "object": "6:8f1d32c2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.o_s0NGR7mrH-4JKpvYS6Ip14hbm4w5o_5:head", "version": "80588'26369" }, { "object": "6:8f1d32c3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WV1Ja5V21BKfmI5e1W90a7iQXdcTnSG_6:head", "version": "80588'26370" }, { "object": "6:8f1d32c4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.96tkpnCzKmtnPTLewpai2cXSr_z_QcR_1:head", "version": "80588'26371" }, { "object": "6:8f1d331d:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.InACTSlwV7irTe0g9cLnadT8EbQr4Cl_5:head", "version": "80588'26372" }, { "object": "6:8f1d3326:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_233-155987:head", "version": "80588'26373" }, { "object": "6:8f1d3332:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.5aipEOj0vV3h4jdSf6tkAUn64yJmvoH_1:head", "version": "80588'26374" }, { "object": "6:8f1d3343:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QTzZJhDV6JLC5wJ5BDxHvK4sfp3Hr6o_6:head", "version": "80588'26375" }, { "object": "6:8f1d334b:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QgMCmHWV87HlAjqkfvbO9X5OUZrgIPc_2:head", "version": "80588'26376" }, { "object": "6:8f1d3355:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.H4j6DWgQa0jsrl-xzgugsW22GHCOgGL_7:head", "version": "80588'26377" }, { "object": "6:8f1d3379:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.25g7c0w-VnGqX-z_m53YmnoWUO0R56N_5:head", "version": "80588'26378" }, { "object": "6:8f1d337c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WMmV4BoLMm4q6BztoZ1uPc4boiJ-zhW_1:head", "version": "80588'26379" }, { "object": "6:8f1d3385:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_184-108817:head", "version": "80588'26380" }, { "object": "6:8f1d3389:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sGNiGV9K-TGyTYRqWSOcSIXkBBOT93X_3:head", "version": "80588'26381" }, { "object": "6:8f1d339c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.NkMm_dhUxRyxXhBDYqGCuOaSvwNvSuJ_3:head", "version": "80588'26382" }, { "object": "6:8f1d33a3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.BV__z8jIWqcjMsTf40MvaARU1dVtCv2_4:head", "version": "80588'26383" }, { "object": "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.XJLSyRSnrDupnOjWzHKc4wykPJjqR5g_1:head", "version": "80588'26384" }, { "object": "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.oYSnMakOIpFmfLdym2oCx-fLBDCLDmr_4:head", "version": "80588'26385" }, { "object": "6:8f1d33b4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_135-2779:head", "version": "80588'26386" }, { "object": "6:8f1d33b6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sEHqthyEGg3FQP4-vdL9zkhSJmQEaIc_2:head", "version": "80588'26387" }, { "object": "6:8f1d33ec:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.wL36lNEfHGE6B4V-YT6fMuAI-n2uV30_1:head", "version": "80588'26388" }, { "object": "6:8f1d33f3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.j1UloV7DpRwG0-4AKULi3CXOTC0CrdU_4:head", "version": "80588'26389" }, { "object": "6:8f1d33f5:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.q3ExUB7Mp-LggBIsRBBKyIob574rNpG_3:head", "version": "80588'26390" }, { "object": "6:8f1d33f9:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_72-13403:head", "version": "80588'26391" }, { "object": "6:8f1d33fd:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_157-172390:head", "version": "80588'26392" }, { "object": "6:8f1d3403:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Wn4ed97PvQp_oDnk_Y4xlZMke0pQ4z1_3:head", "version": "80588'26393" }, { "object": "6:8f1d3405:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.uFg6eFSD3BCEqzJLCZlw546nDqqg80U_4:head", "version": "80588'26394" }, { "object": "6:8f1d3436:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xPwN1sJsNFAg2Nr_Vnu4ItFgHFyu_Nx_7:head", "version": "80588'26395" }, { "object": "6:8f1d3438:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Qgd9EXWAraBWnYcZ3OhFvepsT8r0Ksi_1:head", "version": "80588'26396" }, { "object": "6:8f1d3464:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.1KnQBzx5u-CkXROQVUzUPfJiuTq52zH_3:head", "version": "80588'26397" }, { "object": "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Aw2QRjM0ZXkf20MtQlhyq31hwVMEWwe_1:head", "version": "80588'26398" }, { "object": "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xprsKnjtYu1ITOsgDtkPup9Nk6xzGTN_6:head", "version": "80588'26399" }, { "object": "6:8f1d3475:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.jo4-jEQV1kB9kHewUYdbU1UUNX60EUm_1:head", "version": "80588'26400" }, { "object": "6:8f1d3488:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.f9EE8lnu2FzW1_vGo9J41KY-iFpI9-f_1:head", "version": "80588'26401" }, { "object": "6:8f1d3491:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_11-138415:head", "version": "80588'26402" }, { "object": "6:8f1d34ac:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_33-123891:head", "version": "80588'26403" }, { "object": "6:8f1d34ad:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.x3tIcVCAeRnOJmmHhJ2hxCjsUZ3GBsK_3:head", "version": "80588'26404" }, { "object": "6:8f1d34bc:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.9yQELiJhCNs5Z-foJHH3vEfQ_1TZdBF_1:head", "version": "80588'26405" }, { "object": "6:8f1d34d1:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Me9Xl2_j9z2_yE1laHWpHdJ2fXvwZDZ_1:head", "version": "80588'26406" }, { "object": "6:8f1d34fa:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Y5VD6lU4UqjXEh5Yb9GVwusKpsqGzuG_4:head", "version": "80588'26407" }, { "object": "6:8f1d3503:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_58-178651:head", "version": "80588'26408" }, { "object": "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.FFyoywHasB0i44Y9ZtCi_YgtYhm8cP6_6:head", "version": "80588'26409" }, { "object": "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.lHmY-Z-Q3Ewx7kHwCoLFoFU70Xtrgzu_1:head", "version": "80588'26410" }, { "object": "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.ETPvUyLCwz5Bfmh22s_odAwEIMCC_Vn_2:head", "version": "80588'26411" }, { "object": "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.EdYbzDvA-7HpfrM0DMfHRgalz8ZiLhJ_1:head", "version": "80588'26412" }, { "object": "6:8f1d3529:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1186045.1_164-1500436:head", "version": "80588'26413" } ] }, "peer_backfill_info": [ "727", { "begin": "MAX", "end": "MAX", "objects": [] } ], "backfills_in_flight": [ "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" ], "recovering": [ "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" ], "pg_backend": { "pull_from_peer": [], "pushing": [ { "pushing": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", "pushing_to": [ { "pushing_to": "727", "push_info": { "recovery_progress": { "first?": 0, "data_complete?": 1, "data_recovered_to": 4194304, "omap_complete?": 1, "omap_recovered_to": "" }, "recovery_info": { "object": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", "at_version": "80588'26366", "size": "4194304", "object_info": { "oid": { "oid": "d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605", "key": "", "snapid": -2, "hash": 4182554865, "max": 0, "pool": 6, "namespace": "" }, "version": "80588'26366", "prior_version": "6896'10987", "last_reqid": "osd.313.0:4380013", "user_version": 10987, "size": 4194304, "mtime": "2023-12-19T07:17:04.766167+0000", "local_mtime": "2023-12-19T07:17:04.799712+0000", "lost": 0, "flags": [ "dirty", "data_digest", "omap_digest" ], "truncate_seq": 0, "truncate_size": 0, "data_digest": "0x9cdfa22f", "omap_digest": "0xffffffff", "expected_object_size": 0, "expected_write_size": 0, "alloc_hint_flags": 0, "manifest": { "type": 0 }, "watchers": {} }, "snapset": { "seq": 0, "clones": [] }, "copy_subset": "[0~4194304]", "clone_subset": "{}", "object_exist": "0" } } } ] } ] } } }, { "name": "Started", "enter_time": "2024-02-03T22:31:13.882074+0000" } ], "scrubber": { "active": false, "must_scrub": false, "must_deep_scrub": false, "must_repair": false, "need_auto": false, "scrub_reg_stamp": "2024-01-30T15:39:19.044093+0000", "schedule": "queued for deep scrub" }, "agent_state": {} }

2 months, 3 weeks

5
7
0 0

list topic shows endpoint url and username e password

by Giada Malatesta

Hello everyone, we are facing a problem regarding the topic operations to send notification, particularly when using amqp protocol. We are using Ceph version 18.2.1. We have created a topic by giving as attributes all needed information and so the push-endpoint (in our case a rabbit endpoint that is used to collect notification messages). Then we have configured all the buckets in our cluster Ceph so that it is possible to send notification when some changes occur. The problem regards particularly the list_topic operation: we noticed that any authenticated user is able to get a full list of the created topics and with them to get all the information, including endpoint, and so username and password and IP and port, when using the boto3.set_stream_logger(), which is not good for our goal since we do not want the users to know implementation details. There is the possibility to solve this problem? Any help would be useful. Thanks and best regards. GM.

2 months, 3 weeks

2
1
0 0

MDS in ReadOnly and 2 MDS behind on trimming

by Edouard FAZENDA

Dear Ceph Community, I am having an issue with my Ceph Cluster , there were several osd crashing but now active and recovery finished and now the CephFS filesystem cannot be access by clients in RW (K8S worklod) as the 1 MDS is in Read-Only and 2 are being on trimming The cephfs seems to have volume OK The trimming process seems not going further, maybe stuck ? We are running 3 hosts using ceph Pacific version 16.2.1 Here some logs on the situation : ceph versions { "mon": { "ceph version 16.2.1 (afb9061ab4117f798c858c741efa6390e48ccf10) pacific (stable)": 3 }, "mgr": { "ceph version 16.2.1 (afb9061ab4117f798c858c741efa6390e48ccf10) pacific (stable)": 3 }, "osd": { "ceph version 16.2.1 (afb9061ab4117f798c858c741efa6390e48ccf10) pacific (stable)": 18 }, "mds": { "ceph version 16.2.1 (afb9061ab4117f798c858c741efa6390e48ccf10) pacific (stable)": 3 }, "rgw": { "ceph version 16.2.1 (afb9061ab4117f798c858c741efa6390e48ccf10) pacific (stable)": 6 }, "overall": { "ceph version 16.2.1 (afb9061ab4117f798c858c741efa6390e48ccf10) pacific (stable)": 33 } } ceph orch ps NAME HOST STATUS REFRESHED AGE PORTS VERSION IMAGE ID CONTAINER ID crash.rke-sh1-1 rke-sh1-1 running (21h) 36s ago 21h - 16.2.1 c757e4a3636b e8652edb2b49 crash.rke-sh1-2 rke-sh1-2 running (21h) 3m ago 20M - 16.2.1 c757e4a3636b a1249a605ee0 crash.rke-sh1-3 rke-sh1-3 running (17h) 36s ago 17h - 16.2.1 c757e4a3636b 026667bc1776 mds.cephfs.rke-sh1-1.ojmpnk rke-sh1-1 running (18h) 36s ago 4M - 16.2.1 c757e4a3636b 9b4c2b08b759 mds.cephfs.rke-sh1-2.isqjza rke-sh1-2 running (18h) 3m ago 23M - 16.2.1 c757e4a3636b 71681a5f34d3 mds.cephfs.rke-sh1-3.vdicdn rke-sh1-3 running (17h) 36s ago 3M - 16.2.1 c757e4a3636b e89946ad6b7e mgr.rke-sh1-1.qskoyj rke-sh1-1 running (21h) 36s ago 2y *:8082 *:9283 16.2.1 c757e4a3636b 7ce7cfbb3e55 mgr.rke-sh1-2.lxmguj rke-sh1-2 running (21h) 3m ago 22M *:8082 *:9283 16.2.1 c757e4a3636b 5a0025adfd46 mgr.rke-sh1-3.ckunvo rke-sh1-3 running (17h) 36s ago 6M *:8082 *:9283 16.2.1 c757e4a3636b 2fcaf18f3218 mon.rke-sh1-1 rke-sh1-1 running (20h) 36s ago 20h - 16.2.1 c757e4a3636b c0a90103cabc mon.rke-sh1-2 rke-sh1-2 running (21h) 3m ago 3M - 16.2.1 c757e4a3636b f4b32ba4466b mon.rke-sh1-3 rke-sh1-3 running (17h) 36s ago 17h - 16.2.1 c757e4a3636b d5e44c245998 osd.0 rke-sh1-2 running (20h) 3m ago 2y - 16.2.1 c757e4a3636b 7b0e69942c15 osd.1 rke-sh1-3 running (17h) 36s ago 2y - 16.2.1 c757e4a3636b 4451654d9a2d osd.10 rke-sh1-3 running (17h) 36s ago 2y - 16.2.1 c757e4a3636b 3f9d5f95e284 osd.11 rke-sh1-1 running (21h) 36s ago 2y - 16.2.1 c757e4a3636b db1cc6d2e37f osd.12 rke-sh1-2 running (21h) 3m ago 2y - 16.2.1 c757e4a3636b de416c1ef766 osd.13 rke-sh1-3 running (17h) 36s ago 2y - 16.2.1 c757e4a3636b 25a281cc5a9b osd.14 rke-sh1-1 running (21h) 36s ago 2y - 16.2.1 c757e4a3636b 62f25ba61667 osd.15 rke-sh1-2 running (21h) 3m ago 2y - 16.2.1 c757e4a3636b d3514d823c45 osd.16 rke-sh1-3 running (17h) 36s ago 2y - 16.2.1 c757e4a3636b bba857759bfe osd.17 rke-sh1-1 running (21h) 36s ago 2y - 16.2.1 c757e4a3636b 59281d4bb3d0 osd.2 rke-sh1-1 running (21h) 36s ago 2y - 16.2.1 c757e4a3636b 418041b5e60d osd.3 rke-sh1-2 running (21h) 3m ago 2y - 16.2.1 c757e4a3636b 04a0e29d5623 osd.4 rke-sh1-1 running (20h) 36s ago 2y - 16.2.1 c757e4a3636b 1cc78a5153d3 osd.5 rke-sh1-3 running (17h) 36s ago 2y - 16.2.1 c757e4a3636b 39a4b11e31fb osd.6 rke-sh1-2 running (21h) 3m ago 2y - 16.2.1 c757e4a3636b 2f218ffb566e osd.7 rke-sh1-1 running (20h) 36s ago 2y - 16.2.1 c757e4a3636b cf761fbe4d5f osd.8 rke-sh1-3 running (17h) 36s ago 2y - 16.2.1 c757e4a3636b f9f85480e800 osd.9 rke-sh1-2 running (21h) 3m ago 2y - 16.2.1 c757e4a3636b 664c54ff46d2 rgw.default.rke-sh1-1.dgucwl rke-sh1-1 running (21h) 36s ago 22M *:8000 16.2.1 c757e4a3636b f03212b955a7 rgw.default.rke-sh1-1.vylchc rke-sh1-1 running (21h) 36s ago 22M *:8001 16.2.1 c757e4a3636b da486ce43fe5 rgw.default.rke-sh1-2.dfhhfw rke-sh1-2 running (21h) 3m ago 2y *:8000 16.2.1 c757e4a3636b ef4089d0aef2 rgw.default.rke-sh1-2.efkbum rke-sh1-2 running (21h) 3m ago 2y *:8001 16.2.1 c757e4a3636b 9e053d5a2f7b rgw.default.rke-sh1-3.krfgey rke-sh1-3 running (17h) 36s ago 9M *:8001 16.2.1 c757e4a3636b 45cd3d75edd3 rgw.default.rke-sh1-3.pwdbmp rke-sh1-3 running (17h) 36s ago 9M *:8000 16.2.1 c757e4a3636b e2710265a7f4 ceph health detail HEALTH_WARN 1 MDSs are read only; 2 MDSs behind on trimming [WRN] MDS_READ_ONLY: 1 MDSs are read only mds.cephfs.rke-sh1-2.isqjza(mds.0): MDS in read-only mode [WRN] MDS_TRIM: 2 MDSs behind on trimming mds.cephfs.rke-sh1-2.isqjza(mds.0): Behind on trimming (2149/128) max_segments: 128, num_segments: 2149 mds.cephfs.rke-sh1-1.ojmpnk(mds.0): Behind on trimming (2149/128) max_segments: 128, num_segments: 2149 root@rke-sh1-1:~# ceph fs status cephfs - 27 clients ====== RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active cephfs.rke-sh1-2.isqjza Reqs: 8 /s 85.2k 53.2k 1742 101 0-s standby-replay cephfs.rke-sh1-1.ojmpnk Evts: 0 /s 52.2k 20.2k 1737 0 POOL TYPE USED AVAIL cephfs_metadata metadata 1109G 6082G cephfs_data data 8419G 6082G STANDBY MDS cephfs.rke-sh1-3.vdicdn MDS version: ceph version 16.2.1 (afb9061ab4117f798c858c741efa6390e48ccf10) pacific (stable) ceph status cluster: id: fcb373ce-7aaa-11eb-984f-e7c6e0038e87 health: HEALTH_WARN 1 MDSs are read only 2 MDSs behind on trimming services: mon: 3 daemons, quorum rke-sh1-2,rke-sh1-1,rke-sh1-3 (age 17h) mgr: rke-sh1-1.qskoyj(active, since 17h), standbys: rke-sh1-2.lxmguj, rke-sh1-3.ckunvo mds: 1/1 daemons up, 1 standby, 1 hot standby osd: 18 osds: 18 up (since 17h), 18 in (since 20h) rgw: 6 daemons active (3 hosts, 1 zones) data: volumes: 1/1 healthy pools: 11 pools, 849 pgs objects: 10.10M objects, 5.3 TiB usage: 11 TiB used, 15 TiB / 26 TiB avail pgs: 849 active+clean io: client: 35 KiB/s rd, 1.0 MiB/s wr, 302 op/s rd, 165 op/s wr # ceph mds stat cephfs:1 {0=cephfs.rke-sh1-2.isqjza=up:active} 1 up:standby-replay 1 up:standby Have you got an idea on what could be my next steps to bring the cluster healthy ? Help will very be appreciated. Thank a lot for your feedback. Best Regards, Edouard FAZENDA Technical Support Chemin du Curé-Desclouds 2, CH-1226 THONEX +41 (0)22 869 04 40 <https://www.csti.ch/> www.csti.ch

2 months, 3 weeks

2
4
0 0

Query Regarding Calculating Ingress/Egress Traffic for Buckets via API

by asad.siddiqui＠rapidcompute.com

Hi, I am currently working on Ceph object storage and would like to inquire about how we can calculate the ingress and egress traffic for buckets/tenant via API.

2 months, 3 weeks

2
1
0 0

Issue with Setting Public/Private Permissions for Bucket

by asad.siddiqui＠rapidcompute.com

Hi Team, I'm currently working with Ceph object storage and would like to understand how to set permissions to private or public on buckets/objects in Ceph object storage.

2 months, 3 weeks

2
1
0 0

Re: cephadm purge cluster does not work

by Eugen Block

It works for me on 17.2.6 as well. Could you be more specific what doesn't work for you? Running that command only removes the cluster configs etc. on that host, it does not orchestrate a removal on all hosts, not sure if you're aware of that. Zitat von Vahideh Alinouri <vahideh.alinouri(a)gmail.com>: > The version that has been installed is 17.2.5. But this method does not > work at all. > > On Fri, Feb 23, 2024, 10:23 AM Eugen Block <eblock(a)nde.ag> wrote: > >> Which ceph version is this? In a small Reef test cluster this works as >> expected: >> >> # cephadm rm-cluster --fsid 2851404a-d09a-11ee-9aaa-fa163e2de51a >> --zap-osds --force >> Using recent ceph image >> >> registry.cloud.hh.nde.ag/ebl/ceph-upstream@sha256:057e08bf8d2d20742173a571bc28b65674b055bebe5f4c6cd488c1a6fd51f685 >> Zapping /dev/sdb... >> Zapping /dev/sdc... >> Zapping /dev/sdd... >> >> and lsblk shows empty drives. >> >> Zitat von Vahideh Alinouri <vahideh.alinouri(a)gmail.com>: >> >> > Hi Guys, >> > >> > I faced an issue. When I wanted to purge, the cluster was not purged >> > using the below command: >> > >> > ceph mgr module disable cephadm >> > cephadm rm-cluster --force --zap-osds --fsid <fsid> >> > >> > The OSDs will remain. There should be some cleanup methods for the >> > whole cluster, not just MON nodes. Is there anything related to this? >> > >> > Regards >> > _______________________________________________ >> > ceph-users mailing list -- ceph-users(a)ceph.io >> > To unsubscribe send an email to ceph-users-leave(a)ceph.io >> >> >> _______________________________________________ >> ceph-users mailing list -- ceph-users(a)ceph.io >> To unsubscribe send an email to ceph-users-leave(a)ceph.io >>

2 months, 3 weeks

1
0
0 0

cephadm purge cluster does not work

by Vahideh Alinouri

Hi Guys, I faced an issue. When I wanted to purge, the cluster was not purged using the below command: ceph mgr module disable cephadm cephadm rm-cluster --force --zap-osds --fsid <fsid> The OSDs will remain. There should be some cleanup methods for the whole cluster, not just MON nodes. Is there anything related to this? Regards

2 months, 3 weeks

2
1
0 0

Re-linking subdirectories with root inodes in CephFS

by caskd

Hello Ceph users, we've had a incident with CephFS recently which resulted in the MDSs crashing for one of the filesystems. First it was a journal corruption which was easily recoverable with no real damage but following that the MDSs fail to start due to a crash related to snapshots. -10> 2024-02-18T20:03:51.656+0000 7f08725bfb38 1 mds.0.234010 handle_mds_map state change up:rejoin --> up:active -9> 2024-02-18T20:03:51.656+0000 7f08725bfb38 1 mds.0.234010 recovery_done -- successful recovery! -8> 2024-02-18T20:03:51.656+0000 7f087271fb38 10 monclient: get_auth_request con 0x7f0871836380 auth_method 0 -7> 2024-02-18T20:03:51.656+0000 7f08726d5b38 10 monclient: get_auth_request con 0x7f08718353c0 auth_method 0 -6> 2024-02-18T20:03:51.656+0000 7f08726fab38 10 monclient: get_auth_request con 0x7f0871895280 auth_method 0 -5> 2024-02-18T20:03:51.656+0000 7f08725bfb38 1 mds.0.234010 active_start -4> 2024-02-18T20:03:51.656+0000 7f08725bfb38 1 mds.0.cache dump_cache to cachedump.234013.mds0 -3> 2024-02-18T20:03:51.736+0000 7f08725bfb38 1 mds.0.234010 cluster recovered. -2> 2024-02-18T20:03:51.736+0000 7f08725bfb38 4 mds.0.234010 set_osd_epoch_barrier: epoch=75012 -1> 2024-02-18T20:03:51.736+0000 7f08722f9b38 -1 /home/buildozer/aports/community/ceph18/src/ceph-18.2.1/src/mds/MDCache.cc: In function 'void MDCache::journal_cow_dentry(MutationImpl*, EMetaBlob*, CDentry*, snapid_t, CInode**, CDentry::linkage_t*)' thread 7f08722f9b38 time 2024-02-18T20:03:51.747600+0000 /home/buildozer/aports/community/ceph18/src/ceph-18.2.1/src/mds/MDCache.cc: 1638: FAILED ceph_assert(follows >= realm->get_newest_seq()) ceph version 18.2.1 (e3fce6809130d78ac0058fc87e537ecd926cd213) reef (stable) 0> 2024-02-18T20:03:51.736+0000 7f08722f9b38 -1 *** Caught signal (Aborted) ** in thread 7f08722f9b38 thread_name:MR_Finisher ceph version 18.2.1 (e3fce6809130d78ac0058fc87e537ecd926cd213) reef (stable) We've sadly attempted to restore the root, which made the entire filesystem tree stray. 1. Is there a way to re-link a directory as the root (or dentry of root) manually? 2. Could snapshots of objects be purged completly manually? Thanks in advance. -- Alex D. RedXen System & Infrastructure Administration https://redxen.eu/

2 months, 3 weeks

1
3
0 0

FS down - mds degraded

by Sake Ceph

Starting a new thread, forgot subject in the previous. So our FS down. Got the following error, what can I do? # ceph health detail HEALTH_ERR 1 filesystem is degraded; 1 mds daemon damaged [WRN] FS_DEGRADED: 1 filesystem is degraded fs atlassian/prod is degraded [ERR] MDS_DAMAGE: 1 mds daemon damaged fs atlassian-prod mds.1 is damaged # ceph fs get atlassian-prod Filesystem 'atlassian-prod' (2) fs_name atlassian-prod epoch 43440 flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay created 2023-05-10T08:45:46.911064+0000 modified 2023-12-21T06:47:19.291154+0000 tableserver 0 root 0 session_timeout 60 session_autoclose 300 max_file_size 1099511627776 required_client_features {} last_failure 0 last_failure_osd_epoch 29480 compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} max_mds 3 in 0,1,2 up {0=1073573,2=1073583} failed damaged 1 stopped data_pools [5] metadata_pool 4 inline_data disabled balancer standby_count_wanted 1 [mds.atlassian-prod.pwsoel13142.egsdfl{0:1073573} state up:resolve seq 573 join_fscid=2 addr [v2:10.233.127.22:6800/61692284,v1:10.233.127.22:6801/61692284] compat {c=[1],r=[1],i=[7ff]}] [mds.atlassian-prod.pwsoel13143.qlvypn{2:1073583} state up:resolve seq 571 join_fscid=2 addr [v2:10.233.127.18:6800/3627858294,v1:10.233.127.18:6801/3627858294] compat {c=[1],r=[1],i=[7ff]}] Best regards, Sake

2 months, 3 weeks

4
7
0 0

High IO utilization for bstore_kv_sync

by Work Ceph

Hello guys, We are running Ceph Octopus on Ubuntu 18.04, and we are noticing spikes of IO utilization for bstore_kv_sync thread during processes such as adding a new pool and increasing/reducing the number of PGs in a pool. It is funny though that the IO utilization (reported with IOTOP) is 99.99%, but the reading for R/W speeds are slow. The devices where we are seeing these issues are all SSDs systems. We are not using high end SSDs devices though. Have you guys seen such behavior? Also, do you guys have any clues on why the IO utilization would be high, when there is such a small amount of data being read and written to the OSD/disks?

2 months, 3 weeks

3
4
0 0

2024

2023

2022

2021

2020

2019

ceph-users February 2024