Here you have. Perf dump: { "AsyncMessenger::Worker-0": { "msgr_recv_messages": 12239872, "msgr_send_messages": 12284221, "msgr_recv_bytes": 43759275160, "msgr_send_bytes": 61268769426, "msgr_created_connections": 754, "msgr_active_connections": 100, "msgr_running_total_time": 939.476931816, "msgr_running_send_time": 337.873686715, "msgr_running_recv_time": 360.728238752, "msgr_running_fast_dispatch_time": 183.737116872, "msgr_send_messages_queue_lat": { "avgcount": 12284206, "sum": 1538.989479364, "avgtime": 0.000125281 }, "msgr_handle_ack_lat": { "avgcount": 5258403, "sum": 1.005075918, "avgtime": 0.000000191 } }, "AsyncMessenger::Worker-1": { "msgr_recv_messages": 12099771, "msgr_send_messages": 12138795, "msgr_recv_bytes": 56967534605, "msgr_send_bytes": 130548664272, "msgr_created_connections": 647, "msgr_active_connections": 91, "msgr_running_total_time": 977.277996439, "msgr_running_send_time": 362.155959231, "msgr_running_recv_time": 365.376281473, "msgr_running_fast_dispatch_time": 191.186643292, "msgr_send_messages_queue_lat": { "avgcount": 12138818, "sum": 1557.187685700, "avgtime": 0.000128281 }, "msgr_handle_ack_lat": { "avgcount": 6155265, "sum": 1.096270527, "avgtime": 0.000000178 } }, "AsyncMessenger::Worker-2": { "msgr_recv_messages": 11858354, "msgr_send_messages": 11960404, "msgr_recv_bytes": 60727084610, "msgr_send_bytes": 168534726650, "msgr_created_connections": 1043, "msgr_active_connections": 103, "msgr_running_total_time": 937.324084772, "msgr_running_send_time": 351.174710644, "msgr_running_recv_time": 2744.276782474, "msgr_running_fast_dispatch_time": 172.960322050, "msgr_send_messages_queue_lat": { "avgcount": 11960392, "sum": 1763.762581924, "avgtime": 0.000147466 }, "msgr_handle_ack_lat": { "avgcount": 2651457, "sum": 0.538495450, "avgtime": 0.000000203 } }, "bluefs": { "db_total_bytes": 128005955584, "db_used_bytes": 3271557120, "wal_total_bytes": 0, "wal_used_bytes": 0, "slow_total_bytes": 10000810377216, "slow_used_bytes": 0, "num_files": 70, "log_bytes": 13045760, "log_compactions": 58, "logged_bytes": 922333184, "files_written_wal": 2, "files_written_sst": 13, "bytes_written_wal": 1988489216, "bytes_written_sst": 268890112, "bytes_written_slow": 0, "max_bytes_wal": 0, "max_bytes_db": 3271557120, "max_bytes_slow": 0, "read_random_count": 577484, "read_random_bytes": 2879541532, "read_random_disk_count": 284290, "read_random_disk_bytes": 1540394118, "read_random_buffer_count": 319088, "read_random_buffer_bytes": 1339147414, "read_count": 1086625, "read_bytes": 15054317429, "read_prefetch_count": 1069462, "read_prefetch_bytes": 14506469332, "read_zeros_candidate": 0, "read_zeros_errors": 0 }, "bluestore": { "kv_flush_lat": { "avgcount": 225099, "sum": 526.605165277, "avgtime": 0.002339438 }, "kv_commit_lat": { "avgcount": 225099, "sum": 61.412175620, "avgtime": 0.000272822 }, "kv_sync_lat": { "avgcount": 225099, "sum": 588.017340897, "avgtime": 0.002612261 }, "kv_final_lat": { "avgcount": 225096, "sum": 6.516869320, "avgtime": 0.000028951 }, "state_prepare_lat": { "avgcount": 241063, "sum": 173.705759592, "avgtime": 0.000720582 }, "state_aio_wait_lat": { "avgcount": 241063, "sum": 1008.936150524, "avgtime": 0.004185362 }, "state_io_done_lat": { "avgcount": 241063, "sum": 2.923457351, "avgtime": 0.000012127 }, "state_kv_queued_lat": { "avgcount": 241063, "sum": 560.050193021, "avgtime": 0.002323252 }, "state_kv_commiting_lat": { "avgcount": 241063, "sum": 68.355225981, "avgtime": 0.000283557 }, "state_kv_done_lat": { "avgcount": 241063, "sum": 0.097836444, "avgtime": 0.000000405 }, "state_deferred_queued_lat": { "avgcount": 47230, "sum": 70711.960011607, "avgtime": 1.497183146 }, "state_deferred_aio_wait_lat": { "avgcount": 47230, "sum": 70.931594687, "avgtime": 0.001501833 }, "state_deferred_cleanup_lat": { "avgcount": 47229, "sum": 333702.384485631, "avgtime": 7.065624605 }, "state_finishing_lat": { "avgcount": 241062, "sum": 0.033475853, "avgtime": 0.000000138 }, "state_done_lat": { "avgcount": 241062, "sum": 15258.375631838, "avgtime": 0.063296478 }, "throttle_lat": { "avgcount": 241063, "sum": 67.464167288, "avgtime": 0.000279861 }, "submit_lat": { "avgcount": 241063, "sum": 190.468701838, "avgtime": 0.000790120 }, "commit_lat": { "avgcount": 241063, "sum": 1814.026687331, "avgtime": 0.007525114 }, "read_lat": { "avgcount": 1590088, "sum": 8928.843945649, "avgtime": 0.005615314 }, "read_onode_meta_lat": { "avgcount": 3106127, "sum": 18.873640476, "avgtime": 0.000006076 }, "read_wait_aio_lat": { "avgcount": 1516039, "sum": 8830.114048334, "avgtime": 0.005824463 }, "compress_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "decompress_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "csum_lat": { "avgcount": 7047778, "sum": 52.140862362, "avgtime": 0.000007398 }, "compress_success_count": 0, "compress_rejected_count": 0, "write_pad_bytes": 194870362, "deferred_write_ops": 50669, "deferred_write_bytes": 388481024, "write_penalty_read_ops": 2067, "bluestore_allocated": 294756155392, "bluestore_stored": 292495538258, "bluestore_compressed": 0, "bluestore_compressed_allocated": 0, "bluestore_compressed_original": 0, "bluestore_onodes": 266002, "bluestore_pinned_onodes": 1, "onode_hits": 8782379, "onode_misses": 6047682, "onode_shard_hits": 1277688, "bluestore_onode_shard_misses": 947296, "bluestore_extents": 840549, "bluestore_blobs": 840549, "bluestore_buffers": 3591, "bluestore_buffer_bytes": 224182272, "bluestore_buffer_hit_bytes": 110213431882, "bluestore_buffer_miss_bytes": 406742926152, "bluestore_write_big": 39721, "bluestore_write_big_bytes": 94625427456, "bluestore_write_big_blobs": 1450312, "bluestore_write_big_deferred": 0, "bluestore_write_small": 60521, "bluestore_write_small_bytes": 48842873, "bluestore_write_small_unused": 926, "bluestore_write_deferred": 62388, "bluestore_write_deferred_bytes": 398618624, "bluestore_write_small_pre_read": 7434, "bluestore_write_new": 1448445, "bluestore_txc": 241063, "bluestore_onode_reshard": 27825, "bluestore_blob_split": 5, "bluestore_extent_compress": 3758, "bluestore_gc_merged": 0, "bluestore_read_eio": 0, "bluestore_reads_with_retries": 0, "bluestore_fragmentation_micros": 0, "omap_seek_to_first_lat": { "avgcount": 759176, "sum": 8.351645295, "avgtime": 0.000011000 }, "omap_upper_bound_lat": { "avgcount": 308, "sum": 0.002216099, "avgtime": 0.000007195 }, "omap_lower_bound_lat": { "avgcount": 87154, "sum": 1.192486653, "avgtime": 0.000013682 }, "omap_next_lat": { "avgcount": 308598, "sum": 0.211502859, "avgtime": 0.000000685 }, "omap_get_keys_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "omap_get_values_lat": { "avgcount": 19348, "sum": 0.032918565, "avgtime": 0.000001701 }, "clist_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "remove_lat": { "avgcount": 239512, "sum": 121.690074965, "avgtime": 0.000508075 } }, "bluestore-pricache": { "target_bytes": 6713193267, "mapped_bytes": 6718742528, "unmapped_bytes": 467025920, "heap_bytes": 7185768448, "cache_bytes": 4161537138 }, "bluestore-pricache:data": { "pri0_bytes": 0, "pri1_bytes": 224182272, "pri2_bytes": 0, "pri3_bytes": 0, "pri4_bytes": 0, "pri5_bytes": 0, "pri6_bytes": 0, "pri7_bytes": 0, "pri8_bytes": 0, "pri9_bytes": 0, "pri10_bytes": 0, "pri11_bytes": 30742099, "reserved_bytes": 80619949, "committed_bytes": 335544320 }, "bluestore-pricache:kv": { "pri0_bytes": 0, "pri1_bytes": 176540224, "pri2_bytes": 0, "pri3_bytes": 0, "pri4_bytes": 0, "pri5_bytes": 0, "pri6_bytes": 0, "pri7_bytes": 0, "pri8_bytes": 0, "pri9_bytes": 0, "pri10_bytes": 0, "pri11_bytes": 230565745, "reserved_bytes": 79433295, "committed_bytes": 486539264 }, "bluestore-pricache:kv_onode": { "pri0_bytes": 0, "pri1_bytes": 1977148816, "pri2_bytes": 0, "pri3_bytes": 0, "pri4_bytes": 0, "pri5_bytes": 0, "pri6_bytes": 0, "pri7_bytes": 0, "pri8_bytes": 0, "pri9_bytes": 0, "pri10_bytes": 0, "pri11_bytes": 20494732, "reserved_bytes": 82731236, "committed_bytes": 2080374784 }, "bluestore-pricache:meta": { "pri0_bytes": 0, "pri1_bytes": 942217320, "pri2_bytes": 0, "pri3_bytes": 0, "pri4_bytes": 0, "pri5_bytes": 0, "pri6_bytes": 0, "pri7_bytes": 0, "pri8_bytes": 0, "pri9_bytes": 0, "pri10_bytes": 0, "pri11_bytes": 230565745, "reserved_bytes": 68730919, "committed_bytes": 1241513984 }, "cct": { "total_workers": 5, "unhealthy_workers": 0 }, "finisher-commit_finisher": { "queue_len": 0, "complete_latency": { "avgcount": 1507, "sum": 0.568110372, "avgtime": 0.000376981 } }, "finisher-objecter-finisher-0": { "queue_len": 0, "complete_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluestore_alloc_bytes": 12461120, "bluestore_alloc_items": 155764, "bluestore_cache_data_bytes": 233998707, "bluestore_cache_data_items": 229226, "bluestore_cache_onode_bytes": 163857232, "bluestore_cache_onode_items": 266002, "bluestore_cache_meta_bytes": 456521298, "bluestore_cache_meta_items": 84112717, "bluestore_cache_other_bytes": 91560856, "bluestore_cache_other_items": 11308804, "bluestore_Buffer_bytes": 513888, "bluestore_Buffer_items": 5353, "bluestore_Extent_bytes": 40346352, "bluestore_Extent_items": 840549, "bluestore_Blob_bytes": 94141488, "bluestore_Blob_items": 840549, "bluestore_SharedBlob_bytes": 94141488, "bluestore_SharedBlob_items": 840549, "bluestore_inline_bl_bytes": 1134718, "bluestore_inline_bl_items": 8758, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_txc_bytes": 784, "bluestore_txc_items": 1, "bluestore_writing_deferred_bytes": 1918388, "bluestore_writing_deferred_items": 81, "bluestore_writing_bytes": 0, "bluestore_writing_items": 0, "bluefs_bytes": 36800, "bluefs_items": 1443, "bluefs_file_reader_bytes": 1842688, "bluefs_file_reader_items": 95, "bluefs_file_writer_bytes": 576, "bluefs_file_writer_items": 3, "buffer_anon_bytes": 10746933, "buffer_anon_items": 25967, "buffer_meta_bytes": 2541352, "buffer_meta_items": 28879, "osd_bytes": 916272, "osd_items": 81, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 164733192, "osd_pglog_items": 363565, "osdmap_bytes": 2182392, "osdmap_items": 53839, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "mds_co_bytes": 0, "mds_co_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 0, "op_send_bytes": 0, "op_resend": 0, "op_reply": 0, "oplen_avg": { "avgcount": 0, "sum": 0 }, "op": 0, "op_r": 0, "op_w": 0, "op_rmw": 0, "op_pg": 0, "osdop_stat": 0, "osdop_create": 0, "osdop_read": 0, "osdop_write": 0, "osdop_writefull": 0, "osdop_writesame": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_call": 0, "osdop_watch": 0, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 0, "linger_active": 0, "linger_send": 0, "linger_resend": 0, "linger_ping": 0, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 63038, "map_full": 2, "map_inc": 1511, "osd_sessions": 0, "osd_session_open": 0, "osd_session_close": 0, "osd_laggy": 0, "omap_wr": 0, "omap_rd": 0, "omap_del": 0 }, "osd": { "op_wip": 0, "op": 272274, "op_in_bytes": 39148504708, "op_out_bytes": 181035613199, "op_latency": { "avgcount": 272274, "sum": 2312.558437554, "avgtime": 0.008493497 }, "op_process_latency": { "avgcount": 272274, "sum": 2151.138778914, "avgtime": 0.007900639 }, "op_prepare_latency": { "avgcount": 280169, "sum": 1068.077633101, "avgtime": 0.003812262 }, "op_r": 191667, "op_r_out_bytes": 181035613199, "op_r_latency": { "avgcount": 191667, "sum": 1035.691613777, "avgtime": 0.005403599 }, "op_r_process_latency": { "avgcount": 191667, "sum": 959.303890513, "avgtime": 0.005005055 }, "op_r_prepare_latency": { "avgcount": 191712, "sum": 962.370649266, "avgtime": 0.005019876 }, "op_w": 34260, "op_w_in_bytes": 39121651122, "op_w_latency": { "avgcount": 34260, "sum": 1117.333684176, "avgtime": 0.032613359 }, "op_w_process_latency": { "avgcount": 34260, "sum": 1059.202697781, "avgtime": 0.030916599 }, "op_w_prepare_latency": { "avgcount": 36798, "sum": 91.420473715, "avgtime": 0.002484387 }, "op_rw": 46347, "op_rw_in_bytes": 26853586, "op_rw_out_bytes": 0, "op_rw_latency": { "avgcount": 46347, "sum": 159.533139601, "avgtime": 0.003442145 }, "op_rw_process_latency": { "avgcount": 46347, "sum": 132.632190620, "avgtime": 0.002861721 }, "op_rw_prepare_latency": { "avgcount": 51659, "sum": 14.286510120, "avgtime": 0.000276554 }, "op_before_queue_op_lat": { "avgcount": 897996, "sum": 154.309110719, "avgtime": 0.000171837 }, "op_before_dequeue_op_lat": { "avgcount": 901696, "sum": 622.443106332, "avgtime": 0.000690302 }, "subop": 119722, "subop_in_bytes": 55578084121, "subop_latency": { "avgcount": 119722, "sum": 1159.173264760, "avgtime": 0.009682207 }, "subop_w": 119722, "subop_w_in_bytes": 55578084121, "subop_w_latency": { "avgcount": 119722, "sum": 1159.173264760, "avgtime": 0.009682207 }, "subop_pull": 0, "subop_pull_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "subop_push": 0, "subop_push_in_bytes": 0, "subop_push_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "pull": 0, "push": 86911, "push_out_bytes": 34318722925, "recovery_ops": 86922, "recovery_bytes": 34314397051, "loadavg": 62, "cached_crc": 0, "cached_crc_adjusted": 0, "missed_crc": 0, "numpg": 81, "numpg_primary": 32, "numpg_replica": 49, "numpg_stray": 0, "numpg_removing": 0, "heartbeat_to_peers": 53, "map_messages": 3924, "map_message_epochs": 4465, "map_message_epoch_dups": 2957, "messages_delayed_for_map": 0, "osd_map_cache_hit": 127507, "osd_map_cache_miss": 22, "osd_map_cache_miss_low": 2, "osd_map_cache_miss_low_avg": { "avgcount": 2, "sum": 21 }, "osd_map_bl_cache_hit": 4254, "osd_map_bl_cache_miss": 2798, "stat_bytes": 10128816332800, "stat_bytes_used": 422762119168, "stat_bytes_avail": 9706054213632, "copyfrom": 0, "tier_promote": 0, "tier_flush": 0, "tier_flush_fail": 0, "tier_try_flush": 0, "tier_try_flush_fail": 0, "tier_evict": 0, "tier_whiteout": 0, "tier_dirty": 29567, "tier_clean": 0, "tier_delay": 0, "tier_proxy_read": 0, "tier_proxy_write": 0, "agent_wake": 0, "agent_skip": 0, "agent_flush": 0, "agent_evict": 0, "object_ctx_cache_hit": 288613, "object_ctx_cache_total": 488855, "op_cache_hit": 0, "osd_tier_flush_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "osd_tier_promote_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "osd_tier_r_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "osd_pg_info": 238380, "osd_pg_fastinfo": 207321, "osd_pg_biginfo": 1634 }, "recoverystate_perf": { "initial_latency": { "avgcount": 83, "sum": 1.762260823, "avgtime": 0.021232058 }, "started_latency": { "avgcount": 597, "sum": 12091840.383300855, "avgtime": 20254.339000503 }, "reset_latency": { "avgcount": 678, "sum": 105.680233720, "avgtime": 0.155870551 }, "start_latency": { "avgcount": 678, "sum": 0.013440001, "avgtime": 0.000019823 }, "primary_latency": { "avgcount": 229, "sum": 3916876.117159230, "avgtime": 17104.262520345 }, "peering_latency": { "avgcount": 261, "sum": 242.487618037, "avgtime": 0.929071333 }, "backfilling_latency": { "avgcount": 11, "sum": 28355.181184224, "avgtime": 2577.743744020 }, "waitremotebackfillreserved_latency": { "avgcount": 11, "sum": 0.010010103, "avgtime": 0.000910009 }, "waitlocalbackfillreserved_latency": { "avgcount": 11, "sum": 0.000914005, "avgtime": 0.000083091 }, "notbackfilling_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "repnotrecovering_latency": { "avgcount": 236, "sum": 8174666.478144644, "avgtime": 34638.417280273 }, "repwaitrecoveryreserved_latency": { "avgcount": 13, "sum": 0.000544600, "avgtime": 0.000041892 }, "repwaitbackfillreserved_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "reprecovering_latency": { "avgcount": 13, "sum": 5.236229915, "avgtime": 0.402786916 }, "activating_latency": { "avgcount": 226, "sum": 7.792077960, "avgtime": 0.034478221 }, "waitlocalrecoveryreserved_latency": { "avgcount": 6, "sum": 0.000532092, "avgtime": 0.000088682 }, "waitremoterecoveryreserved_latency": { "avgcount": 6, "sum": 0.005132390, "avgtime": 0.000855398 }, "recovering_latency": { "avgcount": 6, "sum": 2.188644385, "avgtime": 0.364774064 }, "recovered_latency": { "avgcount": 225, "sum": 0.004724875, "avgtime": 0.000020999 }, "clean_latency": { "avgcount": 193, "sum": 3888298.606566839, "avgtime": 20146.624904491 }, "active_latency": { "avgcount": 194, "sum": 3916662.571342664, "avgtime": 20188.982326508 }, "replicaactive_latency": { "avgcount": 223, "sum": 8174668.635187419, "avgtime": 36657.706884248 }, "stray_latency": { "avgcount": 417, "sum": 343.521093120, "avgtime": 0.823791590 }, "getinfo_latency": { "avgcount": 261, "sum": 26.466818426, "avgtime": 0.101405434 }, "getlog_latency": { "avgcount": 229, "sum": 0.313088774, "avgtime": 0.001367199 }, "waitactingchange_latency": { "avgcount": 3, "sum": 2.787477889, "avgtime": 0.929159296 }, "incomplete_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "down_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "getmissing_latency": { "avgcount": 226, "sum": 0.070469958, "avgtime": 0.000311813 }, "waitupthru_latency": { "avgcount": 226, "sum": 215.630016592, "avgtime": 0.954115117 }, "notrecovering_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "rocksdb": { "get": 3900573, "get_latency": { "avgcount": 3900573, "sum": 46.497747616, "avgtime": 0.000011920 }, "submit_latency": { "avgcount": 241063, "sum": 14.648569398, "avgtime": 0.000060766 }, "submit_sync_latency": { "avgcount": 225099, "sum": 45.107860324, "avgtime": 0.000200391 }, "compact": 0, "compact_range": 0, "compact_queue_merge": 0, "compact_queue_len": 0, "rocksdb_write_wal_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "rocksdb_write_memtable_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "rocksdb_write_delay_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "rocksdb_write_pre_and_post_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-bluestore_throttle_bytes": { "val": 0, "max": 67108864, "get_started": 241063, "get": 241063, "get_sum": 1227078527883, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 225099, "put_sum": 1227078527883, "wait": { "avgcount": 2039, "sum": 65.096143721, "avgtime": 0.031925524 } }, "throttle-bluestore_throttle_deferred_bytes": { "val": 0, "max": 201326592, "get_started": 0, "get": 47230, "get_sum": 32111982926, "get_or_fail_fail": 0, "get_or_fail_success": 47230, "take": 0, "take_sum": 0, "put": 44540, "put_sum": 32111982926, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-client": { "val": 0, "max": 104857600, "get_started": 0, "get": 292633, "get_sum": 39290356304, "get_or_fail_fail": 0, "get_or_fail_success": 292633, "take": 0, "take_sum": 0, "put": 292633, "put_sum": 39290356304, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-cluster": { "val": 0, "max": 104857600, "get_started": 0, "get": 7182670, "get_sum": 60512426404, "get_or_fail_fail": 0, "get_or_fail_success": 7182670, "take": 0, "take_sum": 0, "put": 7182670, "put_sum": 60512426404, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_back_client": { "val": 0, "max": 104857600, "get_started": 0, "get": 7382217, "get_sum": 15008047161, "get_or_fail_fail": 0, "get_or_fail_success": 7382217, "take": 0, "take_sum": 0, "put": 7382217, "put_sum": 15008047161, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0, "max": 104857600, "get_started": 0, "get": 6979126, "get_sum": 14188562814, "get_or_fail_fail": 0, "get_or_fail_success": 6979126, "take": 0, "take_sum": 0, "put": 6979126, "put_sum": 14188562814, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_front_client": { "val": 0, "max": 104857600, "get_started": 0, "get": 7382217, "get_sum": 15008047161, "get_or_fail_fail": 0, "get_or_fail_success": 7382217, "take": 0, "take_sum": 0, "put": 7382217, "put_sum": 15008047161, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0, "max": 104857600, "get_started": 0, "get": 6979126, "get_sum": 14188562814, "get_or_fail_fail": 0, "get_or_fail_success": 6979126, "take": 0, "take_sum": 0, "put": 6979126, "put_sum": 14188562814, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-ms_objecter": { "val": 0, "max": 104857600, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_bytes": { "val": 0, "max": 104857600, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_ops": { "val": 0, "max": 1024, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-osd_client_bytes": { "val": 47, "max": 524288000, "get_started": 0, "get": 291042, "get_sum": 39289027913, "get_or_fail_fail": 0, "get_or_fail_success": 291042, "take": 0, "take_sum": 0, "put": 1164143, "put_sum": 39289027866, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-osd_client_messages": { "val": 1, "max": 256, "get_started": 0, "get": 291042, "get_sum": 291042, "get_or_fail_fail": 0, "get_or_fail_success": 291042, "take": 0, "take_sum": 0, "put": 291041, "put_sum": 291041, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } } } and dump_mempools: { "mempool": { "by_pool": { "bloom_filter": { "items": 0, "bytes": 0 }, "bluestore_alloc": { "items": 155779, "bytes": 12462320 }, "bluestore_cache_data": { "items": 228476, "bytes": 233092536 }, "bluestore_cache_onode": { "items": 265318, "bytes": 163435888 }, "bluestore_cache_meta": { "items": 83890049, "bytes": 455300708 }, "bluestore_cache_other": { "items": 11355469, "bytes": 91930988 }, "bluestore_Buffer": { "items": 5325, "bytes": 511200 }, "bluestore_Extent": { "items": 842524, "bytes": 40441152 }, "bluestore_Blob": { "items": 842524, "bytes": 94362688 }, "bluestore_SharedBlob": { "items": 842524, "bytes": 94362688 }, "bluestore_inline_bl": { "items": 8842, "bytes": 1142714 }, "bluestore_fsck": { "items": 0, "bytes": 0 }, "bluestore_txc": { "items": 0, "bytes": 0 }, "bluestore_writing_deferred": { "items": 77, "bytes": 1798719 }, "bluestore_writing": { "items": 0, "bytes": 0 }, "bluefs": { "items": 1443, "bytes": 36800 }, "bluefs_file_reader": { "items": 96, "bytes": 2104832 }, "bluefs_file_writer": { "items": 3, "bytes": 576 }, "buffer_anon": { "items": 25898, "bytes": 10464534 }, "buffer_meta": { "items": 28808, "bytes": 2535104 }, "osd": { "items": 81, "bytes": 916272 }, "osd_mapbl": { "items": 0, "bytes": 0 }, "osd_pglog": { "items": 363769, "bytes": 164806408 }, "osdmap": { "items": 53839, "bytes": 2182392 }, "osdmap_mapping": { "items": 0, "bytes": 0 }, "pgmap": { "items": 0, "bytes": 0 }, "mds_co": { "items": 0, "bytes": 0 }, "unittest_1": { "items": 0, "bytes": 0 }, "unittest_2": { "items": 0, "bytes": 0 } }, "total": { "items": 98910844, "bytes": 1371888519 } } } Luis Domingues Proton AG ------- Original Message ------- On Tuesday, July 11th, 2023 at 14:59, Mark Nelson <mark.nelson@clyso.com> wrote:

Hi Luis,

Can you do a "ceph tell osd.<num> perf dump" and "ceph daemon osd.<num>

dump_mempools"? Those should help us understand how much memory is being used by different parts of the OSD/bluestore and how much memory the priority cache thinks it has to work with.

Mark

On 7/11/23 4:57 AM, Luis Domingues wrote:

...

Hi everyone,

We recently migrate a cluster from ceph-ansible to cephadm. Everything went as expected. But now we have some alerts on high memory usage. Cluster is running ceph 16.2.13.

Of course, after adoption OSDs ended up in the <unmanaged> zone:

NAME PORTS RUNNING REFRESHED AGE PLACEMENT osd 88 7m ago - <unmanaged>

But the weirdest thing I observed, is that the OSDs seem to use more memory that the mem limit:

NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID osd.0 <node> running (5d) 2m ago 5d 19.7G 6400M 16.2.13 327f301eff51 ca07fe74a0fa osd.1 <node> running (5d) 2m ago 5d 7068M 6400M 16.2.13 327f301eff51 6223ed8e34e9 osd.10 <node> running (5d) 10m ago 5d 7235M 6400M 16.2.13 327f301eff51 073ddc0d7391 osd.100 <node> running (5d) 2m ago 5d 7118M 6400M 16.2.13 327f301eff51 b7f9238c0c24

Does anybody knows why OSDs would use more memory than the limit?

Thanks

Luis Domingues Proton AG _______________________________________________ ceph-users mailing list -- ceph-users@ceph.io To unsubscribe send an email to ceph-users-leave@ceph.io

-- Best Regards, Mark Nelson Head of R&D (USA)

Clyso GmbH p: +49 89 21552391 12 a: Loristraße 8 | 80335 München | Germany w: https://clyso.com | e: mark.nelson@clyso.com

We are hiring: https://www.clyso.com/jobs/ _______________________________________________ ceph-users mailing list -- ceph-users@ceph.io To unsubscribe send an email to ceph-users-leave@ceph.io

Hi, Thanks for your hints. I tries to play a little bit with the configs. And now I want to put the 0.7 value as default. So I configured ceph: mgr advanced mgr/cephadm/autotune_memory_target_ratio 0.700000 * osd advanced osd_memory_target_autotune true And I ended up having this configs osd host:st10-cbosd-001 basic osd_memory_target 7219293672 osd host:st10-cbosd-002 basic osd_memory_target 7219293672 osd host:st10-cbosd-004 basic osd_memory_target 7219293672 osd host:st10-cbosd-005 basic osd_memory_target 7219293451 osd host:st10-cbosd-006 basic osd_memory_target 7219293451 osd host:st11-cbosd-007 basic osd_memory_target 7216821484 osd host:st11-cbosd-008 basic osd_memory_target 7216825454 And running a ceph orch ps gaves me: osd.0 st11-cbosd-007.plabs.ch running (2d) 10m ago 10d 25.8G 6882M 16.2.13 327f301eff51 29a075f2f925 osd.1 st10-cbosd-001.plabs.ch running (19m) 8m ago 10d 2115M 6884M 16.2.13 327f301eff51 df5067bde5ce osd.10 st10-cbosd-005.plabs.ch running (2d) 10m ago 10d 5524M 6884M 16.2.13 327f301eff51 f7bc0641ee46 osd.100 st11-cbosd-008.plabs.ch running (2d) 10m ago 10d 5234M 6882M 16.2.13 327f301eff51 74efa243b953 osd.101 st11-cbosd-008.plabs.ch running (2d) 10m ago 10d 4741M 6882M 16.2.13 327f301eff51 209671007c65 osd.102 st11-cbosd-008.plabs.ch running (2d) 10m ago 10d 5174M 6882M 16.2.13 327f301eff51 63691d557732 So far so good. But when I took a look on the memory usage of my OSDs, I was below of that value, by quite a bite. Looking at the OSDs themselves, I have: "bluestore-pricache": { "target_bytes": 4294967296, "mapped_bytes": 1343455232, "unmapped_bytes": 16973824, "heap_bytes": 1360429056, "cache_bytes": 2845415832 }, And if I get the running config: "osd_memory_target": "4294967296", "osd_memory_target_autotune": "true", "osd_memory_target_cgroup_limit_ratio": "0.800000", Which is not the value I observe from the config. I have 4294967296 instead of something around 7219293672. Did I miss something? Luis Domingues Proton AG ------- Original Message ------- On Tuesday, July 11th, 2023 at 18:10, Mark Nelson <mark.nelson@clyso.com> wrote:

On 7/11/23 09:44, Luis Domingues wrote:

...

"bluestore-pricache": { "target_bytes": 6713193267, "mapped_bytes": 6718742528, "unmapped_bytes": 467025920, "heap_bytes": 7185768448, "cache_bytes": 4161537138 },

Hi Luis,

Looks like the mapped bytes for this OSD process is very close to (just a little over) the target bytes that has been set when you did the perf dump. There is some unmapped memory that can be reclaimed by the kernel, but we can't force the kernel to reclaim it. It could be that the kernel is being a little lazy if there isn't memory pressure.

The way the memory autotuning works in Ceph is that periodically the prioritycache system will look at the mapped memory usage of the process, then grow/shrink the aggregate size of the in-memory caches to try and stay near the target. It's reactive in nature, meaning that it can't completely control for spikes. It also can't shrink the caches below a small minimum size, so if there is a memory leak it will help to an extent but can't completely fix it. Once the aggregate memory size is decided on, it goes through a process of looking at how hot the different caches are and assigns memory based on where it thinks the memory would be most useful. Again this is based on mapped memory though. It can't force the kernel to reclaim memory that has already been released.

Thanks,

Mark

-- Best Regards, Mark Nelson Head of R&D (USA)

Clyso GmbH p: +49 89 21552391 12 a: Loristraße 8 | 80335 München | Germany w: https://clyso.com | e: mark.nelson@clyso.com

We are hiring: https://www.clyso.com/jobs/ _______________________________________________ ceph-users mailing list -- ceph-users@ceph.io To unsubscribe send an email to ceph-users-leave@ceph.io

It looks indeed to be that bug that I hit. Thanks. Luis Domingues Proton AG ------- Original Message ------- On Monday, July 17th, 2023 at 07:45, Sridhar Seshasayee <sseshasa@redhat.com> wrote:

Hello Luis,

Please see my response below:

But when I took a look on the memory usage of my OSDs, I was below of that

...

value, by quite a bite. Looking at the OSDs themselves, I have:

"bluestore-pricache": { "target_bytes": 4294967296, "mapped_bytes": 1343455232, "unmapped_bytes": 16973824, "heap_bytes": 1360429056, "cache_bytes": 2845415832 },

And if I get the running config: "osd_memory_target": "4294967296", "osd_memory_target_autotune": "true", "osd_memory_target_cgroup_limit_ratio": "0.800000",

Which is not the value I observe from the config. I have 4294967296 instead of something around 7219293672. Did I miss something?

This is very likely due to https://tracker.ceph.com/issues/48750. The fix was recently merged into the main branch and should be backported soon all the way to pacific.

Until then, the workaround would be to set the osd_memory_target on each OSD individually to the desired value.

-Sridhar _______________________________________________ ceph-users mailing list -- ceph-users@ceph.io To unsubscribe send an email to ceph-users-leave@ceph.io