Hello List, any ideas why my OSDs are that unbalanced ? root@ceph01:~# ceph -s cluster: id: 5436dd5d-83d4-4dc8-a93b-60ab5db145df health: HEALTH_WARN 1 nearfull osd(s) 4 pool(s) nearfull services: mon: 3 daemons, quorum ceph03,ceph01,ceph02 (age 2w) mgr: ceph03(active, since 4M), standbys: ceph02.jwvivm mds: backup:1 {0=backup.ceph06.hdjehi=up:active} 3 up:standby osd: 56 osds: 56 up (since 29h), 56 in (since 3d) task status: scrub status: mds.backup.ceph06.hdjehi: idle data: pools: 4 pools, 1185 pgs objects: 24.29M objects, 44 TiB usage: 151 TiB used, 55 TiB / 206 TiB avail pgs: 675 active+clean 476 active+clean+snaptrim_wait 30 active+clean+snaptrim 4 active+clean+scrubbing+deep root@ceph01:~# ceph osd df tree ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME -1 206.79979 - 206 TiB 151 TiB 151 TiB 36 GiB 503 GiB 55 TiB 73.23 1.00 - root default -2 28.89995 - 29 TiB 20 TiB 20 TiB 5.5 GiB 74 GiB 8.9 TiB 69.19 0.94 - host ceph01 0 hdd 2.70000 1.00000 2.7 TiB 1.8 TiB 1.8 TiB 590 MiB 6.9 GiB 908 GiB 66.81 0.91 44 up osd.0 1 hdd 2.70000 1.00000 2.7 TiB 1.6 TiB 1.6 TiB 411 MiB 6.5 GiB 1.1 TiB 60.43 0.83 39 up osd.1 4 hdd 2.70000 1.00000 2.7 TiB 1.8 TiB 1.8 TiB 501 MiB 6.8 GiB 898 GiB 67.15 0.92 43 up osd.4 8 hdd 2.70000 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 453 MiB 7.0 GiB 700 GiB 74.39 1.02 47 up osd.8 11 hdd 1.70000 1.00000 1.7 TiB 1.3 TiB 1.3 TiB 356 MiB 5.6 GiB 433 GiB 75.39 1.03 31 up osd.11 12 hdd 2.70000 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 471 MiB 7.0 GiB 591 GiB 78.40 1.07 48 up osd.12 14 hdd 2.70000 1.00000 2.7 TiB 1.6 TiB 1.6 TiB 448 MiB 6.0 GiB 1.1 TiB 59.68 0.82 38 up osd.14 18 hdd 2.70000 1.00000 2.7 TiB 1.7 TiB 1.7 TiB 515 MiB 6.2 GiB 980 GiB 64.15 0.88 41 up osd.18 22 hdd 1.70000 1.00000 1.7 TiB 1.2 TiB 1.2 TiB 360 MiB 4.2 GiB 491 GiB 72.06 0.98 29 up osd.22 30 hdd 1.70000 1.00000 1.7 TiB 1.2 TiB 1.2 TiB 366 MiB 4.7 GiB 558 GiB 68.26 0.93 28 up osd.30 33 hdd 1.59999 1.00000 1.6 TiB 1.2 TiB 1.2 TiB 406 MiB 4.9 GiB 427 GiB 74.28 1.01 29 up osd.33 64 hdd 3.29999 1.00000 3.3 TiB 2.4 TiB 2.4 TiB 736 MiB 8.6 GiB 915 GiB 73.22 1.00 60 up osd.64 -3 29.69995 - 30 TiB 22 TiB 22 TiB 5.4 GiB 81 GiB 7.9 TiB 73.20 1.00 - host ceph02 2 hdd 1.70000 1.00000 1.7 TiB 1.3 TiB 1.2 TiB 402 MiB 5.2 GiB 476 GiB 72.93 1.00 30 up osd.2 3 hdd 2.70000 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 653 MiB 7.8 GiB 652 GiB 76.15 1.04 49 up osd.3 7 hdd 2.70000 1.00000 2.7 TiB 2.5 TiB 2.5 TiB 456 MiB 7.7 GiB 209 GiB 92.36 1.26 56 up osd.7 9 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 434 MiB 7.2 GiB 781 GiB 71.46 0.98 46 up osd.9 13 hdd 2.39999 1.00000 2.4 TiB 1.6 TiB 1.6 TiB 451 MiB 6.1 GiB 823 GiB 66.28 0.91 38 up osd.13 16 hdd 2.70000 1.00000 2.7 TiB 1.6 TiB 1.6 TiB 375 MiB 6.4 GiB 1.1 TiB 59.84 0.82 39 up osd.16 19 hdd 1.70000 1.00000 1.7 TiB 1.1 TiB 1.1 TiB 323 MiB 4.7 GiB 601 GiB 65.80 0.90 27 up osd.19 23 hdd 2.70000 1.00000 2.7 TiB 2.2 TiB 2.2 TiB 471 MiB 7.7 GiB 520 GiB 80.99 1.11 50 up osd.23 24 hdd 1.70000 1.00000 1.7 TiB 1.4 TiB 1.4 TiB 371 MiB 5.5 GiB 273 GiB 84.44 1.15 32 up osd.24 28 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 428 MiB 7.4 GiB 818 GiB 70.07 0.96 44 up osd.28 31 hdd 2.70000 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 516 MiB 7.4 GiB 660 GiB 75.85 1.04 48 up osd.31 32 hdd 3.29999 1.00000 3.3 TiB 2.2 TiB 2.2 TiB 661 MiB 7.9 GiB 1.2 TiB 64.86 0.89 52 up osd.32 -4 26.29996 - 26 TiB 18 TiB 18 TiB 4.3 GiB 73 GiB 8.0 TiB 69.58 0.95 - host ceph03 5 hdd 1.70000 1.00000 1.7 TiB 1.2 TiB 1.2 TiB 298 MiB 5.2 GiB 541 GiB 69.21 0.95 29 up osd.5 6 hdd 1.70000 1.00000 1.7 TiB 1.0 TiB 1.0 TiB 321 MiB 4.4 GiB 697 GiB 60.34 0.82 25 up osd.6 10 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 431 MiB 7.5 GiB 796 GiB 70.89 0.97 46 up osd.10 15 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 500 MiB 6.6 GiB 805 GiB 70.55 0.96 44 up osd.15 17 hdd 1.59999 1.00000 1.6 TiB 1.1 TiB 1.1 TiB 377 MiB 4.9 GiB 530 GiB 68.05 0.93 27 up osd.17 20 hdd 1.70000 1.00000 1.7 TiB 1.0 TiB 1.0 TiB 223 MiB 4.7 GiB 685 GiB 61.03 0.83 25 up osd.20 21 hdd 2.70000 1.00000 2.7 TiB 1.7 TiB 1.7 TiB 392 MiB 6.7 GiB 951 GiB 65.23 0.89 42 up osd.21 25 hdd 1.70000 1.00000 1.7 TiB 1.1 TiB 1.1 TiB 157 MiB 5.1 GiB 601 GiB 65.83 0.90 27 up osd.25 26 hdd 2.70000 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 512 MiB 7.6 GiB 573 GiB 79.06 1.08 50 up osd.26 27 hdd 2.70000 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 473 MiB 7.6 GiB 805 GiB 70.55 0.96 46 up osd.27 29 hdd 2.70000 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 478 MiB 7.3 GiB 539 GiB 80.29 1.10 50 up osd.29 63 hdd 1.70000 1.00000 1.7 TiB 1.1 TiB 1.1 TiB 195 MiB 5.1 GiB 646 GiB 63.23 0.86 26 up osd.63 -11 24.79999 - 25 TiB 18 TiB 18 TiB 4.1 GiB 59 GiB 6.3 TiB 74.51 1.02 - host ceph04 34 hdd 5.20000 1.00000 5.2 TiB 3.9 TiB 3.8 TiB 954 MiB 13 GiB 1.4 TiB 73.48 1.00 94 up osd.34 42 hdd 5.20000 1.00000 5.2 TiB 3.9 TiB 3.8 TiB 841 MiB 13 GiB 1.4 TiB 73.43 1.00 94 up osd.42 44 hdd 7.20000 1.00000 7.2 TiB 5.5 TiB 5.5 TiB 1.2 GiB 17 GiB 1.6 TiB 77.54 1.06 133 up osd.44 45 hdd 7.20000 1.00000 7.2 TiB 5.2 TiB 5.2 TiB 1.2 GiB 16 GiB 1.9 TiB 73.03 1.00 125 up osd.45 -13 30.09998 - 30 TiB 22 TiB 22 TiB 5.1 GiB 72 GiB 8.0 TiB 73.48 1.00 - host ceph05 39 hdd 7.20000 1.00000 7.2 TiB 5.6 TiB 5.6 TiB 1.4 GiB 17 GiB 1.6 TiB 77.89 1.06 126 up osd.39 40 hdd 7.20000 1.00000 7.2 TiB 5.3 TiB 5.3 TiB 1.2 GiB 17 GiB 1.9 TiB 73.87 1.01 124 up osd.40 41 hdd 7.20000 1.00000 7.2 TiB 5.4 TiB 5.3 TiB 1.1 GiB 17 GiB 1.8 TiB 74.92 1.02 128 up osd.41 43 hdd 5.20000 1.00000 5.2 TiB 3.7 TiB 3.7 TiB 853 MiB 13 GiB 1.5 TiB 71.28 0.97 91 up osd.43 60 hdd 3.29999 1.00000 3.3 TiB 2.1 TiB 2.1 TiB 573 MiB 7.9 GiB 1.2 TiB 63.53 0.87 52 up osd.60 -9 17.59999 - 18 TiB 12 TiB 12 TiB 3.0 GiB 40 GiB 5.3 TiB 70.10 0.96 - host ceph06 35 hdd 7.20000 1.00000 7.2 TiB 5.2 TiB 5.2 TiB 1.3 GiB 16 GiB 1.9 TiB 72.80 0.99 125 up osd.35 36 hdd 5.20000 1.00000 5.2 TiB 3.5 TiB 3.5 TiB 804 MiB 12 GiB 1.8 TiB 66.50 0.91 85 up osd.36 38 hdd 5.20000 1.00000 5.2 TiB 3.7 TiB 3.7 TiB 978 MiB 12 GiB 1.6 TiB 70.02 0.96 88 up osd.38 -15 24.89998 - 25 TiB 18 TiB 18 TiB 4.3 GiB 58 GiB 6.5 TiB 73.75 1.01 - host ceph07 66 hdd 7.20000 1.00000 7.2 TiB 5.3 TiB 5.3 TiB 1.1 GiB 17 GiB 1.8 TiB 74.74 1.02 126 up osd.66 67 hdd 7.20000 1.00000 7.2 TiB 5.3 TiB 5.3 TiB 1.2 GiB 17 GiB 1.8 TiB 74.51 1.02 121 up osd.67 68 hdd 3.29999 1.00000 3.3 TiB 2.3 TiB 2.3 TiB 720 MiB 7.9 GiB 1.0 TiB 68.63 0.94 55 up osd.68 69 hdd 7.20000 1.00000 7.2 TiB 5.3 TiB 5.3 TiB 1.2 GiB 17 GiB 1.8 TiB 74.40 1.02 129 up osd.69 -17 24.50000 - 24 TiB 20 TiB 20 TiB 4.1 GiB 47 GiB 4.4 TiB 82.08 1.12 - host ceph08 37 hdd 9.50000 1.00000 9.5 TiB 7.8 TiB 7.7 TiB 1.5 GiB 18 GiB 1.8 TiB 81.39 1.11 166 up osd.37 46 hdd 5.00000 1.00000 5.0 TiB 4.0 TiB 3.9 TiB 889 MiB 9.3 GiB 1.0 TiB 79.67 1.09 87 up osd.46 47 hdd 5.00000 1.00000 5.0 TiB 4.2 TiB 4.2 TiB 863 MiB 9.7 GiB 817 GiB 83.90 1.15 90 up osd.47 48 hdd 5.00000 1.00000 5.0 TiB 4.2 TiB 4.2 TiB 969 MiB 10 GiB 813 GiB 83.99 1.15 91 up osd.48 TOTAL 206 TiB 151 TiB 151 TiB 36 GiB 503 GiB 55 TiB 73.23 MIN/MAX VAR: 0.82/1.26 STDDEV: 7.00 root@ceph01:~# ceph balancer status { "active": true, "last_optimize_duration": "0:00:00.016174", "last_optimize_started": "Fri Apr 16 12:54:47 2021", "mode": "upmap", "optimize_result": "Unable to find further optimization, or pool(s) pg_num is decreasing, or distribution is already perfect", "plans": [] } root@ceph01:~# ceph versions { "mon": { "ceph version 15.2.5 (2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 3 }, "mgr": { "ceph version 15.2.5 (2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 2 }, "osd": { "ceph version 15.2.5 (2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 56 }, "mds": { "ceph version 15.2.5 (2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 4 }, "overall": { "ceph version 15.2.5 (2c93eff00150f0cc5f106a559557a58d3d7b6f1f) octopus (stable)": 65 } } root@ceph01:~# ceph osd crush rule ls replicated_ruleset root@ceph01:~# ceph osd crush rule dump replicated_ruleset { "rule_id": 0, "rule_name": "replicated_ruleset", "ruleset": 0, "type": 1, "min_size": 1, "max_size": 10, "steps": [ { "op": "take", "item": -1, "item_name": "default" }, { "op": "chooseleaf_firstn", "num": 0, "type": "host" }, { "op": "emit" } ] } Cheers, Michael