4 Jun
2020
4 Jun
'20
3:32 p.m.
Hello,
any idea why it´s so bad balanced?
e.g.: osd.52 (82%) vs osd.34 (29%)
I did run "/usr/bin/ceph osd reweight-by-utilization " by cron for
some time, since i was low on space for some time and that helped a
bit.
What should i do next?
Here is some info:
root@ceph01:~# ceph -s
cluster:
id: 5436dd5d-83d4-4dc8-a93b-60ab5db145df
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph01,ceph02,ceph03 (age 4h)
mgr: ceph01(active, since 4h), standbys: ceph03, ceph02
osd: 55 osds: 55 up (since 4h), 54 in (since 4h); 10 remapped pgs
data:
pools: 3 pools, 2049 pgs
objects: 8.17M objects, 29 TiB
usage: 89 TiB used, 45 TiB / 133 TiB avail
pgs: 27740/24497208 objects misplaced (0.113%)
2034 active+clean
9 active+remapped+backfilling
5 active+clean+scrubbing+deep
1 active+remapped+backfill_wait
io:
recovery: 172 MiB/s, 45 objects/s
root@ceph01:~# ceph balancer status
{
"active": true,
"last_optimize_duration": "0:00:00.017034",
"last_optimize_started": "Thu Jun 4 15:26:28 2020",
"mode": "upmap",
"optimize_result": "Unable to find further optimization, or
pool(s) pg_num is decreasing, or distribution is already perfect",
"plans": []
}
root@ceph01:~# ceph osd df tree
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP
META AVAIL %USE VAR PGS STATUS TYPE NAME
-1 111.70750 - 133 TiB 89 TiB 88 TiB 4.9 GiB
240 GiB 44 TiB 66.79 1.00 - root default
-2 25.44400 - 25 TiB 19 TiB 19 TiB 1.0 GiB
49 GiB 6.1 TiB 75.99 1.14 - host ceph01
0 hdd 2.39999 1.00000 2.7 TiB 1.9 TiB 1.8 TiB 107 MiB
5.2 GiB 835 GiB 69.45 1.04 130 up osd.0
1 hdd 2.29999 1.00000 2.7 TiB 1.8 TiB 1.8 TiB 79 MiB
5.4 GiB 920 GiB 66.36 0.99 124 up osd.1
4 hdd 2.67029 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 127 MiB
5.6 GiB 620 GiB 77.31 1.16 144 up osd.4
8 hdd 2.39999 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 111 MiB
5.3 GiB 755 GiB 72.38 1.08 135 up osd.8
11 hdd 1.71660 1.00000 1.7 TiB 1.3 TiB 1.3 TiB 56 MiB
2.6 GiB 418 GiB 76.22 1.14 92 up osd.11
12 hdd 2.67029 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 122 MiB
5.3 GiB 625 GiB 77.14 1.16 144 up osd.12
14 hdd 2.67029 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 114 MiB
5.6 GiB 642 GiB 76.53 1.15 144 up osd.14
18 hdd 2.79999 1.00000 2.7 TiB 2.3 TiB 2.3 TiB 143 MiB
6.0 GiB 411 GiB 84.95 1.27 158 up osd.18
22 hdd 1.71660 1.00000 1.7 TiB 1.3 TiB 1.3 TiB 59 MiB
2.4 GiB 388 GiB 77.94 1.17 94 up osd.22
30 hdd 1.79999 0.95001 1.7 TiB 1.4 TiB 1.4 TiB 71 MiB
2.9 GiB 324 GiB 81.58 1.22 96 up osd.30
32 hdd 0.50000 0 0 B 0 B 0 B 0 B
0 B 0 B 0 0 0 up osd.32
33 hdd 1.79999 0.95001 1.6 TiB 1.3 TiB 1.3 TiB 54 MiB
2.4 GiB 323 GiB 80.55 1.21 91 up osd.33
-3 24.52742 - 26 TiB 19 TiB 19 TiB 1.1 GiB
54 GiB 6.8 TiB 74.10 1.11 - host ceph02
2 hdd 1.00000 1.00000 1.7 TiB 827 GiB 825 GiB 29 MiB
2.2 GiB 931 GiB 47.05 0.70 56 up osd.2
3 hdd 2.89999 0.95001 2.7 TiB 2.2 TiB 2.2 TiB 123 MiB
6.1 GiB 454 GiB 83.38 1.25 154 up osd.3
7 hdd 2.67029 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 152 MiB
5.8 GiB 540 GiB 80.25 1.20 150 up osd.7
9 hdd 2.67029 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 172 MiB
6.0 GiB 553 GiB 79.79 1.19 149 up osd.9
13 hdd 1.79999 1.00000 2.4 TiB 1.4 TiB 1.4 TiB 70 MiB
4.6 GiB 979 GiB 59.91 0.90 100 up osd.13
16 hdd 2.89999 0.95001 2.7 TiB 2.2 TiB 2.2 TiB 140 MiB
6.0 GiB 479 GiB 82.46 1.23 153 up osd.16
19 hdd 1.39999 1.00000 1.7 TiB 1.1 TiB 1.1 TiB 41 MiB
2.8 GiB 616 GiB 64.96 0.97 78 up osd.19
23 hdd 2.00000 1.00000 2.7 TiB 1.6 TiB 1.6 TiB 96 MiB
5.3 GiB 1.1 TiB 59.58 0.89 111 up osd.23
24 hdd 1.71660 1.00000 1.7 TiB 1.4 TiB 1.4 TiB 52 MiB
3.3 GiB 334 GiB 81.02 1.21 97 up osd.24
28 hdd 2.79999 1.00000 2.7 TiB 2.2 TiB 2.2 TiB 143 MiB
6.2 GiB 440 GiB 83.90 1.26 155 up osd.28
31 hdd 2.67029 1.00000 2.7 TiB 2.2 TiB 2.2 TiB 116 MiB
6.0 GiB 523 GiB 80.87 1.21 149 up osd.31
-4 20.40346 - 25 TiB 16 TiB 16 TiB 932 MiB
46 GiB 8.2 TiB 66.48 1.00 - host ceph03
5 hdd 1.71660 1.00000 1.7 TiB 1.4 TiB 1.4 TiB 61 MiB
2.8 GiB 363 GiB 79.37 1.19 95 up osd.5
6 hdd 1.71660 1.00000 1.7 TiB 1.4 TiB 1.4 TiB 67 MiB
2.9 GiB 329 GiB 81.26 1.22 97 up osd.6
10 hdd 2.50000 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 123 MiB
5.9 GiB 714 GiB 73.89 1.11 139 up osd.10
15 hdd 2.29999 1.00000 2.7 TiB 1.8 TiB 1.8 TiB 102 MiB
5.7 GiB 860 GiB 68.55 1.03 127 up osd.15
17 hdd 1.39999 1.00000 1.6 TiB 1.1 TiB 1.1 TiB 48 MiB
2.5 GiB 524 GiB 68.42 1.02 78 up osd.17
20 hdd 1.59999 1.00000 1.7 TiB 1.3 TiB 1.3 TiB 52 MiB
3.0 GiB 444 GiB 74.73 1.12 90 up osd.20
21 hdd 2.00000 1.00000 2.7 TiB 1.6 TiB 1.6 TiB 103 MiB
5.4 GiB 1.1 TiB 59.96 0.90 112 up osd.21
25 hdd 1.00000 1.00000 1.7 TiB 847 GiB 845 GiB 45 MiB
2.2 GiB 911 GiB 48.20 0.72 57 up osd.25
26 hdd 2.50000 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 126 MiB
5.7 GiB 719 GiB 73.70 1.10 138 up osd.26
27 hdd 1.00000 1.00000 2.7 TiB 820 GiB 816 GiB 61 MiB
3.6 GiB 1.9 TiB 29.97 0.45 56 up osd.27
29 hdd 2.67029 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 145 MiB
6.0 GiB 540 GiB 80.24 1.20 149 up osd.29
-11 12.96458 - 20 TiB 11 TiB 11 TiB 581 MiB
28 GiB 9.0 TiB 54.04 0.81 - host ceph04
34 hdd 1.00000 1.00000 2.7 TiB 819 GiB 815 GiB 70 MiB
3.6 GiB 1.9 TiB 29.96 0.45 55 up osd.34
35 hdd 1.00000 1.00000 2.7 TiB 860 GiB 857 GiB 59 MiB
3.3 GiB 1.8 TiB 31.46 0.47 57 up osd.35
44 hdd 1.00000 1.00000 2.7 TiB 856 GiB 853 GiB 56 MiB
3.6 GiB 1.8 TiB 31.31 0.47 57 up osd.44
45 hdd 1.00000 1.00000 2.7 TiB 816 GiB 812 GiB 65 MiB
3.6 GiB 1.9 TiB 29.84 0.45 55 up osd.45
48 hdd 1.71660 1.00000 1.7 TiB 1.4 TiB 1.4 TiB 50 MiB
2.7 GiB 330 GiB 81.21 1.22 95 up osd.48
51 hdd 3.62399 1.00000 3.6 TiB 3.0 TiB 2.9 TiB 143 MiB
5.3 GiB 685 GiB 81.55 1.22 201 up osd.51
52 hdd 3.62399 1.00000 3.6 TiB 3.0 TiB 3.0 TiB 138 MiB
5.4 GiB 644 GiB 82.64 1.24 204 up osd.52
-13 11.24799 - 16 TiB 9.2 TiB 9.2 TiB 490 MiB
22 GiB 6.6 TiB 58.02 0.87 - host ceph05
46 hdd 1.00000 1.00000 2.6 TiB 856 GiB 853 GiB 56 MiB
3.1 GiB 1.7 TiB 32.48 0.49 57 up osd.46
47 hdd 1.00000 1.00000 2.6 TiB 857 GiB 854 GiB 86 MiB
3.4 GiB 1.7 TiB 32.51 0.49 57 up osd.47
49 hdd 1.00000 1.00000 1.7 TiB 830 GiB 829 GiB 36 MiB
1.9 GiB 927 GiB 47.24 0.71 56 up osd.49
50 hdd 1.00000 1.00000 1.7 TiB 826 GiB 824 GiB 30 MiB
1.8 GiB 932 GiB 47.00 0.70 55 up osd.50
53 hdd 3.62399 1.00000 3.6 TiB 2.9 TiB 2.9 TiB 159 MiB
5.9 GiB 696 GiB 81.25 1.22 200 up osd.53
54 hdd 3.62399 1.00000 3.6 TiB 2.9 TiB 2.9 TiB 123 MiB
5.7 GiB 691 GiB 81.38 1.22 200 up osd.54
-9 17.12006 - 21 TiB 14 TiB 14 TiB 843 MiB
42 GiB 7.3 TiB 65.39 0.98 - host ceph06
36 hdd 2.50000 1.00000 2.7 TiB 2.0 TiB 2.0 TiB 125 MiB
5.8 GiB 677 GiB 75.24 1.13 140 up osd.36
37 hdd 2.39999 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 120 MiB
5.8 GiB 769 GiB 71.89 1.08 134 up osd.37
38 hdd 2.67029 1.00000 2.7 TiB 2.1 TiB 2.1 TiB 125 MiB
5.9 GiB 551 GiB 79.85 1.20 149 up osd.38
39 hdd 2.39999 1.00000 2.7 TiB 1.9 TiB 1.9 TiB 121 MiB
5.6 GiB 773 GiB 71.74 1.07 134 up osd.39
40 hdd 2.57489 1.00000 2.6 TiB 2.0 TiB 2.0 TiB 91 MiB
5.6 GiB 543 GiB 79.40 1.19 143 up osd.40
41 hdd 1.00000 1.00000 2.6 TiB 850 GiB 846 GiB 85 MiB
3.9 GiB 1.7 TiB 32.23 0.48 57 up osd.41
42 hdd 2.57489 1.00000 2.6 TiB 2.0 TiB 2.0 TiB 90 MiB
5.6 GiB 544 GiB 79.38 1.19 142 up osd.42
43 hdd 1.00000 1.00000 2.6 TiB 844 GiB 840 GiB 87 MiB
3.9 GiB 1.8 TiB 31.99 0.48 57 up osd.43
TOTAL 133 TiB 89 TiB 88 TiB 4.9 GiB
241 GiB 45 TiB 66.79
MIN/MAX VAR: 0.45/1.27 STDDEV: 18.42