[ceph-users] Large STDDEV in pg per osd

I have been digging for a while on how to minimize STDDEV of the distribution of data on my OSDs and I can't seem to get it to get below 12. I have other clusters that have a STDDEV of 1 which is my goal. But this cluster is really giving me fits. This cluster started off on the Emperor. Might have even ran Dumpling originally. I'm not sure I have some crud laying around that is preventing better balancing. I have tried the ceph balancer and I even tried jj balancer. As you can see below My lowest osd percent used is 37% and highest is 64% If anyone has any ideas on how to crack this nut I would greatly appreciate it. Thanks, Joe Here is my data: ceph osd tree ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF -1 167.63680 root default -3 57.62895 rack marack -4 10.47656 host gallo 0 ssd 3.49219 osd.0 up 1.00000 1.00000 1 ssd 3.49219 osd.1 up 1.00000 1.00000 8 ssd 3.49219 osd.8 up 1.00000 1.00000 -8 7.85712 host jeep 18 ssd 0.87299 osd.18 up 1.00000 1.00000 19 ssd 0.87299 osd.19 up 1.00000 1.00000 20 ssd 0.87299 osd.20 up 1.00000 1.00000 21 ssd 0.87299 osd.21 up 1.00000 1.00000 22 ssd 3.49219 osd.22 up 1.00000 1.00000 45 ssd 0.87299 osd.45 up 1.00000 1.00000 -5 7.85712 host joy 5 ssd 0.87299 osd.5 up 1.00000 1.00000 12 ssd 0.87299 osd.12 up 1.00000 1.00000 13 ssd 0.87299 osd.13 up 1.00000 1.00000 16 ssd 0.87299 osd.16 up 1.00000 1.00000 17 ssd 3.49219 osd.17 up 1.00000 1.00000 43 ssd 0.87299 osd.43 up 1.00000 1.00000 -37 6.98625 host kc01 82 ssd 0.87329 osd.82 up 1.00000 1.00000 83 ssd 0.87329 osd.83 up 1.00000 1.00000 84 ssd 0.87329 osd.84 up 1.00000 1.00000 85 ssd 3.49309 osd.85 up 1.00000 1.00000 86 ssd 0.87329 osd.86 up 1.00000 1.00000 -50 6.98625 host ks02 97 ssd 0.87329 osd.97 up 1.00000 1.00000 98 ssd 0.87329 osd.98 up 1.00000 1.00000 99 ssd 0.87329 osd.99 up 1.00000 1.00000 100 ssd 0.87329 osd.100 up 1.00000 1.00000 101 ssd 3.49309 osd.101 up 1.00000 1.00000 -27 3.49319 host lc02 52 ssd 1.74660 osd.52 up 1.00000 1.00000 73 ssd 1.74660 osd.73 up 1.00000 1.00000 -55 13.97246 host lx01 14 ssd 3.49309 osd.14 up 1.00000 1.00000 27 ssd 3.49309 osd.27 up 1.00000 1.00000 29 ssd 3.49309 osd.29 up 1.00000 1.00000 48 ssd 1.74660 osd.48 up 1.00000 1.00000 49 ssd 1.74660 osd.49 up 1.00000 1.00000 -2 55.00392 rack marack2 -13 13.96875 host helm 11 ssd 3.49219 osd.11 up 1.00000 1.00000 28 ssd 3.49219 osd.28 up 1.00000 1.00000 30 ssd 3.49219 osd.30 up 1.00000 1.00000 31 ssd 3.49219 osd.31 up 1.00000 1.00000 -17 7.85712 host jazz 23 ssd 0.87299 osd.23 up 1.00000 1.00000 24 ssd 0.87299 osd.24 up 1.00000 1.00000 25 ssd 0.87299 osd.25 up 1.00000 1.00000 26 ssd 0.87299 osd.26 up 1.00000 1.00000 36 ssd 3.49219 osd.36 up 1.00000 1.00000 44 ssd 0.87299 osd.44 up 1.00000 1.00000 -6 6.98438 host john 2 ssd 3.49219 osd.2 up 1.00000 1.00000 3 ssd 3.49219 osd.3 up 1.00000 1.00000 -49 7.85712 host jolt 34 ssd 3.49219 osd.34 up 1.00000 1.00000 63 ssd 0.87299 osd.63 up 1.00000 1.00000 64 ssd 0.87299 osd.64 up 1.00000 1.00000 65 ssd 0.87299 osd.65 up 1.00000 1.00000 66 ssd 0.87299 osd.66 up 1.00000 1.00000 67 ssd 0.87299 osd.67 up 1.00000 1.00000 -19 7.85712 host juju 37 ssd 0.87299 osd.37 up 1.00000 1.00000 38 ssd 0.87299 osd.38 up 1.00000 1.00000 39 ssd 0.87299 osd.39 up 1.00000 1.00000 40 ssd 0.87299 osd.40 up 1.00000 1.00000 41 ssd 3.49219 osd.41 up 1.00000 1.00000 46 ssd 0.87299 osd.46 up 1.00000 1.00000 -44 6.98625 host ks01 92 ssd 0.87329 osd.92 up 1.00000 1.00000 93 ssd 0.87329 osd.93 up 1.00000 1.00000 94 ssd 0.87329 osd.94 up 1.00000 1.00000 95 ssd 0.87329 osd.95 up 1.00000 1.00000 96 ssd 3.49309 osd.96 up 1.00000 1.00000 -22 3.49319 host lc01 50 ssd 1.74660 osd.50 up 1.00000 1.00000 51 ssd 1.74660 osd.51 up 1.00000 1.00000 -11 55.00392 rack marack3 -10 13.96875 host gold 7 ssd 3.49219 osd.7 up 1.00000 1.00000 9 ssd 3.49219 osd.9 up 1.00000 1.00000 10 ssd 3.49219 osd.10 up 1.00000 1.00000 15 ssd 3.49219 osd.15 up 1.00000 1.00000 -12 3.49319 host innes 42 ssd 1.74660 osd.42 up 1.00000 1.00000 47 ssd 1.74660 osd.47 up 1.00000 1.00000 -16 6.98438 host jack 4 ssd 3.49219 osd.4 up 1.00000 1.00000 6 ssd 3.49219 osd.6 up 1.00000 1.00000 -43 7.85712 host jam 32 ssd 3.49219 osd.32 up 1.00000 1.00000 53 ssd 0.87299 osd.53 up 1.00000 1.00000 54 ssd 0.87299 osd.54 up 1.00000 1.00000 55 ssd 0.87299 osd.55 up 1.00000 1.00000 56 ssd 0.87299 osd.56 up 1.00000 1.00000 57 ssd 0.87299 osd.57 up 1.00000 1.00000 -46 7.85712 host jet 33 ssd 3.49219 osd.33 up 1.00000 1.00000 58 ssd 0.87299 osd.58 up 1.00000 1.00000 59 ssd 0.87299 osd.59 up 1.00000 1.00000 60 ssd 0.87299 osd.60 up 1.00000 1.00000 61 ssd 0.87299 osd.61 up 1.00000 1.00000 62 ssd 0.87299 osd.62 up 1.00000 1.00000 -52 7.85712 host jug 35 ssd 3.49219 osd.35 up 1.00000 1.00000 68 ssd 0.87299 osd.68 up 1.00000 1.00000 69 ssd 0.87299 osd.69 up 1.00000 1.00000 70 ssd 0.87299 osd.70 up 1.00000 1.00000 71 ssd 0.87299 osd.71 up 1.00000 1.00000 72 ssd 0.87299 osd.72 up 1.00000 1.00000 -41 6.98625 host kc02 87 ssd 0.87329 osd.87 up 1.00000 1.00000 88 ssd 0.87329 osd.88 up 1.00000 1.00000 89 ssd 0.87329 osd.89 up 1.00000 1.00000 90 ssd 3.49309 osd.90 up 1.00000 1.00000 91 ssd 0.87329 osd.91 up 1.00000 1.00000 ceph osd df tree ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME -1 167.63680 - 168 TiB 80 TiB 79 TiB 331 MiB 282 GiB 88 TiB 47.56 1.00 - root default -3 57.62895 - 58 TiB 26 TiB 26 TiB 106 MiB 96 GiB 31 TiB 45.78 0.96 - rack marack -4 10.47656 - 10 TiB 4.3 TiB 4.3 TiB 30 MiB 14 GiB 6.1 TiB 41.46 0.87 - host gallo 0 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.4 TiB 9.9 MiB 4.8 GiB 2.0 TiB 41.53 0.87 141 up osd.0 1 ssd 3.49219 1.00000 3.5 TiB 1.4 TiB 1.4 TiB 3.7 MiB 4.7 GiB 2.1 TiB 40.42 0.85 136 up osd.1 8 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 16 MiB 4.8 GiB 2.0 TiB 42.44 0.89 132 up osd.8 -8 7.85712 - 7.9 TiB 4.1 TiB 4.1 TiB 14 MiB 14 GiB 3.7 TiB 52.30 1.10 - host jeep 18 ssd 0.87299 1.00000 894 GiB 569 GiB 567 GiB 987 KiB 2.0 GiB 325 GiB 63.65 1.34 35 up osd.18 19 ssd 0.87299 1.00000 894 GiB 552 GiB 550 GiB 991 KiB 1.9 GiB 342 GiB 61.72 1.30 32 up osd.19 20 ssd 0.87299 1.00000 894 GiB 549 GiB 548 GiB 945 KiB 1.8 GiB 345 GiB 61.46 1.29 69 up osd.20 21 ssd 0.87299 1.00000 894 GiB 577 GiB 575 GiB 1.0 MiB 1.8 GiB 317 GiB 64.57 1.36 33 up osd.21 22 ssd 3.49219 1.00000 3.5 TiB 1.4 TiB 1.4 TiB 8.8 MiB 4.8 GiB 2.1 TiB 39.21 0.82 133 up osd.22 45 ssd 0.87299 1.00000 894 GiB 558 GiB 556 GiB 938 KiB 1.9 GiB 336 GiB 62.41 1.31 35 up osd.45 -5 7.85712 - 7.9 TiB 4.2 TiB 4.1 TiB 16 MiB 14 GiB 3.7 TiB 52.85 1.11 - host joy 5 ssd 0.87299 1.00000 894 GiB 569 GiB 567 GiB 968 KiB 1.9 GiB 325 GiB 63.60 1.34 32 up osd.5 12 ssd 0.87299 1.00000 894 GiB 576 GiB 575 GiB 1006 KiB 1.9 GiB 318 GiB 64.49 1.36 33 up osd.12 13 ssd 0.87299 1.00000 894 GiB 578 GiB 576 GiB 1.0 MiB 1.9 GiB 316 GiB 64.65 1.36 33 up osd.13 16 ssd 0.87299 1.00000 894 GiB 569 GiB 567 GiB 1015 KiB 1.9 GiB 325 GiB 63.68 1.34 35 up osd.16 17 ssd 3.49219 1.00000 3.5 TiB 1.4 TiB 1.4 TiB 11 MiB 4.6 GiB 2.1 TiB 39.36 0.83 130 up osd.17 43 ssd 0.87299 1.00000 894 GiB 553 GiB 551 GiB 968 KiB 1.9 GiB 341 GiB 61.81 1.30 33 up osd.43 -37 6.98625 - 7.0 TiB 3.3 TiB 3.3 TiB 9.0 MiB 13 GiB 3.7 TiB 46.81 0.98 - host kc01 82 ssd 0.87329 1.00000 894 GiB 497 GiB 495 GiB 984 KiB 1.8 GiB 397 GiB 55.58 1.17 34 up osd.82 83 ssd 0.87329 1.00000 894 GiB 496 GiB 494 GiB 1.6 MiB 1.9 GiB 399 GiB 55.42 1.17 36 up osd.83 84 ssd 0.87329 1.00000 894 GiB 513 GiB 511 GiB 1.0 MiB 1.9 GiB 381 GiB 57.34 1.21 35 up osd.84 85 ssd 3.49309 1.00000 3.5 TiB 1.3 TiB 1.3 TiB 4.5 MiB 5.7 GiB 2.2 TiB 37.68 0.79 140 up osd.85 86 ssd 0.87329 1.00000 894 GiB 495 GiB 493 GiB 931 KiB 1.9 GiB 399 GiB 55.37 1.16 43 up osd.86 -50 6.98625 - 7.0 TiB 3.3 TiB 3.3 TiB 18 MiB 13 GiB 3.7 TiB 47.32 0.99 - host ks02 97 ssd 0.87329 1.00000 894 GiB 493 GiB 491 GiB 934 KiB 1.9 GiB 401 GiB 55.11 1.16 100 up osd.97 98 ssd 0.87329 1.00000 894 GiB 518 GiB 516 GiB 1.5 MiB 1.9 GiB 376 GiB 57.92 1.22 33 up osd.98 99 ssd 0.87329 1.00000 894 GiB 514 GiB 512 GiB 1019 KiB 1.7 GiB 380 GiB 57.46 1.21 38 up osd.99 100 ssd 0.87329 1.00000 894 GiB 514 GiB 512 GiB 1.0 MiB 1.9 GiB 381 GiB 57.44 1.21 35 up osd.100 101 ssd 3.49309 1.00000 3.5 TiB 1.3 TiB 1.3 TiB 13 MiB 5.3 GiB 2.2 TiB 37.65 0.79 140 up osd.101 -27 3.49319 - 3.5 TiB 1.6 TiB 1.6 TiB 4.2 MiB 6.9 GiB 1.9 TiB 45.58 0.96 - host lc02 52 ssd 1.74660 1.00000 1.7 TiB 799 GiB 796 GiB 2.0 MiB 3.4 GiB 989 GiB 44.70 0.94 70 up osd.52 73 ssd 1.74660 1.00000 1.7 TiB 831 GiB 828 GiB 2.2 MiB 3.5 GiB 957 GiB 46.47 0.98 66 up osd.73 -55 13.97246 - 14 TiB 5.6 TiB 5.6 TiB 16 MiB 20 GiB 8.4 TiB 40.15 0.84 - host lx01 14 ssd 3.49309 1.00000 3.5 TiB 1.3 TiB 1.3 TiB 3.8 MiB 3.9 GiB 2.2 TiB 37.83 0.80 140 up osd.14 27 ssd 3.49309 1.00000 3.5 TiB 1.3 TiB 1.3 TiB 3.9 MiB 4.7 GiB 2.2 TiB 37.65 0.79 140 up osd.27 29 ssd 3.49309 1.00000 3.5 TiB 1.4 TiB 1.4 TiB 4.1 MiB 4.8 GiB 2.1 TiB 40.47 0.85 133 up osd.29 48 ssd 1.74660 1.00000 1.7 TiB 817 GiB 813 GiB 2.1 MiB 3.2 GiB 972 GiB 45.66 0.96 71 up osd.48 49 ssd 1.74660 1.00000 1.7 TiB 780 GiB 776 GiB 2.0 MiB 3.5 GiB 1009 GiB 43.61 0.92 69 up osd.49 -2 55.00392 - 55 TiB 27 TiB 27 TiB 118 MiB 94 GiB 28 TiB 48.50 1.02 - rack marack2 -13 13.96875 - 14 TiB 5.9 TiB 5.9 TiB 20 MiB 20 GiB 8.0 TiB 42.41 0.89 - host helm 11 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 3.9 MiB 4.8 GiB 2.0 TiB 42.34 0.89 132 up osd.11 28 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 7.3 MiB 5.0 GiB 2.0 TiB 42.48 0.89 132 up osd.28 30 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 4.1 MiB 5.1 GiB 2.0 TiB 42.44 0.89 132 up osd.30 31 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 4.7 MiB 5.1 GiB 2.0 TiB 42.40 0.89 132 up osd.31 -17 7.85712 - 7.9 TiB 4.3 TiB 4.3 TiB 20 MiB 15 GiB 3.6 TiB 54.33 1.14 - host jazz 23 ssd 0.87299 1.00000 894 GiB 570 GiB 568 GiB 1003 KiB 1.9 GiB 324 GiB 63.81 1.34 51 up osd.23 24 ssd 0.87299 1.00000 894 GiB 572 GiB 570 GiB 1.0 MiB 2.0 GiB 322 GiB 63.99 1.35 34 up osd.24 25 ssd 0.87299 1.00000 894 GiB 576 GiB 574 GiB 1.0 MiB 1.9 GiB 318 GiB 64.39 1.35 33 up osd.25 26 ssd 0.87299 1.00000 894 GiB 571 GiB 569 GiB 1018 KiB 2.0 GiB 323 GiB 63.91 1.34 49 up osd.26 36 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 15 MiB 5.0 GiB 2.0 TiB 42.23 0.89 131 up osd.36 44 ssd 0.87299 1.00000 894 GiB 572 GiB 570 GiB 1.0 MiB 1.9 GiB 322 GiB 63.96 1.34 49 up osd.44 -6 6.98438 - 7.0 TiB 2.9 TiB 2.9 TiB 27 MiB 9.9 GiB 4.1 TiB 41.83 0.88 - host john 2 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 16 MiB 4.8 GiB 2.0 TiB 42.33 0.89 132 up osd.2 3 ssd 3.49219 1.00000 3.5 TiB 1.4 TiB 1.4 TiB 11 MiB 5.1 GiB 2.0 TiB 41.33 0.87 141 up osd.3 -49 7.85712 - 7.9 TiB 4.2 TiB 4.2 TiB 17 MiB 14 GiB 3.6 TiB 53.92 1.13 - host jolt 34 ssd 3.49219 1.00000 3.5 TiB 1.4 TiB 1.4 TiB 9.6 MiB 4.8 GiB 2.0 TiB 41.43 0.87 142 up osd.34 63 ssd 0.87299 1.00000 894 GiB 571 GiB 569 GiB 1023 KiB 1.9 GiB 323 GiB 63.88 1.34 45 up osd.63 64 ssd 0.87299 1.00000 894 GiB 571 GiB 569 GiB 1.0 MiB 2.0 GiB 323 GiB 63.91 1.34 34 up osd.64 65 ssd 0.87299 1.00000 894 GiB 571 GiB 569 GiB 983 KiB 1.9 GiB 323 GiB 63.86 1.34 39 up osd.65 66 ssd 0.87299 1.00000 894 GiB 571 GiB 569 GiB 992 KiB 1.9 GiB 323 GiB 63.91 1.34 41 up osd.66 67 ssd 0.87299 1.00000 894 GiB 572 GiB 570 GiB 3.6 MiB 1.9 GiB 322 GiB 64.00 1.35 34 up osd.67 -19 7.85712 - 7.9 TiB 4.3 TiB 4.3 TiB 14 MiB 15 GiB 3.6 TiB 54.34 1.14 - host juju 37 ssd 0.87299 1.00000 894 GiB 574 GiB 572 GiB 1.0 MiB 1.9 GiB 320 GiB 64.23 1.35 33 up osd.37 38 ssd 0.87299 1.00000 894 GiB 570 GiB 569 GiB 1003 KiB 1.8 GiB 324 GiB 63.80 1.34 42 up osd.38 39 ssd 0.87299 1.00000 894 GiB 577 GiB 575 GiB 1.1 MiB 1.7 GiB 317 GiB 64.51 1.36 33 up osd.39 40 ssd 0.87299 1.00000 894 GiB 577 GiB 575 GiB 1004 KiB 1.9 GiB 317 GiB 64.51 1.36 33 up osd.40 41 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 9.2 MiB 5.4 GiB 2.0 TiB 41.97 0.88 135 up osd.41 46 ssd 0.87299 1.00000 894 GiB 573 GiB 571 GiB 1003 KiB 1.8 GiB 321 GiB 64.09 1.35 33 up osd.46 -44 6.98625 - 7.0 TiB 3.4 TiB 3.4 TiB 16 MiB 13 GiB 3.5 TiB 49.37 1.04 - host ks01 92 ssd 0.87329 1.00000 894 GiB 522 GiB 520 GiB 1.0 MiB 2.0 GiB 372 GiB 58.35 1.23 33 up osd.92 93 ssd 0.87329 1.00000 894 GiB 520 GiB 518 GiB 1004 KiB 1.8 GiB 375 GiB 58.10 1.22 33 up osd.93 94 ssd 0.87329 1.00000 894 GiB 515 GiB 513 GiB 1.7 MiB 2.0 GiB 379 GiB 57.61 1.21 48 up osd.94 95 ssd 0.87329 1.00000 894 GiB 515 GiB 514 GiB 1.0 MiB 1.8 GiB 379 GiB 57.63 1.21 35 up osd.95 96 ssd 3.49309 1.00000 3.5 TiB 1.4 TiB 1.4 TiB 11 MiB 5.7 GiB 2.1 TiB 40.83 0.86 132 up osd.96 -22 3.49319 - 3.5 TiB 1.6 TiB 1.6 TiB 4.1 MiB 7.1 GiB 1.9 TiB 45.95 0.97 - host lc01 50 ssd 1.74660 1.00000 1.7 TiB 820 GiB 817 GiB 2.1 MiB 3.5 GiB 968 GiB 45.87 0.96 76 up osd.50 51 ssd 1.74660 1.00000 1.7 TiB 823 GiB 819 GiB 2.0 MiB 3.6 GiB 965 GiB 46.02 0.97 78 up osd.51 -11 55.00392 - 55 TiB 27 TiB 27 TiB 106 MiB 92 GiB 28 TiB 48.49 1.02 - rack marack3 -10 13.96875 - 14 TiB 5.9 TiB 5.8 TiB 32 MiB 20 GiB 8.1 TiB 41.90 0.88 - host gold 7 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 4.0 MiB 4.8 GiB 2.0 TiB 42.24 0.89 131 up osd.7 9 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.4 TiB 9.3 MiB 4.9 GiB 2.0 TiB 41.61 0.87 140 up osd.9 10 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 4.0 MiB 5.0 GiB 2.0 TiB 42.20 0.89 131 up osd.10 15 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.4 TiB 14 MiB 4.8 GiB 2.0 TiB 41.54 0.87 143 up osd.15 -12 3.49319 - 3.5 TiB 1.6 TiB 1.6 TiB 4.2 MiB 6.6 GiB 1.9 TiB 46.48 0.98 - host innes 42 ssd 1.74660 1.00000 1.7 TiB 834 GiB 830 GiB 2.1 MiB 3.2 GiB 955 GiB 46.61 0.98 66 up osd.42 47 ssd 1.74660 1.00000 1.7 TiB 829 GiB 826 GiB 2.1 MiB 3.4 GiB 959 GiB 46.36 0.97 67 up osd.47 -16 6.98438 - 7.0 TiB 3.0 TiB 2.9 TiB 15 MiB 10 GiB 4.0 TiB 42.30 0.89 - host jack 4 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 11 MiB 5.1 GiB 2.0 TiB 42.31 0.89 131 up osd.4 6 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 4.1 MiB 4.9 GiB 2.0 TiB 42.30 0.89 132 up osd.6 -43 7.85712 - 7.9 TiB 4.3 TiB 4.3 TiB 14 MiB 14 GiB 3.6 TiB 54.53 1.15 - host jam 32 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.5 TiB 9.1 MiB 5.0 GiB 2.0 TiB 42.58 0.90 133 up osd.32 53 ssd 0.87299 1.00000 894 GiB 574 GiB 572 GiB 1011 KiB 1.9 GiB 320 GiB 64.17 1.35 34 up osd.53 54 ssd 0.87299 1.00000 894 GiB 574 GiB 572 GiB 1.0 MiB 2.0 GiB 320 GiB 64.19 1.35 33 up osd.54 55 ssd 0.87299 1.00000 894 GiB 572 GiB 571 GiB 984 KiB 1.8 GiB 322 GiB 64.02 1.35 48 up osd.55 56 ssd 0.87299 1.00000 894 GiB 572 GiB 570 GiB 1003 KiB 1.9 GiB 322 GiB 64.03 1.35 103 up osd.56 57 ssd 0.87299 1.00000 894 GiB 573 GiB 571 GiB 995 KiB 1.7 GiB 321 GiB 64.06 1.35 35 up osd.57 -46 7.85712 - 7.9 TiB 4.3 TiB 4.2 TiB 19 MiB 14 GiB 3.6 TiB 54.20 1.14 - host jet 33 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.4 TiB 13 MiB 5.0 GiB 2.0 TiB 41.61 0.87 142 up osd.33 58 ssd 0.87299 1.00000 894 GiB 573 GiB 571 GiB 1004 KiB 2.0 GiB 321 GiB 64.05 1.35 35 up osd.58 59 ssd 0.87299 1.00000 894 GiB 577 GiB 575 GiB 1.3 MiB 1.8 GiB 317 GiB 64.55 1.36 33 up osd.59 60 ssd 0.87299 1.00000 894 GiB 573 GiB 571 GiB 1019 KiB 1.9 GiB 321 GiB 64.11 1.35 34 up osd.60 61 ssd 0.87299 1.00000 894 GiB 575 GiB 573 GiB 1011 KiB 1.9 GiB 319 GiB 64.29 1.35 33 up osd.61 62 ssd 0.87299 1.00000 894 GiB 575 GiB 573 GiB 1.0 MiB 1.7 GiB 319 GiB 64.31 1.35 33 up osd.62 -52 7.85712 - 7.9 TiB 4.3 TiB 4.2 TiB 13 MiB 14 GiB 3.6 TiB 54.23 1.14 - host jug 35 ssd 3.49219 1.00000 3.5 TiB 1.5 TiB 1.4 TiB 8.2 MiB 4.9 GiB 2.0 TiB 41.64 0.88 142 up osd.35 68 ssd 0.87299 1.00000 894 GiB 576 GiB 574 GiB 1.0 MiB 1.9 GiB 318 GiB 64.42 1.35 33 up osd.68 69 ssd 0.87299 1.00000 894 GiB 574 GiB 572 GiB 1000 KiB 2.0 GiB 320 GiB 64.17 1.35 34 up osd.69 70 ssd 0.87299 1.00000 894 GiB 573 GiB 571 GiB 1011 KiB 1.9 GiB 321 GiB 64.07 1.35 35 up osd.70 71 ssd 0.87299 1.00000 894 GiB 574 GiB 572 GiB 988 KiB 1.9 GiB 320 GiB 64.15 1.35 33 up osd.71 72 ssd 0.87299 1.00000 894 GiB 578 GiB 576 GiB 1.0 MiB 2.0 GiB 316 GiB 64.67 1.36 33 up osd.72 -41 6.98625 - 7.0 TiB 3.4 TiB 3.4 TiB 9.4 MiB 13 GiB 3.5 TiB 49.23 1.04 - host kc02 87 ssd 0.87329 1.00000 894 GiB 521 GiB 519 GiB 1.0 MiB 1.9 GiB 374 GiB 58.23 1.22 33 up osd.87 88 ssd 0.87329 1.00000 894 GiB 521 GiB 519 GiB 1012 KiB 1.8 GiB 373 GiB 58.29 1.23 33 up osd.88 89 ssd 0.87329 1.00000 894 GiB 519 GiB 517 GiB 1.4 MiB 1.9 GiB 376 GiB 57.99 1.22 33 up osd.89 90 ssd 3.49309 1.00000 3.5 TiB 1.4 TiB 1.4 TiB 5.0 MiB 5.2 GiB 2.1 TiB 40.31 0.85 148 up osd.90 91 ssd 0.87329 1.00000 894 GiB 519 GiB 517 GiB 997 KiB 2.0 GiB 375 GiB 58.07 1.22 33 up osd.91 TOTAL 168 TiB 80 TiB 79 TiB 331 MiB 282 GiB 88 TiB 47.56 MIN/MAX VAR: 0.79/1.36 STDDEV: 12.03 placementoptimizer.py show --osds --sort-utilization poolid name type size min pg_num stored used avail shrdsize crush 0 data repl 3 1 32 118.5M 355.9M 17.0T 3.7M 0:data default*1.000 1 metadata repl 3 1 32 13.2K 1.1M 17.0T 423.9B 1:metadata default*1.000 2 rbd repl 3 1 32 0.0B 0.0B 17.0T 0.0B 2:rbd default*1.000 3 images repl 3 1 2048 19.0T 56.9T 17.0T 9.5G 0:data default*1.000 4 locks repl 3 1 32 595.2M 1.8G 17.0T 18.6M 0:data default*1.000 24 device_health_metrics repl 3 2 1 6.7M 20.0M 17.0T 6.7M 0:data default*1.000 25 k8s-101v repl 3 1 32 113.3M 341.3M 17.0T 3.5M 0:data default*1.000 26 k8s-102pm repl 3 2 32 50.4G 151.6G 17.0T 1.6G 0:data default*1.000 27 k8s-103pm repl 3 2 32 25.0B 64.0K 17.0T 0.8B 0:data default*1.000 default 19.02T 57.06T sum 19.02T 57.06T osdid hostname cls devsize weight cweight util pg_num pools 27 lx01 ssd 3.5T 1.00 3.5T 37.6% 140 0 1 2 3 4 25 26 27 101 ks02 ssd 3.5T 1.00 3.5T 37.7% 140 0 1 2 3 4 25 26 27 85 kc01 ssd 3.5T 1.00 3.5T 37.7% 140 0 1 2 3 4 25 26 27 14 lx01 ssd 3.5T 1.00 3.5T 37.8% 140 0 1 2 3 4 25 26 27 22 jeep ssd 3.5T 1.00 3.5T 39.2% 133 0 1 2 3 4 25 26 27 17 joy ssd 3.5T 1.00 3.5T 39.4% 130 0 1 2 3 4 25 26 27 90 kc02 ssd 3.5T 1.00 3.5T 40.3% 148 0 1 2 3 4 25 26 27 1 gallo ssd 3.5T 1.00 3.5T 40.4% 136 0 1 2 3 4 25 26 27 29 lx01 ssd 3.5T 1.00 3.5T 40.5% 133 3 26 27 96 ks01 ssd 3.5T 1.00 3.5T 40.8% 132 3 3 john ssd 3.5T 1.00 3.5T 41.3% 141 0 1 2 3 4 25 26 27 34 jolt ssd 3.5T 1.00 3.5T 41.4% 142 0 1 2 3 4 24 25 26 27 0 gallo ssd 3.5T 1.00 3.5T 41.5% 141 0 1 2 3 4 25 26 27 15 gold ssd 3.5T 1.00 3.5T 41.5% 143 0 1 2 3 4 24 25 26 27 33 jet ssd 3.5T 1.00 3.5T 41.6% 142 0 1 2 3 4 25 26 27 9 gold ssd 3.5T 1.00 3.5T 41.6% 140 0 1 2 3 4 25 26 27 35 jug ssd 3.5T 1.00 3.5T 41.6% 142 0 1 2 3 4 25 26 27 41 juju ssd 3.5T 1.00 3.5T 42.0% 135 0 3 4 25 26 27 10 gold ssd 3.5T 1.00 3.5T 42.2% 131 3 36 jazz ssd 3.5T 1.00 3.5T 42.2% 131 3 7 gold ssd 3.5T 1.00 3.5T 42.2% 131 3 6 jack ssd 3.5T 1.00 3.5T 42.3% 132 3 27 4 jack ssd 3.5T 1.00 3.5T 42.3% 131 3 2 john ssd 3.5T 1.00 3.5T 42.3% 132 3 11 helm ssd 3.5T 1.00 3.5T 42.3% 132 3 31 helm ssd 3.5T 1.00 3.5T 42.4% 132 3 30 helm ssd 3.5T 1.00 3.5T 42.4% 132 3 8 gallo ssd 3.5T 1.00 3.5T 42.4% 132 3 28 helm ssd 3.5T 1.00 3.5T 42.5% 132 3 32 jam ssd 3.5T 1.00 3.5T 42.6% 133 3 49 lx01 ssd 1.7T 1.00 1.7T 43.6% 69 0 1 2 3 4 25 26 27 52 lc02 ssd 1.7T 1.00 1.7T 44.7% 70 0 1 2 3 4 25 26 27 48 lx01 ssd 1.7T 1.00 1.7T 45.6% 71 0 1 2 3 4 25 26 27 50 lc01 ssd 1.7T 1.00 1.7T 45.9% 76 0 1 2 3 4 25 26 27 51 lc01 ssd 1.7T 1.00 1.7T 46.0% 78 0 1 2 3 4 25 26 27 47 innes ssd 1.7T 1.00 1.7T 46.3% 67 3 26 73 lc02 ssd 1.7T 1.00 1.7T 46.5% 66 3 42 innes ssd 1.7T 1.00 1.7T 46.6% 66 3 97 ks02 ssd 894.2G 1.00 894.2G 55.1% 100 0 1 2 3 4 25 26 27 86 kc01 ssd 894.2G 1.00 894.2G 55.4% 43 1 2 3 24 26 83 kc01 ssd 894.2G 1.00 894.2G 55.4% 36 1 2 3 4 25 26 27 82 kc01 ssd 894.2G 1.00 894.2G 55.6% 34 1 3 25 26 27 84 kc01 ssd 894.2G 1.00 894.2G 57.3% 35 0 3 25 26 100 ks02 ssd 894.2G 1.00 894.2G 57.4% 35 1 2 3 27 99 ks02 ssd 894.2G 1.00 894.2G 57.5% 38 0 1 2 3 4 25 26 94 ks01 ssd 894.2G 1.00 894.2G 57.6% 48 0 3 4 25 26 27 95 ks01 ssd 894.2G 1.00 894.2G 57.6% 35 0 3 25 26 98 ks02 ssd 894.2G 1.00 894.2G 57.9% 33 3 89 kc02 ssd 894.2G 1.00 894.2G 58.0% 33 3 91 kc02 ssd 894.2G 1.00 894.2G 58.1% 33 3 93 ks01 ssd 894.2G 1.00 894.2G 58.1% 33 3 87 kc02 ssd 894.2G 1.00 894.2G 58.2% 33 3 88 kc02 ssd 894.2G 1.00 894.2G 58.3% 33 3 92 ks01 ssd 894.2G 1.00 894.2G 58.3% 33 3 20 jeep ssd 894.0G 1.00 893.9G 61.5% 69 1 2 3 4 26 19 jeep ssd 894.0G 1.00 893.9G 61.7% 32 3 25 26 43 joy ssd 894.0G 1.00 893.9G 61.8% 33 0 3 4 26 45 jeep ssd 894.0G 1.00 893.9G 62.4% 35 0 1 2 3 4 5 joy ssd 894.0G 1.00 893.9G 63.6% 32 3 18 jeep ssd 894.0G 1.00 893.9G 63.6% 35 1 3 26 27 16 joy ssd 894.0G 1.00 893.9G 63.7% 35 1 3 26 27 23 jazz ssd 894.0G 1.00 893.9G 63.8% 51 0 3 4 25 26 27 38 juju ssd 894.0G 1.00 893.9G 63.8% 42 0 3 4 25 26 27 65 jolt ssd 894.0G 1.00 893.9G 63.9% 39 0 1 2 3 4 25 26 63 jolt ssd 894.0G 1.00 893.9G 63.9% 45 0 3 4 25 26 27 26 jazz ssd 894.0G 1.00 893.9G 63.9% 49 0 3 4 25 26 27 64 jolt ssd 894.0G 1.00 893.9G 63.9% 34 3 26 27 66 jolt ssd 894.0G 1.00 893.9G 63.9% 41 0 1 2 3 4 25 26 44 jazz ssd 894.0G 1.00 893.9G 63.9% 49 0 3 4 25 26 27 24 jazz ssd 894.0G 1.00 893.9G 64.0% 34 3 26 67 jolt ssd 894.0G 1.00 893.9G 64.0% 34 0 3 26 55 jam ssd 894.0G 1.00 893.9G 64.0% 48 0 3 4 25 26 58 jet ssd 894.0G 1.00 893.9G 64.0% 35 3 26 56 jam ssd 894.0G 1.00 893.9G 64.0% 103 0 3 4 25 26 27 70 jug ssd 894.0G 1.00 893.9G 64.1% 35 3 26 57 jam ssd 894.0G 1.00 893.9G 64.1% 35 3 26 46 juju ssd 894.0G 1.00 893.9G 64.1% 33 3 26 60 jet ssd 894.0G 1.00 893.9G 64.1% 34 3 26 71 jug ssd 894.0G 1.00 893.9G 64.1% 33 3 69 jug ssd 894.0G 1.00 893.9G 64.2% 34 3 26 53 jam ssd 894.0G 1.00 893.9G 64.2% 34 3 26 54 jam ssd 894.0G 1.00 893.9G 64.2% 33 3 37 juju ssd 894.0G 1.00 893.9G 64.2% 33 3 26 61 jet ssd 894.0G 1.00 893.9G 64.3% 33 3 62 jet ssd 894.0G 1.00 893.9G 64.3% 33 3 25 jazz ssd 894.0G 1.00 893.9G 64.4% 33 3 68 jug ssd 894.0G 1.00 893.9G 64.4% 33 3 12 joy ssd 894.0G 1.00 893.9G 64.5% 33 3 40 juju ssd 894.0G 1.00 893.9G 64.5% 33 3 39 juju ssd 894.0G 1.00 893.9G 64.5% 33 3 59 jet ssd 894.0G 1.00 893.9G 64.6% 33 3 21 jeep ssd 894.0G 1.00 893.9G 64.6% 33 3 13 joy ssd 894.0G 1.00 893.9G 64.6% 33 3 72 jug ssd 894.0G 1.00 893.9G 64.7% 33 3 ceph osd crush dump { "devices": [ { "id": 0, "name": "osd.0", "class": "ssd" }, { "id": 1, "name": "osd.1", "class": "ssd" }, { "id": 2, "name": "osd.2", "class": "ssd" }, { "id": 3, "name": "osd.3", "class": "ssd" }, { "id": 4, "name": "osd.4", "class": "ssd" }, { "id": 5, "name": "osd.5", "class": "ssd" }, { "id": 6, "name": "osd.6", "class": "ssd" }, { "id": 7, "name": "osd.7", "class": "ssd" }, { "id": 8, "name": "osd.8", "class": "ssd" }, { "id": 9, "name": "osd.9", "class": "ssd" }, { "id": 10, "name": "osd.10", "class": "ssd" }, { "id": 11, "name": "osd.11", "class": "ssd" }, { "id": 12, "name": "osd.12", "class": "ssd" }, { "id": 13, "name": "osd.13", "class": "ssd" }, { "id": 14, "name": "osd.14", "class": "ssd" }, { "id": 15, "name": "osd.15", "class": "ssd" }, { "id": 16, "name": "osd.16", "class": "ssd" }, { "id": 17, "name": "osd.17", "class": "ssd" }, { "id": 18, "name": "osd.18", "class": "ssd" }, { "id": 19, "name": "osd.19", "class": "ssd" }, { "id": 20, "name": "osd.20", "class": "ssd" }, { "id": 21, "name": "osd.21", "class": "ssd" }, { "id": 22, "name": "osd.22", "class": "ssd" }, { "id": 23, "name": "osd.23", "class": "ssd" }, { "id": 24, "name": "osd.24", "class": "ssd" }, { "id": 25, "name": "osd.25", "class": "ssd" }, { "id": 26, "name": "osd.26", "class": "ssd" }, { "id": 27, "name": "osd.27", "class": "ssd" }, { "id": 28, "name": "osd.28", "class": "ssd" }, { "id": 29, "name": "osd.29", "class": "ssd" }, { "id": 30, "name": "osd.30", "class": "ssd" }, { "id": 31, "name": "osd.31", "class": "ssd" }, { "id": 32, "name": "osd.32", "class": "ssd" }, { "id": 33, "name": "osd.33", "class": "ssd" }, { "id": 34, "name": "osd.34", "class": "ssd" }, { "id": 35, "name": "osd.35", "class": "ssd" }, { "id": 36, "name": "osd.36", "class": "ssd" }, { "id": 37, "name": "osd.37", "class": "ssd" }, { "id": 38, "name": "osd.38", "class": "ssd" }, { "id": 39, "name": "osd.39", "class": "ssd" }, { "id": 40, "name": "osd.40", "class": "ssd" }, { "id": 41, "name": "osd.41", "class": "ssd" }, { "id": 42, "name": "osd.42", "class": "ssd" }, { "id": 43, "name": "osd.43", "class": "ssd" }, { "id": 44, "name": "osd.44", "class": "ssd" }, { "id": 45, "name": "osd.45", "class": "ssd" }, { "id": 46, "name": "osd.46", "class": "ssd" }, { "id": 47, "name": "osd.47", "class": "ssd" }, { "id": 48, "name": "osd.48", "class": "ssd" }, { "id": 49, "name": "osd.49", "class": "ssd" }, { "id": 50, "name": "osd.50", "class": "ssd" }, { "id": 51, "name": "osd.51", "class": "ssd" }, { "id": 52, "name": "osd.52", "class": "ssd" }, { "id": 53, "name": "osd.53", "class": "ssd" }, { "id": 54, "name": "osd.54", "class": "ssd" }, { "id": 55, "name": "osd.55", "class": "ssd" }, { "id": 56, "name": "osd.56", "class": "ssd" }, { "id": 57, "name": "osd.57", "class": "ssd" }, { "id": 58, "name": "osd.58", "class": "ssd" }, { "id": 59, "name": "osd.59", "class": "ssd" }, { "id": 60, "name": "osd.60", "class": "ssd" }, { "id": 61, "name": "osd.61", "class": "ssd" }, { "id": 62, "name": "osd.62", "class": "ssd" }, { "id": 63, "name": "osd.63", "class": "ssd" }, { "id": 64, "name": "osd.64", "class": "ssd" }, { "id": 65, "name": "osd.65", "class": "ssd" }, { "id": 66, "name": "osd.66", "class": "ssd" }, { "id": 67, "name": "osd.67", "class": "ssd" }, { "id": 68, "name": "osd.68", "class": "ssd" }, { "id": 69, "name": "osd.69", "class": "ssd" }, { "id": 70, "name": "osd.70", "class": "ssd" }, { "id": 71, "name": "osd.71", "class": "ssd" }, { "id": 72, "name": "osd.72", "class": "ssd" }, { "id": 73, "name": "osd.73", "class": "ssd" }, { "id": 74, "name": "device74" }, { "id": 75, "name": "device75" }, { "id": 76, "name": "device76" }, { "id": 77, "name": "device77" }, { "id": 78, "name": "device78" }, { "id": 79, "name": "device79" }, { "id": 80, "name": "device80" }, { "id": 81, "name": "device81" }, { "id": 82, "name": "osd.82", "class": "ssd" }, { "id": 83, "name": "osd.83", "class": "ssd" }, { "id": 84, "name": "osd.84", "class": "ssd" }, { "id": 85, "name": "osd.85", "class": "ssd" }, { "id": 86, "name": "osd.86", "class": "ssd" }, { "id": 87, "name": "osd.87", "class": "ssd" }, { "id": 88, "name": "osd.88", "class": "ssd" }, { "id": 89, "name": "osd.89", "class": "ssd" }, { "id": 90, "name": "osd.90", "class": "ssd" }, { "id": 91, "name": "osd.91", "class": "ssd" }, { "id": 92, "name": "osd.92", "class": "ssd" }, { "id": 93, "name": "osd.93", "class": "ssd" }, { "id": 94, "name": "osd.94", "class": "ssd" }, { "id": 95, "name": "osd.95", "class": "ssd" }, { "id": 96, "name": "osd.96", "class": "ssd" }, { "id": 97, "name": "osd.97", "class": "ssd" }, { "id": 98, "name": "osd.98", "class": "ssd" }, { "id": 99, "name": "osd.99", "class": "ssd" }, { "id": 100, "name": "osd.100", "class": "ssd" }, { "id": 101, "name": "osd.101", "class": "ssd" } ], "types": [ { "type_id": 0, "name": "osd" }, { "type_id": 1, "name": "host" }, { "type_id": 2, "name": "rack" }, { "type_id": 3, "name": "row" }, { "type_id": 4, "name": "room" }, { "type_id": 5, "name": "datacenter" }, { "type_id": 6, "name": "root" } ], "buckets": [ { "id": -1, "name": "default", "type_id": 6, "type_name": "root", "weight": 10986245, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": -3, "weight": 3776771, "pos": 0 }, { "id": -2, "weight": 3604737, "pos": 1 }, { "id": -11, "weight": 3604737, "pos": 2 } ] }, { "id": -2, "name": "marack2", "type_id": 2, "type_name": "rack", "weight": 3604737, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": -13, "weight": 915456, "pos": 0 }, { "id": -19, "weight": 514924, "pos": 1 }, { "id": -49, "weight": 514924, "pos": 2 }, { "id": -17, "weight": 514924, "pos": 3 }, { "id": -44, "weight": 457851, "pos": 4 }, { "id": -6, "weight": 457728, "pos": 5 }, { "id": -22, "weight": 228930, "pos": 6 } ] }, { "id": -3, "name": "marack", "type_id": 2, "type_name": "rack", "weight": 3776771, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": -4, "weight": 686592, "pos": 0 }, { "id": -37, "weight": 457851, "pos": 1 }, { "id": -55, "weight": 915699, "pos": 2 }, { "id": -5, "weight": 514924, "pos": 3 }, { "id": -8, "weight": 514924, "pos": 4 }, { "id": -50, "weight": 457851, "pos": 5 }, { "id": -27, "weight": 228930, "pos": 6 } ] }, { "id": -4, "name": "gallo", "type_id": 1, "type_name": "host", "weight": 686592, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 0, "weight": 228864, "pos": 0 }, { "id": 1, "weight": 228864, "pos": 1 }, { "id": 8, "weight": 228864, "pos": 2 } ] }, { "id": -5, "name": "joy", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 5, "weight": 57212, "pos": 0 }, { "id": 12, "weight": 57212, "pos": 1 }, { "id": 13, "weight": 57212, "pos": 2 }, { "id": 16, "weight": 57212, "pos": 3 }, { "id": 17, "weight": 228864, "pos": 4 }, { "id": 43, "weight": 57212, "pos": 5 } ] }, { "id": -6, "name": "john", "type_id": 1, "type_name": "host", "weight": 457728, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 2, "weight": 228864, "pos": 0 }, { "id": 3, "weight": 228864, "pos": 1 } ] }, { "id": -7, "name": "joy~ssd", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 5, "weight": 57212, "pos": 0 }, { "id": 12, "weight": 57212, "pos": 1 }, { "id": 13, "weight": 57212, "pos": 2 }, { "id": 16, "weight": 57212, "pos": 3 }, { "id": 17, "weight": 228864, "pos": 4 }, { "id": 43, "weight": 57212, "pos": 5 } ] }, { "id": -8, "name": "jeep", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 18, "weight": 57212, "pos": 0 }, { "id": 19, "weight": 57212, "pos": 1 }, { "id": 20, "weight": 57212, "pos": 2 }, { "id": 21, "weight": 57212, "pos": 3 }, { "id": 22, "weight": 228864, "pos": 4 }, { "id": 45, "weight": 57212, "pos": 5 } ] }, { "id": -9, "name": "john~ssd", "type_id": 1, "type_name": "host", "weight": 457728, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 2, "weight": 228864, "pos": 0 }, { "id": 3, "weight": 228864, "pos": 1 } ] }, { "id": -10, "name": "gold", "type_id": 1, "type_name": "host", "weight": 915456, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 7, "weight": 228864, "pos": 0 }, { "id": 9, "weight": 228864, "pos": 1 }, { "id": 10, "weight": 228864, "pos": 2 }, { "id": 15, "weight": 228864, "pos": 3 } ] }, { "id": -11, "name": "marack3", "type_id": 2, "type_name": "rack", "weight": 3604737, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": -52, "weight": 514924, "pos": 0 }, { "id": -46, "weight": 514924, "pos": 1 }, { "id": -43, "weight": 514924, "pos": 2 }, { "id": -10, "weight": 915456, "pos": 3 }, { "id": -41, "weight": 457851, "pos": 4 }, { "id": -16, "weight": 457728, "pos": 5 }, { "id": -12, "weight": 228930, "pos": 6 } ] }, { "id": -12, "name": "innes", "type_id": 1, "type_name": "host", "weight": 228930, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 42, "weight": 114465, "pos": 0 }, { "id": 47, "weight": 114465, "pos": 1 } ] }, { "id": -13, "name": "helm", "type_id": 1, "type_name": "host", "weight": 915456, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 28, "weight": 228864, "pos": 0 }, { "id": 30, "weight": 228864, "pos": 1 }, { "id": 31, "weight": 228864, "pos": 2 }, { "id": 11, "weight": 228864, "pos": 3 } ] }, { "id": -14, "name": "gallo~ssd", "type_id": 1, "type_name": "host", "weight": 686592, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 0, "weight": 228864, "pos": 0 }, { "id": 1, "weight": 228864, "pos": 1 }, { "id": 8, "weight": 228864, "pos": 2 } ] }, { "id": -15, "name": "jeep~ssd", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 18, "weight": 57212, "pos": 0 }, { "id": 19, "weight": 57212, "pos": 1 }, { "id": 20, "weight": 57212, "pos": 2 }, { "id": 21, "weight": 57212, "pos": 3 }, { "id": 22, "weight": 228864, "pos": 4 }, { "id": 45, "weight": 57212, "pos": 5 } ] }, { "id": -16, "name": "jack", "type_id": 1, "type_name": "host", "weight": 457728, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 4, "weight": 228864, "pos": 0 }, { "id": 6, "weight": 228864, "pos": 1 } ] }, { "id": -17, "name": "jazz", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 23, "weight": 57212, "pos": 0 }, { "id": 24, "weight": 57212, "pos": 1 }, { "id": 25, "weight": 57212, "pos": 2 }, { "id": 26, "weight": 57212, "pos": 3 }, { "id": 36, "weight": 228864, "pos": 4 }, { "id": 44, "weight": 57212, "pos": 5 } ] }, { "id": -18, "name": "jazz~ssd", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 23, "weight": 57212, "pos": 0 }, { "id": 24, "weight": 57212, "pos": 1 }, { "id": 25, "weight": 57212, "pos": 2 }, { "id": 26, "weight": 57212, "pos": 3 }, { "id": 36, "weight": 228864, "pos": 4 }, { "id": 44, "weight": 57212, "pos": 5 } ] }, { "id": -19, "name": "juju", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 37, "weight": 57212, "pos": 0 }, { "id": 38, "weight": 57212, "pos": 1 }, { "id": 39, "weight": 57212, "pos": 2 }, { "id": 40, "weight": 57212, "pos": 3 }, { "id": 41, "weight": 228864, "pos": 4 }, { "id": 46, "weight": 57212, "pos": 5 } ] }, { "id": -20, "name": "juju~ssd", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 37, "weight": 57212, "pos": 0 }, { "id": 38, "weight": 57212, "pos": 1 }, { "id": 39, "weight": 57212, "pos": 2 }, { "id": 40, "weight": 57212, "pos": 3 }, { "id": 41, "weight": 228864, "pos": 4 }, { "id": 46, "weight": 57212, "pos": 5 } ] }, { "id": -21, "name": "innes~ssd", "type_id": 1, "type_name": "host", "weight": 228930, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 42, "weight": 114465, "pos": 0 }, { "id": 47, "weight": 114465, "pos": 1 } ] }, { "id": -22, "name": "lc01", "type_id": 1, "type_name": "host", "weight": 228930, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 50, "weight": 114465, "pos": 0 }, { "id": 51, "weight": 114465, "pos": 1 } ] }, { "id": -24, "name": "jack~ssd", "type_id": 1, "type_name": "host", "weight": 457728, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 4, "weight": 228864, "pos": 0 }, { "id": 6, "weight": 228864, "pos": 1 } ] }, { "id": -26, "name": "lc01~ssd", "type_id": 1, "type_name": "host", "weight": 228930, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 50, "weight": 114465, "pos": 0 }, { "id": 51, "weight": 114465, "pos": 1 } ] }, { "id": -27, "name": "lc02", "type_id": 1, "type_name": "host", "weight": 228930, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 52, "weight": 114465, "pos": 0 }, { "id": 73, "weight": 114465, "pos": 1 } ] }, { "id": -28, "name": "gold~ssd", "type_id": 1, "type_name": "host", "weight": 915456, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 7, "weight": 228864, "pos": 0 }, { "id": 9, "weight": 228864, "pos": 1 }, { "id": 10, "weight": 228864, "pos": 2 }, { "id": 15, "weight": 228864, "pos": 3 } ] }, { "id": -29, "name": "lc02~ssd", "type_id": 1, "type_name": "host", "weight": 228930, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 52, "weight": 114465, "pos": 0 }, { "id": 73, "weight": 114465, "pos": 1 } ] }, { "id": -30, "name": "marack~ssd", "type_id": 2, "type_name": "rack", "weight": 3776771, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": -14, "weight": 686592, "pos": 0 }, { "id": -40, "weight": 457851, "pos": 1 }, { "id": -56, "weight": 915699, "pos": 2 }, { "id": -7, "weight": 514924, "pos": 3 }, { "id": -15, "weight": 514924, "pos": 4 }, { "id": -53, "weight": 457851, "pos": 5 }, { "id": -29, "weight": 228930, "pos": 6 } ] }, { "id": -34, "name": "marack2~ssd", "type_id": 2, "type_name": "rack", "weight": 3604737, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": -35, "weight": 915456, "pos": 0 }, { "id": -20, "weight": 514924, "pos": 1 }, { "id": -51, "weight": 514924, "pos": 2 }, { "id": -18, "weight": 514924, "pos": 3 }, { "id": -47, "weight": 457851, "pos": 4 }, { "id": -9, "weight": 457728, "pos": 5 }, { "id": -26, "weight": 228930, "pos": 6 } ] }, { "id": -35, "name": "helm~ssd", "type_id": 1, "type_name": "host", "weight": 915456, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 28, "weight": 228864, "pos": 0 }, { "id": 30, "weight": 228864, "pos": 1 }, { "id": 31, "weight": 228864, "pos": 2 }, { "id": 11, "weight": 228864, "pos": 3 } ] }, { "id": -37, "name": "kc01", "type_id": 1, "type_name": "host", "weight": 457851, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 82, "weight": 57232, "pos": 0 }, { "id": 83, "weight": 57232, "pos": 1 }, { "id": 84, "weight": 57232, "pos": 2 }, { "id": 85, "weight": 228923, "pos": 3 }, { "id": 86, "weight": 57232, "pos": 4 } ] }, { "id": -38, "name": "marack3~ssd", "type_id": 2, "type_name": "rack", "weight": 3604737, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": -54, "weight": 514924, "pos": 0 }, { "id": -48, "weight": 514924, "pos": 1 }, { "id": -45, "weight": 514924, "pos": 2 }, { "id": -28, "weight": 915456, "pos": 3 }, { "id": -42, "weight": 457851, "pos": 4 }, { "id": -24, "weight": 457728, "pos": 5 }, { "id": -21, "weight": 228930, "pos": 6 } ] }, { "id": -39, "name": "default~ssd", "type_id": 6, "type_name": "root", "weight": 10986245, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": -30, "weight": 3776771, "pos": 0 }, { "id": -34, "weight": 3604737, "pos": 1 }, { "id": -38, "weight": 3604737, "pos": 2 } ] }, { "id": -40, "name": "kc01~ssd", "type_id": 1, "type_name": "host", "weight": 457851, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 82, "weight": 57232, "pos": 0 }, { "id": 83, "weight": 57232, "pos": 1 }, { "id": 84, "weight": 57232, "pos": 2 }, { "id": 85, "weight": 228923, "pos": 3 }, { "id": 86, "weight": 57232, "pos": 4 } ] }, { "id": -41, "name": "kc02", "type_id": 1, "type_name": "host", "weight": 457851, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 87, "weight": 57232, "pos": 0 }, { "id": 88, "weight": 57232, "pos": 1 }, { "id": 89, "weight": 57232, "pos": 2 }, { "id": 90, "weight": 228923, "pos": 3 }, { "id": 91, "weight": 57232, "pos": 4 } ] }, { "id": -42, "name": "kc02~ssd", "type_id": 1, "type_name": "host", "weight": 457851, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 87, "weight": 57232, "pos": 0 }, { "id": 88, "weight": 57232, "pos": 1 }, { "id": 89, "weight": 57232, "pos": 2 }, { "id": 90, "weight": 228923, "pos": 3 }, { "id": 91, "weight": 57232, "pos": 4 } ] }, { "id": -43, "name": "jam", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 53, "weight": 57212, "pos": 0 }, { "id": 54, "weight": 57212, "pos": 1 }, { "id": 55, "weight": 57212, "pos": 2 }, { "id": 56, "weight": 57212, "pos": 3 }, { "id": 57, "weight": 57212, "pos": 4 }, { "id": 32, "weight": 228864, "pos": 5 } ] }, { "id": -44, "name": "ks01", "type_id": 1, "type_name": "host", "weight": 457851, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 92, "weight": 57232, "pos": 0 }, { "id": 93, "weight": 57232, "pos": 1 }, { "id": 94, "weight": 57232, "pos": 2 }, { "id": 95, "weight": 57232, "pos": 3 }, { "id": 96, "weight": 228923, "pos": 4 } ] }, { "id": -45, "name": "jam~ssd", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 53, "weight": 57212, "pos": 0 }, { "id": 54, "weight": 57212, "pos": 1 }, { "id": 55, "weight": 57212, "pos": 2 }, { "id": 56, "weight": 57212, "pos": 3 }, { "id": 57, "weight": 57212, "pos": 4 }, { "id": 32, "weight": 228864, "pos": 5 } ] }, { "id": -46, "name": "jet", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 58, "weight": 57212, "pos": 0 }, { "id": 59, "weight": 57212, "pos": 1 }, { "id": 60, "weight": 57212, "pos": 2 }, { "id": 61, "weight": 57212, "pos": 3 }, { "id": 62, "weight": 57212, "pos": 4 }, { "id": 33, "weight": 228864, "pos": 5 } ] }, { "id": -47, "name": "ks01~ssd", "type_id": 1, "type_name": "host", "weight": 457851, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 92, "weight": 57232, "pos": 0 }, { "id": 93, "weight": 57232, "pos": 1 }, { "id": 94, "weight": 57232, "pos": 2 }, { "id": 95, "weight": 57232, "pos": 3 }, { "id": 96, "weight": 228923, "pos": 4 } ] }, { "id": -48, "name": "jet~ssd", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 58, "weight": 57212, "pos": 0 }, { "id": 59, "weight": 57212, "pos": 1 }, { "id": 60, "weight": 57212, "pos": 2 }, { "id": 61, "weight": 57212, "pos": 3 }, { "id": 62, "weight": 57212, "pos": 4 }, { "id": 33, "weight": 228864, "pos": 5 } ] }, { "id": -49, "name": "jolt", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 63, "weight": 57212, "pos": 0 }, { "id": 64, "weight": 57212, "pos": 1 }, { "id": 65, "weight": 57212, "pos": 2 }, { "id": 66, "weight": 57212, "pos": 3 }, { "id": 67, "weight": 57212, "pos": 4 }, { "id": 34, "weight": 228864, "pos": 5 } ] }, { "id": -50, "name": "ks02", "type_id": 1, "type_name": "host", "weight": 457851, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 97, "weight": 57232, "pos": 0 }, { "id": 98, "weight": 57232, "pos": 1 }, { "id": 99, "weight": 57232, "pos": 2 }, { "id": 100, "weight": 57232, "pos": 3 }, { "id": 101, "weight": 228923, "pos": 4 } ] }, { "id": -51, "name": "jolt~ssd", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 63, "weight": 57212, "pos": 0 }, { "id": 64, "weight": 57212, "pos": 1 }, { "id": 65, "weight": 57212, "pos": 2 }, { "id": 66, "weight": 57212, "pos": 3 }, { "id": 67, "weight": 57212, "pos": 4 }, { "id": 34, "weight": 228864, "pos": 5 } ] }, { "id": -52, "name": "jug", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 68, "weight": 57212, "pos": 0 }, { "id": 69, "weight": 57212, "pos": 1 }, { "id": 70, "weight": 57212, "pos": 2 }, { "id": 71, "weight": 57212, "pos": 3 }, { "id": 72, "weight": 57212, "pos": 4 }, { "id": 35, "weight": 228864, "pos": 5 } ] }, { "id": -53, "name": "ks02~ssd", "type_id": 1, "type_name": "host", "weight": 457851, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 97, "weight": 57232, "pos": 0 }, { "id": 98, "weight": 57232, "pos": 1 }, { "id": 99, "weight": 57232, "pos": 2 }, { "id": 100, "weight": 57232, "pos": 3 }, { "id": 101, "weight": 228923, "pos": 4 } ] }, { "id": -54, "name": "jug~ssd", "type_id": 1, "type_name": "host", "weight": 514924, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 68, "weight": 57212, "pos": 0 }, { "id": 69, "weight": 57212, "pos": 1 }, { "id": 70, "weight": 57212, "pos": 2 }, { "id": 71, "weight": 57212, "pos": 3 }, { "id": 72, "weight": 57212, "pos": 4 }, { "id": 35, "weight": 228864, "pos": 5 } ] }, { "id": -55, "name": "lx01", "type_id": 1, "type_name": "host", "weight": 915699, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 14, "weight": 228923, "pos": 0 }, { "id": 27, "weight": 228923, "pos": 1 }, { "id": 29, "weight": 228923, "pos": 2 }, { "id": 48, "weight": 114465, "pos": 3 }, { "id": 49, "weight": 114465, "pos": 4 } ] }, { "id": -56, "name": "lx01~ssd", "type_id": 1, "type_name": "host", "weight": 915699, "alg": "straw2", "hash": "rjenkins1", "items": [ { "id": 14, "weight": 228923, "pos": 0 }, { "id": 27, "weight": 228923, "pos": 1 }, { "id": 29, "weight": 228923, "pos": 2 }, { "id": 48, "weight": 114465, "pos": 3 }, { "id": 49, "weight": 114465, "pos": 4 } ] } ], "rules": [ { "rule_id": 0, "rule_name": "data", "ruleset": 0, "type": 1, "min_size": 1, "max_size": 10, "steps": [ { "op": "take", "item": -1, "item_name": "default" }, { "op": "chooseleaf_firstn", "num": 0, "type": "rack" }, { "op": "emit" } ] }, { "rule_id": 1, "rule_name": "metadata", "ruleset": 1, "type": 1, "min_size": 1, "max_size": 10, "steps": [ { "op": "take", "item": -1, "item_name": "default" }, { "op": "chooseleaf_firstn", "num": 0, "type": "host" }, { "op": "emit" } ] }, { "rule_id": 2, "rule_name": "rbd", "ruleset": 2, "type": 1, "min_size": 1, "max_size": 10, "steps": [ { "op": "take", "item": -1, "item_name": "default" }, { "op": "chooseleaf_firstn", "num": 0, "type": "host" }, { "op": "emit" } ] } ], "tunables": { "choose_local_tries": 0, "choose_local_fallback_tries": 0, "choose_total_tries": 50, "chooseleaf_descend_once": 1, "chooseleaf_vary_r": 1, "chooseleaf_stable": 1, "straw_calc_version": 1, "allowed_bucket_algs": 54, "profile": "jewel", "optimal_tunables": 1, "legacy_tunables": 0, "minimum_required_version": "jewel", "require_feature_tunables": 1, "require_feature_tunables2": 1, "has_v2_rules": 0, "require_feature_tunables3": 1, "has_v3_rules": 0, "has_v4_buckets": 1, "require_feature_tunables5": 1, "has_v5_rules": 0 }, "choose_args": {} } -- Joe Ryner