Hello, ceph is stuck since 4 days with 0.064% misplaced and i dunno why. Can anyone help me to get it fixed? I did restart some OSDs and reweight them again to get some data moving but that did not help. root@node01:~ # ceph -s cluster: id: 251c937e-0b55-48c1-8f34-96e84e4023d4 health: HEALTH_WARN 1803/2799972 objects misplaced (0.064%) mon node02 is low on available space services: mon: 3 daemons, quorum node01,node02,node03 mgr: node03(active), standbys: node01, node02 osd: 16 osds: 16 up, 16 in; 1 remapped pgs data: pools: 1 pools, 512 pgs objects: 933.32k objects, 2.68TiB usage: 9.54TiB used, 5.34TiB / 14.9TiB avail pgs: 1803/2799972 objects misplaced (0.064%) 511 active+clean 1 active+clean+remapped io: client: 131KiB/s rd, 8.57MiB/s wr, 28op/s rd, 847op/s wr root@node01:~ # ceph health detail HEALTH_WARN 1803/2800179 objects misplaced (0.064%); mon node02 is low on available space OBJECT_MISPLACED 1803/2800179 objects misplaced (0.064%) MON_DISK_LOW mon node02 is low on available space mon.node02 has 28% avail root@node01:~ # ceph versions { "mon": { "ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd) luminous (stable)": 3 }, "mgr": { "ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd) luminous (stable)": 3 }, "osd": { "ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd) luminous (stable)": 16 }, "mds": {}, "overall": { "ceph version 12.2.13 (98af9a6b9a46b2d562a0de4b09263d70aeb1c9dd) luminous (stable)": 22 } } root@node02:~ # df -h Filesystem Size Used Avail Use% Mounted on udev 63G 0 63G 0% /dev tmpfs 13G 1.3G 12G 11% /run /dev/sda3 46G 31G 14G 70% / tmpfs 63G 57M 63G 1% /dev/shm tmpfs 5.0M 0 5.0M 0% /run/lock tmpfs 63G 0 63G 0% /sys/fs/cgroup /dev/sda1 922M 206M 653M 24% /boot /dev/fuse 30M 144K 30M 1% /etc/pve /dev/sde1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-11 /dev/sdf1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-14 /dev/sdc1 889G 676G 214G 77% /var/lib/ceph/osd/ceph-3 /dev/sdb1 889G 667G 222G 76% /var/lib/ceph/osd/ceph-2 /dev/sdd1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-7 tmpfs 13G 0 13G 0% /run/user/0 root@node02:~ # ceph osd tree ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF -1 14.34781 root default -2 4.25287 host node01 0 hdd 0.85999 osd.0 up 0.80005 1.00000 1 hdd 0.86749 osd.1 up 0.85004 1.00000 6 hdd 0.87270 osd.6 up 0.90002 1.00000 12 hdd 0.78000 osd.12 up 0.95001 1.00000 13 hdd 0.87270 osd.13 up 0.95001 1.00000 -3 3.91808 host node02 2 hdd 0.70000 osd.2 up 0.80005 1.00000 3 hdd 0.59999 osd.3 up 0.85004 1.00000 7 hdd 0.87270 osd.7 up 0.85004 1.00000 11 hdd 0.87270 osd.11 up 0.75006 1.00000 14 hdd 0.87270 osd.14 up 0.85004 1.00000 -4 6.17686 host node03 4 hdd 0.87000 osd.4 up 1.00000 1.00000 5 hdd 0.87000 osd.5 up 1.00000 1.00000 8 hdd 0.87270 osd.8 up 1.00000 1.00000 10 hdd 0.87270 osd.10 up 1.00000 1.00000 15 hdd 0.87270 osd.15 up 1.00000 1.00000 16 hdd 1.81879 osd.16 up 1.00000 1.00000 root@node01:~ # ceph osd df tree ID CLASS WEIGHT REWEIGHT SIZE USE DATA OMAP META AVAIL %USE VAR PGS TYPE NAME -1 14.55780 - 14.9TiB 9.45TiB 7.46TiB 1.47GiB 23.2GiB 5.43TiB 63.52 1.00 - root default -2 4.27286 - 4.35TiB 3.15TiB 2.41TiB 486MiB 7.62GiB 1.21TiB 72.32 1.14 - host node01 0 hdd 0.85999 0.80005 888GiB 619GiB 269GiB 92.3MiB 0B 269GiB 69.72 1.10 89 osd.0 1 hdd 0.86749 0.85004 888GiB 641GiB 248GiB 109MiB 0B 248GiB 72.12 1.14 92 osd.1 6 hdd 0.87270 0.90002 894GiB 634GiB 632GiB 98.9MiB 2.65GiB 259GiB 70.99 1.12 107 osd.6 12 hdd 0.79999 0.95001 894GiB 664GiB 661GiB 94.4MiB 2.52GiB 230GiB 74.31 1.17 112 osd.12 13 hdd 0.87270 0.95001 894GiB 665GiB 663GiB 91.7MiB 2.46GiB 229GiB 74.43 1.17 112 osd.13 -3 4.10808 - 4.35TiB 3.17TiB 2.18TiB 479MiB 6.99GiB 1.18TiB 72.86 1.15 - host node02 2 hdd 0.78999 0.75006 888GiB 654GiB 235GiB 95.6MiB 0B 235GiB 73.57 1.16 94 osd.2 3 hdd 0.70000 0.80005 888GiB 737GiB 151GiB 114MiB 0B 151GiB 82.98 1.31 105 osd.3 7 hdd 0.87270 0.85004 894GiB 612GiB 610GiB 88.9MiB 2.43GiB 281GiB 68.50 1.08 103 osd.7 11 hdd 0.87270 0.75006 894GiB 576GiB 574GiB 81.8MiB 2.19GiB 317GiB 64.47 1.01 97 osd.11 14 hdd 0.87270 0.85004 894GiB 669GiB 666GiB 98.8MiB 2.37GiB 225GiB 74.85 1.18 112 osd.14 -4 6.17686 - 6.17TiB 3.13TiB 2.86TiB 541MiB 8.58GiB 3.04TiB 50.73 0.80 - host node03 4 hdd 0.87000 1.00000 888GiB 504GiB 384GiB 124MiB 0B 384GiB 56.72 0.89 72 osd.4 5 hdd 0.87000 1.00000 888GiB 520GiB 368GiB 96.2MiB 0B 368GiB 58.57 0.92 75 osd.5 8 hdd 0.87270 1.00000 894GiB 508GiB 505GiB 80.2MiB 2.07GiB 386GiB 56.80 0.89 85 osd.8 10 hdd 0.87270 1.00000 894GiB 374GiB 373GiB 51.9MiB 1.73GiB 519GiB 41.88 0.66 63 osd.10 15 hdd 0.87270 1.00000 894GiB 504GiB 502GiB 60.1MiB 1.99GiB 390GiB 56.37 0.89 84 osd.15 16 hdd 1.81879 1.00000 1.82TiB 797GiB 794GiB 129MiB 2.79GiB 1.04TiB 42.77 0.67 134 osd.16 TOTAL 14.9TiB 9.45TiB 7.46TiB 1.47GiB 23.2GiB 5.43TiB 63.52 MIN/MAX VAR: 0.66/1.31 STDDEV: 11.59 root@node02:~ # df -h Filesystem Size Used Avail Use% Mounted on udev 63G 0 63G 0% /dev tmpfs 13G 1.4G 12G 11% /run /dev/sda3 46G 31G 14G 70% / tmpfs 63G 63M 63G 1% /dev/shm tmpfs 5.0M 0 5.0M 0% /run/lock tmpfs 63G 0 63G 0% /sys/fs/cgroup /dev/sda1 922M 206M 653M 24% /boot /dev/fuse 30M 140K 30M 1% /etc/pve /dev/sde1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-11 /dev/sdf1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-14 /dev/sdc1 889G 738G 152G 83% /var/lib/ceph/osd/ceph-3 /dev/sdb1 889G 654G 235G 74% /var/lib/ceph/osd/ceph-2 /dev/sdd1 93M 5.4M 88M 6% /var/lib/ceph/osd/ceph-7 tmpfs 13G 0 13G 0% /run/user/0 Any idea? Thanks, Michael