These repeat for every host, only after upgrading from prev release Quincy to 17.2.7. As a result, the cluster is always warned, never indicates healthy. root@noc1:~# ceph health detail HEALTH_WARN failed to probe daemons or devices [WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices host sysmon1 `cephadm ceph-volume` failed: cephadm exited with an error code: 1, stderr: Inferring config /var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/mon.sysmon1/config Non-zero exit code 1 from /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.io/ceph/ceph@sha2 56:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e -e NODE_NAME=sysmon1 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v /var/run/ceph/4067126d-01cb-40af-824a-881c130140f8:/var /run/ceph:z -v /var/log/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/log/ceph:z -v /var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/crash:/var/lib/ceph/crash:z -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/lv m -v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v /tmp/ceph-tmpl1e27bun:/etc/ceph/ceph.conf:z quay.io/ceph/ceph@sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e inventory --format=json-pretty --filter-for-batch /usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or absolute path in /dev/ or /sys expected. /usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or absolute path in /dev/ or /sys expected. /usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or absolute path in /dev/ or /sys expected. /usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or absolute path in /dev/ or /sys expected. /usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or absolute path in /dev/ or /sys expected. /usr/bin/docker: stderr stderr: Unknown device, --name=, --path=, or absolute path in /dev/ or /sys expected. /usr/bin/docker: stderr Traceback (most recent call last): /usr/bin/docker: stderr File "/usr/sbin/ceph-volume", line 11, in <module> /usr/bin/docker: stderr load_entry_point('ceph-volume==1.0.0', 'console_scripts', 'ceph-volume')() /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 41, in __init__ /usr/bin/docker: stderr self.main(self.argv) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59, in newfunc /usr/bin/docker: stderr return f(*a, **kw) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main /usr/bin/docker: stderr terminal.dispatch(self.mapper, subcommand_args) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch /usr/bin/docker: stderr instance.main() /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/inventory/main.py", line 60, in main /usr/bin/docker: stderr list_all=self.args.list_all)) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 50, in __init__ /usr/bin/docker: stderr sys_info.devices.keys()] /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 49, in <listcomp> /usr/bin/docker: stderr all_devices_vgs=all_devices_vgs) for k in /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 147, in __init__ /usr/bin/docker: stderr self.available_lvm, self.rejected_reasons_lvm = self._check_lvm_reject_reasons() /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 646, in _check_lvm_reject_reasons /usr/bin/docker: stderr rejected.extend(self._check_generic_reject_reasons()) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 601, in _check_generic_reject_reasons /usr/bin/docker: stderr if self.is_acceptable_device: /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 502, in is_acceptable_device /usr/bin/docker: stderr return self.is_device or self.is_partition or self.is_lv /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 482, in is_partition /usr/bin/docker: stderr return self.blkid_api['TYPE'] == 'part' /usr/bin/docker: stderr KeyError: 'TYPE' Traceback (most recent call last): File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 9679, in <module> main() File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 9667, in main r = ctx.func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 2116, in _infer_config return func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 2032, in _infer_fsid return func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 2144, in _infer_image return func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 2019, in _validate_fsid return func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 6272, in command_ceph_volume out, err, code = call_throws(ctx, c.run_cmd(), verbosity=CallVerbosity.QUIET_UNLESS_ERROR) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 1807, in call_throws raise RuntimeError('Failed command: %s' % ' '.join(command)) RuntimeError: Failed command: /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.io/ceph/ceph@ sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e -e NODE_NAME=sysmon1 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v /var/run/ceph/4067126d-01cb-40af-824a-881c130140f8: /var/run/ceph:z -v /var/log/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/log/ceph:z -v /var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/crash:/var/lib/ceph/crash:z -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/ru n/lvm -v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v /tmp/ceph-tmpl1e27bun:/etc/ceph/ceph.conf:z quay.io/ceph/ceph@sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e inventory --format=json-pretty --filter-for-b atch host noc3 `cephadm ceph-volume` failed: cephadm exited with an error code: 1, stderr: Inferring config /var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/mon.noc3/config Non-zero exit code 1 from /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.io/ceph/ceph@sha2 56:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e -e NODE_NAME=noc3 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v /var/run/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/ru n/ceph:z -v /var/log/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/log/ceph:z -v /var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/crash:/var/lib/ceph/crash:z -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/lvm - v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v /tmp/ceph-tmpjjmsgag7:/etc/ceph/ceph.conf:z quay.io/ceph/ceph@sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e inventory --format=json-pretty --filter-for-batch /usr/bin/docker: stderr Traceback (most recent call last): /usr/bin/docker: stderr File "/usr/sbin/ceph-volume", line 11, in <module> /usr/bin/docker: stderr load_entry_point('ceph-volume==1.0.0', 'console_scripts', 'ceph-volume')() /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 41, in __init__ /usr/bin/docker: stderr self.main(self.argv) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59, in newfunc /usr/bin/docker: stderr return f(*a, **kw) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 153, in main /usr/bin/docker: stderr terminal.dispatch(self.mapper, subcommand_args) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch /usr/bin/docker: stderr instance.main() /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/inventory/main.py", line 60, in main /usr/bin/docker: stderr list_all=self.args.list_all)) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 50, in __init__ /usr/bin/docker: stderr sys_info.devices.keys()] /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 49, in <listcomp> /usr/bin/docker: stderr all_devices_vgs=all_devices_vgs) for k in /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 147, in __init__ /usr/bin/docker: stderr self.available_lvm, self.rejected_reasons_lvm = self._check_lvm_reject_reasons() /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 646, in _check_lvm_reject_reasons /usr/bin/docker: stderr rejected.extend(self._check_generic_reject_reasons()) /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 601, in _check_generic_reject_reasons /usr/bin/docker: stderr if self.is_acceptable_device: /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 502, in is_acceptable_device /usr/bin/docker: stderr return self.is_device or self.is_partition or self.is_lv /usr/bin/docker: stderr File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 482, in is_partition /usr/bin/docker: stderr return self.blkid_api['TYPE'] == 'part' /usr/bin/docker: stderr KeyError: 'TYPE' Traceback (most recent call last): File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 9679, in <module> main() File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 9667, in main r = ctx.func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 2116, in _infer_config return func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 2032, in _infer_fsid return func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 2144, in _infer_image return func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 2019, in _validate_fsid return func(ctx) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 6272, in command_ceph_volume out, err, code = call_throws(ctx, c.run_cmd(), verbosity=CallVerbosity.QUIET_UNLESS_ERROR) File "/var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/cephadm.8b92cafd937eb89681ee011f9e70f85937fd09c4bd61ed4a59981d275a1f255b", line 1807, in call_throws raise RuntimeError('Failed command: %s' % ' '.join(command)) RuntimeError: Failed command: /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.io/ceph/ceph@ sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e -e NODE_NAME=noc3 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v /var/run/ceph/4067126d-01cb-40af-824a-881c130140f8:/va r/run/ceph:z -v /var/log/ceph/4067126d-01cb-40af-824a-881c130140f8:/var/log/ceph:z -v /var/lib/ceph/4067126d-01cb-40af-824a-881c130140f8/crash:/var/lib/ceph/crash:z -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/l vm -v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v /tmp/ceph-tmpjjmsgag7:/etc/ceph/ceph.conf:z quay.io/ceph/ceph@sha256:92e8fa7d8ca17a7a5bbfde6e596fdfecc8e165fcb94d86493f4e6c7b1f326e4e inventory --format=json-pretty --filter-for-batc h ....