On 6/7/23 14:22, Frank Schilder wrote: Requirements: Kernel version requirement and higher: 5.9 cryptsetup: 2.3.4 and higher. Preferably 2.4.x (automatic alignment of sector size based on physical disk properties). RAW device: cryptsetup luksFormat /dev/device LVM device: cryptsetup luksFormat /dev/LVM/LV-here (this is what ceph-volume uses) Afterwards unlock the device with "cryptsetup open": cryptsetup open /dev/sdz name_that_makes_sense_for_you To use it with ceph-volume, the following changes can be made, here a diff against 16.2.11: --- before: /usr/lib/python3/dist-packages/ceph_volume/util/disk.py +++ after: /home/stefan/git/bit-ceph/configs/osd/usr/lib/python3/dist-packages/ceph_volume/util/disk.py @@ -194,6 +194,26 @@ devices.append(_lsblk_parser(line)) return devices + +def is_rotational(device): + """ + Returns true if a device is rotational or not. + """ + labels = ['ROTA'] + command = ['lsblk', '-P', '-p', '-o', ','.join(labels), device] + out, err, rc = process.call(command) + device = [] + for line in out: + device.append(_lsblk_parser(line)) + + for i in device: + if type(i) is dict and "ROTA" in i.keys(): + if i['ROTA'] == '1': + rotational = True + else: + rotational = False + + return rotational def udevadm_property(device, properties=[]): @@ -900,6 +920,10 @@ metadata['sectorsize'] = get_file_contents(sysdir + "/queue/logical_block_size", fallback_sectorsize) + fallback_rotational = '1' + metadata['rotational'] = get_file_contents(sysdir + + "/queue/rotational", + fallback_rotational) metadata['size'] = float(size) * 512 metadata['human_readable_size'] = human_readable_size(metadata['size']) metadata['path'] = diskname --- before: /usr/lib/python3/dist-packages/ceph_volume/util/encryption.py +++ after: /home/stefan/git/bit-ceph/configs/osd/usr/lib/python3/dist-packages/ceph_volume/util/encryption.py @@ -1,11 +1,12 @@ import base64 import os import logging +import subprocess from ceph_volume import process, conf, terminal from ceph_volume.util import constants, system from ceph_volume.util.device import Device from .prepare import write_keyring -from .disk import lsblk, device_family, get_part_entry_type +from .disk import lsblk, device_family, get_part_entry_type, is_rotational logger = logging.getLogger(__name__) mlogger = terminal.MultiLogger(__name__) @@ -23,10 +24,52 @@ if key_size not in ['256', '512']: logger.warning(("Invalid value set for osd_dmcrypt_key_size ({}). " - "Falling back to {}bits".format(key_size, default_key_size))) + "Falling back to {} bits".format(key_size, default_key_size))) return default_key_size return key_size + +def execCmd(cmdline): + cmd = subprocess.Popen(cmdline, shell=True, stdout=subprocess.PIPE) + stdout, _ = cmd.communicate() + return stdout.decode("utf-8") + +def bypass_workqueues(device): + """ + 1) Check if cryptsetup has support for no_read_workqueue. + If so, it also supports no_write_workqueue. + cryptsetup has support for this since 2.3.4 + + 2) Also check if we are using a flash (non-rotational) device + or not. I.e. Spinning media might still benefit from queuing. + + Only if both 1) and 2) are true we do _not_ want to use work queues + Note: + crypt version 1.22 and higher have support for this. Example: + modprobe dm-crypt + dmsetup targets + + integrity v1.10.0 + crypt v1.23.0 + striped v1.6.0 + linear v1.4.0 + error v1.5.0 + + If dm-crypt does not have support for no_read_workqueue / + no_write_workqueue the options will be discarded and + logged by the linux kernel: + + device-mapper: table: major:minor: crypt: Invalid feature arguments + device-mapper: ioctl: error adding target to table + + The encrypted device will come online without the options active. + """ + cryptsetup_help = execCmd("cryptsetup --help 2>/dev/null") + + if '--perf-no_read_workqueue' not in cryptsetup_help or is_rotational(device): + return [] + + return ['--perf-no_read_workqueue', '--perf-no_write_workqueue'] def create_dmcrypt_key(): """ @@ -78,7 +121,8 @@ '--type', 'plain', '--key-size', '256', ] - + for extra_opts in bypass_workqueues(device): + command.insert(1, extra_opts) process.call(command, stdin=key, terminal_verbose=True, show_command=True) @@ -103,6 +147,8 @@ device, mapping, ] + for extra_opts in bypass_workqueues(device): + command.insert(1, extra_opts) process.call(command, stdin=key, terminal_verbose=True, show_command=True) But in your case you might want to change the logic a bit so regardless of rotational media or not the queues are bypassed (make the ROTA function always return true for example). To change Ceph container images ... I used this quick and dirty procedure: docker run -it ceph-image bash Make the above changes to ceph-volume exit from container docker ps -a Look for the id of the most recently used container (that you just exited) docker commit container-hash alternative_name_of_container_image if you want to upload to registry: docker image tag alternative_name_of_container_image docker_registry_url:5000/alternative_name_of_container_image docker image push docker_registry_url:5000/alternative_name_of_container_image for cephadm users: docker image ls --digests <- look for the sha256 hash of the image ceph config-key set config/global/container_image docker_registry_url:5000/alternative_name_of_container_image:latest@sha256:the-hash-here redeploy containers with this new image Let me know if you need other info to make this work for you. Looking forward to the results. Gr. Stefan