Hi: We have a cluster (v13.2.4), and we do some tests on a EC k=2, m=1 pool "VMPool0". We deploy some VMs (Windows, CentOS7) on the pool and then use IOMeter to write data to these VMs. After a period of time, we observe a strange thing that pool actual usage is much larger than stored data * 1.5 (stored_raw). [root@Sim-str-R6-4 ~]# ceph df GLOBAL: CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 3.5 TiB 1.8 TiB 1.7 TiB 1.7 TiB 48.74 TOTAL 3.5 TiB 1.8 TiB 1.7 TiB 1.7 TiB 48.74 POOLS: NAME ID USED %USED MAX AVAIL OBJECTS cephfs_data 1 29 GiB 100.00 0 B 2597 cephfs_md 2 831 MiB 100.00 0 B 133 erasure_meta_hdd 3 22 MiB 100.00 0 B 170 VMPool0 4 1.2 TiB 56.77 644 GiB 116011 stresspool 5 2.6 MiB 100.00 0 B 32 [root@Sim-str-R6-4 ~]# ceph df detail -f json-pretty -----snippet----- { "name": "VMPool0", "id": 4, "stats": { "kb_used": 1328888832, "bytes_used": 1360782163968, <---------------- "percent_used": 0.567110, "max_avail": 692481687552, "objects": 116011, "quota_objects": 0, "quota_bytes": 0, "dirty": 116011, "rd": 27449034, "rd_bytes": 126572760064, "wr": 20675381, "wr_bytes": 1006460652544, "comp_ratio": 1.000000, "stored": 497657610240, "stored_raw": 746486431744, <---------------- } }, The perf counters of all osds (all hdd) used by VMPool0 also show that bluestore_allocated is much larger than bluestore_stored. [root@Sim-str-R6-4 ~]# for i in {0..3}; do echo $i; ceph daemon osd.$i perf dump | grep bluestore | head -6; done 0 "bluestore": { "bluestore_allocated": 175032369152, <---------------- "bluestore_stored": 83557936482, <---------------- "bluestore_compressed": 958795770, "bluestore_compressed_allocated": 6431965184, "bluestore_compressed_original": 18576584704, 1 "bluestore": { "bluestore_allocated": 119943593984, <---------------- "bluestore_stored": 53325238866, <---------------- "bluestore_compressed": 670158436, "bluestore_compressed_allocated": 4751818752, "bluestore_compressed_original": 13752328192, 2 "bluestore": { "bluestore_allocated": 155444707328, <---------------- "bluestore_stored": 69067116553, <---------------- "bluestore_compressed": 565170876, "bluestore_compressed_allocated": 4614324224, "bluestore_compressed_original": 13469696000, 3 "bluestore": { "bluestore_allocated": 128179240960, <---------------- "bluestore_stored": 60884752114, <---------------- "bluestore_compressed": 1653455847, "bluestore_compressed_allocated": 9741795328, "bluestore_compressed_original": 27878768640, [root@Sim-str-R6-5 osd]# for i in {4..7}; do echo $i; sh -c "ceph daemon osd.$i perf dump | grep bluestore | head -6"; done 4 "bluestore": { "bluestore_allocated": 165950652416, <---------------- "bluestore_stored": 80255191687, <---------------- "bluestore_compressed": 1526871060, "bluestore_compressed_allocated": 8900378624, "bluestore_compressed_original": 25324142592, 5 admin_socket: exception getting command descriptions: [Errno 111] Connection refused 6 "bluestore": { "bluestore_allocated": 166022152192, <---------------- "bluestore_stored": 84645390708, <---------------- "bluestore_compressed": 1169055606, "bluestore_compressed_allocated": 8647278592, "bluestore_compressed_original": 25135091712, 7 "bluestore": { "bluestore_allocated": 204633604096, <---------------- "bluestore_stored": 100116382041, <---------------- "bluestore_compressed": 1081260422, "bluestore_compressed_allocated": 6510018560, "bluestore_compressed_original": 18654052352, [root@Sim-str-R6-6 osd]# for i in {8..12}; do echo $i; ceph daemon osd.$i perf dump | grep bluestore | head -6; done 8 "bluestore": { "bluestore_allocated": 106330193920, <---------------- "bluestore_stored": 45282848089, <---------------- "bluestore_compressed": 1136610231, "bluestore_compressed_allocated": 7248609280, "bluestore_compressed_original": 20882960384, 9 "bluestore": { "bluestore_allocated": 120657412096, <---------------- "bluestore_stored": 52550745942, <---------------- "bluestore_compressed": 1321632665, "bluestore_compressed_allocated": 7401504768, "bluestore_compressed_original": 21073027072, 10 "bluestore": { "bluestore_allocated": 155985772544, <---------------- "bluestore_stored": 73236910054, <---------------- "bluestore_compressed": 98351920, "bluestore_compressed_allocated": 772210688, "bluestore_compressed_original": 2242043904, 11 "bluestore": { "bluestore_allocated": 106040524800, <---------------- "bluestore_stored": 45353612134, <---------------- "bluestore_compressed": 874216443, "bluestore_compressed_allocated": 4962844672, "bluestore_compressed_original": 14160310272, 12 "bluestore": { "bluestore_allocated": 118751363072, <---------------- "bluestore_stored": 52194408691, <---------------- "bluestore_compressed": 782919969, "bluestore_compressed_allocated": 5546311680, "bluestore_compressed_original": 16043233280, The config min_alloc_size_hdd of all osds are 64K and rados objects of VMPool0 are all 4M rbd_data.x.xxxxxxx.xxxxxxx objects. It's kind of strange that allocation space is much larger than stored data. Can anyone explains this?