Greetings, I am having trouble with (the same) PGs stuck in snaptrrim and snaptrrim_wait for over half a year now. I don’t know how to debug/fix that, I tried bringing down the cluster, rebooted everything, failed the mds, mgr, upgraded to a new mentor version, upgraded to a new major version, all to no avail. Ceph version is 19.2.0 # ceph pg dump|grep snaptrim 9.7d 123817 0 0 0 0 479247022395 0 0 4375 346 4375 active+clean+snaptrim_wait 2024-10-13T00:00:12.050817+0000 134416'449058 134416:4144441 [9,14,0,6] 9 [9,14,0,6] 9 134054'434172 2024-10-11T01:26:12.751712+0000 134054'434172 2024-10-11T01:26:12.751712+0000 1256 138366 user requested deep scrub scheduled @ 2024-12-27T08:47:58.847491+0000 114488 0 9.74 123271 0 0 0 0 478810964688 0 0 4440 1140 4440 active+clean+snaptrim 2024-10-13T00:02:12.299691+0000 134416'451014 134416:4037366 [9,8,6,14] 9 [9,8,6,14] 9 134049'436191 2024-10-10T09:25:49.744261+0000 134049'436191 2024-10-10T09:25:49.744261+0000 1256 80746 user requested deep scrub scheduled @ 2024-12-27T08:48:59.094636+0000 114101 0 9.71 124359 0 0 0 0 482220121204 0 0 4374 878 4374 active+clean+snaptrim_wait 2024-10-13T00:02:05.517504+0000 134416'454057 134416:4334146 [9,8,7,3] 9 [9,8,7,3] 9 134049'438914 2024-10-10T05:16:04.290745+0000 134049'438914 2024-10-10T05:16:04.290745+0000 1256 65761 user requested deep scrub scheduled @ 2024-12-27T08:48:03.954808+0000 115086 0 9.6b 123942 0 0 0 0 480451675897 0 0 4358 345 4358 active+clean+snaptrim_wait 2024-10-13T00:00:12.540600+0000 134416'450439 134416:4705616 [9,4,0,11] 9 [9,4,0,11] 9 134049'435438 2024-10-10T18:35:30.564215+0000 134049'435438 2024-10-10T18:35:30.564215+0000 1256 113715 user requested deep scrub scheduled @ 2024-12-27T08:47:37.794121+0000 114709 0 9.6a 123870 0 0 0 0 480164531666 0 0 4364 879 4364 active+clean+snaptrim 2024-10-13T00:02:05.652753+0000 134416'456701 134416:4003484 [9,4,0,13] 9 [9,4,0,13] 9 134049'441302 2024-10-10T12:42:55.289279+0000 134049'441302 2024-10-10T12:42:55.289279+0000 1256 92560 user requested deep scrub scheduled @ 2024-12-27T08:48:09.914747+0000 114496 0 9.5f 123879 0 0 0 0 479941485759 0 0 4431 1141 4431 active+clean+snaptrim_wait 2024-10-13T00:00:13.340335+0000 134416'450791 134416:4123574 [9,3,2,6] 9 [9,3,2,6] 9 134054'435775 2024-10-11T01:27:38.427214+0000 134054'435775 2024-10-11T01:27:38.427214+0000 1256 138452 user requested deep scrub scheduled @ 2024-12-27T08:49:04.452674+0000 114542 0 27.19 7017 0 0 0 0 29241743329 0 0 4358 0 4358 active+clean+snaptrim_wait 2024-10-13T00:02:05.652729+0000 134416'5924654 134416:11859363 [9,7,3,4] 9 [9,7,3,4] 9 133910'5800741 2024-10-09T13:55:08.732933+0000 133910'5800741 2024-10-09T13:55:08.732933+0000 1256 10492 user requested deep scrub scheduled @ 2024-12-27T08:49:10.053054+0000 6293 0 27.4 7072 0 0 0 0 29496160608 0 0 4401 0 4401 active+clean+snaptrim_wait 2024-10-13T00:00:13.740349+0000 134416'5920493 134416:12128517 [9,0,14,2] 9 [9,0,14,2] 9 133972'5796609 2024-10-09T12:20:39.099453+0000 133972'5796609 2024-10-09T12:20:39.099453+0000 1256 4833 user requested deep scrub scheduled @ 2024-12-27T08:47:47.863907+0000 6249 0 9.1c 124329 0 0 0 0 482320315530 0 0 4368 878 4368 active+clean+snaptrim_wait 2024-10-13T00:02:05.518217+0000 134416'453725 134416:3954611 [9,7,11,0] 9 [9,7,11,0] 9 134049'438455 2024-10-10T06:40:15.938302+0000 134049'438455 2024-10-10T06:40:15.938302+0000 1256 70803 user requested deep scrub scheduled @ 2024-12-27T08:49:10.053068+0000 114948 0 9.19 123790 0 0 0 0 480077901246 0 0 4411 218 4411 active+clean+snaptrim_wait 2024-10-13T00:00:12.539642+0000 134416'451736 134416:3826333 [9,6,7,1] 9 [9,6,7,1] 9 134049'436829 2024-10-10T21:35:24.519252+0000 134049'436829 2024-10-10T21:35:24.519252+0000 1256 138616 user requested deep scrub scheduled @ 2024-12-27T08:49:10.053063+0000 114595 0 dumped all 9.13 124027 0 0 0 0 480656352004 0 0 4370 0 4370 active+clean+snaptrim_wait 2024-10-13T00:00:06.753470+0000 134416'447134 134416:3903589 [9,14,13,0] 9 [9,14,13,0] 9 134049'431887 2024-10-10T03:55:32.120382+0000 134049'431887 2024-10-10T03:55:32.120382+0000 1256 60916 user requested deep scrub scheduled @ 2024-12-27T08:49:10.053062+0000 114776 0 9.28 123470 0 0 0 0 478607727659 0 0 4355 0 4355 active+clean+snaptrim_wait 2024-10-13T00:02:12.298971+0000 134416'450238 134416:3500480 [9,0,1,3] 9 [9,0,1,3] 9 134054'435412 2024-10-11T01:23:22.978541+0000 134054'435412 2024-10-11T01:23:22.978541+0000 1256 138187 user requested deep scrub scheduled @ 2024-12-27T08:48:20.260350+0000 114182 0 9.37 123338 0 0 0 0 478217011616 0 0 4391 248 4391 active+clean+snaptrim_wait 2024-10-13T00:02:06.016700+0000 134416'448180 134416:3783995 [9,14,7,13] 9 [9,14,7,13] 9 134054'433338 2024-10-11T01:26:24.029875+0000 134054'433338 2024-10-11T01:26:24.029875+0000 1256 138368 user requested deep scrub scheduled @ 2024-12-27T08:49:10.053069+0000 114085 0 9.42 123550 0 0 0 0 479437295594 0 0 4385 415 4385 active+clean+snaptrim_wait 2024-10-13T00:00:13.264210+0000 134416'446881 134416:3886466 [9,7,1,6] 9 [9,7,1,6] 9 134054'431973 2024-10-11T04:24:50.076511+0000 134054'431973 2024-10-11T04:24:50.076511+0000 1256 163177 user requested deep scrub scheduled @ 2024-12-27T08:49:10.053050+0000 114369 0 9.48 123504 0 0 0 0 478672624191 0 0 4370 0 4370 active+clean+snaptrim_wait 2024-10-13T00:00:15.366907+0000 134416'447888 134416:3476913 [9,2,13,7] 9 [9,2,13,7] 9 134049'432764 2024-10-10T18:43:14.290802+0000 134049'432764 2024-10-10T18:43:14.290802+0000 1256 114183 user requested deep scrub scheduled @ 2024-12-27T08:48:59.094642+0000 114094 0 9.4c 123799 0 0 0 0 480058080747 0 0 4367 298 4367 active+clean+snaptrim_wait 2024-10-13T00:00:15.365742+0000 134416'450187 134416:4460709 [9,7,4,0] 9 [9,7,4,0] 9 134049'435272 2024-10-10T12:45:20.588451+0000 134049'435272 2024-10-10T12:45:20.588451+0000 1256 92705 user requested deep scrub scheduled @ 2024-12-27T08:48:09.914749+0000 114623 0 9.4d 124344 0 0 0 0 480982998237 0 0 4349 878 4349 active+clean+snaptrim_wait 2024-10-13T00:02:12.299642+0000 134416'445413 134416:4170713 [9,14,0,7] 9 [9,14,0,7] 9 134049'430041 2024-10-10T18:35:55.877861+0000 134049'430041 2024-10-10T18:35:55.877861+0000 1255 113735 user requested deep scrub scheduled @ 2024-12-27T08:48:59.094640+0000 114850 0 9.5c 123379 0 0 0 0 478151706144 0 0 4377 0 4377 active+clean+snaptrim_wait 2024-10-13T00:02:06.017489+0000 134416'452726 134416:4446619 [9,11,6,13] 9 [9,11,6,13] 9 134049'437743 2024-10-10T18:43:28.133384+0000 134049'437743 2024-10-10T18:43:28.133384+0000 1255 114188 user requested deep scrub scheduled @ 2024-12-27T08:48:59.094632+0000 114239 0 pgs: 206 active+clean 16 active+clean+snaptrim_wait 2 active+clean+snaptrim 1 active+clean+scrubbing+deep Please advice. Daniel