Hi all, at the University of Zurich we run a cephfs cluster of ~12PB raw size. We currently run Pacific 16.2.15 and our clients (Ubuntu 20.04) mount cephfs using the kernel driver. The cluster was deployed in Mimic and subsequently to Nautilus (14.2.22) and then Pacific (16.2.15). Last Wednesday (4 Dec.) the kernel clients have been upgraded from 5.15.0-124 to 5.15.0-126. We run 6 active MDSs and 2 standby with static pinning of directory subtrees. Yesterday (actually it happened on Friday evening) we experienced a problem where an MDS serving rank.4 crashed and then the two standby MDS in turn crashed when trying to replay the journal. The other ranks are not affected. We would appreciate any help to debug and recover the failed filesystem. Thank you. Following some info on the crash and status of the cluster. The oldest and initial crash info is: { "assert_condition": "batch_ops.empty()", "assert_file": "/build/ceph-16.2.15/src/mds/CDentry.h", "assert_func": "virtual CDentry::~CDentry()", "assert_line": 130, "assert_msg": "/build/ceph-16.2.15/src/mds/CDentry.h: In function 'virtual CDentry::~CDentry()' thread 7f14fb70e700 time 2024-12-06T19:08:09.292770+0100\n/build/ceph-16.2.15/src/mds/CDentry.h: 130: FAILED ceph_assert(batch_ops.empty())\n", "assert_thread_name": "ms_dispatch", "backtrace": [ "/lib/x86_64-linux-gnu/libpthread.so.0(+0x12980) [0x7f1507109980]", "gsignal()", "abort()", "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x19c) [0x7f15077b4a2e]", "(ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7f15077b4bb8]", "(CDentry::~CDentry()+0x4d9) [0x564e03dec519]", "(CDir::remove_dentry(CDentry*)+0x17e) [0x564e03df672e]", "(MDCache::trim_dentry(CDentry*, std::map<int, boost::intrusive_ptr<MCacheExpire>, std::less<int>, std::allocator<std::pair<int const, boost::intrusive_ptr<MCacheExpire> > > >&)+0xd4) [0x564e03cb9b54]", "(MDCache::trim_lru(unsigned long, std::map<int, boost::intrusive_ptr<MCacheExpire>, std::less<int>, std::allocator<std::pair<int const, boost::intrusive_ptr<MCacheExpire> > > >&)+0x91a) [0x564e03cbb36a]", "(MDCache::trim(unsigned long)+0xc1) [0x564e03cbbde1]", "(MDCache::upkeep_main()+0x94a) [0x564e03cf8aea]", "/usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xbd6df) [0x7f1506c276df]", "/lib/x86_64-linux-gnu/libpthread.so.0(+0x76db) [0x7f15070fe6db]", "clone()" ], "ceph_version": "16.2.15", "crash_id": "2024-12-06T18:08:09.295408Z_e6f3560d-21aa-425d-a1b2-f0b5fb5bd103", "entity_name": "mds.mds-l20-33", "os_id": "ubuntu", "os_name": "Ubuntu", "os_version": "18.04.6 LTS (Bionic Beaver)", "os_version_id": "18.04", "process_name": "ceph-mds", "stack_sig": "5e65f2db93b88e8fb398b21684d658a9ef1cec94845c09c581a0d8a80722960a", "timestamp": "2024-12-06T18:08:09.295408Z", "utsname_hostname": "mds-l20-33", "utsname_machine": "x86_64", "utsname_release": "5.4.0-146-generic", "utsname_sysname": "Linux", "utsname_version": "#163~18.04.1-Ubuntu SMP Mon Mar 20 15:02:59 UTC 2023" } and it looks similar to https://tracker.ceph.com/issues/66967 even though the "assert_line" is different. But all the following crash are different and look like: { "assert_condition": "p->first <= start", "assert_file": "/build/ceph-16.2.15/src/include/interval_set.h", "assert_func": "void interval_set<T, C>::erase(T, T, std::function<bool(T, T)>) [with T = inodeno_t; C = std::map]", "assert_line": 568, "assert_msg": "/build/ceph-16.2.15/src/include/interval_set.h: In function 'void interval_set<T, C>::erase(T, T, std::function<bool(T, T)>) [with T = inodeno_t; C = std::map]' thread 7fdd353b9700 time 2024-12-06T19:08:24.550104+0100\n/build/ceph-16.2.15/src/include/interval_set.h: 568: FAILED ceph_assert(p->first <= start)\n", "assert_thread_name": "md_log_replay", "backtrace": [ "/lib/x86_64-linux-gnu/libpthread.so.0(+0x12980) [0x7fdd445bb980]", "gsignal()", "abort()", "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x19c) [0x7fdd44c66a2e]", "(ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7fdd44c66bb8]", "(interval_set<inodeno_t, std::map>::erase(inodeno_t, inodeno_t, std::function<bool (inodeno_t, inodeno_t)>)+0x296) [0x55876572c9f6]", "(interval_set<inodeno_t, std::map>::erase(inodeno_t)+0x2f) [0x55876572ca3f]", "(EMetaBlob::replay(MDSRank*, LogSegment*, int, MDPeerUpdate*)+0x588b) [0x558765a0868b]", "(EUpdate::replay(MDSRank*)+0x5d) [0x558765a0a0bd]", "(MDLog::_replay_thread()+0x78e) [0x55876598d64e]", "(MDLog::ReplayThread::entry()+0xd) [0x55876567f4ad]", "/lib/x86_64-linux-gnu/libpthread.so.0(+0x76db) [0x7fdd445b06db]", "clone()" ], "ceph_version": "16.2.15", "crash_id": "2024-12-06T18:08:24.552401Z_35179c40-1fb8-473e-84d6-c013b27f341f", "entity_name": "mds.mds-l20-33", "os_id": "ubuntu", "os_name": "Ubuntu", "os_version": "18.04.6 LTS (Bionic Beaver)", "os_version_id": "18.04", "process_name": "ceph-mds", "stack_sig": "96ed41e917b880ad1183c7acd926e3259e37e1511eb6de7730a21d14121679b2", "timestamp": "2024-12-06T18:08:24.552401Z", "utsname_hostname": "mds-l20-33", "utsname_machine": "x86_64", "utsname_release": "5.4.0-146-generic", "utsname_sysname": "Linux", "utsname_version": "#163~18.04.1-Ubuntu SMP Mon Mar 20 15:02:59 UTC 2023" } that looks like https://tracker.ceph.com/issues/61009 even if it has not been reported for Pacific. Our current status is: root@mon-l19-31:~# ceph -s cluster: id: 6ef3c8fe-33e9-4db3-80a7-c64d8991b9f0 health: HEALTH_WARN 1054 OSD(s) reporting legacy (not per-pool) BlueStore omap usage stats 1 filesystem is degraded insufficient standby MDS daemons available 4 MDSs report slow requests 2 MDSs behind on trimming 4 pgs not deep-scrubbed in time 2 pools have too many placement groups 33 daemons have recently crashed services: mon: 3 daemons, quorum mon-l19-31,mon-l20-41,mon-l15-41 (age 2M) mgr: mon-l20-41(active, since 2M), standbys: mon-l19-31, mon-l15-41 mds: 6/6 daemons up osd: 1216 osds: 1213 up (since 11d), 1213 in (since 11d) data: volumes: 0/1 healthy, 1 recovering pools: 4 pools, 9729 pgs objects: 2.78G objects, 5.3 PiB usage: 8.4 PiB used, 3.3 PiB / 12 PiB avail pgs: 9655 active+clean 74 active+clean+scrubbing+deep root@mon-l19-31:~# ceph fs status spindlefs - 611 clients ========= RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active mds-l21-34-2 Reqs: 1 /s 248k 222k 119k 5107 1 active mds-l19-33 Reqs: 1 /s 15.5M 14.5M 2049k 1836k 2 active mds-l21-33 Reqs: 0 /s 25.7M 20.5M 2765k 65.5k 3 active mds-l19-33-2 Reqs: 0 /s 20.4M 20.4M 582k 10.3k 4 replay(laggy) mds-l21-33-2 0 0 0 0 5 active mds-l21-34 Reqs: 1 /s 304k 274k 139k 9112 POOL TYPE USED AVAIL spindle_metadata metadata 592G 4321G spindle data 8384T 1092T vhp data 119T 10.2T MDS version: ceph version 16.2.15 (618f440892089921c3e944a991122ddc44e60516) pacific (stable) root@mon-l19-31:~# ceph health detail HEALTH_WARN 1054 OSD(s) reporting legacy (not per-pool) BlueStore omap usage stats; 1 filesystem is degraded; insufficient standby MDS daemons available; 4 MDSs report slow requests; 2 MDSs behind on trimming; 4 pgs not deep-scrubbed in time; 2 pools have too many placement groups; 33 daemons have recently crashed [WRN] BLUESTORE_NO_PER_POOL_OMAP: 1054 OSD(s) reporting legacy (not per-pool) BlueStore omap usage stats osd.13 legacy (not per-pool) omap detected, suggest to run store repair to benefit from per-pool omap usage statistics ... ... ... osd.1226 legacy (not per-pool) omap detected, suggest to run store repair to benefit from per-pool omap usage statistics [WRN] FS_DEGRADED: 1 filesystem is degraded fs spindlefs is degraded [WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available have 0; want 1 more [WRN] MDS_SLOW_REQUEST: 4 MDSs report slow requests mds.mds-l21-33(mds.2): 953 slow requests are blocked > 30 secs mds.mds-l19-33(mds.1): 35 slow requests are blocked > 30 secs mds.mds-l21-34-2(mds.0): 19 slow requests are blocked > 30 secs mds.mds-l19-33-2(mds.3): 3 slow requests are blocked > 30 secs [WRN] MDS_TRIM: 2 MDSs behind on trimming mds.mds-l19-33(mds.1): Behind on trimming (449/128) max_segments: 128, num_segments: 449 mds.mds-l19-33-2(mds.3): Behind on trimming (7099/128) max_segments: 128, num_segments: 7099 [WRN] PG_NOT_DEEP_SCRUBBED: 4 pgs not deep-scrubbed in time pg 11.140a not deep-scrubbed since 2024-11-28T08:56:45.425135+0100 pg 11.694 not deep-scrubbed since 2024-11-27T22:18:56.908953+0100 pg 11.21b not deep-scrubbed since 2024-11-28T04:48:14.734599+0100 pg 11.ad8 not deep-scrubbed since 2024-11-27T19:59:32.851964+0100 [WRN] POOL_TOO_MANY_PGS: 2 pools have too many placement groups Pool spindle has 8192 placement groups, should have 8192 Pool vhp has 1024 placement groups, should have 1024 [WRN] RECENT_CRASH: 33 daemons have recently crashed mds.mds-l20-33 crashed on host mds-l20-33 at 2024-12-06T18:08:50.420719Z mds.mds-l20-33-2 crashed on host mds-l20-33 at 2024-12-06T18:08:33.409537Z mds.mds-l20-33-2 crashed on host mds-l20-33 at 2024-12-06T18:08:27.335401Z mds.mds-l20-33 crashed on host mds-l20-33 at 2024-12-06T18:08:53.156894Z mds.mds-l20-33-2 crashed on host mds-l20-33 at 2024-12-06T18:08:31.359872Z mds.mds-l20-33-2 crashed on host mds-l20-33 at 2024-12-06T18:08:29.331676Z mds.mds-l20-33 crashed on host mds-l20-33 at 2024-12-06T18:08:09.295408Z mds.mds-l20-33 crashed on host mds-l20-33 at 2024-12-06T18:08:24.552401Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-06T18:09:13.029070Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-06T18:09:10.641714Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-06T18:09:18.205911Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-06T18:09:15.176043Z mds.mds-l20-33 crashed on host mds-l20-33 at 2024-12-08T13:33:16.196251Z mds.mds-l20-33 crashed on host mds-l20-33 at 2024-12-08T13:33:18.242351Z mds.mds-l20-33 crashed on host mds-l20-33 at 2024-12-08T13:33:13.762433Z mds.mds-l20-33-2 crashed on host mds-l20-33 at 2024-12-08T15:22:20.143608Z mds.mds-l20-33-2 crashed on host mds-l20-33 at 2024-12-08T15:22:18.083712Z mds.mds-l20-33-2 crashed on host mds-l20-33 at 2024-12-08T15:22:15.248451Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:19:10.277110Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:19:06.202423Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:23:24.469758Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:23:22.372727Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:19:08.271763Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:23:26.559707Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:50:21.508069Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:50:19.417250Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:50:17.469305Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:59:42.693519Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:59:40.494913Z mds.mds-l21-33-2 crashed on host mds-l21-33 at 2024-12-09T09:59:44.763196Z and 3 more Best regards, Enrico -- Enrico Favero S3IT Services and Support for Science IT Office Y11 F 52 University of Zürich Winterthurerstrasse 190, CH-8057 Zürich (Switzerland) Tel: +41 44 635 42 22