mgr down : OSDMap.cc: 5155: FAILED ceph_assert(osd_weight.count(oid)

24 Aug 2023

When ceph osd reweight <osd> 0 is executed, mgrs go down and after several retries,
mgrs eventually go down.
In mgr.log and crashinfo, the following logs are displayed.

BUILD/ceph-17.2.3/src/osd/OSDMap.cc: 5155: FAILED ceph_assert(osd_weight.count(oid))

There seems to be a problem with the ceph_assert(osd_weight.count(oid)) part.
Anyone helps me.

# ceph crash info 
{
    "assert_condition": "osd_weight.count(oid)",
    "assert_file":
"/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.2.3/rpm/el8/BUILD/ceph-17.2.3/src/osd/OSDMap.cc",
    "assert_func": "float
OSDMap::calc_deviations(ceph::common::CephContext*, const std::map<int,
std::set<pg_t> >&, const std::map<int, float>&, float,
std::map<int, float>&, std::multimap<float, int>&, float&)",
    "assert_line": 5155,
    "assert_msg":
"/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.2.3/rpm/el8/BUILD/ceph-17.2.3/src/osd/OSDMap.cc:
In function 'float OSDMap::calc_deviations(ceph::common::CephContext*, const
std::map<int, std::set<pg_t> >&, const std::map<int, float>&,
float, std::map<int, float>&, std::multimap<float, int>&,
float&)' thread 7febe3a6f700 time
2023-08-22T10:27:57.207040+0000\n/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.2.3/rpm/el8/BUILD/ceph-17.2.3/src/osd/OSDMap.cc:
5155: FAILED ceph_assert(osd_weight.count(oid))\n",
    "assert_thread_name": "balancer",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12ce0) [0x7fec4fb2bce0]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1b0)
[0x7fec50d0ccd2]",
        "/usr/lib64/ceph/libceph-common.so.2(+0x283e95) [0x7fec50d0ce95]",
        "(OSDMap::calc_deviations(ceph::common::CephContext*, std::map<int,
std::set<pg_t, std::less<pg_t>, std::allocator<pg_t> >,
std::less<int>, std::allocator<std::pair<int const, std::set<pg_t,
std::less<pg_t>, std::allocator<pg_t> > > > > const&,
std::map<int, float, std::less<int>, std::allocator<std::pair<int const,
float> > > const&, float, std::map<int, float, std::less<int>,
std::allocator<std::pair<int const, float> > >&,
std::multimap<float, int, std::less<float>, std::allocator<std::pair<float
const, int> > >&, float&)+0xe0) [0x7fec511d9490]",
        "(OSDMap::calc_pg_upmaps(ceph::common::CephContext*, unsigned int, int,
std::set<long, std::less<long>, std::allocator<long> > const&,
OSDMap::Incremental*, unsigned int*)+0x389) [0x7fec511dd829]",
        "/usr/bin/ceph-mgr(+0x299406) [0x55e77ab23406]",
        "/lib64/libpython3.6m.so.1.0(+0x19d0d7) [0x7fec51c400d7]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0xf9984) [0x7fec51b9c984]",
        "/lib64/libpython3.6m.so.1.0(+0x17a030) [0x7fec51c1d030]",
        "/lib64/libpython3.6m.so.1.0(+0x19d377) [0x7fec51c40377]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0x179e48) [0x7fec51c1ce48]",
        "/lib64/libpython3.6m.so.1.0(+0x19d377) [0x7fec51c40377]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0x179e48) [0x7fec51c1ce48]",
        "/lib64/libpython3.6m.so.1.0(+0x19d377) [0x7fec51c40377]",
        "_PyEval_EvalFrameDefault()",
        "/lib64/libpython3.6m.so.1.0(+0xfa2f6) [0x7fec51b9d2f6]",
        "_PyFunction_FastCallDict()",
        "_PyObject_FastCallDict()",
        "/lib64/libpython3.6m.so.1.0(+0x10db30) [0x7fec51bb0b30]",
        "_PyObject_FastCallDict()",
        "PyObject_CallMethod()",
        "(PyModuleRunner::serve()+0x66) [0x55e77ab1ee06]",
        "(PyModuleRunner::PyModuleRunnerThread::entry()+0x3e3)
[0x55e77ab20443]",
        "/lib64/libpthread.so.0(+0x81ca) [0x7fec4fb211ca]",
        "clone()"
    ],
    "ceph_version": "17.2.3",

# mgr.log
2023-08-22T11:06:56.450+0000 7fbf43d5e700 -1
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.2.3/rpm/el8/BUILD/ceph-17.2.3/src/osd/OSDMap.cc:
In function 'float OSDMap::calc_deviations(ceph::common::CephContext*, const
std::map<int, std::set<pg_t> >&, const std::map<int, float>&,
float, std::map<int, float>&, std::multimap<float, int>&,
float&)' thread 7fbf43d5e700 time 2023-08-22T11:06:56.450708+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/17.2.3/rpm/el8/BUILD/ceph-17.2.3/src/osd/OSDMap.cc:
5155: FAILED ceph_assert(osd_weight.count(oid))

 ceph version 17.2.3 (dff484dfc9e19a9819f375586300b3b79d80034d) quincy (stable)
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152)
[0x7fbfadff5c74]
 2: /usr/lib64/ceph/libceph-common.so.2(+0x283e95) [0x7fbfadff5e95]
 3: (OSDMap::calc_deviations(ceph::common::CephContext*, std::map<int,
std::set<pg_t, std::less<pg_t>, std::allocator<pg_t> >,
std::less<int>, std::allocator<std::pair<int const, std::set<pg_t,
std::less<pg_t>, std::allocator<pg_t> > > > > const&,
std::map<int, float, std::less<int>, std::allocator<std::pair<int const,
float> > > const&, float, std::map<int, float, std::less<int>,
std::allocator<std::pair<int const, float> > >&,
std::multimap<float, int, std::less<float>, std::allocator<std::pair<float
const, int> > >&, float&)+0xe0) [0x7fbfae4c2490]
 4: (OSDMap::calc_pg_upmaps(ceph::common::CephContext*, unsigned int, int,
std::set<long, std::less<long>, std::allocator<long> > const&,
OSDMap::Incremental*, unsigned int*)+0x389) [0x7fbfae4c6829]
 5: /usr/bin/ceph-mgr(+0x299406) [0x55e50358a406]
 6: /lib64/libpython3.6m.so.1.0(+0x19d0d7) [0x7fbfaef290d7]
 7: _PyEval_EvalFrameDefault()
 8: /lib64/libpython3.6m.so.1.0(+0xf9984) [0x7fbfaee85984]
 9: /lib64/libpython3.6m.so.1.0(+0x17a030) [0x7fbfaef06030]
 10: /lib64/libpython3.6m.so.1.0(+0x19d377) [0x7fbfaef29377]
 11: _PyEval_EvalFrameDefault()
 12: /lib64/libpython3.6m.so.1.0(+0x179e48) [0x7fbfaef05e48]
 13: /lib64/libpython3.6m.so.1.0(+0x19d377) [0x7fbfaef29377]
 14: _PyEval_EvalFrameDefault()
 15: /lib64/libpython3.6m.so.1.0(+0x179e48) [0x7fbfaef05e48]
 16: /lib64/libpython3.6m.so.1.0(+0x19d377) [0x7fbfaef29377]
 17: _PyEval_EvalFrameDefault()
 18: /lib64/libpython3.6m.so.1.0(+0xfa2f6) [0x7fbfaee862f6]
 19: _PyFunction_FastCallDict()
 20: _PyObject_FastCallDict()
 21: /lib64/libpython3.6m.so.1.0(+0x10db30) [0x7fbfaee99b30]
 22: _PyObject_FastCallDict()
 23: PyObject_CallMethod()
 24: (PyModuleRunner::serve()+0x66) [0x55e503585e06]
 25: (PyModuleRunner::PyModuleRunnerThread::entry()+0x3e3) [0x55e503587443]
 26: /lib64/libpthread.so.0(+0x81ca) [0x7fbface0a1ca]
 27: clone()

2023-08-22T11:06:56.450+0000 7fbf43d5e700 -1 *** Caught signal (Aborted) **
 in thread 7fbf43d5e700 thread_name:balancer

 ceph version 17.2.3 (dff484dfc9e19a9819f375586300b3b79d80034d) quincy (stable)
 1: /lib64/libpthread.so.0(+0x12ce0) [0x7fbface14ce0]
 2: gsignal()
 3: abort()
 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1b0)
[0x7fbfadff5cd2]
 5: /usr/lib64/ceph/libceph-common.so.2(+0x283e95) [0x7fbfadff5e95]
 6: (OSDMap::calc_deviations(ceph::common::CephContext*, std::map<int,
std::set<pg_t, std::less<pg_t>, std::allocator<pg_t> >,
std::less<int>, std::allocator<std::pair<int const, std::set<pg_t,
std::less<pg_t>, std::allocator<pg_t> > > > > const&,
std::map<int, float, std::less<int>, std::allocator<std::pair<int const,
float> > > const&, float, std::map<int, float, std::less<int>,
std::allocator<std::pair<int const, float> > >&,
std::multimap<float, int, std::less<float>, std::allocator<std::pair<float
const, int> > >&, float&)+0xe0) [0x7fbfae4c2490]
 7: (OSDMap::calc_pg_upmaps(ceph::common::CephContext*, unsigned int, int,
std::set<long, std::less<long>, std::allocator<long> > const&,
OSDMap::Incremental*, unsigned int*)+0x389) [0x7fbfae4c6829]
 8: /usr/bin/ceph-mgr(+0x299406) [0x55e50358a406]
 9: /lib64/libpython3.6m.so.1.0(+0x19d0d7) [0x7fbfaef290d7]
 10: _PyEval_EvalFrameDefault()
 11: /lib64/libpython3.6m.so.1.0(+0xf9984) [0x7fbfaee85984]
 12: /lib64/libpython3.6m.so.1.0(+0x17a030) [0x7fbfaef06030]
 13: /lib64/libpython3.6m.so.1.0(+0x19d377) [0x7fbfaef29377]
 14: _PyEval_EvalFrameDefault()
 15: /lib64/libpython3.6m.so.1.0(+0x179e48) [0x7fbfaef05e48]
 16: /lib64/libpython3.6m.so.1.0(+0x19d377) [0x7fbfaef29377]
 17: _PyEval_EvalFrameDefault()
 18: /lib64/libpython3.6m.so.1.0(+0x179e48) [0x7fbfaef05e48]
 19: /lib64/libpython3.6m.so.1.0(+0x19d377) [0x7fbfaef29377]
 20: _PyEval_EvalFrameDefault()
 21: /lib64/libpython3.6m.so.1.0(+0xfa2f6) [0x7fbfaee862f6]
 22: _PyFunction_FastCallDict()
 23: _PyObject_FastCallDict()
 24: /lib64/libpython3.6m.so.1.0(+0x10db30) [0x7fbfaee99b30]
 25: _PyObject_FastCallDict()
 26: PyObject_CallMethod()
 27: (PyModuleRunner::serve()+0x66) [0x55e503585e06]
 28: (PyModuleRunner::PyModuleRunnerThread::entry()+0x3e3) [0x55e503587443]
 29: /lib64/libpthread.so.0(+0x81ca) [0x7fbface0a1ca]
 30: clone()
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to
interpret this.

2024

2023

2022

2021

2020

2019

mgr down : OSDMap.cc: 5155: FAILED ceph_assert(osd_weight.count(oid)