[ceph-users] Re: Ceph Health error right after starting balancer

This is the output of OSD.270 that remains with slow requests blocked even after restarting. What's the interpretation of it? root@ld5507:~# ceph daemon osd.270 dump_blocked_ops [330/1857] {     "ops": [         {             "description": "osd_pg_create(e293649 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:22:13.563017",             "age": 2785.2698560419999,             "duration": 2785.2699056299998,             "type_data": {                 "flag_point": "started",                 "events": [                     {                         "time": "2019-10-31 19:22:13.563017",                         "event": "initiated"                     },                     {                         "time": "2019-10-31 19:22:13.563017",                         "event": "header_read"                     },                     {                         "time": "2019-10-31 19:22:13.563011",                         "event": "throttled"                     },                     {                         "time": "2019-10-31 19:22:13.563024",                         "event": "all_read"                     },                     {                         "time": "2019-10-31 20:07:43.881441",                         "event": "dispatched" }, [300/1857]                     {                         "time": "2019-10-31 20:07:43.881472",                         "event": "wait for new map"                     },                     {                         "time": "2019-10-31 20:07:44.665714",                         "event": "started"                     }                 ]             }         },         {             "description": "osd_pg_create(e293650 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:23:16.150040",             "age": 2722.6828331659999,             "duration": 2722.683007228,             "type_data": {                 "flag_point": "delayed",                 "events": [                     {                         "time": "2019-10-31 19:23:16.150040",                         "event": "initiated"                     },                     {                         "time": "2019-10-31 19:23:16.150040",                         "event": "header_read"                     },                     {                         "time": "2019-10-31 19:23:16.150035",                         "event": "throttled" }, [269/1857]                     {                         "time": "2019-10-31 19:23:16.150055",                         "event": "all_read"                     },                     {                         "time": "2019-10-31 20:07:43.882197",                         "event": "dispatched"                     },                     {                         "time": "2019-10-31 20:07:43.882198",                         "event": "wait for new map"                     }                 ]             }         },         {             "description": "osd_pg_create(e293651 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:23:17.779034",             "age": 2721.0538393319998,             "duration": 2721.0541152350002,             "type_data": {                 "flag_point": "delayed",                 "events": [                     {                         "time": "2019-10-31 19:23:17.779034",                         "event": "initiated"                     },                     {                         "time": "2019-10-31 19:23:17.779034",                         "event": "header_read" }, [238/1857]                     {                         "time": "2019-10-31 19:23:17.779027",                         "event": "throttled"                     },                     {                         "time": "2019-10-31 19:23:17.779044",                         "event": "all_read"                     },                     {                         "time": "2019-10-31 20:07:43.882326",                         "event": "dispatched"                     },                     {                         "time": "2019-10-31 20:07:43.882328",                         "event": "wait for new map"                     }                 ]             }         },         {             "description": "osd_pg_create(e293652 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:23:22.178466",             "age": 2716.6544071140002,             "duration": 2716.654784073,             "type_data": {                 "flag_point": "delayed",                 "events": [                     {                         "time": "2019-10-31 19:23:22.178466",                         "event": "initiated" }, [207/1857]                     {                         "time": "2019-10-31 19:23:22.178466",                         "event": "header_read"                     },                     {                         "time": "2019-10-31 19:23:22.178463",                         "event": "throttled"                     },                     {                         "time": "2019-10-31 19:23:22.178471",                         "event": "all_read"                     },                     {                         "time": "2019-10-31 20:07:43.882355",                         "event": "dispatched"                     },                     {                         "time": "2019-10-31 20:07:43.882356",                         "event": "wait for new map"                     }                 ]             }         },         {             "description": "osd_pg_create(e293653 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:23:28.227563",             "age": 2710.6053095460002,             "duration": 2710.6057835810002,             "type_data": {                 "flag_point": "delayed",                 "events": [ [176/1857]                     {                         "time": "2019-10-31 19:23:28.227563",                         "event": "initiated"                     },                     {                         "time": "2019-10-31 19:23:28.227563",                         "event": "header_read"                     },                     {                         "time": "2019-10-31 19:23:28.227559",                         "event": "throttled"                     },                     {                         "time": "2019-10-31 19:23:28.227570",                         "event": "all_read"                     },                     {                         "time": "2019-10-31 20:07:43.882383",                         "event": "dispatched"                     },            "age": 2574.9672041270001, [145/1857]             "duration": 2574.9677752090001,             "type_data": {                 "flag_point": "delayed",                 "events": [                     {                         "time": "2019-10-31 19:25:43.865669",                         "event": "initiated"                     },                     {                         "time": "2019-10-31 19:25:43.865669",                         "event": "header_read"                     },                     {                         "time": "2019-10-31 19:25:43.865663",                         "event": "throttled"                     },                     {                         "time": "2019-10-31 19:25:43.865682",                         "event": "all_read"                     },                     {                         "time": "2019-10-31 20:07:43.883843",                         "event": "dispatched"                     },                     {                         "time": "2019-10-31 20:07:43.883846",                         "event": "wait for new map" } [117/1857]                 ]             }         },         {             "description": "osd_pg_create(e293655 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:26:16.129187",             "age": 2542.703685858,             "duration": 2542.7043550879998,             "type_data": {                 "flag_point": "delayed",                 "events": [                     {                         "time": "2019-10-31 19:26:16.129187",                         "event": "initiated"                     },                     {                         "time": "2019-10-31 19:26:16.129187",                         "event": "header_read"                     },                     {                         "time": "2019-10-31 19:26:16.129181",                         "event": "throttled"                     },                     {                         "time": "2019-10-31 19:26:16.129197",                         "event": "all_read"                     },                     {                         "time": "2019-10-31 20:07:43.883928",                         "event": "dispatched" }, [86/1857]                     {                         "time": "2019-10-31 20:07:43.883929",                         "event": "wait for new map"                     }                 ]             }         },         {             "description": "osd_pg_create(e293656 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:30:47.890777",             "age": 2270.9420963450002,             "duration": 2270.9428696479999,             "type_data": {                 "flag_point": "delayed",                 "events": [                     {                         "time": "2019-10-31 19:30:47.890777",                         "event": "initiated"                     },                     {                         "time": "2019-10-31 19:30:47.890777",                         "event": "header_read"                     },                     {                         "time": "2019-10-31 19:30:47.890772",                         "event": "throttled"                     },                     {                         "time": "2019-10-31 19:30:47.890788",                         "event": "all_read" }, [55/1857]                     {                         "time": "2019-10-31 20:07:43.913783",                         "event": "dispatched"                     },                     {                         "time": "2019-10-31 20:07:43.913786",                         "event": "wait for new map"                     }                 ]             }         },         {             "description": "osd_pg_create(e293657 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:35:32.323775",             "age": 1986.5090976690001,             "duration": 1986.509968458,             "type_data": {                 "flag_point": "delayed",                 "events": [                     {                         "time": "2019-10-31 19:35:32.323775",                         "event": "initiated"                     },                     {                         "time": "2019-10-31 19:35:32.323775",                         "event": "header_read"                     },                     {                         "time": "2019-10-31 19:35:32.323769",                         "event": "throttled" }, [24/1857]                     {                         "time": "2019-10-31 19:35:32.323802",                         "event": "all_read"                     },                     {                         "time": "2019-10-31 20:07:43.917500",                         "event": "dispatched"                     },                     {                         "time": "2019-10-31 20:07:43.917503",                         "event": "wait for new map"                     }                 ]             }         },         {             "description": "osd_pg_create(e293658 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:36:32.783376",             "age": 1926.0494967300001,             "duration": 1926.050464765,             "type_data": {                 "flag_point": "delayed",                 "events": [                     {                         "time": "2019-10-31 19:36:32.783376",                         "event": "initiated"                     },                     {                         "time": "2019-10-31 19:36:32.783376",                         "event": "header_read"                     },                     {                         "time": "2019-10-31 19:36:32.783373",                         "event": "throttled"                     },                     {                         "time": "2019-10-31 19:36:32.783388",                         "event": "all_read"                     },                     {                         "time": "2019-10-31 20:07:43.919280",                         "event": "dispatched"                     },                     {                         "time": "2019-10-31 20:07:43.919282",                         "event": "wait for new map"                     }                 ]             }         }     ],     "complaint_time": 30,     "num_blocked_ops": 10 }                     {                         "time": "2019-10-31 20:07:43.882384",                         "event": "wait for new map"                     }                 ]             }         },         {             "description": "osd_pg_create(e293654 59.b:267033 59.2c:267033)",             "initiated_at": "2019-10-31 19:25:43.865669", Am 31.10.2019 um 18:27 schrieb Paul Emmerich:

ceph daemon osd.X dump_blocked_ops