[rook@rook-ceph-tools-5c55f45cb8-kzpl8 ~]$ ceph health detail HEALTH_WARN 1 filesystem is degraded; Reduced data availability: 70 pgs inactive, 70 pgs incomplete; Degraded data redundancy: 24181/26522400 objects degraded (0.091%), 3 pgs degraded, 3 pgs undersized; 1 slow ops, oldest one blocked for 9169 sec, osd.17 has slow ops [WRN] FS_DEGRADED: 1 filesystem is degraded     fs cephfs is degraded [WRN] PG_AVAILABILITY: Reduced data availability: 70 pgs inactive, 70 pgs incomplete     pg 2.0 is incomplete, acting [17,23]     pg 3.1 is incomplete, acting [13,17]     pg 3.24 is incomplete, acting [14,11]     pg 3.39 is incomplete, acting [7,22]     pg 3.3e is incomplete, acting [23,8]     pg 3.55 is incomplete, acting [3,7]     pg 3.79 is incomplete, acting [8,14]     pg 3.86 is incomplete, acting [14,6]     pg 3.b2 is incomplete, acting [12,21]     pg 3.be is incomplete, acting [20,5]     pg 3.c7 is incomplete, acting [15,17]     pg 3.f8 is incomplete, acting [6,22]     pg 3.108 is incomplete, acting [13,10]     pg 3.11b is incomplete, acting [8,13]     pg 3.11d is incomplete, acting [15,6]     pg 3.125 is incomplete, acting [1,9]     pg 3.139 is incomplete, acting [11,0]     pg 3.146 is incomplete, acting [24,3]     pg 20.7 is incomplete, acting [2,19]     pg 20.13 is incomplete, acting [6,0]     pg 20.19 is incomplete, acting [18,24]     pg 20.26 is incomplete, acting [12,19]     pg 20.2b is incomplete, acting [5,15]     pg 20.2d is incomplete, acting [8,23]     pg 20.2f is incomplete, acting [19,9]     pg 20.3a is incomplete, acting [14,6]     pg 20.4f is incomplete, acting [6,19]     pg 20.5e is incomplete, acting [11,13]     pg 20.6f is incomplete, acting [13,4]     pg 20.77 is incomplete, acting [18,2]     pg 20.79 is incomplete, acting [16,4]     pg 20.91 is incomplete, acting [14,24]     pg 20.9e is incomplete, acting [11,18]     pg 20.b1 is incomplete, acting [7,18]     pg 20.c3 is incomplete, acting [20,12]     pg 20.c4 is incomplete, acting [8,22]     pg 20.c5 is incomplete, acting [6,14]     pg 20.c7 is incomplete, acting [9,19]     pg 20.d1 is incomplete, acting [22,2]     pg 20.e5 is incomplete, acting [13,6]     pg 20.e6 is incomplete, acting [24,20]     pg 20.f0 is incomplete, acting [10,22]     pg 20.f1 is incomplete, acting [16,9]     pg 20.f9 is incomplete, acting [13,4]     pg 20.103 is incomplete, acting [10,23]     pg 20.118 is incomplete, acting [10,16]     pg 20.122 is incomplete, acting [18,12]     pg 20.131 is incomplete, acting [20,10]     pg 20.135 is incomplete, acting [8,0]     pg 20.153 is incomplete, acting [20,9]     pg 20.15c is stuck inactive since forever, current state incomplete, last acting [17,0] [WRN] PG_DEGRADED: Degraded data redundancy: 24181/26522400 objects degraded (0.091%), 3 pgs degraded, 3 pgs undersized     pg 20.9d is stuck undersized for 2h, current state active+undersized+degraded+remapped+backfilling, last acting [22]     pg 20.e8 is stuck undersized for 2h, current state active+undersized+degraded+remapped+backfilling, last acting [22]     pg 20.115 is stuck undersized for 2h, current state active+undersized+degraded+remapped+backfilling, last acting [22] [WRN] SLOW_OPS: 1 slow ops, oldest one blocked for 9169 sec, osd.17 has slow ops [rook@rook-ceph-tools-5c55f45cb8-kzpl8 ~]$ ceph osd df tree ID  CLASS  WEIGHT    REWEIGHT  SIZE     RAW USE  DATA     OMAP     META     AVAIL    %USE   VAR   PGS  STATUS  TYPE NAME -1         13.50000         -   14 TiB  5.7 TiB  5.7 TiB   16 GiB   71 GiB  7.8 TiB  42.56  1.00    -          root default -3          6.50000         -  6.5 TiB  2.9 TiB  2.8 TiB  7.7 GiB   35 GiB  3.6 TiB  44.17  1.04    -              host cephosd01  0    hdd   0.50000   1.00000  512 GiB  264 GiB  260 GiB  1.7 GiB  2.9 GiB  248 GiB  51.61  1.21   99      up          osd.0  1    hdd   0.50000   1.00000  512 GiB  236 GiB  233 GiB  172 MiB  2.8 GiB  276 GiB  46.16  1.08   83      up          osd.1  3    hdd   0.50000   1.00000  512 GiB  237 GiB  233 GiB  1.2 GiB  3.1 GiB  275 GiB  46.32  1.09   84      up          osd.3 13    hdd   0.50000   1.00000  512 GiB  181 GiB  180 GiB    5 KiB  1.9 GiB  331 GiB  35.43  0.83   84      up          osd.13 14    hdd   0.50000   1.00000  512 GiB  194 GiB  191 GiB   54 MiB  2.5 GiB  318 GiB  37.80  0.89   88      up          osd.14 15    hdd   0.50000   1.00000  512 GiB  291 GiB  287 GiB  176 MiB  3.3 GiB  221 GiB  56.79  1.33  107      up          osd.15 16    hdd   0.50000   1.00000  512 GiB  247 GiB  243 GiB  1.1 GiB  3.1 GiB  265 GiB  48.29  1.13   88      up          osd.16 18    hdd   0.50000   1.00000  512 GiB  220 GiB  217 GiB   65 MiB  2.7 GiB  292 GiB  42.88  1.01   83      up          osd.18 19    hdd   0.50000   1.00000  512 GiB  175 GiB  172 GiB   57 MiB  2.1 GiB  337 GiB  34.10  0.80   70      up          osd.19 20    hdd   0.50000   1.00000  512 GiB  265 GiB  262 GiB  629 MiB  2.8 GiB  247 GiB  51.75  1.22   97      up          osd.20 21    hdd   0.50000   1.00000  512 GiB  235 GiB  232 GiB  957 MiB  2.7 GiB  277 GiB  45.96  1.08   91      up          osd.21 22    hdd   0.50000   1.00000  512 GiB  182 GiB  178 GiB  1.6 GiB  2.4 GiB  330 GiB  35.61  0.84   67      up          osd.22 23    hdd   0.50000   1.00000  512 GiB  213 GiB  210 GiB   20 MiB  2.7 GiB  299 GiB  41.54  0.98   80      up          osd.23 -5          7.00000         -    7 TiB  2.9 TiB  2.8 TiB  8.0 GiB   36 GiB  4.1 TiB  41.07  0.96    -              host cephosd02  2    hdd   0.50000   1.00000  512 GiB  230 GiB  226 GiB  1.1 GiB  2.7 GiB  282 GiB  44.83  1.05   80      up          osd.2  4    hdd   0.50000   1.00000  512 GiB  256 GiB  252 GiB  686 MiB  2.9 GiB  256 GiB  49.97  1.17  102      up          osd.4  5    hdd   0.50000   1.00000  512 GiB  220 GiB  216 GiB  573 MiB  2.7 GiB  292 GiB  42.89  1.01   73      up          osd.5  6    hdd   0.50000   1.00000  512 GiB  239 GiB  236 GiB  118 MiB  3.0 GiB  273 GiB  46.71  1.10   94      up          osd.6  7    hdd   0.50000   1.00000  512 GiB  173 GiB  171 GiB   56 MiB  1.8 GiB  339 GiB  33.83  0.79   65      up          osd.7  8    hdd   0.50000   1.00000  512 GiB  226 GiB  222 GiB  927 MiB  2.7 GiB  286 GiB  44.07  1.04   88      up          osd.8  9    hdd   0.50000   1.00000  512 GiB  201 GiB  198 GiB  645 MiB  2.7 GiB  311 GiB  39.27  0.92   76      up          osd.9 10    hdd   0.50000   1.00000  512 GiB  180 GiB  178 GiB  116 MiB  2.3 GiB  332 GiB  35.24  0.83   68      up          osd.10 11    hdd   0.50000   1.00000  512 GiB  169 GiB  167 GiB  321 MiB  2.3 GiB  343 GiB  33.09  0.78   66      up          osd.11 12    hdd   0.50000   1.00000  512 GiB  236 GiB  232 GiB  1.3 GiB  2.7 GiB  276 GiB  46.14  1.08   91      up          osd.12 17    hdd   0.50000   1.00000  512 GiB  177 GiB  174 GiB   56 MiB  2.4 GiB  335 GiB  34.49  0.81   74      up          osd.17 24    hdd   0.50000   1.00000  512 GiB  221 GiB  217 GiB  1.4 GiB  2.3 GiB  291 GiB  43.21  1.02   85      up          osd.24 25    hdd   0.50000   1.00000  512 GiB  208 GiB  204 GiB  834 MiB  2.8 GiB  304 GiB  40.54  0.95   78      up          osd.25 26    hdd   0.50000   1.00000  512 GiB  208 GiB  205 GiB   58 MiB  2.9 GiB  304 GiB  40.69  0.96   78      up          osd.26                         TOTAL   14 TiB  5.7 TiB  5.7 TiB   16 GiB   71 GiB  7.8 TiB  42.56 MIN/MAX VAR: 0.78/1.33  STDDEV: 6.18 [rook@rook-ceph-tools-5c55f45cb8-kzpl8 ~]$ ceph osd pool ls detail pool 1 '.mgr' replicated size 2 min_size 1 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 115786 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr pool 2 'cephfs-metadata' replicated size 2 min_size 1 crush_rule 1 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode on last_change 119072 lfor 0/0/44 flags hashpspool stripe_width 0 compression_mode none pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs pool 3 'cephfs-replicated' replicated size 2 min_size 1 crush_rule 2 object_hash rjenkins pg_num 512 pgp_num 512 pg_num_target 128 pgp_num_target 128 autoscale_mode on last_change 119074 lfor 0/70140/80822 flags hashpspool stripe_width 0 compression_mode none application cephfs pool 6 'rbdpool' replicated size 2 min_size 1 crush_rule 4 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 119069 lfor 0/0/3824 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd pool 14 'ceph-store.rgw.buckets.index' replicated size 2 min_size 1 crush_rule 7 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode on last_change 119071 flags hashpspool stripe_width 0 compression_mode none pg_num_min 8 application rook-ceph-rgw pool 15 'ceph-store.rgw.meta' replicated size 2 min_size 1 crush_rule 5 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode on last_change 119071 flags hashpspool stripe_width 0 compression_mode none pg_num_min 8 application rook-ceph-rgw pool 16 'ceph-store.rgw.control' replicated size 2 min_size 1 crush_rule 9 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode on last_change 119071 flags hashpspool stripe_width 0 compression_mode none pg_num_min 8 application rook-ceph-rgw pool 17 'ceph-store.rgw.buckets.non-ec' replicated size 2 min_size 1 crush_rule 8 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode on last_change 119071 flags hashpspool stripe_width 0 compression_mode none pg_num_min 8 application rook-ceph-rgw pool 18 '.rgw.root' replicated size 2 min_size 1 crush_rule 6 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode on last_change 119071 flags hashpspool stripe_width 0 compression_mode none pg_num_min 8 application rook-ceph-rgw pool 19 'ceph-store.rgw.log' replicated size 2 min_size 1 crush_rule 10 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode on last_change 119071 flags hashpspool stripe_width 0 compression_mode none pg_num_min 8 application rook-ceph-rgw pool 20 'ceph-store.rgw.buckets.data' replicated size 2 min_size 1 crush_rule 11 object_hash rjenkins pg_num 512 pgp_num 512 autoscale_mode on last_change 119073 lfor 0/63287/80822 flags hashpspool stripe_width 0 compression_mode none application rook-ceph-rgw [rook@rook-ceph-tools-5c55f45cb8-kzpl8 ~]$ ceph -s   cluster:     id:     asdf9334-e04c-4b2d-9a56-134fc66fasdf     health: HEALTH_WARN             1 filesystem is degraded             Reduced data availability: 70 pgs inactive, 70 pgs incomplete             Degraded data redundancy: 23389/26522400 objects degraded (0.088%), 3 pgs degraded, 3 pgs undersized             1 slow ops, oldest one blocked for 9369 sec, osd.17 has slow ops   services:     mon: 3 daemons, quorum a,b,d (age 2h)     mgr: a(active, since 4h), standbys: b     mds: 1/1 daemons up, 1 standby     osd: 27 osds: 27 up (since 2h), 27 in (since 2h); 5 remapped pgs     rgw: 1 daemon active (1 hosts, 1 zones)   data:     volumes: 0/1 healthy, 1 recovering     pools:   11 pools, 1121 pgs     objects: 13.26M objects, 2.8 TiB     usage:   5.7 TiB used, 7.8 TiB / 14 TiB avail     pgs:     6.244% pgs not active              23389/26522400 objects degraded (0.088%)              30172/26522400 objects misplaced (0.114%)              1045 active+clean              70   incomplete              3    active+undersized+degraded+remapped+backfilling              2    active+remapped+backfilling              1    active+clean+scrubbing+deep   io:     recovery: 2.4 MiB/s, 13 objects/s [rook@rook-ceph-tools-5c55f45cb8-kzpl8 ~]$ ceph pg ls incomplete PG      OBJECTS  DEGRADED  MISPLACED  UNFOUND  BYTES       OMAP_BYTES*  OMAP_KEYS*  LOG   STATE       SINCE  VERSION           REPORTED     UP          ACTING      SCRUB_STAMP                      DEEP_SCRUB_STAMP                 LAST_SCRUB_DURATION  SCRUB_SCHEDULING 2.0           0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:471  [17,23]p17  [17,23]p17  2024-06-15T23:48:05.932624+0000  2024-06-15T23:48:05.932624+0000                    0  queued for scrub 3.1           0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2707  [13,17]p13  [13,17]p13  2024-06-16T23:39:24.417646+0000  2024-06-15T16:59:41.457407+0000                    0  periodic scrub scheduled @ 2024-06-18T04:50:01.780496+0000 3.24          0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:598  [14,11]p14  [14,11]p14  2024-06-16T18:22:30.298436+0000  2024-06-16T18:22:30.298436+0000                    0  periodic scrub scheduled @ 2024-06-18T03:41:12.619561+0000 3.39          0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2842    [7,22]p7    [7,22]p7  2024-06-12T09:53:58.885238+0000  2024-06-10T07:10:02.324876+0000                    0  queued for deep scrub 3.3e          0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2867   [23,8]p23   [23,8]p23  2024-06-16T21:00:21.815343+0000  2024-06-11T16:52:34.115626+0000                    0  periodic scrub scheduled @ 2024-06-18T03:16:15.380066+0000 3.55          0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2877     [3,7]p3     [3,7]p3  2024-06-16T17:46:14.088532+0000  2024-06-11T18:55:27.747854+0000                    0  periodic scrub scheduled @ 2024-06-17T18:09:31.324587+0000 3.79         44         0          0        0    35778866            0           0  1428  incomplete     2h  114670'225050853  119549:4029    [8,14]p8    [8,14]p8  2024-06-16T23:37:19.133531+0000  2024-06-16T23:37:19.133531+0000                    0  periodic scrub scheduled @ 2024-06-18T11:27:54.161459+0000 3.86          0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:595   [14,6]p14   [14,6]p14  2024-06-16T17:27:05.983294+0000  2024-06-12T20:40:12.980177+0000                    0  periodic scrub scheduled @ 2024-06-18T04:33:05.073855+0000 3.b2         24         0          0        0    25436605            0           0  2447  incomplete     2h  114758'224343196  119549:3203  [12,21]p12  [12,21]p12  2024-06-16T21:10:26.470142+0000  2024-06-15T16:46:09.233527+0000                    0  periodic scrub scheduled @ 2024-06-18T01:25:54.474637+0000 3.be          0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2808   [20,5]p20   [20,5]p20  2024-06-16T19:49:12.451351+0000  2024-06-12T11:12:44.716544+0000                    0  periodic scrub scheduled @ 2024-06-17T23:39:49.148445+0000 3.c7          0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2873  [15,17]p15  [15,17]p15  2024-06-16T18:43:12.732859+0000  2024-06-16T18:43:12.732859+0000                    0  periodic scrub scheduled @ 2024-06-18T05:10:54.788724+0000 3.f8        732         0          0        0   669465066            0           0  2032  incomplete     2h  114758'223302998  119549:4038    [6,22]p6    [6,22]p6  2024-06-16T23:56:44.923218+0000  2024-06-15T16:44:58.217628+0000                    0  periodic scrub scheduled @ 2024-06-18T03:34:35.536597+0000 3.108         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2695  [13,10]p13  [13,10]p13  2024-06-16T23:37:31.238636+0000  2024-06-15T17:11:43.866008+0000                    0  periodic scrub scheduled @ 2024-06-18T01:39:06.036207+0000 3.11b        24         0          0        0    20593314            0           0  2227  incomplete     2h  114758'220614552  119549:3797    [8,13]p8    [8,13]p8  2024-06-16T17:59:09.982621+0000  2024-06-15T16:47:29.166887+0000                    0  periodic scrub scheduled @ 2024-06-17T19:22:27.977582+0000 3.11d         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2872   [15,6]p15   [15,6]p15  2024-06-16T23:37:02.950901+0000  2024-06-16T23:37:02.950901+0000                    0  periodic scrub scheduled @ 2024-06-18T08:58:30.145323+0000 3.125         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2920     [1,9]p1     [1,9]p1  2024-06-16T18:12:06.604249+0000  2024-06-15T17:18:58.503926+0000                    0  periodic scrub scheduled @ 2024-06-17T23:03:00.713407+0000 3.139        24         0          0        0    23395686            0           0  3066  incomplete     2h  111688'223354803  119549:3359   [11,0]p11   [11,0]p11  2024-06-16T20:11:56.894641+0000  2024-06-12T13:37:05.221928+0000                    0  periodic scrub scheduled @ 2024-06-18T07:57:26.910262+0000 3.146         0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:469   [24,3]p24   [24,3]p24  2024-06-16T21:44:45.164670+0000  2024-06-12T17:42:51.040928+0000                    0  periodic scrub scheduled @ 2024-06-18T04:56:01.606021+0000 3.14c        36         0          0        0    33925449            0           0  2612  incomplete     2h  116105'224249735  119549:3700  [10,13]p10  [10,13]p10  2024-06-16T23:57:51.454127+0000  2024-06-16T23:57:51.454127+0000                    0  periodic scrub scheduled @ 2024-06-18T07:18:49.988378+0000 3.193         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2637    [0,24]p0    [0,24]p0  2024-06-16T23:56:31.806282+0000  2024-06-15T17:17:32.182612+0000                    0  periodic scrub scheduled @ 2024-06-18T10:47:44.692999+0000 3.195         0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:471   [17,0]p17   [17,0]p17  2024-06-16T21:46:38.179629+0000  2024-06-11T13:16:32.236167+0000                    0  periodic scrub scheduled @ 2024-06-18T04:09:57.287460+0000 3.1ad         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2891   [19,5]p19   [19,5]p19  2024-06-16T21:22:57.390864+0000  2024-06-16T21:22:57.390864+0000                    0  periodic scrub scheduled @ 2024-06-18T08:21:19.332244+0000 3.1e1         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2859   [23,8]p23   [23,8]p23  2024-06-16T18:44:25.140463+0000  2024-06-11T19:38:27.447780+0000                    0  periodic scrub scheduled @ 2024-06-18T01:29:07.185995+0000 3.1ee        24         0          0        0    19306070            0           0  3216  incomplete     2h  111661'223425659  119549:4221    [7,16]p7    [7,16]p7  2024-06-16T19:59:18.223402+0000  2024-06-11T03:14:16.569649+0000                    0  periodic scrub scheduled @ 2024-06-18T02:10:56.126225+0000 3.1f4         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2701  [13,17]p13  [13,17]p13  2024-06-15T17:08:03.277404+0000  2024-06-15T17:08:03.277404+0000                    0  queued for scrub 20.7       9512         0          0        0  1688792716            0           0  2368  incomplete     2h    115974'3084182  119549:3903    [2,19]p2    [2,19]p2  2024-06-16T19:08:50.553680+0000  2024-06-16T19:08:50.553680+0000                    0  periodic scrub scheduled @ 2024-06-18T06:36:07.460002+0000 20.13       360         0          0        0    73738676            0           0  2165  incomplete     2h    116105'3181872  119549:4085     [6,0]p6     [6,0]p6  2024-06-16T23:56:19.979015+0000  2024-06-16T23:56:19.979015+0000                    0  periodic scrub scheduled @ 2024-06-18T11:48:06.834496+0000 20.19         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2892  [18,24]p18  [18,24]p18  2024-06-17T02:39:42.749548+0000  2024-06-11T17:59:43.432987+0000                    0  periodic scrub scheduled @ 2024-06-18T09:35:34.839103+0000 20.26        24         0          0        0     5539541            0           0  3051  incomplete     2h    114583'3195836  119549:3308  [12,19]p12  [12,19]p12  2024-06-16T23:54:48.493125+0000  2024-06-15T14:19:41.916324+0000                    0  periodic scrub scheduled @ 2024-06-18T01:09:46.433224+0000 20.2b      3256         0          0        0   607303207            0           0  3547  incomplete     2h    114581'2462360  119549:4270    [5,15]p5    [5,15]p5  2024-06-16T18:38:47.430188+0000  2024-06-15T14:18:58.050325+0000                    0  periodic scrub scheduled @ 2024-06-18T01:28:25.927361+0000 20.2d        20         0          0        0     1263047            0           0  2227  incomplete     2h    114605'2588714  119549:4085    [8,23]p8    [8,23]p8  2024-06-16T23:54:23.393056+0000  2024-06-15T14:35:32.588709+0000                    0  periodic scrub scheduled @ 2024-06-18T02:46:26.512465+0000 20.2f         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2881   [19,9]p19   [19,9]p19  2024-06-16T21:24:09.919836+0000  2024-06-15T16:42:42.055242+0000                    0  periodic scrub scheduled @ 2024-06-18T04:44:57.515744+0000 20.3a         0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:595   [14,6]p14   [14,6]p14  2024-06-16T23:39:23.598808+0000  2024-06-16T23:39:23.598808+0000                    0  periodic scrub scheduled @ 2024-06-18T06:57:34.989447+0000 20.4f       368         0          0        0    83117388            0           0  2012  incomplete     2h    114694'2564891  119549:3913    [6,19]p6    [6,19]p6  2024-06-16T18:31:06.714144+0000  2024-06-12T09:01:16.964537+0000                    0  periodic scrub scheduled @ 2024-06-17T22:50:04.609254+0000 20.5e        24         0          0        0     5877539            0           0  2942  incomplete     2h    114758'3167133  119549:3374  [11,13]p11  [11,13]p11  2024-06-16T21:49:21.000257+0000  2024-06-15T16:45:52.739151+0000                    0  periodic scrub scheduled @ 2024-06-18T03:14:55.646492+0000 20.6f         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2695   [13,4]p13   [13,4]p13  2024-06-16T16:58:57.092273+0000  2024-06-10T06:53:27.121528+0000                    0  periodic deep scrub scheduled @ 2024-06-17T21:49:59.318204+0000 20.77         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2880   [18,2]p18   [18,2]p18  2024-06-16T17:27:56.749699+0000  2024-06-15T17:16:31.652171+0000                    0  periodic scrub scheduled @ 2024-06-17T23:40:36.649994+0000 20.79         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2860   [16,4]p16   [16,4]p16  2024-06-16T15:51:10.501388+0000  2024-06-15T15:08:47.984644+0000                    0  periodic scrub scheduled @ 2024-06-17T19:24:57.749803+0000 20.91         0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:600  [14,24]p14  [14,24]p14  2024-06-16T23:57:20.363629+0000  2024-06-12T18:04:36.913692+0000                    0  periodic scrub scheduled @ 2024-06-18T08:25:44.689817+0000 20.9e        24         0          0        0     6859327            0           0  3470  incomplete     2h    111598'3475260  119549:3342  [11,18]p11  [11,18]p11  2024-06-15T17:13:49.917370+0000  2024-06-09T02:25:18.007276+0000                    0  queued for deep scrub 20.b1        36         0          0        0    11052770            0           0  3215  incomplete     2h    115974'2684907  119549:4011    [7,18]p7    [7,18]p7  2024-06-16T18:33:38.646452+0000  2024-06-16T18:33:38.646452+0000                    0  periodic scrub scheduled @ 2024-06-18T02:59:13.231417+0000 20.c3         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2805  [20,12]p20  [20,12]p20  2024-06-16T21:51:48.159492+0000  2024-06-10T23:51:40.489888+0000                    0  periodic scrub scheduled @ 2024-06-18T07:08:26.021365+0000 20.c4        40         0          0        0     4861197            0           0  2459  incomplete     2h    115971'2585570  119549:3764    [8,22]p8    [8,22]p8  2024-06-16T17:42:09.247301+0000  2024-06-16T17:42:09.247301+0000                    0  periodic scrub scheduled @ 2024-06-17T22:34:03.531309+0000 20.c5       592         0          0        0    89115402            0           0  2029  incomplete     2h    114728'3601658  119549:4069    [6,14]p6    [6,14]p6  2024-06-15T16:34:01.112524+0000  2024-06-11T19:13:07.311754+0000                    0  queued for scrub 20.c7        24         0          0        0    12044375            0           0  2701  incomplete     2h    111596'3465587  119549:3627    [9,19]p9    [9,19]p9  2024-06-16T19:19:44.103540+0000  2024-06-10T13:14:23.142995+0000                    0  periodic deep scrub scheduled @ 2024-06-17T21:13:24.790888+0000 20.d1         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2917   [22,2]p22   [22,2]p22  2024-06-16T17:58:58.099198+0000  2024-06-15T16:55:02.678844+0000                    0  periodic scrub scheduled @ 2024-06-17T20:13:05.807250+0000 20.e5         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2691   [13,6]p13   [13,6]p13  2024-06-16T21:06:28.142540+0000  2024-06-11T02:21:23.502208+0000                    0  periodic scrub scheduled @ 2024-06-18T01:54:05.307906+0000 20.e6         0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:469  [24,20]p24  [24,20]p24  2024-06-16T21:02:54.534391+0000  2024-06-15T17:13:52.182217+0000                    0  periodic scrub scheduled @ 2024-06-18T08:26:28.967830+0000 20.f0        24         0          0        0     9285607            0           0  2214  incomplete     2h    114677'2190115  119549:3489  [10,22]p10  [10,22]p10  2024-06-16T22:15:50.011371+0000  2024-06-15T16:04:29.594234+0000                    0  periodic scrub scheduled @ 2024-06-18T06:52:49.523501+0000 20.f1         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2860   [16,9]p16   [16,9]p16  2024-06-16T17:57:55.241940+0000  2024-06-10T18:27:14.019838+0000                    0  periodic scrub scheduled @ 2024-06-17T21:05:56.570556+0000 20.f9         0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2695   [13,4]p13   [13,4]p13  2024-06-16T15:57:29.726923+0000  2024-06-10T02:08:02.584253+0000                    0  periodic deep scrub scheduled @ 2024-06-17T17:22:06.428216+0000 20.103       24         0          0        0     2855787            0           0  3115  incomplete     2h    114677'3141513  119549:3667  [10,23]p10  [10,23]p10  2024-06-16T23:39:48.008120+0000  2024-06-15T16:07:43.067676+0000                    0  periodic scrub scheduled @ 2024-06-18T11:31:34.863601+0000 20.118       24         0          0        0     5130995            0           0  2523  incomplete     2h    114758'3435860  119549:3661  [10,16]p10  [10,16]p10  2024-06-16T23:58:46.678486+0000  2024-06-15T16:58:03.535460+0000                    0  periodic scrub scheduled @ 2024-06-18T10:48:37.181682+0000 20.122        0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2888  [18,12]p18  [18,12]p18  2024-06-16T22:28:42.397537+0000  2024-06-11T23:45:02.000866+0000                    0  periodic scrub scheduled @ 2024-06-18T01:16:31.425324+0000 20.131        0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2805  [20,10]p20  [20,10]p20  2024-06-16T19:44:42.443673+0000  2024-06-11T19:48:02.225886+0000                    0  periodic scrub scheduled @ 2024-06-18T01:43:36.198377+0000 20.135       24         0          0        0     3409951            0           0  2702  incomplete     2h    111648'2627621  119549:4066     [8,0]p8     [8,0]p8  2024-06-16T23:56:42.996730+0000  2024-06-12T19:15:51.153250+0000                    0  periodic scrub scheduled @ 2024-06-18T01:50:15.706512+0000 20.153        0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2794   [20,9]p20   [20,9]p20  2024-06-16T23:54:07.088581+0000  2024-06-12T11:36:55.715158+0000                    0  periodic scrub scheduled @ 2024-06-18T09:30:44.451613+0000 20.15c        0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:471   [17,0]p17   [17,0]p17  2024-06-15T16:04:44.382656+0000  2024-06-10T02:53:33.553050+0000                    0  queued for deep scrub 20.170       28         0          0        0     2961436            0           0  2984  incomplete     2h    114758'2190806  119549:4159    [7,14]p7    [7,14]p7  2024-06-16T18:03:49.902925+0000  2024-06-15T16:37:41.831937+0000                    0  periodic scrub scheduled @ 2024-06-18T05:39:37.221506+0000 20.17a       24         0          0        0     6033231            0           0  2143  incomplete     2h    113716'3122701  119549:3198  [12,19]p12  [12,19]p12  2024-06-16T23:33:38.562814+0000  2024-06-11T14:39:24.438089+0000                    0  periodic scrub scheduled @ 2024-06-18T07:02:02.556125+0000 20.181        0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2856  [16,24]p16  [16,24]p16  2024-06-16T22:51:45.934424+0000  2024-06-15T15:06:13.179919+0000                    0  periodic scrub scheduled @ 2024-06-18T07:42:24.823481+0000 20.18d        0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:595   [14,6]p14   [14,6]p14  2024-06-16T19:38:47.985314+0000  2024-06-15T17:02:56.794207+0000                    0  periodic scrub scheduled @ 2024-06-18T02:54:31.549356+0000 20.18e        0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2873   [23,4]p23   [23,4]p23  2024-06-16T23:36:43.177607+0000  2024-06-16T23:36:43.177607+0000                    0  periodic scrub scheduled @ 2024-06-18T07:23:38.830409+0000 20.191       44         0          0        0     5862907            0           0  1888  incomplete     2h    112277'3316940  119549:3762     [8,1]p8     [8,1]p8  2024-06-16T23:56:30.274132+0000  2024-06-11T13:38:36.484704+0000                    0  periodic scrub scheduled @ 2024-06-18T10:34:49.442736+0000 20.1b1        0         0          0        0           0            0           0     0  incomplete     2h               0'0   119549:471   [17,0]p17   [17,0]p17  2024-06-16T21:06:55.996335+0000  2024-06-15T16:10:42.337949+0000                    0  periodic scrub scheduled @ 2024-06-17T21:36:47.404885+0000 20.1b3        0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2892   [19,7]p19   [19,7]p19  2024-06-16T21:17:11.689950+0000  2024-06-12T15:31:29.057101+0000                    0  periodic scrub scheduled @ 2024-06-18T02:10:48.084226+0000 20.1ce     1928         0          0        0   318383333            0           0  1927  incomplete     2h    111648'2311876  119549:3949    [5,14]p5    [5,14]p5  2024-06-15T16:01:37.133233+0000  2024-06-12T11:14:16.383536+0000                    0  queued for scrub 20.1d3        0         0          0        0           0            0           0     0  incomplete     2h               0'0  119549:2867  [23,12]p23  [23,12]p23  2024-06-17T01:29:18.466650+0000  2024-06-17T01:29:18.466650+0000                    0  periodic scrub scheduled @ 2024-06-18T10:45:13.027329+0000 20.1ec       36         0          0        0     4881293            0           0  2246  incomplete     2h    114758'2815274  119549:3756    [8,19]p8    [8,19]p8  2024-06-16T19:34:53.640250+0000  2024-06-15T16:33:40.664040+0000                    0  periodic scrub scheduled @ 2024-06-18T01:53:21.185330+0000 20.1f1       36         0          0        0    10313830            0           0  2375  incomplete     2h    111602'2994531  119549:3593  [11,16]p11  [11,16]p11  2024-06-16T19:16:31.521408+0000  2024-06-10T15:51:29.338840+0000                    0  periodic scrub scheduled @ 2024-06-17T22:52:42.098717+0000 * NOTE: Omap statistics are gathered during deep scrub and may be inaccurate soon afterwards depending on utilization. See http://docs.ceph.com/en/latest/dev/placement-group/#omap-statistics for further details. [rook@rook-ceph-tools-5c55f45cb8-kzpl8 ~]$ ceph pg 2.0 query { "snap_trimq": "[]", "snap_trimq_len": 0, "state": "incomplete", "epoch": 119554, "up": [ 17, 23 ], "acting": [ 17, 23 ], "info": { "pgid": "2.0", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 34, "epoch_pool_created": 34, "last_epoch_started": 117670, "last_interval_started": 117669, "last_epoch_clean": 116838, "last_interval_clean": 116837, "last_epoch_split": 44, "last_epoch_marked_full": 0, "same_up_since": 119081, "same_interval_since": 119081, "same_primary_since": 119081, "last_scrub": "114758'73829265", "last_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_deep_scrub": "114758'73829265", "last_deep_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_clean_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 476, "reported_epoch": 119554, "state": "incomplete", "last_fresh": "2024-06-17T15:15:58.659723+0000", "last_change": "2024-06-17T12:34:27.962503+0000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "2024-06-17T15:15:58.659723+0000", "last_undegraded": "2024-06-17T15:15:58.659723+0000", "last_fullsized": "2024-06-17T15:15:58.659723+0000", "mapping_epoch": 119081, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 34, "last_epoch_clean": 116838, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "114758'73829265", "last_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_deep_scrub": "114758'73829265", "last_deep_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_clean_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "objects_scrubbed": 0, "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "queued for scrub", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 17, 23 ], "acting": [ 17, 23 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": 17, "acting_primary": 17, "purged_snaps": [] }, "empty": 1, "dne": 0, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, "peer_info": [ { "peer": "2", "pgid": "2.0", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 0, "epoch_pool_created": 0, "last_epoch_started": 0, "last_interval_started": 0, "last_epoch_clean": 0, "last_interval_clean": 0, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 0, "same_interval_since": 0, "same_primary_since": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [], "acting": [], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1, "purged_snaps": [] }, "empty": 1, "dne": 1, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "8", "pgid": "2.0", "last_update": "116105'73881472", "last_complete": "116105'73881472", "log_tail": "115937'73877640", "last_user_version": 0, "last_backfill": "2:00053b81:::100867c0201.00000000:head", "purged_snaps": [], "history": { "epoch_created": 34, "epoch_pool_created": 34, "last_epoch_started": 117670, "last_interval_started": 117669, "last_epoch_clean": 116838, "last_interval_clean": 116837, "last_epoch_split": 44, "last_epoch_marked_full": 0, "same_up_since": 119081, "same_interval_since": 119081, "same_primary_since": 119081, "last_scrub": "114758'73829265", "last_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_deep_scrub": "114758'73829265", "last_deep_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_clean_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "116105'73881472", "reported_seq": 2773, "reported_epoch": 119077, "state": "incomplete", "last_fresh": "2024-06-17T12:34:19.229020+0000", "last_change": "2024-06-17T12:34:00.641115+0000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "2024-06-17T12:34:19.229020+0000", "last_undegraded": "2024-06-17T12:34:19.229020+0000", "last_fullsized": "2024-06-17T12:34:19.229020+0000", "mapping_epoch": 119081, "log_start": "115937'73877640", "ondisk_log_start": "115937'73877640", "created": 34, "last_epoch_clean": 116838, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "114758'73829265", "last_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_deep_scrub": "114758'73829265", "last_deep_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_clean_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "objects_scrubbed": 0, "log_size": 3832, "ondisk_log_size": 3832, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "queued for scrub", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 20, "num_object_clones": 0, "num_object_copies": 40, "num_objects_missing_on_primary": 0, "num_objects_missing": 23240, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 20, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 20, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 17, 23 ], "acting": [ 17, 23 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [ 17, 24 ], "up_primary": 17, "acting_primary": 17, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 117670, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "19", "pgid": "2.0", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 0, "epoch_pool_created": 0, "last_epoch_started": 0, "last_interval_started": 0, "last_epoch_clean": 0, "last_interval_clean": 0, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 0, "same_interval_since": 0, "same_primary_since": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [], "acting": [], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1, "purged_snaps": [] }, "empty": 1, "dne": 1, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "23", "pgid": "2.0", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 34, "epoch_pool_created": 34, "last_epoch_started": 117670, "last_interval_started": 117669, "last_epoch_clean": 116838, "last_interval_clean": 116837, "last_epoch_split": 44, "last_epoch_marked_full": 0, "same_up_since": 119081, "same_interval_since": 119081, "same_primary_since": 119081, "last_scrub": "114758'73829265", "last_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_deep_scrub": "114758'73829265", "last_deep_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_clean_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 4, "reported_epoch": 119080, "state": "down", "last_fresh": "2024-06-17T12:34:22.345391+0000", "last_change": "2024-06-17T12:34:20.242281+0000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "2024-06-17T12:34:22.345391+0000", "last_undegraded": "2024-06-17T12:34:22.345391+0000", "last_fullsized": "2024-06-17T12:34:22.345391+0000", "mapping_epoch": 119081, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 34, "last_epoch_clean": 116838, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "114758'73829265", "last_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_deep_scrub": "114758'73829265", "last_deep_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "last_clean_scrub_stamp": "2024-06-15T23:48:05.932624+0000", "objects_scrubbed": 0, "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "queued for scrub", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [ 17, 23 ], "acting": [ 17, 23 ], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [ 17, 24 ], "up_primary": 17, "acting_primary": 17, "purged_snaps": [] }, "empty": 1, "dne": 0, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } }, { "peer": "24", "pgid": "2.0", "last_update": "0'0", "last_complete": "0'0", "log_tail": "0'0", "last_user_version": 0, "last_backfill": "MAX", "purged_snaps": [], "history": { "epoch_created": 0, "epoch_pool_created": 0, "last_epoch_started": 0, "last_interval_started": 0, "last_epoch_clean": 0, "last_interval_clean": 0, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 0, "same_interval_since": 0, "same_primary_since": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "prior_readable_until_ub": 0 }, "stats": { "version": "0'0", "reported_seq": 0, "reported_epoch": 0, "state": "unknown", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 0, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "0.000000", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "0.000000", "last_clean_scrub_stamp": "0.000000", "objects_scrubbed": 0, "log_size": 0, "ondisk_log_size": 0, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "last_scrub_duration": 0, "scrub_schedule": "--", "scrub_duration": 0, "objects_trimmed": 0, "snaptrim_duration": 0, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0, "num_objects_repaired": 0 }, "up": [], "acting": [], "avail_no_missing": [], "object_location_counts": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1, "purged_snaps": [] }, "empty": 1, "dne": 1, "incomplete": 0, "last_epoch_started": 0, "hit_set_history": { "current_last_update": "0'0", "history": [] } } ], "recovery_state": [ { "name": "Started/Primary/Peering/Incomplete", "enter_time": "2024-06-17T12:34:27.962496+0000", "comment": "not enough complete instances of this PG" }, { "name": "Started/Primary/Peering", "enter_time": "2024-06-17T12:34:27.485271+0000", "past_intervals": [ { "first": "116837", "last": "119080", "all_participants": [ { "osd": 2 }, { "osd": 8 }, { "osd": 17 }, { "osd": 19 }, { "osd": 23 }, { "osd": 24 } ], "intervals": [ { "first": "117563", "last": "117564", "acting": "17" }, { "first": "117582", "last": "117585", "acting": "2" }, { "first": "117669", "last": "117678", "acting": "24" }, { "first": "118995", "last": "119011", "acting": "8" }, { "first": "119078", "last": "119080", "acting": "23" } ] } ], "probing_osds": [ "2", "8", "17", "19", "23", "24" ], "down_osds_we_would_probe": [], "peering_blocked_by": [], "peering_blocked_by_detail": [ { "detail": "peering_blocked_by_history_les_bound" } ] }, { "name": "Started", "enter_time": "2024-06-17T12:34:27.485245+0000" } ], "agent_state": {} }