模拟ceph集群ERROR状态

root 🌐 summer133-112 in  ~ 
 ♥ 10:37 > summer -s
    cluster 0be48747-efac-4ece-8cbe-9a5d06baccab
     health HEALTH_OK
     monmap e5: 3 mons at {summer133-112=192.168.133.112:6789/0,summer133-113=192.168.133.113:6789/0,summer133-114=192.168.133.114:6789/0}
            election epoch 1098, quorum 0,1,2 summer133-112,summer133-113,summer133-114
     mdsmap e30: 1/1/1 up {0=summer133-112=up:active}, 2 up:standby-replay
     osdmap e156: 6 osds: 6 up, 6 in
      pgmap v16473: 1296 pgs, 4 pools, 690 MB data, 8843 objects
            7953 MB used, 110 GB / 118 GB avail
                1296 active+clean

root 🌐 summer133-112 in  ~ 
 ♥ 10:38 > summer df
GLOBAL:
    SIZE     AVAIL     RAW USED     %RAW USED 
    118G      110G        7953M          6.56 
POOLS:
    NAME                                   ID     USED      %USED     MAX AVAIL     TOTAL     TOTAL AVAIL     OBJECTS 
    mTR-00000-magtools-dp                  0      56470         0        53557M      118G          60657M         189 
    mTR-220913204427-f2_x-LTXkOFl34483     2       689M      1.27        53557M      118G          60657M        8637 
    mTR-220913204512-r2_x-lg2qJNQJ1197     3      28672         0        53557M      118G          60657M          16 
    mTR-220913204555-r2_1-lMtsKMnN8141     4      1924k         0        71410M      118G          80876M           1 

root 🌐 summer133-112 in  ~ 
 ♥ 10:38 > dd if=/dev/zero of=summer  bs=4M count=10
10+0 records in
10+0 records out
41943040 bytes (42 MB) copied, 0.0430584 s, 974 MB/s

root 🌐 summer133-112 in  ~ 
 ♥ 10:38 > rados -p  mTR-220913204512-r2_x-lg2qJNQJ1197  put summer summer 

root 🌐 summer133-112 in  ~ 
 ♥ 10:38 > summer osd map  mTR-220913204512-r2_x-lg2qJNQJ1197 summer
osdmap e156 pool 'mTR-220913204512-r2_x-lg2qJNQJ1197' (3) object 'summer' -> pg 3.90cae7f1 (3.f1) -> up ([2,3], p2) acting ([2,3], p2)

root 🌐 summer133-112 in  ~ 
 ♥ 10:38 > lsblk
NAME                      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
loop0                       7:0    0  100G  0 loop 
└─docker-8:2-1058534-pool 252:0    0  100G  0 dm   
loop1                       7:1    0    2G  0 loop 
└─docker-8:2-1058534-pool 252:0    0  100G  0 dm   
sda                         8:0    0  100G  0 disk 
├─sda1                      8:1    0    1G  0 part /boot
└─sda2                      8:2    0   99G  0 part /
sdb                         8:16   0   30G  0 disk 
├─sdb1                      8:17   0 10.2G  0 part 
└─sdb2                      8:18   0 19.8G  0 part /var/lib/summer/osd/summer-4
sdc                         8:32   0   30G  0 disk 
├─sdc1                      8:33   0 10.2G  0 part 
└─sdc2                      8:34   0 19.8G  0 part /var/lib/summer/osd/summer-2
sr0                        11:0    1  4.1G  0 rom  

root 🌐 summer133-112 in  ~ 
 ♥ 10:39 > cd /var/lib/summer/osd/summer-2/current/3.f1_head/

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:39 > ll
total 4100
-rw-r--r-- 1 root root       0 Sep 13 20:45 __head_000000F1__3
-rw-r--r-- 1 root root 1970176 Sep 14 10:38 summer__head_90CAE7F1__3

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:39 > rm -rf summer__head_90CAE7F1__3 

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:39 > /etc/init.d/summer restart osd.2
=== osd.2 === 
=== osd.2 === 
Stopping summer osd.2 on summer133-112...kill 17858...kill 17858...done
=== osd.2 === 
Mounting xfs on summer133-112:/var/lib/summer/osd/summer-2
create-or-move updated item name 'osd.2' weight 0.02 at location {host=summer133-112,root=default} to crush map
Starting summer osd.2 on summer133-112...

root 🌐 summer133-113 in  ~ 
 ♥ 10:39 > cd /var/lib/summer/osd/summer-3/current/3.1
3.102_head/ 3.104_head/ 3.106_head/ 3.108_head/ 3.10d_head/ 3.10_head/  3.11_head/  3.15_head/  3.18_head/  3.1b_head/  3.1e_head/  

root 🌐 summer133-113 in  ~ 
 ♥ 10:39 > cd /var/lib/summer/osd/summer-3/current/3.f1_head/

root 🌐 summer133-113 in  …/summer-3/current/3.f1_head 
 ♥ 10:40 > ll
total 4100
-rw-r--r-- 1 root root       0 Sep 13 20:45 __head_000000F1__3
-rw-r--r-- 1 root root 1970176 Sep 14 10:38 summer__head_90CAE7F1__3

root 🌐 summer133-113 in  …/summer-3/current/3.f1_head 
 ♥ 10:40 > rm  -rf summer__head_90CAE7F1__3 

root 🌐 summer133-113 in  …/summer-3/current/3.f1_head 
 ♥ 10:40 > /etc/init.d/summer restart osd.3
=== osd.3 === 
=== osd.3 === 
Stopping summer osd.3 on summer133-113...kill 19484...kill 19484...done
=== osd.3 === 
Mounting xfs on summer133-113:/var/lib/summer/osd/summer-3
create-or-move updated item name 'osd.3' weight 0.02 at location {host=summer133-113,root=default} to crush map
Starting summer osd.3 on summer133-113...

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head took 5s 
 ♥ 10:40 > summer pg scrub 3.f1
instructing pg 3.f1 on osd.2 to scrub

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:41 > summer pg deep-scrub 3.f1
instructing pg 3.f1 on osd.2 to deep-scrub

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:41 > summer -s
    cluster 0be48747-efac-4ece-8cbe-9a5d06baccab
     health HEALTH_ERR
            1 pgs inconsistent
            1 scrub errors
     monmap e5: 3 mons at {summer133-112=192.168.133.112:6789/0,summer133-113=192.168.133.113:6789/0,summer133-114=192.168.133.114:6789/0}
            election epoch 1098, quorum 0,1,2 summer133-112,summer133-113,summer133-114
     mdsmap e30: 1/1/1 up {0=summer133-112=up:active}, 2 up:standby-replay
     osdmap e164: 6 osds: 6 up, 6 in
      pgmap v16511: 1296 pgs, 4 pools, 692 MB data, 8844 objects
            8291 MB used, 110 GB / 118 GB avail
                1295 active+clean
                   1 active+clean+inconsistent
  client io 2039 B/s rd, total ops: 1, write ops: 0, read ops: 1

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:41 > 

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:48 > summer pg dump | grep inconsistent
dumped all in format plain
3.f1	1	0	0	0	0	1970176	10	10	active+clean+inconsistent	2022-09-14 10:41:25.812995	156'10	164:44	[2,3]	2	[2,3]	2	156'10	2022-09-14 10:41:25.812938	156'10	2022-09-14 10:41:25.812938

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:48 > summer pg repair 3.f1
instructing pg 3.f1 on osd.2 to repair

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:48 > summer -s
    cluster 0be48747-efac-4ece-8cbe-9a5d06baccab
     health HEALTH_OK
     monmap e5: 3 mons at {summer133-112=192.168.133.112:6789/0,summer133-113=192.168.133.113:6789/0,summer133-114=192.168.133.114:6789/0}
            election epoch 1098, quorum 0,1,2 summer133-112,summer133-113,summer133-114
     mdsmap e30: 1/1/1 up {0=summer133-112=up:active}, 2 up:standby-replay
     osdmap e164: 6 osds: 6 up, 6 in
      pgmap v16602: 1296 pgs, 4 pools, 690 MB data, 8843 objects
            7931 MB used, 110 GB / 118 GB avail
                1296 active+clean
  client io 1705 B/s rd, total ops: 1, write ops: 0, read ops: 1

root 🌐 summer133-112 in  …/summer-2/current/3.f1_head 
 ♥ 10:48 >