# 系统故障排查的主要内容
# 系统启动故障排查：排查系统启动问题
# 网络故障排查：排查网络连接问题
# 磁盘故障排查：排查磁盘I/O问题
# 进程故障排查：排查进程异常问题
# 内存故障排查：排查内存不足问题
# CPU故障排查：排查CPU负载高问题

# 系统启动故障排查

# 1. 查看启动日志
[root@localhost ~]# journalctl -b
Apr 03 09:00:00 localhost systemd[1]: Starting system…
Apr 03 09:00:00 localhost kernel: Linux version 5.14.0-362.el9.x86_64 (mockbuild@x86-05.bsys.centos.org) (gcc version 11.2.1 20220127 (Red Hat 11.2.1-9), GNU ld version 2.35-9.el9) #1 SMP PREEMPT Wed Jan 11 18:35:18 UTC 2023
Apr 03 09:00:00 localhost kernel: Command line: BOOT_IMAGE=/vmlinuz-5.14.0-362.el9.x86_64 root=/dev/mapper/rootvg-rootlv ro crashkernel=auto resume=/dev/mapper/rootvg-swaplv rd.lvm.lv=rootvg/rootlv rd.lvm.lv=rootvg/swaplv rhgb quiet
Apr 03 09:00:00 localhost kernel: x86/fpu: Supporting XSAVE feature 0x001: ‘x87 floating point registers’

# 2. 查看启动服务状态
[root@localhost ~]# systemctl list-units –failed
UNIT LOAD ACTIVE SUB DESCRIPTION
● mysqld.service loaded failed active MySQL Database Server

LOAD = Reflects whether the unit definition was properly loaded.
ACTIVE = The high-level unit activation state, i.e. generalization of SUB.
SUB = The low-level unit activation state, values depend on unit type.

1 loaded units listed.

# 3. 查看失败服务详情
[root@localhost ~]# systemctl status mysqld.service
● mysqld.service – MySQL Database Server
Loaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; preset: disabled)
Active: failed (Result: exit-code) since Fri 2026-04-03 10:00:00 CST; 5s ago
Process: 1234 ExecStartPre=/usr/libexec/mysql-check-socket (code=exited, status=0/SUCCESS)
Process: 1235 ExecStartPre=/usr/libexec/mysql-check-upgrade (code=exited, status=0/SUCCESS)
Process: 1236 ExecStart=/usr/libexec/mysqld –daemonize –pid-file=/var/run/mysqld/mysqld.pid $MYSQLD_OPTS (code=exited, status=1/FAILURE)
Main PID: 1236 (code=exited, status=1/FAILURE)

Apr 03 10:00:00 localhost systemd[1]: Starting MySQL Database Server…
Apr 03 10:00:00 localhost mysqld[1236]: 2026-04-03T10:00:00.000Z 0 [ERROR] [MY-000000] [Server] Can’t start server: Bind on TCP/IP port: Address already in use
Apr 03 10:00:00 localhost mysqld[1236]: 2026-04-03T10:00:00.000Z 0 [ERROR] [MY-010119] [Server] Aborting
Apr 03 10:00:00 localhost systemd[1]: mysqld.service: Main process exited, code=exited, status=1/FAILURE
Apr 03 10:00:00 localhost systemd[1]: mysqld.service: Failed with result ‘exit-code’.
Apr 03 10:00:00 localhost systemd[1]: Failed to start MySQL Database Server.

# 4. 查看系统资源
[root@localhost ~]# free -h
total used free shared buff/cache available
Mem: 7.6Gi 2.0Gi 4.0Gi 100Mi 1.6Gi 5.3Gi
Swap: 4.0Gi 0B 4.0Gi

# 5. 查看磁盘空间
[root@localhost ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 4.0G 0 4.0G 0% /dev
tmpfs 4.0G 0 4.0G 0% /dev/shm
tmpfs 4.0G 100M 3.9G 3% /run
/dev/mapper/rootvg-rootlv 100G 50G 50G 50% /
/dev/sda1 500M 100M 400M 20% /boot
/dev/mapper/rootvg-datalv 500G 400G 100G 80% /data

# 6. 查看启动时间
[root@localhost ~]# systemd-analyze
Startup finished in 2.123s (kernel) + 5.456s (userspace) = 7.579s
graphical.target reached after 5.456s in userspace

# 7. 查看启动服务耗时
[root@localhost ~]# systemd-analyze blame
5.456s mysqld.service
2.123s network.service
1.234s sshd.service
0.567s crond.service
0.123s systemd-logind.service

# 网络故障排查

# 1. 检查网络接口状态
[root@localhost ~]# ip addr show
1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:11:22:33:44:55 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.100/24 brd 192.168.1.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet6 fe80::211:22ff:fe33:4455/64 scope link
valid_lft forever preferred_lft forever

# 2. 检查网络连接
[root@localhost ~]# ping -c 4 192.168.1.1
PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.
64 bytes from 192.168.1.1: icmp_seq=1 ttl=64 time=0.123 ms
64 bytes from 192.168.1.1: icmp_seq=2 ttl=64 time=0.123 ms
64 bytes from 192.168.1.1: icmp_seq=3 ttl=64 time=0.123 ms
64 bytes from 192.168.1.1: icmp_seq=4 ttl=64 time=0.123 ms

— 192.168.1.1 ping statistics —
4 packets transmitted, 4 received, 0% packet loss
rtt min/avg/max/mdev = 0.123/0.123/0.123/0.000 ms

# 3. 检查路由表
[root@localhost ~]# ip route show
default via 192.168.1.1 dev eth0 proto static metric 100
192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.100 metric 100

# 4. 检查DNS解析
[root@localhost ~]# nslookup www.fgedu.net.cn
Server: 192.168.1.1
Address: 192.168.1.1#53

Non-authoritative answer:
Name: www.fgedu.net.cn
Address: 93.184.216.34

# 5. 检查端口监听
[root@localhost ~]# ss -tlnp
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:((“sshd”,pid=1234,fd=3))
LISTEN 0 128 [::]:22 [::]:* users:((“sshd”,pid=1234,fd=4))

# 6. 检查网络连接状态
[root@localhost ~]# ss -s
Total: 100
TCP: 10 (estab 3, closed 0, orphaned 0, timewait 0)

Transport Total IP IPv6
* 100 – –
RAW 0 0 0
UDP 2 2 0
TCP 10 8 2
INET 12 10 2
FRAG 0 0 0

# 7. 检查防火墙规则
[root@localhost ~]# firewall-cmd –list-all
public (active)
target: default
icmp-block-inversion: no
interfaces: eth0
sources:
services: dhcpv6-client ssh
ports:
protocols:
forward: no
masquerade: no
forward-ports:
source-ports:
icmp-blocks:
rich rules:

# 磁盘故障排查

# 1. 查看磁盘使用情况
[root@localhost ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 4.0G 0 4.0G 0% /dev
tmpfs 4.0G 0 4.0G 0% /dev/shm
tmpfs 4.0G 100M 3.9G 3% /run
/dev/mapper/rootvg-rootlv 100G 50G 50G 50% /
/dev/sda1 500M 100M 400M 20% /boot
/dev/mapper/rootvg-datalv 500G 400G 100G 80% /data

# 2. 查看磁盘I/O
[root@localhost ~]# iostat -x 1 3
Linux 5.14.0-362.el9.x86_64 (localhost.localdomain) 04/03/2026 _x86_64_ (4 CPU)

avg-cpu: %user %nice %system %iowait %steal %idle
2.50 0.00 1.00 0.50 0.00 96.00

Device r/s w/s rkB/s wkB/s rrqm/s wrqm/s %rrqm %wrqm r_await w_await aqu-sz rareq-sz wareq-sz svctm %util
sda 0.50 0.50 5.00 2.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 10.00 5.00 0.00 0.00
sdb 1.00 1.00 10.00 5.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 10.00 5.00 0.00 0.00
sdc 1.50 1.50 15.00 7.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 10.00 5.00 0.00 0.00

# 3. 查看磁盘健康状态
[root@localhost ~]# smartctl -H /dev/sda
smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-362.el9.x86_64] (local build)
Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED

# 4. 查看磁盘详细信息
[root@localhost ~]# smartctl -i /dev/sda
smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-362.el9.x86_64] (local build)
Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF INFORMATION SECTION ===
Model Family: Seagate Barracuda 3.5
Device Model: ST1000DM003-1CH162
Serial Number: Z4Z12345
LU WWN Device Id: 5 000c50 0abcdef12
Firmware Version: CC46
User Capacity: 1,000,204,886,016 bytes [1.00 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 7200 rpm
Device is: In smartctl database [for details use: -P show]
ATA Version is: ATA8-ACS T13/1699-D revision 4
SATA Version is: SATA 3.0, 6.0 Gb/s (current: 6.0 Gb/s)

# 5. 查看磁盘错误日志
[root@localhost ~]# dmesg | grep -i “sda”
[12345.678901] sd 0:0:0:0: [sda] 1953525168 512-byte logical blocks: (1.00 TB/931 GiB)
[12345.678902] sd 0:0:0:0: [sda] Write cache: enabled, read cache: enabled, doesn’t support DPO or FUA
[12345.678903] sda: sda1
[12345.678904] sd 0:0:0:0: [sda] Attached SCSI disk

# 6. 查看文件系统错误
[root@localhost ~]# xfs_repair -n /dev/sda1
Phase 1 – find and verify superblock…
Phase 2 – using internal log
– zero log…
– scan filesystem freespace and inode maps…
– found root inode chunk
Phase 3 – for each AG…
– scan and clear agi unlinked lists…
– process known inodes and perform inode discovery…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
Phase 4 – check for duplicate blocks…
– setting up duplicate extent list…
– check for inodes claiming duplicate blocks…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
Phase 5 – check AG free lists and block maps…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
Phase 6 – check inode allocation…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
Phase 7 – check inode block maps…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
Phase 8 – check directory linkage…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
Phase 9 – check for duplicate blocks…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
Phase 10 – check inode allocation…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
Phase 11 – check inode block maps…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
Phase 12 – check directory linkage…
– agno = 0
– agno = 1
– agno = 2
– agno = 3
done

风哥提示：

5. 进程故障排查

排查进程异常问题。

# 进程故障排查

# 1. 查看进程状态
[root@localhost ~]# ps aux
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 1 0.0 0.1 123456 10240 ? Ss 09:00 0:01 /usr/lib/systemd/systemd –switched-root –system –deserialize 17
root 123 0.0 0.0 65536 5120 ? S< 09:00 0:00 [kworker/0:0H] root 3456 0.5 6.3 2097152 524288 ? S 09:00 1:23 /usr/libexec/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid root 4567 0.3 0.6 262144 51200 ? S 09:00 0:12 nginx: master process /usr/sbin/nginx root 5678 0.2 0.1 131072 10240 pts/0 R+ 10:00 0:00 ps aux # 2. 查看进程树 [root@localhost ~]# pstree systemd─┬─NetworkManager───2*[{NetworkManager}] ├─auditd───{auditd} ├─chronyd ├─crond ├─dbus-daemon───{dbus-daemon} ├─mysqld───26*[{mysqld}] ├─nginx───nginx ├─sshd───sshd───bash───pstree ├─systemd-journal ├─systemd-logind └─systemd-udevd # 3. 查看进程打开的文件 [root@localhost ~]# lsof -p 3456 COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME mysqld 3456 mysql cwd DIR 8,1 4096 2 / mysqld 3456 mysql rtd DIR 8,1 4096 2 / mysqld 3456 mysql txt REG 8,1 12345678 1234567 /usr/libexec/mysqld mysqld 3456 mysql mem REG 8,1 1234567 1234568 /usr/lib64/libpthread.so.0 mysqld 3456 mysql mem REG 8,1 1234567 1234569 /usr/lib64/libc.so.6 mysqld 3456 mysql mem REG 8,1 123456 1234570 /usr/lib64/libdl.so.2 mysqld 3456 mysql mem REG 8,1 123456 1234571 /usr/lib64/librt.so.1 mysqld 3456 mysql mem REG 8,1 123456 1234572 /usr/lib64/libm.so.6 mysqld 3456 mysql mem REG 8,1 123456 1234573 /usr/lib64/libgcc_s.so.1 mysqld 3456 mysql mem REG 8,1 1234567 1234574 /usr/lib64/libstdc++.so.6 mysqld 3456 mysql 0u CHR 1,3 0t0 4 /dev/null mysqld 3456 mysql 1u CHR 1,3 0t0 4 /dev/null mysqld 3456 mysql 2u CHR 1,3 0t0 4 /dev/null mysqld 3456 mysql 3u unix 0xffff880012345678 0t0 12345678 type=STREAM mysqld 3456 mysql 4u unix 0xffff880012345679 0t0 12345679 type=STREAM mysqld 3456 mysql 5u REG 8,1 12345678 1234575 /var/lib/mysql/ibdata1 mysqld 3456 mysql 6u REG 8,1 12345678 1234576 /var/lib/mysql/ib_logfile0 mysqld 3456 mysql 7u REG 8,1 12345678 1234577 /var/lib/mysql/ib_logfile1 # 4. 查看进程网络连接 [root@localhost ~]# netstat -anp | grep 3456 tcp 0 0 0.0.0.0:3306 0.0.0.0:* LISTEN 3456/mysqld tcp 0 0 192.168.1.100:3306 192.168.1.200:12345 ESTABLISHED 3456/mysqld tcp 0 0 192.168.1.100:3306 192.168.1.201:12346 ESTABLISHED 3456/mysqld # 5. 查看进程资源限制 [root@localhost ~]# cat /proc/3456/limits Limit Soft Limit Hard Limit Units Max cpu time unlimited unlimited seconds Max file size unlimited unlimited bytes Max data size unlimited unlimited bytes Max stack size 8388608 unlimited bytes Max core file size 0 unlimited bytes Max resident set unlimited unlimited bytes Max processes 12345 12345 processes Max open files 1024 4096 files Max locked memory 65536 65536 bytes Max address space unlimited unlimited bytes Max file locks unlimited unlimited locks Max pending signals 12345 12345 signals Max msgqueue size 819200 819200 bytes Max nice priority 0 0 Max realtime priority 0 0 Max realtime timeout unlimited unlimited us # 6. 查看进程内存映射 [root@localhost ~]# cat /proc/3456/maps 00400000-004a8000 r-xp 00000000 08:01 1234567 /usr/libexec/mysqld 006a7000-006a8000 r--p 000a7000 08:01 1234567 /usr/libexec/mysqld 006a8000-006aa000 rw-p 000a8000 08:01 1234567 /usr/libexec/mysqld 006aa000-006ab000 rw-p 00000000 00:00 0 7f1234567000-7f1234727000 r-xp 00000000 08:01 1234568 /usr/lib64/libpthread.so.0 7f1234727000-7f1234728000 r--p 001c0000 08:01 1234568 /usr/lib64/libpthread.so.0 7f1234728000-7f1234729000 rw-p 001c1000 08:01 1234568 /usr/lib64/libpthread.so.0 <

更多学习教程公众号风哥教程itpux_com

/div>

6. 实战案例

系统故障排查实战案例。

# 实战案例：系统故障排查

# 1. 创建故障排查脚本
[root@localhost ~]# cat > /usr/local/bin/troubleshoot.sh << 'EOF' #!/bin/bash # script.sh # from:www.itpux.com.qq113257174.wx:itpux-com # web: http://www.fgedu.net.cn # 故障排查脚本 LOG_FILE="/var/log/troubleshoot.log" REPORT_FILE="/var/log/troubleshoot-report.txt" # 记录日志函数 log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >> $LOG_FILE
}

# 检查系统启动
check_system_boot() {
log “Checking system boot…”

echo “=== System Boot Check ===” >> $REPORT_FILE
echo “” >> $REPORT_FILE

# 检查启动时间
BOOT_TIME=$(systemd-analyze | grep “Startup finished” | awk ‘{print $4, $5, $6}’)
echo “Boot time: $BOOT_TIME” >> $REPORT_FILE

# 检查失败服务
FAILED_SERVICES=$(systemctl list-units –failed | grep “loaded failed” | wc -l)
echo “Failed services: $FAILED_SERVICES” >> $REPORT_FILE

echo “” >> $REPORT_FILE
}

# 检查网络
check_network() {
log “Checking network…”

echo “=== Network Check ===” >> $REPORT_FILE
echo “” >> $REPORT_FILE

# 检查网络接口
ip addr show >> $REPORT_FILE

# 检查网络连接
ping -c 4 192.168.1.1 >> $REPORT_FILE

# 检查DNS解析
nslookup www.fgedu.net.cn >> $REPORT_FILE

echo “” >> $REPORT_FILE
}

# 检查磁盘
check_disk() {
log “Checking disk…”

echo “=== Disk Check ===” >> $REPORT_FILE
echo “” >> $REPORT_FILE

# 检查磁盘使用
df -h >> $REPORT_FILE

# 检查磁盘I/O
iostat -x 1 2 >> $REPORT_FILE

# 检查磁盘健康
smartctl -H /dev/sda >> $REPORT_FILE

echo “” >> $REPORT_FILE
}

# 检查进程
check_processes() {
log “Checking processes…”

echo “=== Process Check ===” >> $REPORT_FILE
echo “” >> $REPORT_FILE

# 检查进程状态
ps aux >> $REPORT_FILE

# 检查僵尸进程
ZOMBIE_PROCESSES=$(ps aux | grep -c “defunct”)
echo “Zombie processes: $ZOMBIE_PROCESSES” >> $REPORT_FILE

echo “” >> $REPORT_FILE
}

# 主函数
main() {
log “Starting troubleshooting…”

# 清空报告文件
> $REPORT_FILE

# 检查各项
check_system_boot
check_network
check_disk
check_processes

log “Troubleshooting completed. Report saved to $REPORT_FILE”
}

# 执行主函数
main
EOF

# 2. 设置脚本执行权限
[root@localhost ~]# chmod +x /usr/local/bin/troubleshoot.sh

# 3. 创建日志文件
[root@localhost ~]# touch /var/log/troubleshoot.log
[root@localhost ~]# chmod 644 /var/log/troubleshoot.log

# 4. 测试故障排查脚本
[root@localhost ~]# /usr/local/bin/troubleshoot.sh

# 5. 查看报告
[root@localhost ~]# cat /var/log/troubleshoot-report.txt
=== System Boot Check ===

Boot time: 7.579s
Failed services: 1

=== Network Check ===

1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:11:22:33:44:55 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.100/24 brd 192.168.1.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet6 fe80::211:22ff:fe33:4455/64 scope link
valid_lft forever preferred_lft forever

PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.
64 bytes from 192.168.1.1: icmp_seq=1 ttl=64 time=0.123 ms
64 bytes from 192.168.1.1: icmp_seq=2 ttl=64 time=0.123 ms
64 bytes from 192.168.1.1: icmp_seq=3 ttl=64 time=0.123 ms
64 bytes from 192.168.1.1: icmp_seq=4 ttl=64 time=0.123 ms

— 192.168.1.1 ping statistics —
4 packets transmitted, 4 received, 0% packet loss
rtt min/avg/max/mdev = 0.123/0.123/0.123/0.000 ms

Server: 192.168.1.1
Address: 192.168.1.1#53

Non-authoritative answer:
Name: www.fgedu.net.cn
Address: 93.184.216.34

=== Disk Check ===

Filesystem Size Used Avail Use% Mounted on
devtmpfs 4.0G 0 4.0G 0% /dev
tmpfs 4.0G 0 4.0G 0% /dev/shm
tmpfs 4.0G 100M 3.9G 3% /run
/dev/mapper/rootvg-rootlv 100G 50G 50G 50% /
/dev/sda1 500M 100M 400M 20% /boot
/dev/mapper/rootvg-datalv 500G 400G 100G 80% /data

=== Process Check ===

USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 1 0.0 0.1 123456 10240 ? Ss 09:00 0:01 /usr/lib/systemd/systemd –switched-root –system –deserialize 17
root 123 0.0 0.0 65536 5120 ? S< 09:00 0:00 [kworker/0:0H] root 3456 0.5 6.3 2097152 524288 ? S 09:00 1:23 /usr/libexec/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid root 4567 0.3 0.6 262144 51200 ? S 09:00 0:12 nginx: master process /usr/sbin/nginx root 5678 0.2 0.1 131072 10240 pts/0 R+ 10:00 0:00 ps aux Zombie processes: 0 # 6. 查看日志 [root@localhost ~]# tail -f /var/log/troubleshoot.log [2026-04-03 10:00:00] Starting troubleshooting... [2026-04-03 10:00:01] Checking system boot... [2026-04-03 10:00:02] Checking network... [2026-04-03 10:00:03] Checking disk... [2026-04-03 10:00:04] Checking processes... [2026-04-03 10:00:05] Troubleshooting completed. Report saved to /var/log/troubleshoot-report.txt