# 1. 发现故障
# 监控告警
# 业务反馈HDFS不可用

# 2. 检查状态
echo “=== Checking NameNode Status ===”
hdfs haadmin -getServiceState nn1
hdfs haadmin -getServiceState nn2

# 输出示例：
# nn1: standby
# nn2: active (如果自动故障转移成功)
# 或者
# nn1: unknown
# nn2: standby

# 3. 检查ZKFC
jps | grep DFSZKFailoverController

# 4. 如果自动故障转移没成功，手动执行
if [ “$(hdfs haadmin -getServiceState nn2)” != “active” ]; then
echo “Manual failover…”
hdfs haadmin -failover nn1 nn2
fi

# 5. 验证
echo “=== Verifying HDFS ===”
hdfs dfs -ls /
hdfs dfsadmin -report

# 6. 恢复原NameNode
echo “=== Recovering nn1 ===”
# 检查日志
tail -n 100 /bigdata/app/hadoop/logs/hadoop-hdfs-namenode-fgedu-nn01.log

# 启动
ssh fgedu-nn01 “hdfs –daemon start namenode”
sleep 30

# 检查状态
hdfs haadmin -getServiceState nn1
hdfs haadmin -getServiceState nn2

# 7. 最终验证
echo “=== Final Verification ===”
hadoop jar /bigdata/app/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar pi 2 100

# 1. 发现故障
hdfs dfsadmin -report

# 输出示例：
# Live datanodes (2):
# …
# Dead datanodes (1):
# fgedu-dn01

# 2. 检查节点
ssh fgedu-dn01
uptime
jps

# 3. 启动DataNode
echo “Starting DataNode…”
hdfs –daemon start datanode
sleep 60

# 4. 检查
jps
hdfs dfsadmin -report

# 5. 观察块恢复
echo “Checking block recovery…”
hdfs fsck /

# 6. 验证
hdfs dfs -ls /
hdfs dfs -put /etc/hosts /test/
hdfs dfs -cat /test/hosts

# 1. 检查服务器状态
# 逐个检查服务器
for host in fgedu-nn01 fgedu-nn02 fgedu-dn01 fgedu-dn02 fgedu-dn03; do
echo “=== Checking $host ===”
ssh $host “uptime; jps”
done

# 2. 启动ZooKeeper
echo “=== Starting ZooKeeper ===”
for host in fgedu-zk01 fgedu-zk02 fgedu-zk03; do
ssh $host “/bigdata/app/zookeeper/bin/zkServer.sh start”
done
sleep 30

# 检查ZK
/bigdata/app/zookeeper/bin/zkServer.sh status

# 3. 启动JournalNode
echo “=== Starting JournalNode ===”
for host in fgedu-jn01 fgedu-jn02 fgedu-jn03; do
ssh $host “hdfs –daemon start journalnode”
done
sleep 10

# 4. 启动NameNode
echo “=== Starting NameNode ===”
ssh fgedu-nn01 “hdfs –daemon start namenode”
ssh fgedu-nn02 “hdfs –daemon start namenode”
sleep 30

# 检查NN
hdfs haadmin -getServiceState nn1
hdfs haadmin -getServiceState nn2

# 5. 启动DataNode
echo “=== Starting DataNode ===”
for host in fgedu-dn01 fgedu-dn02 fgedu-dn03; do
ssh $host “hdfs –daemon start datanode”
done
sleep 60

# 6. 启动ResourceManager
echo “=== Starting ResourceManager ===”
ssh fgedu-rm01 “yarn –daemon start resourcemanager”
ssh fgedu-rm02 “yarn –daemon start resourcemanager”
sleep 30

# 7. 启动NodeManager
echo “=== Starting NodeManager ===”
for host in fgedu-dn01 fgedu-dn02 fgedu-dn03; do
ssh $host “yarn –daemon start nodemanager”
done
sleep 30

# 8. 启动其他组件
# Hive、HBase等

# 9. 验证
echo “=== Verifying Cluster ===”
jps
hdfs dfsadmin -report
yarn node -list
hdfs dfs -ls /
hadoop jar /bigdata/app/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar pi 2 100