# 配置HDFS复制因子
# vi $HADOOP_HOME/etc/hadoop/hdfs-site.xml

dfs.replication
3 dfs.namenode.name.dir
file:///hadoop/dfs/name dfs.datanode.data.dir
file:///hadoop/dfs/data

# 检查HDFS复制状态
# hdfs fsck / -files -blocks -locations
Connecting to namenode via http://namenode:9870/fsck?ugi=root&path=%2F&files=1&blocks=1&locations=1
FSCK started by root (auth:SIMPLE) from /192.168.1.100 for path / at 2026-03-30 10:00:00
……………..
Status: HEALTHY
Total size: 1024000 B
Total dirs: 3
Total files: 5
Total symlinks: 0
Total blocks (validated): 5 (avg. block size 204800 B)
Minimally replicated blocks: 5 (100.0 %)
Over-replicated blocks: 0 (0.0 %)
Under-replicated blocks: 0 (0.0 %)
Mis-replicated blocks: 0 (0.0 %)
Default replication factor: 3
Average block replication: 3.0
Corrupt blocks: 0
Missing replicas: 0 (0.0 %)
Number of data-nodes: 3
Number of racks: 1
FSCK ended at 2026-03-30 10:00:30 in 30 milliseconds

# 配置跨数据中心复制
# hdfs dfs -mkdir -p hdfs://dc2-namenode:8020/data

# 启动跨数据中心复制
# hadoop distcp hdfs://dc1-namenode:8020/data hdfs://dc2-namenode:8020/data

# 检查复制状态
# hadoop distcp -status hdfs://dc1-namenode:8020/data hdfs://dc2-namenode:8020/data
DistCp job_1234567890_0001 completed successfully

# 配置Spark作业恢复
# vi $SPARK_HOME/conf/spark-defaults.conf

spark.eventLog.enabled true
spark.eventLog.dir hdfs://namenode:8020/spark/eventLogs
spark.history.fs.logDirectory hdfs://namenode:8020/spark/eventLogs

# 检查Spark历史服务器状态
# start-history-server.sh
starting org.apache.spark.deploy.history.HistoryServer, logging to /spark/logs/spark-root-org.apache.spark.deploy.history.HistoryServer-1-node1.out

# 查看Spark作业历史
# curl http://node1:18080/api/v1/applications
[
{
“id”: “application_1234567890_0001”,
“name”: “Spark Pi”,
“attempts”: [
{
“attemptId”: “appattempt_1234567890_0001_1”,
“startTime”: 1617033600000,
“endTime”: 1617033660000,
“duration”: 60000,
“sparkUser”: “root”,
“completed”: true
}
]
}
]

# 配置MongoDB副本集
# mongo
> rs.initiate()
> rs.add(“mongo2:27017”)
> rs.add(“mongo3:27017”)
> rs.status()
{
“set”: “rs0”,
“date”: ISODate(“2026-03-30T10:00:00Z”),
“myState”: 1,
“term”: NumberLong(1),
“syncingTo”: “”,
“syncSourceHost”: “”,
“syncSourceId”: -1,
“heartbeatIntervalMillis”: NumberLong(2000),
“majorityVoteCount”: 2,
“writeMajorityCount”: 2,
“optimes”: {
“lastCommittedOpTime”: {
“ts”: Timestamp(1617033600, 1),
“t”: NumberLong(1)
},
“appliedOpTime”: {
“ts”: Timestamp(1617033600, 1),
“t”: NumberLong(1)
},
“durableOpTime”: {
“ts”: Timestamp(1617033600, 1),
“t”: NumberLong(1)
}
},
“members”: [
{
“_id”: 0,
“name”: “mongo1:27017”,
“health”: 1,
“state”: 1,
“stateStr”: “PRIMARY”,
“uptime”: 3600,
“optime”: {
“ts”: Timestamp(1617033600, 1),
“t”: NumberLong(1)
},
“optimeDate”: ISODate(“2026-03-30T10:00:00Z”),
“syncingTo”: “”,
“syncSourceHost”: “”,
“syncSourceId”: -1,
“infoMessage”: “”,
“electionTime”: Timestamp(1617030000, 1),
“electionDate”: ISODate(“2026-03-30T09:00:00Z”),
“configVersion”: 1,
“self”: true,
“lastHeartbeatMessage”: “”
},
{
“_id”: 1,
“name”: “mongo2:27017”,
“health”: 1,
“state”: 2,
“stateStr”: “SECONDARY”,
“uptime”: 3500,
“optime”: {
“ts”: Timestamp(1617033600, 1),
“t”: NumberLong(1)
},
“optimeDurable”: {
“ts”: Timestamp(1617033600, 1),
“t”: NumberLong(1)
},
“optimeDate”: ISODate(“2026-03-30T10:00:00Z”),
“optimeDurableDate”: ISODate(“2026-03-30T10:00:00Z”),
“lastHeartbeat”: ISODate(“2026-03-30T10:00:00Z”),
“lastHeartbeatRecv”: ISODate(“2026-03-30T10:00:00Z”),
“pingMs”: NumberLong(1),
“lastHeartbeatMessage”: “”,
“syncingTo”: “mongo1:27017”,
“syncSourceHost”: “mongo1:27017”,
“syncSourceId”: 0,
“infoMessage”: “”,
“configVersion”: 1
},
{
“_id”: 2,
“name”: “mongo3:27017”,
“health”: 1,
“state”: 2,
“stateStr”: “SECONDARY”,
“uptime”: 3400,
“optime”: {
“ts”: Timestamp(1617033600, 1),
“t”: NumberLong(1)
},
“optimeDurable”: {
“ts”: Timestamp(1617033600, 1),
“t”: NumberLong(1)
},
“optimeDate”: ISODate(“2026-03-30T10:00:00Z”),
“optimeDurableDate”: ISODate(“2026-03-30T10:00:00Z”),
“lastHeartbeat”: ISODate(“2026-03-30T10:00:00Z”),
“lastHeartbeatRecv”: ISODate(“2026-03-30T10:00:00Z”),
“pingMs”: NumberLong(1),
“lastHeartbeatMessage”: “”,
“syncingTo”: “mongo1:27017”,
“syncSourceHost”: “mongo1:27017”,
“syncSourceId”: 0,
“infoMessage”: “”,
“configVersion”: 1
}
],
“ok”: 1,
“$clusterTime”: {
“clusterTime”: Timestamp(1617033600, 1),
“signature”: {
“hash”: BinData(0, “AAAAAAAAAAAAAAAAAAAAAAAAAAA=”),
“keyId”: NumberLong(0)
}
},
“operationTime”: Timestamp(1617033600, 1)
}

# 配置Cassandra数据中心复制
# vi /etc/cassandra/cassandra.yaml

cluster_name: ‘Test Cluster’
data_file_directories:
– /var/lib/cassandra/data
commitlog_directory: /var/lib/cassandra/commitlog

# 配置复制策略
# cqlsh
> CREATE KEYSPACE test WITH replication = {‘class’: ‘NetworkTopologyStrategy’, ‘dc1’: 3, ‘dc2’: 3};
> USE test;
> CREATE TABLE users (id UUID PRIMARY KEY, name TEXT, email TEXT);
> INSERT INTO users (id, name, email) VALUES (uuid(), ‘John Doe’, ‘john@fgedu.net.cn’);
> SELECT * FROM users;

id | email | name
————————————–+—————-+———-
123e4567-e89b-12d3-a456-426614174000 | john@fgedu.net.cn | John Doe

# 检查数据中心状态
# nodetool status
Datacenter: dc1
==============
Status=Up/Down
|/ State=Normal/Leaving/Joining/Moving
— Address Load Tokens Owns (effective) Host ID Rack
UN 192.168.1.1 100.0 KB 256 33.3% 123e4567-e89b-12d3-a456-426614174000 rack1
UN 192.168.1.2 100.0 KB 256 33.3% 234e5678-e89b-12d3-a456-426614174000 rack1
UN 192.168.1.3 100.0 KB 256 33.3% 345e6789-e89b-12d3-a456-426614174000 rack1

Datacenter: dc2
==============
Status=Up/Down
|/ State=Normal/Leaving/Joining/Moving
— Address Load Tokens Owns (effective) Host ID Rack
UN 192.168.2.1 100.0 KB 256 33.3% 456e7890-e89b-12d3-a456-426614174000 rack1
UN 192.168.2.2 100.0 KB 256 33.3% 567e8901-e89b-12d3-a456-426614174000 rack1
UN 192.168.2.3 100.0 KB 256 33.3% 678e9012-e89b-12d3-a456-426614174000 rack1

# 配置Apache Airflow高可用
# vi airflow.cfg

[core]
executor = CeleryExecutor

[celery]
broker_url = redis://redis:6379/0
result_backend = redis://redis:6379/0

# 启动Airflow服务
# airflow webserver -D
# airflow scheduler -D
# airflow celery worker -D

# 检查Airflow服务状态
# ps aux | grep airflow
root 1234 0.1 0.5 12345 6789 ? Ss 10:00 0:00 airflow webserver
root 1235 0.2 0.6 23456 7890 ? Ss 10:00 0:00 airflow scheduler
root 1236 0.3 0.7 34567 8901 ? Ss 10:00 0:00 airflow celery worker

# 配置Prometheus监控Hadoop
# vi prometheus.yml

scrape_configs:
– job_name: ‘hadoop’
static_configs:
– targets: [‘namenode:9100’, ‘datanode1:9100’, ‘datanode2:9100’, ‘datanode3:9100’]

# 启动Prometheus
# prometheus –config.file=prometheus.yml

# 配置Grafana dashboard
# curl -X POST -H “Content-Type: application/json” -d @hadoop-dashboard.json http://grafana:3000/api/dashboards/db

# 检查监控状态
# curl http://prometheus:9090/api/v1/query?query=up
{
“status”: “success”,
“data”: {
“resultType”: “vector”,
“result”: [
{
“metric”: {
“__name__”: “up”,
“instance”: “namenode:9100”,
“job”: “hadoop”
},
“value”: [
1617033600,
“1”
]
},
{
“metric”: {
“__name__”: “up”,
“instance”: “datanode1:9100”,
“job”: “hadoop”
},
“value”: [
1617033600,
“1”
]
},
{
“metric”: {
“__name__”: “up”,
“instance”: “datanode2:9100”,
“job”: “hadoop”
},
“value”: [
1617033600,
“1”
]
},
{
“metric”: {
“__name__”: “up”,
“instance”: “datanode3:9100”,
“job”: “hadoop”
},
“value”: [
1617033600,
“1”
]
}
]
}
}

# 执行HDFS容灾测试
# hdfs dfs -mkdir /test
# hdfs dfs -put test.txt /test
# hdfs dfs -ls /test
Found 1 items
-rw-r–r– 3 root supergroup 123 2026-03-30 10:00 /test/test.txt

# 模拟主节点故障
# systemctl stop hadoop-hdfs-namenode

# 检查备用节点状态
# hdfs haadmin -getServiceState nn2
active

# 验证数据可用性
# hdfs dfs -ls /test
Found 1 items
-rw-r–r– 3 root supergroup 123 2026-03-30 10:00 /test/test.txt

# 恢复主节点
# systemctl start hadoop-hdfs-namenode
# hdfs haadmin -getServiceState nn1
standby

# 大数据容灾最佳实践

## 1. 数据复制策略
– 跨数据中心复制：确保数据在不同地理位置有副本
– 定期备份：使用快照、导出等方式定期备份数据
– 增量复制：减少复制开销，提高复制效率

## 2. 集群架构
– 多活架构：多个数据中心同时提供服务
– 主备架构：主数据中心故障时切换到备用数据中心
– 混合架构：结合多活和主备的优势

## 3. 监控与告警
– 实时监控：监控集群状态、数据复制状态
– 智能告警：根据阈值自动触发告警
– 故障预测：使用AI技术预测潜在故障

## 4. 测试与演练
– 定期演练：每季度至少进行一次容灾演练
– 模拟故障：模拟各种故障场景，测试容灾方案
– 演练评估：评估演练结果，优化容灾方案

## 5. 自动化
– 自动故障检测：自动检测集群故障
– 自动故障切换：故障发生时自动切换到备用系统
– 自动恢复：故障修复后自动恢复系统