# 1. 配置告警规则
$ cat > /etc/prometheus/rules/spark-alerts.yml << 'EOF' groups: - name: spark-cluster-alerts rules: - alert: SparkWorkerDown expr: spark_worker_alive == 0 for: 5m labels: severity: critical annotations: summary: "Spark Worker宕机" description: "Worker {{ $labels.worker_id }} 已经宕机超过5分钟" - alert: SparkHighMemoryUsage expr: spark_worker_memory_used / spark_worker_memory_total > 0.9
for: 5m
labels:
severity: warning
annotations:
summary: “Spark内存使用率高”
description: “Worker {{ $labels.worker_id }} 内存使用率超过90%”

– alert: SparkApplicationFailed
expr: increase(spark_app_status{status=”failed”}[5m]) > 0
labels:
severity: critical
annotations:
summary: “Spark应用失败”
description: “应用 {{ $labels.app_name }} 执行失败”

– alert: SparkHighGCTime
expr: spark_executor_jvm_gc_time / spark_executor_run_time > 0.1
for: 10m
labels:
severity: warning
annotations:
summary: “Spark GC时间过长”
description: “Executor {{ $labels.executor_id }} GC时间占比超过10%”
EOF

# 2. 配置Alertmanager
$ cat > /etc/prometheus/alertmanager.yml << 'EOF' global: resolve_timeout: 5m smtp_smarthost: 'smtp.fgedu.net.cn:25' smtp_from: 'alert@fgedu.net.cn' smtp_auth_username: 'alert@fgedu.net.cn' smtp_auth_password: 'fgedu123' route: group_by: ['alertname', 'severity'] group_wait: 30s group_interval: 5m repeat_interval: 1h receiver: 'default-receiver' routes: - match: severity: critical receiver: 'critical-receiver' - match: severity: warning receiver: 'warning-receiver' receivers: - name: 'default-receiver' email_configs: - to: 'admin@fgedu.net.cn' - name: 'critical-receiver' email_configs: - to: 'admin@fgedu.net.cn' webhook_configs: - url: 'http://192.168.1.80:8060/dingtalk/webhook/send' - name: 'warning-receiver' email_configs: - to: 'admin@fgedu.net.cn' EOF # 3. 重启服务 $ systemctl restart prometheus $ systemctl restart alertmanager # 4. Grafana Dashboard # 导入Spark Dashboard # Dashboard ID: 7059 (Spark Metrics Dashboard) # Dashboard ID: 13187 (Spark Application Dashboard)

SPARK_MASTER=”192.168.1.60:8080″
LOG_FILE=”/var/log/spark/daily_check.log”

echo “========================================” >> $LOG_FILE
echo “Spark集群日常巡检 – $(date)” >> $LOG_FILE
echo “========================================” >> $LOG_FILE

# 1. 检查Master状态
echo “1. 检查Master状态” >> $LOG_FILE
curl -s http://$SPARK_MASTER/api/v1/applications | python -m json.tool >> $LOG_FILE

# 2. 检查Worker状态
echo “2. 检查Worker状态” >> $LOG_FILE
curl -s http://$SPARK_MASTER/api/v1/workers | python -m json.tool >> $LOG_FILE

# 3. 检查资源使用
echo “3. 检查资源使用” >> $LOG_FILE
curl -s http://$SPARK_MASTER/metrics/master/prometheus | grep -E “spark_master_(memory|cores)” >> $LOG_FILE

# 4. 检查磁盘空间
echo “4. 检查磁盘空间” >> $LOG_FILE
df -h /bigdata >> $LOG_FILE

# 5. 检查HDFS空间
echo “5. 检查HDFS空间” >> $LOG_FILE
hdfs dfs -df -h / >> $LOG_FILE

# 6. 检查日志目录
echo “6. 检查日志目录” >> $LOG_FILE
hdfs dfs -du -h /spark-logs >> $LOG_FILE

echo “巡检完成” >> $LOG_FILE

# 定时任务
# crontab -e
# 0 8 * * * /bigdata/scripts/spark_daily_check.sh

# 排查步骤
# 1. 检查服务状态
$ jps | grep -E “Master|Worker|HistoryServer”

# 2. 检查端口
$ netstat -tlnp | grep -E “8080|8081|18080|4040”

# 3. 检查防火墙
$ firewall-cmd –list-ports

# 解决方案
# 1. 重启服务
$ /bigdata/app/spark/sbin/stop-all.sh
$ /bigdata/app/spark/sbin/start-all.sh

# 2. 开放端口
$ firewall-cmd –add-port=8080/tcp –permanent
$ firewall-cmd –add-port=8081/tcp –permanent
$ firewall-cmd –add-port=18080/tcp –permanent
$ firewall-cmd –reload

# 3. 检查配置
spark.ui.port=8080
spark.ui.enabled=true

# 排查步骤
# 1. 检查事件日志配置
$ cat /bigdata/app/spark/conf/spark-defaults.conf | grep eventLog

# 2. 检查HDFS目录
$ hdfs dfs -ls /spark-logs

# 3. 检查历史服务器配置
$ cat /bigdata/app/spark/conf/spark-history-server.conf

# 解决方案
# 1. 确认事件日志启用
spark.eventLog.enabled=true
spark.eventLog.dir=hdfs://192.168.1.60:9000/spark-logs

# 2. 创建日志目录
$ hdfs dfs -mkdir -p /spark-logs
$ hdfs dfs -chmod 777 /spark-logs

# 3. 重启历史服务器
$ /bigdata/app/spark/sbin/stop-history-server.sh
$ /bigdata/app/spark/sbin/start-history-server.sh