# 创建告警规则文件
$ cat > /bigdata/app/prometheus/kafka_alerts.yml << 'EOF' groups: - name: kafka_alerts rules: # Broker宕机告警 - alert: KafkaBrokerDown expr: up{job="kafka"} == 0 for: 1m labels: severity: critical annotations: summary: "Kafka Broker宕机" description: "Kafka Broker {{ $labels.instance }} 已宕机超过1分钟" # 未充分复制分区告警 - alert: KafkaUnderReplicatedPartitions expr: kafka_server_ReplicaManager_UnderReplicatedPartitions > 0
for: 5m
labels:
severity: warning
annotations:
summary: “Kafka存在未充分复制分区”
description: “Broker {{ $labels.instance }} 有 {{ $value }} 个未充分复制分区”

# 消费延迟告警
– alert: KafkaConsumerLag
expr: kafka_consumer_group_lag > 10000
for: 5m
labels:
severity: warning
annotations:
summary: “Kafka消费延迟过高”
description: “消费者组 {{ $labels.group }} 的延迟为 {{ $value }}”

# 磁盘空间告警
– alert: KafkaDiskSpaceLow
expr: node_filesystem_avail_bytes{mountpoint=”/bigdata”} / node_filesystem_size_bytes{mountpoint=”/bigdata”} < 0.15 for: 5m labels: severity: critical annotations: summary: "Kafka磁盘空间不足" description: "Broker {{ $labels.instance }} 磁盘使用率超过85%" EOF # 在prometheus.yml中添加告警规则 $ vi /bigdata/app/prometheus/prometheus.yml rule_files: - /bigdata/app/prometheus/kafka_alerts.yml # 重启Prometheus $ systemctl restart prometheus

# 下载Alertmanager
$ wget https://github.com/prometheus/alertmanager/releases/download/v0.25.0/alertmanager-0.25.0.linux-amd64.tar.gz

$ tar -zxvf alertmanager-0.25.0.linux-amd64.tar.gz
$ mv alertmanager-0.25.0.linux-amd64 /bigdata/app/alertmanager

# 配置Alertmanager
$ cat > /bigdata/app/alertmanager/alertmanager.yml << 'EOF' global: resolve_timeout: 5m route: group_by: ['alertname', 'severity'] group_wait: 30s group_interval: 5m repeat_interval: 1h receiver: 'default' routes: - match: severity: critical receiver: 'critical' - match: severity: warning receiver: 'warning' receivers: - name: 'default' webhook_configs: - url: 'http://192.168.1.100:8080/alert' - name: 'critical' webhook_configs: - url: 'http://192.168.1.100:8080/alert/critical' - name: 'warning' webhook_configs: - url: 'http://192.168.1.100:8080/alert/warning' EOF # 启动Alertmanager $ /bigdata/app/alertmanager/alertmanager --config.file=/bigdata/app/alertmanager/alertmanager.yml & # 在Prometheus中配置Alertmanager $ vi /bigdata/app/prometheus/prometheus.yml alerting: alertmanagers: - static_configs: - targets: - 192.168.1.51:9093

# 查看消费者组延迟
$ /bigdata/app/kafka/bin/kafka-consumer-groups.sh –describe \
–group fgedu-order-consumer-group \
–bootstrap-server 192.168.1.51:9092

GROUP TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG
fgedu-order-consumer-group fgedu-orders 0 10000 15000 5000
fgedu-order-consumer-group fgedu-orders 1 8000 12000 4000
fgedu-order-consumer-group fgedu-orders 2 12000 18000 6000

# 监控脚本
#!/bin/bash
# lag_monitor.sh
# from:www.itpux.com.qq113257174.wx:itpux-com
# web: http://www.fgedu.net.cn

KAFKA_HOME=/bigdata/app/kafka
BOOTSTRAP_SERVER=192.168.1.51:9092
THRESHOLD=10000

while true; do
echo “=== Consumer Lag Monitor ===”
echo “Time: $(date)”

# 获取所有消费者组
GROUPS=$($KAFKA_HOME/bin/kafka-consumer-groups.sh –list –bootstrap-server $BOOTSTRAP_SERVER)

for GROUP in $GROUPS; do
LAG=$($KAFKA_HOME/bin/kafka-consumer-groups.sh –describe \
–group $GROUP \
–bootstrap-server $BOOTSTRAP_SERVER | \
awk ‘NR>1 {sum+=$6} END {print sum+0}’)

if [ “$LAG” -gt “$THRESHOLD” ]; then
echo “WARNING: Group $GROUP lag is $LAG (threshold: $THRESHOLD)”
fi
done

sleep 60
done

# 安装Burrow
$ git clone https://github.com/linkedin/Burrow.git
$ cd Burrow
$ go build -o burrow ./cmd/burrow

# 配置Burrow
$ cat > /bigdata/app/burrow/burrow.toml << 'EOF' [general] logdir = "/bigdata/app/burrow/logs" logconfig = "/bigdata/app/burrow/logging.cfg" [zookeeper] servers = ["192.168.1.51:2181", "192.168.1.52:2181", "192.168.1.53:2181"] timeout = 6 [client-profile.kafka] client-id = "burrow" kafka-version = "3.6.1" [cluster.kafka] class-name = "kafka" client-profile = "kafka" servers = ["192.168.1.51:9092", "192.168.1.52:9092", "192.168.1.53:9092"] [consumer.kafka] class-name = "kafka" cluster = "kafka" servers = ["192.168.1.51:9092", "192.168.1.52:9092", "192.168.1.53:9092"] [httpserver.default] address = ":8000" EOF # 启动Burrow $ /bigdata/app/burrow/burrow --config-dir /bigdata/app/burrow & # 查看消费者组状态 $ curl http://localhost:8000/v3/kafka/kafka/consumer/fgedu-order-consumer-group/status { "error": false, "message": "consumer group status returned", "status": { "cluster": "kafka", "group": "fgedu-order-consumer-group", "status": "OK", "complete": 1, "maxlag": 5000, "totallag": 15000 } }

#!/bin/bash
# cluster_health_check.sh
# from:www.itpux.com.qq113257174.wx:itpux-com
# web: http://www.fgedu.net.cn

KAFKA_HOME=/bigdata/app/kafka
BOOTSTRAP_SERVER=192.168.1.51:9092

echo “=== Kafka Cluster Health Check ===”
echo “Time: $(date)”
echo “”

# 1. 检查Broker状态
echo “1. Broker Status:”
$KAFKA_HOME/bin/kafka-broker-api-versions.sh –bootstrap-server $BOOTSTRAP_SERVER | head -5

# 2. 检查Controller
echo “”
echo “2. Controller Status:”
$KAFKA_HOME/bin/kafka-metadata.sh \
–snapshot /bigdata/kafka-logs/__cluster_metadata-0/00000000000000000000.log \
–command “controller” 2>/dev/null || echo “Controller check failed”

# 3. 检查Topic状态
echo “”
echo “3. Topic Status:”
TOPICS=$($KAFKA_HOME/bin/kafka-topics.sh –list –bootstrap-server $BOOTSTRAP_SERVER)
echo “Total Topics: $(echo $TOPICS | wc -w)”

# 4. 检查Under-Replicated分区
echo “”
echo “4. Under-Replicated Partitions:”
$KAFKA_HOME/bin/kafka-topics.sh –describe –bootstrap-server $BOOTSTRAP_SERVER | \
grep -E “Isr:” | \
awk -F'[\\t ]+’ ‘{
split($6, isr, “,”)
split($5, replicas, “,”)
if(length(isr) < length(replicas)) print $2 }' | sort -u # 5. 检查磁盘空间 echo "" echo "5. Disk Space:" df -h /bigdata | tail -1 # 执行检查 $ chmod +x /bigdata/scripts/cluster_health_check.sh $ /bigdata/scripts/cluster_health_check.sh === Kafka Cluster Health Check === Time: Mon Apr 8 10:30:00 CST 2026 1. Broker Status: 192.168.1.51:9092 (id: 1 rack: null) -> (Produce(0): 0 to 9, …)

2. Controller Status:
Controller: Broker 1

3. Topic Status:
Total Topics: 5

4. Under-Replicated Partitions:

5. Disk Space:
/dev/sdb1 500G 150G 350G 30% /bigdata

# 问题现象：Grafana图表无数据

# 排查步骤
# 1. 检查JMX Exporter是否正常
$ curl http://192.168.1.51:7071/metrics | head

# 2. 检查Prometheus是否正常采集
$ curl http://localhost:9090/api/v1/targets

# 3. 检查网络连通性
$ telnet 192.168.1.51 7071

# 解决方案
# 1. 重启JMX Exporter（重启Kafka）
$ systemctl restart kafka

# 2. 检查防火墙
$ firewall-cmd –list-ports
$ firewall-cmd –add-port=7071/tcp –permanent

# 3. 检查Prometheus配置
$ /bigdata/app/prometheus/promtool check config /bigdata/app/prometheus/prometheus.yml

# 监控最佳实践
1. 建立完善的监控体系
– 指标采集：JMX Exporter
– 数据存储：Prometheus
– 可视化：Grafana
– 告警：Alertmanager

2. 监控核心指标
– Broker状态和吞吐量
– 分区和副本状态
– 消费延迟
– 系统资源使用

3. 配置合理告警
– 设置合理阈值
– 配置多级告警
– 避免告警风暴

4. 定期巡检
– 每日检查告警
– 每周检查趋势
– 每月评估容量