# 监控架构设计
# 数据采集层 → 数据存储层 → 数据展示层 → 告警层
# 数据采集：Node Exporter、JMX Exporter、Prometheus
# 数据存储：Prometheus、InfluxDB
# 数据展示：Grafana、Zabbix
# 告警：Alertmanager、Zabbix告警

# 告警策略示例
# 1. 紧急告警：集群节点故障、服务不可用
# 2. 重要告警：资源使用率超过80%、作业失败
# 3. 警告告警：资源使用率超过60%、作业执行缓慢
# 4. 通知方式：紧急告警通过短信和电话，重要告警通过邮件和微信，警告告警通过邮件

# 安装Prometheus
[root@fgedu.net.cn ~]# wget
https://github.com/prometheus/prometheus/releases/download/v2.37.0/prometheus-2.37.0.linux-amd64.tar.gz
[root@fgedu.net.cn ~]# tar -zxvf prometheus-2.37.0.linux-amd64.tar.gz
[root@fgedu.net.cn ~]# mv prometheus-2.37.0.linux-amd64 /bigdata/app/prometheus

# 安装Grafana
[root@fgedu.net.cn ~]# yum install -y grafana
[root@fgedu.net.cn ~]# systemctl start grafana-server
[root@fgedu.net.cn ~]# systemctl enable grafana-server

# 安装Node Exporter
[root@fgedu.net.cn ~]# wget
https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
[root@fgedu.net.cn ~]# tar -zxvf node_exporter-1.3.1.linux-amd64.tar.gz
[root@fgedu.net.cn ~]# mv node_exporter-1.3.1.linux-amd64 /bigdata/app/node_exporter
[root@fgedu.net.cn ~]# /bigdata/app/node_exporter/node_exporter &

# 安装JMX Exporter
[root@fgedu.net.cn ~]# wget
https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar
[root@fgedu.net.cn ~]# mv jmx_prometheus_javaagent-0.16.1.jar /bigdata/app/jmx_exporter/

# 配置Prometheus
[root@fgedu.net.cn ~]# vi /bigdata/app/prometheus/prometheus.yml

global:
scrape_interval: 15s

scrape_configs:
– job_name: ‘node’
static_configs:
– targets: [‘fgedu.net.cn:9100’]
– job_name: ‘hadoop’
static_configs:
– targets: [‘fgedu.net.cn:9150’]
– job_name: ‘hdfs’
static_configs:
– targets: [‘fgedu.net.cn:9160’]
– job_name: ‘yarn’
static_configs:
– targets: [‘fgedu.net.cn:9170’]

# 配置Hadoop JMX
[root@fgedu.net.cn ~]# vi /bigdata/app/hadoop/etc/hadoop/hadoop-env.sh

export HADOOP_NAMENODE_OPTS=”$HADOOP_NAMENODE_OPTS
-javaagent:/bigdata/app/jmx_exporter/jmx_prometheus_javaagent-0.16.1.jar=9160:/bigdata/app/jmx_exporter/hdfs.yml”
export HADOOP_DATANODE_OPTS=”$HADOOP_DATANODE_OPTS
-javaagent:/bigdata/app/jmx_exporter/jmx_prometheus_javaagent-0.16.1.jar=9161:/bigdata/app/jmx_exporter/hdfs.yml”
export YARN_RESOURCEMANAGER_OPTS=”$YARN_RESOURCEMANAGER_OPTS
-javaagent:/bigdata/app/jmx_exporter/jmx_prometheus_javaagent-0.16.1.jar=9170:/bigdata/app/jmx_exporter/yarn.yml”
export YARN_NODEMANAGER_OPTS=”$YARN_NODEMANAGER_OPTS
-javaagent:/bigdata/app/jmx_exporter/jmx_prometheus_javaagent-0.16.1.jar=9171:/bigdata/app/jmx_exporter/yarn.yml”

# 配置Alertmanager
[root@fgedu.net.cn ~]# vi /bigdata/app/prometheus/alertmanager.yml

global:
smtp_smarthost: ‘smtp.163.com:25’
smtp_from: ‘alertmanager@fgedu.net.cn’
smtp_auth_username: ‘alertmanager@fgedu.net.cn’
smtp_auth_password: ‘password’

route:
group_by: [‘alertname’]
group_wait: 30s
group_interval: 5m
repeat_interval: 1h
receiver: ’email’

receivers:
– name: ’email’
email_configs:
– to: ‘ops@fgedu.net.cn’
send_resolved: true

# 配置告警规则
[root@fgedu.net.cn ~]# vi /bigdata/app/prometheus/rules/hadoop_rules.yml

groups:
– name: hadoop_alerts
rules:
– alert: NamenodeDown
expr: up{job=”hdfs”} == 0
for: 5m
labels:
severity: critical
annotations:
summary: “Namenode down”
description: “Namenode {{ $labels.instance }} is down”
– alert: DiskUsageHigh
expr: (node_filesystem_size_bytes{mountpoint=”/”} – node_filesystem_free_bytes{mountpoint=”/”}) /
node_filesystem_size_bytes{mountpoint=”/”} * 100 > 80
for: 10m
labels:
severity: warning
annotations:
summary: “Disk usage high”
description: “Disk usage on {{ $labels.instance }} is {{ $value }}%”