## 1.2 配置Ansible主机清单
vi /etc/ansible/hosts
[hadoop-master]
fgedu01 ansible_ssh_user=fgedu ansible_ssh_pass=fgedu

[hadoop-slaves]
fgedu02 ansible_ssh_user=fgedu ansible_ssh_pass=fgedu
fgedu03 ansible_ssh_user=fgedu ansible_ssh_pass=fgedu

[hadoop:children]
hadoop-master
hadoop-slaves

## 1.3 编写Ansible playbook
vi hadoop-deploy.yml
—
– hosts: hadoop
become: yes
tasks:
– name: 安装Java
yum: name=java-1.8.0-openjdk-devel state=present

– name: 创建目录
file: path={{ item }} state=directory owner=fgedu group=fgedu
with_items:
– /bigdata/app
– /bigdata/fgdata
– /bigdata/fgdata/logs

– name: 复制Hadoop安装包
copy: src=/tmp/hadoop-3.3.5.tar.gz dest=/bigdata/app/

– name: 解压Hadoop安装包
unarchive: src=/bigdata/app/hadoop-3.3.5.tar.gz dest=/bigdata/app/ remote_src=yes

– name: 配置环境变量
lineinfile: dest=/etc/profile line={{ item }}
with_items:
– export HADOOP_HOME=/bigdata/app/hadoop-3.3.5
– export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

# 2. 执行Ansible playbook
ansible-playbook hadoop-deploy.yml

– name: 配置hdfs-site.xml
template: src=hdfs-site.xml.j2 dest=/bigdata/app/hadoop-3.3.5/etc/hadoop/hdfs-site.xml

– name: 配置yarn-site.xml
template: src=yarn-site.xml.j2 dest=/bigdata/app/hadoop-3.3.5/etc/hadoop/yarn-site.xml

– name: 配置mapred-site.xml
template: src=mapred-site.xml.j2 dest=/bigdata/app/hadoop-3.3.5/etc/hadoop/mapred-site.xml

# 2. 编写配置模板
## 2.1 core-site.xml.j2
vi core-site.xml.j2
fs.defaultFS
hdfs://{{ groups[‘hadoop-master’][0] }}:9000 hadoop.tmp.dir
/bigdata/fgdata/tmp

## 2.2 hdfs-site.xml.j2
vi hdfs-site.xml.j2
dfs.replication
3 dfs.namenode.name.dir
/bigdata/fgdata/hdfs/namenode dfs.datanode.data.dir
/bigdata/fgdata/hdfs/datanode

## 2.3 yarn-site.xml.j2
vi yarn-site.xml.j2
yarn.resourcemanager.hostname
{{ groups[‘hadoop-master’][0] }} yarn.nodemanager.aux-services
mapreduce_shuffle

## 2.4 mapred-site.xml.j2
vi mapred-site.xml.j2
mapreduce.framework.name
yarn

# 3. 执行配置管理playbook
ansible-playbook hadoop-config.yml

## 1.2 配置Prometheus
vi /bigdata/app/prometheus/prometheus.yml
global:
scrape_interval: 15s
evaluation_interval: 15s

rule_files:
– “rules/*.yml”

alerting:
alertmanagers:
– static_configs:
– targets: [‘localhost:9093’]

scrape_configs:
– job_name: ‘prometheus’
static_configs:
– targets: [‘localhost:9090’]

– job_name: ‘node_exporter’
static_configs:
– targets: [‘fgedu01:9100’, ‘fgedu02:9100’, ‘fgedu03:9100’]

– job_name: ‘hadoop_hdfs’
static_configs:
– targets: [‘fgedu01:9870’]

– job_name: ‘hadoop_yarn’
static_configs:
– targets: [‘fgedu01:8088’]

# 2. 安装Node Exporter
wget https://github.com/prometheus/node_exporter/releases/download/v1.5.0/node_exporter-1.5.0.linux-amd64.tar.gz
tar -xzf node_exporter-1.5.0.linux-amd64.tar.gz
mv node_exporter-1.5.0.linux-amd64 /bigdata/app/node_exporter

# 3. 安装Grafana
yum install -y grafana
systemctl start grafana-server
systemctl enable grafana-server

# 4. 配置告警规则
vi /bigdata/app/prometheus/rules/alert.yml
groups:
– name: node_alerts
rules:
– alert: HighCPUUsage
expr: 100 – (avg by(instance) (irate(node_cpu_seconds_total{mode=”idle”}[5m])) * 100) > 80
for: 5m
labels:
severity: warning
annotations:
summary: “High CPU usage on {{ $labels.instance }}”
description: “CPU usage is above 80% for 5 minutes”

– alert: HighMemoryUsage
expr: (node_memory_MemTotal_bytes – node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 80
for: 5m
labels:
severity: warning
annotations:
summary: “High memory usage on {{ $labels.instance }}”
description: “Memory usage is above 80% for 5 minutes”

– alert: HighDiskUsage
expr: (node_filesystem_size_bytes{mountpoint=”/”} – node_filesystem_free_bytes{mountpoint=”/”}) / node_filesystem_size_bytes{mountpoint=”/”} * 100 > 80
for: 5m
labels:
severity: warning
annotations:
summary: “High disk usage on {{ $labels.instance }}”
description: “Disk usage is above 80% for 5 minutes”