# 2. 配置prometheus.yml
cat > /bigdata/app/prometheus/prometheus.yml << ‘EOF’
global:
scrape_interval: 15s
evaluation_interval: 15s

alerting:
alertmanagers:
– static_configs:
– targets:
– fgedu-alertmanager:9093

rule_files:
– “rules/*.yml”

scrape_configs:
– job_name: ‘prometheus’
static_configs:
– targets: [‘localhost:9090’]

– job_name: ‘node’
file_sd_configs:
– files:
– ‘/bigdata/app/prometheus/targets/node.json’

– job_name: ‘hadoop_jmx’
file_sd_configs:
– files:
– ‘/bigdata/app/prometheus/targets/hadoop_jmx.json’
EOF

# 3. 创建配置目录
mkdir -p /bigdata/app/prometheus/rules
mkdir -p /bigdata/app/prometheus/targets
mkdir -p /bigdata/fgdata/prometheus

# 4. 配置Node Exporter目标
cat > /bigdata/app/prometheus/targets/node.json << ‘EOF’
[
{
“targets”: [
“fgedu-nn:9100”,
“fgedu-dn01:9100”,
“fgedu-dn02:9100”,
“fgedu-dn03:9100”,
“fgedu-rm:9100”,
“fgedu-hive:9100”
],
“labels”: {
“env”: “production”
}
}
]
EOF

# 5. 配置Hadoop JMX目标
cat > /bigdata/app/prometheus/targets/hadoop_jmx.json << ‘EOF’
[
{
“targets”: [“fgedu-nn:9400”],
“labels”: {
“job”: “hdfs_namenode”,
“env”: “production”
}
},
{
“targets”: [“fgedu-rm:9401”],
“labels”: {
“job”: “yarn_resourcemanager”,
“env”: “production”
}
}
]
EOF

# 6. 配置告警规则
cat > /bigdata/app/prometheus/rules/host_rules.yml << ‘EOF’
groups:
– name: host_alerts
rules:
– alert: HostHighCpuUsage
expr: 100 – (avg by(instance) (irate(node_cpu_seconds_total{mode=”idle”}[5m])) * 100) > 80
for: 5m
labels:
severity: warning
annotations:
summary: “Host high CPU usage”
description: “CPU usage is {{ $value }}%”

– alert: HostHighMemoryUsage
expr: (1 – (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
for: 5m
labels:
severity: warning
annotations:
summary: “Host high memory usage”
description: “Memory usage is {{ $value }}%”

– alert: HostDiskSpaceLow
expr: (node_filesystem_avail_bytes{fstype!=”tmpfs”} / node_filesystem_size_bytes{fstype!=”tmpfs”}) * 100 < 20
for: 5m
labels:
severity: critical
annotations:
summary: “Host disk space low”
description: “Disk {{ $labels.mountpoint }} has {{ $value }}% available”
EOF

# 7. 启动Prometheus
cd /bigdata/app/prometheus
nohup ./prometheus \
–config.file=prometheus.yml \
–storage.tsdb.path=/bigdata/fgdata/prometheus \
–web.enable-lifecycle &

# 8. 验证Prometheus
curl http://localhost:9090

# 9. 安装Grafana
cd /bigdata/app
wget https://dl.grafana.com/oss/release/grafana-9.5.2.linux-amd64.tar.gz
tar -zxvf grafana-9.5.2.linux-amd64.tar.gz
ln -s grafana-9.5.2 grafana

# 10. 启动Grafana
cd /bigdata/app/grafana
nohup ./bin/grafana-server &

# 11. 访问Grafana
# http://fgedu-grafana:3000
# 默认用户名: admin
# 默认密码: admin

# 12. 配置Prometheus数据源
# 在Grafana中添加Prometheus数据源
# URL: http://fgedu-prometheus:9090

# 13. 导入Dashboard
# Node Exporter Dashboard: 1860
# JVM Dashboard: 8563
# HDFS Dashboard: 自定义

# 2. 修改系统配置
vi /etc/security/limits.conf
elasticsearch soft nofile 65536
elasticsearch hard nofile 65536
elasticsearch soft nproc 4096
elasticsearch hard nproc 4096

vi /etc/sysctl.conf
vm.max_map_count=262144

sysctl -p

# 3. 配置elasticsearch.yml
cat > /bigdata/app/elasticsearch/config/elasticsearch.yml << ‘EOF’
cluster.name: fgedu-es-cluster
node.name: fgedu-es01
path.data: /bigdata/fgdata/elasticsearch/data
path.logs: /bigdata/fgdata/logs/elasticsearch
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: [“fgedu-es01”, “fgedu-es02”, “fgedu-es03”]
cluster.initial_master_nodes: [“fgedu-es01”, “fgedu-es02”, “fgedu-es03”]
EOF

# 4. 创建目录
mkdir -p /bigdata/fgdata/elasticsearch/data
mkdir -p /bigdata/fgdata/logs/elasticsearch
useradd elasticsearch
chown -R elasticsearch:elasticsearch /bigdata/app/elasticsearch
chown -R elasticsearch:elasticsearch /bigdata/fgdata/elasticsearch
chown -R elasticsearch:elasticsearch /bigdata/fgdata/logs/elasticsearch

# 5. 启动Elasticsearch
su – elasticsearch
cd /bigdata/app/elasticsearch
nohup ./bin/elasticsearch &

# 6. 验证Elasticsearch
curl http://localhost:9200

# 7. 安装Kibana
cd /bigdata/app
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.17.10-linux-x86_64.tar.gz
tar -zxvf kibana-7.17.10-linux-x86_64.tar.gz
ln -s kibana-7.17.10-linux-x86_64 kibana

# 8. 配置kibana.yml
cat > /bigdata/app/kibana/config/kibana.yml << ‘EOF’
server.host: “0.0.0.0”
server.port: 5601
elasticsearch.hosts: [“http://fgedu-es01:9200”]
EOF

# 9. 启动Kibana
cd /bigdata/app/kibana
nohup ./bin/kibana &

# 10. 安装Fluentd
curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent4.sh | sh

# 11. 配置Fluentd
cat > /etc/td-agent/td-agent.conf << ‘EOF’
<source>
@type tail
path /bigdata/fgdata/logs/hadoop/hadoop-*.log
pos_file /var/log/td-agent/hadoop.log.pos
tag hadoop.log
<parse>
@type none
</parse>
</source>

<source>
@type tail
path /bigdata/fgdata/logs/yarn/yarn-*.log
pos_file /var/log/td-agent/yarn.log.pos
tag yarn.log
<parse>
@type none
</parse>
</source>

<match hadoop.log>
@type elasticsearch
host fgedu-es01
port 9200
logstash_format true
logstash_prefix hadoop
flush_interval 5s
</match>

<match yarn.log>
@type elasticsearch
host fgedu-es01
port 9200
logstash_format true
logstash_prefix yarn
flush_interval 5s
</match>
EOF

# 12. 启动Fluentd
systemctl start td-agent
systemctl enable td-agent

# 2. 编译Atlas（跳过测试）
mvn clean -DskipTests package -Pdist,embedded-hbase-solr

# 3. 解压编译后的包
cd distro/target
tar -zxvf apache-atlas-2.3.0-server.tar.gz
mv apache-atlas-2.3.0 /bigdata/app/
ln -s /bigdata/app/apache-atlas-2.3.0 /bigdata/app/atlas

# 4. 配置Atlas
cd /bigdata/app/atlas
vi conf/atlas-application.properties

# 关键配置
atlas.server.address=fgedu-atlas
atlas.server.http.port=21000
atlas.graph.storage.backend=hbase2
atlas.graph.storage.hostname=fgedu-hbase
atlas.graph.storage.hbase.table=atlas_janus
atlas.kafka.zookeeper.connect=fgedu-zk:2181
atlas.notification.embedded=false
atlas.kafka.bootstrap.servers=fgedu-kafka:9092

# 5. 启动Atlas
bin/atlas_start.py

# 6. 访问Atlas
# http://fgedu-atlas:21000
# 默认用户名: admin
# 默认密码: admin

# 7. 配置Hive Hook
# 在hive-site.xml中添加
<property>
<name>hive.exec.post.hooks</name>
<value>org.apache.atlas.hive.hook.HiveHook</value>
</property>

# 复制Atlas配置到Hive
cp /bigdata/app/atlas/conf/atlas-application.properties /bigdata/app/hive/conf/
cp /bigdata/app/atlas/hook/hive/*.jar /bigdata/app/hive/lib/

# 8. 重启Hive
hive –service hiveserver2 –stop
hive –service hiveserver2 –start

# 9. 测试Hive元数据同步
beeline
> CREATE TABLE fgedu_test (id INT, name STRING);
> INSERT INTO fgedu_test VALUES (1, ‘fgedu’);

# 10. 在Atlas中查看元数据
# 访问Atlas UI，搜索fgedu_test表