# 步骤1：准备服务器
$ hostnamectl set-hostname prometheus.fgedu.net.cn
$ ip addr add 192.168.1.50/24 dev eth0
$ systemctl restart network

# 步骤2：安装Prometheus
$ cd /tidb/app
$ wget https://github.com/prometheus/prometheus/releases/download/v2.45.0/prometheus-2.45.0.linux-amd64.tar.gz
$ tar -xzf prometheus-2.45.0.linux-amd64.tar.gz
$ mv prometheus-2.45.0.linux-amd64 prometheus

# 步骤3：创建配置文件
$ mkdir -p /tidb/app/prometheus/alerts
$ cat > /tidb/app/prometheus/prometheus.yml << EOF global: scrape_interval: 15s evaluation_interval: 15s alerting: alertmanagers: - static_configs: - targets: - localhost:9093 rule_files: - "alerts/*.yml" scrape_configs: - job_name: 'tidb' static_configs: - targets: ['192.168.1.10:10080', '192.168.1.11:10080'] - job_name: 'tikv' static_configs: - targets: ['192.168.1.20:20180', '192.168.1.21:20180', '192.168.1.22:20180'] - job_name: 'pd' static_configs: - targets: ['192.168.1.30:2379', '192.168.1.31:2379', '192.168.1.32:2379'] - job_name: 'tiflash' static_configs: - targets: ['192.168.1.40:20292', '192.168.1.41:20292'] - job_name: 'node' static_configs: - targets: ['192.168.1.10:9100', '192.168.1.11:9100', '192.168.1.20:9100', '192.168.1.21:9100', '192.168.1.22:9100', '192.168.1.30:9100', '192.168.1.31:9100', '192.168.1.32:9100', '192.168.1.40:9100', '192.168.1.41:9100'] EOF # 步骤4：创建系统服务 $ cat > /etc/systemd/system/prometheus.service << EOF [Unit] Description=Prometheus After=network.target [Service] Type=simple ExecStart=/tidb/app/prometheus/prometheus --config.file=/tidb/app/prometheus/prometheus.yml --storage.tsdb.path=/tidb/fgdata/prometheus --storage.tsdb.retention.time=15d Restart=on-failure [Install] WantedBy=multi-user.target EOF # 步骤5：启动服务 $ systemctl daemon-reload $ systemctl enable prometheus $ systemctl start prometheus # 步骤6：验证服务状态 $ systemctl status prometheus # 输出示例 ● prometheus.service - Prometheus Loaded: loaded (/etc/systemd/system/prometheus.service; enabled; vendor preset: disabled) Active: active (running) since Wed 2026-04-09 10:00:00 CST; 5min ago Main PID: 12345 (prometheus) Tasks: 10 Memory: 256.0M CPU: 10% CGroup: /system.slice/prometheus.service └─12345 /tidb/app/prometheus/prometheus --config.file=/tidb/app/prometheus/prometheus.yml --storage.tsdb.path=/tidb/fgdata/prometheus --storage.tsdb.retention.time=15d

# 步骤1：准备两台服务器
# Server1: 192.168.1.50
# Server2: 192.168.1.51

# 步骤2：在两台服务器上安装Prometheus（步骤同单实例部署）

# 步骤3：配置负载均衡（使用HAProxy）
$ yum install haproxy -y
$ cat > /etc/haproxy/haproxy.cfg << EOF global log 127.0.0.1 local0 maxconn 4000 user haproxy group haproxy daemon defaults log global mode http option httplog option dontlognull retries 3 timeout connect 5000 timeout client 50000 timeout server 50000 frontend prometheus bind *:9090 default_backend prometheus_servers backend prometheus_servers balance roundrobin server prometheus1 192.168.1.50:9090 check server prometheus2 192.168.1.51:9090 check EOF $ systemctl enable haproxy $ systemctl start haproxy # 步骤4：配置远程存储（使用Thanos） # 安装Thanos $ wget https://github.com/thanos-io/thanos/releases/download/v0.32.2/thanos-0.32.2.linux-amd64.tar.gz $ tar -xzf thanos-0.32.2.linux-amd64.tar.gz $ mv thanos-0.32.2.linux-amd64/thanos /tidb/app/bin/ # 启动Thanos Sidecar $ /tidb/app/bin/thanos sidecar \ --prometheus.url=http://localhost:9090 \ --tsdb.path=/tidb/fgdata/prometheus \ --objstore.config-file=/tidb/app/thanos/objstore.yml # objstore.yml配置示例 type: S3 config: bucket: "thanos" endpoint: "s3.amazonaws.com" access_key: "access-key" secret_key: "secret-key"

# 访问Prometheus UI
http://192.168.1.50:9090

# 常用PromQL查询示例

## 查询TiDB QPS
sum(rate(tidb_server_qps[5m])) by (instance)

## 查询TiKV读写延迟
histogram_quantile(0.99, sum(rate(tikv_server_request_duration_seconds_bucket[5m])) by (instance, le))

## 查询PD集群状态
pd_cluster_status{type=”health”}

## 查询主机CPU使用率
average(cpu_usage{mode=”idle”}) by (instance) * 100

## 查询磁盘使用率
100 – (node_filesystem_free_bytes{mountpoint=”/”} / node_filesystem_size_bytes{mountpoint=”/”} * 100)

## 查询网络流量
rate(node_network_receive_bytes_total[5m]) by (instance)
rate(node_network_transmit_bytes_total[5m]) by (instance)

# 创建告警规则文件
$ cat > /tidb/app/prometheus/alerts/tidb_alerts.yml << EOF groups: - name: tidb rules: - alert: TiDBDown expr: up{job="tidb"} == 0 for: 5m labels: severity: critical annotations: summary: "TiDB down" description: "TiDB instance {{ labels.instance }} is down for more than 5 minutes" - alert: TiDBHighQPS expr: sum(rate(tidb_server_qps[5m])) by (instance) > 10000
for: 5m
labels:
severity: warning
annotations:
summary: “TiDB high QPS”
description: “TiDB instance {{ labels.instance }} has high QPS: {{ value }}”

– alert: TiDBHighErrorRate
expr: sum(rate(tidb_server_errors_total[5m])) by (instance) / sum(rate(tidb_server_qps[5m])) by (instance) > 0.01
for: 5m
labels:
severity: warning
annotations:
summary: “TiDB high error rate”
description: “TiDB instance {{ labels.instance }} has high error rate: {{ value }}”
EOF

# 重新加载配置
$ curl -X POST http://localhost:9090/-/reload

# 查看告警状态
http://192.168.1.50:9090/alerts

# 问题现象：Prometheus启动失败

# 排查步骤：
1. 查看系统日志
$ journalctl -u prometheus

2. 检查配置文件
$ /tidb/app/prometheus/promtool check config /tidb/app/prometheus/prometheus.yml

3. 检查端口占用
$ netstat -tlnp | grep 9090

4. 检查磁盘空间
$ df -h

5. 检查内存使用
$ free -m

# 常见错误及解决方案：
– 配置文件错误：检查配置文件格式，使用promtool验证
– 端口占用：停止占用端口的进程或修改Prometheus端口
– 磁盘空间不足：清理磁盘空间或修改存储路径
– 内存不足：增加服务器内存或调整Prometheus配置

# 问题现象：监控数据丢失

# 排查步骤：
1. 检查Prometheus存储配置
$ grep storage /tidb/app/prometheus/prometheus.yml

2. 检查存储路径权限
$ ls -la /tidb/fgdata/prometheus/

3. 检查磁盘空间
$ df -h /tidb/fgdata

4. 检查Prometheus日志
$ journalctl -u prometheus | grep error

5. 检查采集目标状态
$ curl http://localhost:9090/targets

# 解决方案：
– 调整存储保留时间：修改storage.tsdb.retention.time
– 清理过期数据：手动清理过期的tsdb文件
– 增加磁盘空间：扩展存储容量
– 检查采集目标：确保所有目标都能正常采集

# 问题现象：PromQL查询缓慢

# 排查步骤：
1. 检查Prometheus内存使用
$ top -p $(pgrep prometheus)

2. 检查查询语句复杂度
$ curl http://localhost:9090/api/v1/query?query=sum(rate(tidb_server_qps[5m])) by (instance)

3. 检查存储性能
$ iostat -x

4. 检查采集频率
$ grep scrape_interval /tidb/app/prometheus/prometheus.yml

# 解决方案：
– 优化查询语句：使用更高效的PromQL表达式
– 增加内存：提高Prometheus的内存配置
– 使用SSD存储：提高存储性能
– 调整采集频率：适当降低采集频率
– 使用聚合：在查询中使用聚合函数减少数据量