## 1. 基础配置
$ cat > prometheus.yml << EOF global: scrape_interval: 15s evaluation_interval: 15s alerting: alertmanagers: - static_configs: - targets: ['localhost:9093'] rule_files: - "rules/*.yml" scrape_configs: - job_name: 'tidb' static_configs: - targets: ['192.168.1.100:9090'] - job_name: 'tikv' static_configs: - targets: ['192.168.1.101:9091', '192.168.1.102:9091', '192.168.1.103:9091'] - job_name: 'pd' static_configs: - targets: ['192.168.1.100:9092'] - job_name: 'node' static_configs: - targets: ['192.168.1.100:9100', '192.168.1.101:9100', '192.168.1.102:9100', '192.168.1.103:9100'] EOF ## 2. 启动Prometheus $ prometheus --config.file=prometheus.yml --storage.tsdb.path=/tidb/prometheus/data --storage.tsdb.retention.time=15d ## 3. 验证Prometheus # 访问 http://localhost:9090 # 查看Targets页面，确认所有目标都已成功采集 ## 4. 配置Prometheus高可用 $ cat > prometheus-ha.yml << EOF global: scrape_interval: 15s evaluation_interval: 15s alerting: alertmanagers: - static_configs: - targets: ['localhost:9093'] rule_files: - "rules/*.yml" scrape_configs: - job_name: 'tidb' static_configs: - targets: ['192.168.1.100:9090'] - job_name: 'tikv' static_configs: - targets: ['192.168.1.101:9091', '192.168.1.102:9091', '192.168.1.103:9091'] - job_name: 'pd' static_configs: - targets: ['192.168.1.100:9092'] - job_name: 'node' static_configs: - targets: ['192.168.1.100:9100', '192.168.1.101:9100', '192.168.1.102:9100', '192.168.1.103:9100'] - job_name: 'prometheus' static_configs: - targets: ['localhost:9090', '192.168.1.104:9090'] EOF

## 1. 基础配置
$ cat > grafana.ini << EOF [server] http_port = 3000 domain = localhost [database] type = sqlite3 path = /tidb/grafana/data/grafana.db [session] provider = file [analytics] check_for_updates = true [security] admin_user = admin admin_password = admin [smtp] enabled = true host = smtp.example.com:587 user = admin@example.com password = password from_address = admin@example.com from_name = Grafana EOF ## 2. 启动Grafana $ grafana-server --config=grafana.ini --homepath=/tidb/grafana ## 3. 配置数据源 # 访问 http://localhost:3000 # 登录（默认用户名：admin，密码：admin） # 配置 Prometheus 数据源： # - 名称：Prometheus # - URL：http://localhost:9090 # - 访问：服务器 ## 4. 导入面板 # 导入 TiDB 官方面板： # - TiDB Overview：12900 # - TiDB Details：12901 # - TiKV Details：12902 # - PD Details：12903 # - Node Exporter Full：1860

## 1. 基础配置
$ cat > alertmanager.yml << EOF global: resolve_timeout: 5m smtp_smarthost: 'smtp.example.com:587' smtp_from: 'admin@example.com' smtp_auth_username: 'admin@example.com' smtp_auth_password: 'password' route: group_by: ['alertname', 'cluster', 'service'] group_wait: 30s group_interval: 5m repeat_interval: 1h receiver: 'email' routes: - match: severity: critical receiver: 'email' receivers: - name: 'email' email_configs: - to: 'admin@example.com' send_resolved: true inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'cluster', 'service'] EOF ## 2. 启动Alertmanager $ alertmanager --config.file=alertmanager.yml --storage.path=/tidb/alertmanager/data ## 3. 验证Alertmanager # 访问 http://localhost:9093 # 查看Alerts页面，确认告警规则已加载 ## 4. 配置告警规则 $ mkdir -p /tidb/prometheus/rules $ cat > /tidb/prometheus/rules/tidb.rules.yml << EOF groups: - name: tidb_alerts rules: - alert: TiDBDown expr: up{job="tidb"} == 0 for: 5m labels: severity: critical annotations: summary: "TiDB 实例宕机" description: "TiDB 实例 {{ labels.instance }} 已宕机超过 5 分钟" - alert: TiDBHighCPU expr: (sum by(instance) (rate(process_cpu_seconds_total{job="tidb"}[5m])) / (count by(instance) (count_processes{job="tidb"})) * 100 > 80
for: 5m
labels:
severity: warning
annotations:
summary: “TiDB CPU 使用率过高”
description: “TiDB 实例 {{ labels.instance }} CPU 使用率超过 80%”

– alert: TiDBHighMemory
expr: (sum by(instance) (process_resident_memory_bytes{job=”tidb”})) / (sum by(instance) (machine_memory_bytes{job=”node”})) * 100 > 80
for: 5m
labels:
severity: warning
annotations:
summary: “TiDB 内存使用率过高”
description: “TiDB 实例 {{ labels.instance }} 内存使用率超过 80%”
EOF

## 1. 业务相关告警规则
$ cat > /tidb/prometheus/rules/business.rules.yml << EOF groups: - name: business_alerts rules: - alert: HighQPS expr: sum(rate(tidb_server_qps{job="tidb"}[5m])) by(instance) > 10000
for: 5m
labels:
severity: warning
annotations:
summary: “TiDB QPS 过高”
description: “TiDB 实例 {{ labels.instance }} QPS 超过 10000″

– alert: SlowQuery
expr: increase(tidb_server_slow_query_count{job=”tidb”}[5m]) > 10
for: 5m
labels:
severity: warning
annotations:
summary: “TiDB 慢查询过多”
description: “TiDB 实例 {{ labels.instance }} 5分钟内慢查询数超过 10″

– alert: ConnectionOverflow
expr: sum(tidb_server_connections{job=”tidb”}) by(instance) > 5000
for: 5m
labels:
severity: warning
annotations:
summary: “TiDB 连接数过多”
description: “TiDB 实例 {{ labels.instance }} 连接数超过 5000”
EOF

## 2. 存储相关告警规则
$ cat > /tidb/prometheus/rules/storage.rules.yml << EOF groups: - name: storage_alerts rules: - alert: TiKVDiskFull expr: (1 - (node_filesystem_free_bytes{job="node", mountpoint="/tidb/fgdata"} / node_filesystem_size_bytes{job="node", mountpoint="/tidb/fgdata"})) * 100 > 80
for: 5m
labels:
severity: warning
annotations:
summary: “TiKV 磁盘空间不足”
description: “TiKV 实例 {{ labels.instance }} 磁盘使用率超过 80%”

– alert: TiKVHighIO
expr: sum by(instance) (rate(node_disk_io_time_seconds_total{job=”node”, device=~”sd.*|nvme.*”}[5m])) > 0.8
for: 5m
labels:
severity: warning
annotations:
summary: “TiKV I/O 使用率过高”
description: “TiKV 实例 {{ labels.instance }} I/O 使用率超过 80%”

– alert: TiKVRaftLogGap
expr: tikv_raftstore_raft_log_gap{job=”tikv”} > 1000
for: 5m
labels:
severity: warning
annotations:
summary: “TiKV Raft 日志差距过大”
description: “TiKV 实例 {{ labels.instance }} Raft 日志差距超过 1000”
EOF

## 1. 创建自定义面板
# 访问 http://localhost:3000
# 点击 “+” → “Dashboard”
# 点击 “Add new panel”

## 2. 配置面板
# 面板标题：TiDB 业务指标
# 数据源：Prometheus
# 查询语句：sum(rate(tidb_server_qps{job=”tidb”}[5m])) by(instance)
# 图表类型：Graph
# 轴标签：QPS
# 单位：ops

## 3. 保存面板
# 点击 “Save Dashboard”
# 输入面板名称：TiDB 业务监控
# 点击 “Save”

## 4. 导出面板
# 点击面板右上角的 “Settings”
# 点击 “JSON Model”
# 复制 JSON 内容
# 保存到文件：tidb-business-dashboard.json

## 5. 导入面板
# 点击 “+” → “Import”
# 上传 tidb-business-dashboard.json 文件
# 点击 “Import”

## 1. 配置邮件通知
$ cat > alertmanager.yml << EOF global: resolve_timeout: 5m smtp_smarthost: 'smtp.example.com:587' smtp_from: 'admin@example.com' smtp_auth_username: 'admin@example.com' smtp_auth_password: 'password' route: group_by: ['alertname', 'cluster', 'service'] group_wait: 30s group_interval: 5m repeat_interval: 1h receiver: 'email' receivers: - name: 'email' email_configs: - to: 'admin@example.com' send_resolved: true html: '{{ template "email.default.html" . }}' inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'cluster', 'service'] EOF ## 2. 配置微信通知 $ cat > alertmanager.yml << EOF global: resolve_timeout: 5m route: group_by: ['alertname', 'cluster', 'service'] group_wait: 30s group_interval: 5m repeat_interval: 1h receiver: 'wechat' receivers: - name: 'wechat' wechat_configs: - corp_id: 'your_corp_id' api_url: 'https://qyapi.weixin.qq.com/cgi-bin/' to_party: 'your_party_id' agent_id: 'your_agent_id' api_secret: 'your_api_secret' message: title: '{{ template "wechat.default.title" . }}' description: '{{ template "wechat.default.content" . }}' inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'cluster', 'service'] EOF ## 3. 配置短信通知 $ cat > alertmanager.yml << EOF global: resolve_timeout: 5m route: group_by: ['alertname', 'cluster', 'service'] group_wait: 30s group_interval: 5m repeat_interval: 1h receiver: 'sms' receivers: - name: 'sms' webhook_configs: - url: 'http://localhost:8080/sms' send_resolved: true inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'cluster', 'service'] EOF

## 案例1：Prometheus高可用配置

### 步骤1：配置Prometheus主节点
$ cat > prometheus-primary.yml << EOF global: scrape_interval: 15s evaluation_interval: 15s alerting: alertmanagers: - static_configs: - targets: ['localhost:9093'] rule_files: - "rules/*.yml" scrape_configs: - job_name: 'tidb' static_configs: - targets: ['192.168.1.100:9090'] - job_name: 'tikv' static_configs: - targets: ['192.168.1.101:9091', '192.168.1.102:9091', '192.168.1.103:9091'] - job_name: 'pd' static_configs: - targets: ['192.168.1.100:9092'] - job_name: 'node' static_configs: - targets: ['192.168.1.100:9100', '192.168.1.101:9100', '192.168.1.102:9100', '192.168.1.103:9100'] - job_name: 'prometheus' static_configs: - targets: ['localhost:9090', '192.168.1.104:9090'] EOF ### 步骤2：配置Prometheus从节点 $ cat > prometheus-secondary.yml << EOF global: scrape_interval: 15s evaluation_interval: 15s alerting: alertmanagers: - static_configs: - targets: ['localhost:9093'] rule_files: - "rules/*.yml" scrape_configs: - job_name: 'tidb' static_configs: - targets: ['192.168.1.100:9090'] - job_name: 'tikv' static_configs: - targets: ['192.168.1.101:9091', '192.168.1.102:9091', '192.168.1.103:9091'] - job_name: 'pd' static_configs: - targets: ['192.168.1.100:9092'] - job_name: 'node' static_configs: - targets: ['192.168.1.100:9100', '192.168.1.101:9100', '192.168.1.102:9100', '192.168.1.103:9100'] - job_name: 'prometheus' static_configs: - targets: ['localhost:9090', '192.168.1.100:9090'] EOF ### 步骤3：启动Prometheus节点 # 启动主节点 $ prometheus --config.file=prometheus-primary.yml --storage.tsdb.path=/tidb/prometheus/primary/data --storage.tsdb.retention.time=15d # 启动从节点 $ prometheus --config.file=prometheus-secondary.yml --storage.tsdb.path=/tidb/prometheus/secondary/data --storage.tsdb.retention.time=15d ## 案例2：Grafana自定义面板 ### 步骤1：创建业务监控面板 # 访问 http://localhost:3000 # 点击 "+" → "Dashboard" # 点击 "Add new panel" ### 步骤2：配置面板 # 面板标题：TiDB 业务指标 # 数据源：Prometheus # 查询语句：sum(rate(tidb_server_qps{job="tidb"}[5m])) by(instance) # 图表类型：Graph # 轴标签：QPS # 单位：ops ### 步骤3：添加更多指标 # 点击 "Add Query" # 查询语句：sum(rate(tidb_server_slow_query_count{job="tidb"}[5m])) by(instance) # 轴标签：慢查询数 # 单位：count ### 步骤4：保存面板 # 点击 "Save Dashboard" # 输入面板名称：TiDB 业务监控 # 点击 "Save"

## 案例1：核心组件告警

### 步骤1：创建核心组件告警规则
$ cat > /tidb/prometheus/rules/core.rules.yml << EOF groups: - name: core_alerts rules: - alert: TiDBDown expr: up{job="tidb"} == 0 for: 5m labels: severity: critical annotations: summary: "TiDB 实例宕机" description: "TiDB 实例 {{ labels.instance }} 已宕机超过 5 分钟" - alert: TiKVDown expr: up{job="tikv"} == 0 for: 5m labels: severity: critical annotations: summary: "TiKV 实例宕机" description: "TiKV 实例 {{ labels.instance }} 已宕机超过 5 分钟" - alert: PDDown expr: up{job="pd"} == 0 for: 5m labels: severity: critical annotations: summary: "PD 实例宕机" description: "PD 实例 {{ labels.instance }} 已宕机超过 5 分钟" EOF ### 步骤2：配置告警通知 $ cat > alertmanager.yml << EOF global: resolve_timeout: 5m smtp_smarthost: 'smtp.example.com:587' smtp_from: 'admin@example.com' smtp_auth_username: 'admin@example.com' smtp_auth_password: 'password' route: group_by: ['alertname', 'cluster', 'service'] group_wait: 30s group_interval: 5m repeat_interval: 1h receiver: 'email' routes: - match: severity: critical receiver: 'email' continue: true - match: severity: warning receiver: 'email' receivers: - name: 'email' email_configs: - to: 'admin@example.com' send_resolved: true inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'cluster', 'service'] EOF ## 案例2：性能告警 ### 步骤1：创建性能告警规则 $ cat > /tidb/prometheus/rules/performance.rules.yml << EOF groups: - name: performance_alerts rules: - alert: HighCPU expr: (sum by(instance) (rate(process_cpu_seconds_total{job="tidb"}[5m])) / (count by(instance) (count_processes{job="tidb"})) * 100 > 80
for: 5m
labels:
severity: warning
annotations:
summary: “CPU 使用率过高”
description: “实例 {{ labels.instance }} CPU 使用率超过 80%”

– alert: HighMemory
expr: (sum by(instance) (process_resident_memory_bytes{job=”tidb”})) / (sum by(instance) (machine_memory_bytes{job=”node”})) * 100 > 80
for: 5m
labels:
severity: warning
annotations:
summary: “内存使用率过高”
description: “实例 {{ labels.instance }} 内存使用率超过 80%”

– alert: HighDisk
expr: (1 – (node_filesystem_free_bytes{job=”node”, mountpoint=”/tidb/fgdata”} / node_filesystem_size_bytes{job=”node”, mountpoint=”/tidb/fgdata”})) * 100 > 80
for: 5m
labels:
severity: warning
annotations:
summary: “磁盘使用率过高”
description: “实例 {{ labels.instance }} 磁盘使用率超过 80%”
EOF

## 案例1：自定义监控指标

### 步骤1：创建自定义指标脚本
$ cat > custom_metrics.sh << EOF #!/bin/bash # custom_metrics.sh # from:www.itpux.com.qq113257174.wx:itpux-com # web: http://www.fgedu.net.cn # 采集业务指标 QPS=$(mysql -h 192.168.1.100 -P 4000 -u root -p password -e "SHOW GLOBAL STATUS LIKE 'Queries';" | grep Queries | awk '{print $2}') SLOW_QUERIES=$(mysql -h 192.168.1.100 -P 4000 -u root -p password -e "SHOW GLOBAL STATUS LIKE 'Slow_queries';" | grep Slow_queries | awk '{print $2}') # 输出指标 echo "# HELP tidb_business_qps Total QPS" echo "# TYPE tidb_business_qps counter" echo "tidb_business_qps $QPS" echo "# HELP tidb_business_slow_queries Total slow queries" echo "# TYPE tidb_business_slow_queries counter" echo "tidb_business_slow_queries $SLOW_QUERIES" EOF ### 步骤2：配置Prometheus采集自定义指标 $ cat > prometheus.yml << EOF global: scrape_interval: 15s evaluation_interval: 15s alerting: alertmanagers: - static_configs: - targets: ['localhost:9093'] rule_files: - "rules/*.yml" scrape_configs: - job_name: 'tidb' static_configs: - targets: ['192.168.1.100:9090'] - job_name: 'custom' static_configs: - targets: ['localhost:9101'] metrics_path: '/metrics' EOF ### 步骤3：启动自定义指标服务 $ nohup bash -c 'while true; do ./custom_metrics.sh > /tidb/custom/metrics.prom; sleep 15; done’ &
$ python3 -m http.server 9101 –directory /tidb/custom

## 案例2：自定义告警通知

### 步骤1：创建告警通知脚本
$ cat > alert_notify.py << EOF #!/usr/bin/env python3 # alert_notify.py # from:www.itpux.com.qq113257174.wx:itpux-com # web: http://www.fgedu.net.cn import json import sys import requests # 读取告警数据 alert_data = json.loads(sys.stdin.read()) # 处理告警 for alert in alert_data['alerts']: status = alert['status'] labels = alert['labels'] annotations = alert['annotations'] # 发送通知 print(f"Status: {status}") print(f"Alertname: {labels.get('alertname')}") print(f"Severity: {labels.get('severity')}") print(f"Summary: {annotations.get('summary')}") print(f"Description: {annotations.get('description')}") print("---") EOF ### 步骤2：配置Alertmanager使用自定义通知 $ cat > alertmanager.yml << EOF global: resolve_timeout: 5m route: group_by: ['alertname', 'cluster', 'service'] group_wait: 30s group_interval: 5m repeat_interval: 1h receiver: 'custom' receivers: - name: 'custom' webhook_configs: - url: 'http://localhost:8080/notify' send_resolved: true inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'cluster', 'service'] EOF ### 步骤3：启动通知服务 $ cat > notify_server.py << EOF #!/usr/bin/env python3 from http.server import HTTPServer, BaseHTTPRequestHandler import json import subprocess class NotifyHandler(BaseHTTPRequestHandler): def do_POST(self): content_length = int(self.headers['Content-Length']) post_data = self.rfile.read(content_length) # 调用通知脚本 process = subprocess.Popen(['python3', 'alert_notify.py'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) stdout, stderr = process.communicate(input=post_data.decode('utf-8')) # 响应 self.send_response(200) self.end_headers() self.wfile.write(b'OK') if __name__ == '__main__': server = HTTPServer(('localhost', 8080), NotifyHandler) server.serve_forever() EOF $ nohup python3 notify_server.py &