# 环境准备
– 操作系统：CentOS 7.9 / RHEL 7.9
– CPU：8核及以上
– 内存：16GB及以上
– 存储：500GB及以上
– 网络：千兆网络及以上

# 依赖安装
$ yum install -y python3 python3-pip git docker docker-compose
$ pip3 install prometheus-client grafana-api elasticsearch python-kafka tensorflow scikit-learn

# 代码获取
$ git clone https://github.com/fgedu/tidb-intelligent-ops.git
$ cd tidb-intelligent-ops

# 配置Docker Compose
$ cat > docker-compose.yml << EOF version: '3' services: prometheus: image: prom/prometheus:v2.30.3 volumes: - ./prometheus.yml:/etc/prometheus/prometheus.yml - ./data/prometheus:/prometheus ports: - "9090:9090" restart: always grafana: image: grafana/grafana:8.3.3 volumes: - ./data/grafana:/var/lib/grafana ports: - "3000:3000" restart: always elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 volumes: - ./data/elasticsearch:/usr/share/elasticsearch/data ports: - "9200:9200" - "9300:9300" environment: - discovery.type=single-node - ES_JAVA_OPTS=-Xms4g -Xmx4g restart: always kibana: image: docker.elastic.co/kibana/kibana:7.17.0 ports: - "5601:5601" restart: always alertmanager: image: prom/alertmanager:v0.23.0 volumes: - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml ports: - "9093:9093" restart: always EOF # 启动服务 $ docker-compose up -d

# 配置Prometheus
$ cat > prometheus.yml << EOF global: scrape_interval: 15s evaluation_interval: 15s alerting: alertmanagers: - static_configs: - targets: ['alertmanager:9093'] rule_files: - "rules/*.yml" scrape_configs: - job_name: 'tidb' static_configs: - targets: ['192.168.1.100:10080'] - job_name: 'tikv' static_configs: - targets: ['192.168.1.101:20180', '192.168.1.102:20180', '192.168.1.103:20180'] - job_name: 'pd' static_configs: - targets: ['192.168.1.100:2379'] - job_name: 'node_exporter' static_configs: - targets: ['192.168.1.100:9100', '192.168.1.101:9100', '192.168.1.102:9100', '192.168.1.103:9100'] EOF # 配置告警规则 $ mkdir -p rules $ cat > rules/tidb-alerts.yml << EOF groups: - name: tidb-alerts rules: - alert: TiDBDown expr: up{job="tidb"} == 0 for: 5m labels: severity: critical annotations: summary: "TiDB 实例宕机" description: "TiDB 实例 {{ labels.instance }} 已宕机超过 5 分钟" - alert: TiKVDown expr: up{job="tikv"} == 0 for: 5m labels: severity: critical annotations: summary: "TiKV 实例宕机" description: "TiKV 实例 {{ labels.instance }} 已宕机超过 5 分钟" - alert: PDDown expr: up{job="pd"} == 0 for: 5m labels: severity: critical annotations: summary: "PD 实例宕机" description: "PD 实例 {{ labels.instance }} 已宕机超过 5 分钟" - alert: HighCPUUsage expr: (100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)) > 80
for: 5m
labels:
severity: warning
annotations:
summary: “CPU 使用率过高”
description: “实例 {{ labels.instance }} CPU 使用率超过 80% 已持续 5 分钟”

– alert: HighMemoryUsage
expr: (node_memory_MemTotal_bytes – node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 80
for: 5m
labels:
severity: warning
annotations:
summary: “内存使用率过高”
description: “实例 {{ labels.instance }} 内存使用率超过 80% 已持续 5 分钟”
EOF

# 重启Prometheus
$ docker-compose restart prometheus

# 配置Grafana
# 1. 打开Grafana界面（http://localhost:3000）
# 2. 登录（默认用户名：admin，密码：admin）
# 3. 添加数据源：
# – 名称：Prometheus
# – 类型：Prometheus
# – URL：http://prometheus:9090
# – 点击Save & Test

# 4. 导入仪表盘：
# – 点击+ → Import
# – 输入仪表盘ID：12900（TiDB集群监控仪表盘）
# – 选择数据源：Prometheus
# – 点击Import

# 5. 创建自定义仪表盘：
# – 点击+ → Dashboard
# – 添加面板
# – 配置查询和可视化
# – 保存仪表盘

# 配置异常检测服务
$ cat > detection_service.py << EOF #!/usr/bin/env python3 # detection_service.py # from:www.itpux.com.qq113257174.wx:itpux-com # web: `http://www.fgedu.net.cn` import time import numpy as np import pandas as pd from sklearn.ensemble import IsolationForest from prometheus_api_client import PrometheusConnect # 连接Prometheus prom = PrometheusConnect(url="http://localhost:9090", disable_ssl=True) # 异常检测模型 model = IsolationForest(contamination=0.01, random_state=42) # 采集数据 def collect_data(): # 采集TiDB QPS qps_query = 'rate(tidb_server_qps[5m])' qps_data = prom.custom_query(qps_query) # 采集TiKV操作数 tikv_query = 'rate(tikv_engine_write_total[5m])' tikv_data = prom.custom_query(tikv_query) # 采集PD leader pd_query = 'pd_server_is_leader{instance="192.168.1.100:2379"}' pd_data = prom.custom_query(pd_query) # 处理数据 data = [] for item in qps_data: data.append({ 'timestamp': time.time(), 'metric': 'tidb_qps', 'value': float(item['value'][1]) }) for item in tikv_data: data.append({ 'timestamp': time.time(), 'metric': 'tikv_ops', 'value': float(item['value'][1]) }) for item in pd_data: data.append({ 'timestamp': time.time(), 'metric': 'pd_leader', 'value': float(item['value'][1]) }) return pd.DataFrame(data) # 训练模型 def train_model(): data = collect_data() if len(data) < 100: return False # 准备训练数据 X = data.pivot(index='timestamp', columns='metric', values='value').fillna(0) # 训练模型 model.fit(X) return True # 检测异常 def detect_anomaly(): data = collect_data() if len(data) < 10: return [] # 准备检测数据 X = data.pivot(index='timestamp', columns='metric', values='value').fillna(0) # 检测异常 predictions = model.predict(X) anomalies = X[predictions == -1] # 处理异常结果 anomaly_list = [] for idx, row in anomalies.iterrows(): anomaly_list.append({ 'timestamp': idx, 'metrics': row.to_dict() }) return anomaly_list # 主函数 def main(): # 训练模型 if train_model(): print("模型训练成功") else: print("数据不足，模型训练失败") return # 持续检测 while True: anomalies = detect_anomaly() if anomalies: print(f"发现异常: {anomalies}") # 发送告警 # send_alert(anomalies) time.sleep(60) if __name__ == "__main__": main() EOF # 启动异常检测服务 $ python3 detection_service.py &

# 配置Alertmanager
$ cat > alertmanager.yml << EOF global: resolve_timeout: 5m smtp_smarthost: 'smtp.example.com:587' smtp_from: 'alertmanager@fgedu.net.cn' smtp_auth_username: 'alertmanager' smtp_auth_password: 'password' route: group_by: ['alertname'] group_wait: 30s group_interval: 5m repeat_interval: 1h receiver: 'email' receivers: - name: 'email' email_configs: - to: 'admin@fgedu.net.cn' send_resolved: true inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'instance'] EOF # 重启Alertmanager $ docker-compose restart alertmanager

# 集成测试

# 1. 功能测试
# 测试数据采集
$ curl http://localhost:9090/api/v1/query?query=tidb_server_qps

# 测试异常检测
$ python3 -c “from detection_service import detect_anomaly; print(detect_anomaly())”

# 测试告警通知
$ curl -X POST http://localhost:9093/api/v1/alerts -H “Content-Type: application/json” -d ‘[
{
“status”: “firing”,
“labels”: {
“alertname”: “TestAlert”,
“severity”: “critical”
},
“annotations”: {
“summary”: “测试告警”,
“description”: “这是一个测试告警”
}
}
]’

# 2. 性能测试
# 测试数据采集性能
$ time curl http://localhost:9090/api/v1/query_range?query=tidb_server_qps&start=now-1h&end=now&step=1m

# 测试异常检测性能
$ time python3 -c “from detection_service import detect_anomaly; detect_anomaly()”

# 3. 稳定性测试
# 持续运行24小时，检查系统稳定性
$ nohup python3 detection_service.py > detection.log 2>&1 &
# 24小时后检查日志
$ tail -n 100 detection.log