## 1. 部署Prometheus
$ tiup cluster deploy fgedu-tidb-cluster 7.5.0 topology.yaml –user root -p

## 2. 配置Prometheus
$ tiup cluster edit-config fgedu-tidb-cluster
prometheus_servers:
– host: 192.168.1.100
config:
global:
scrape_interval: 15s
evaluation_interval: 15s
alerting:
alertmanagers:
– static_configs:
– targets: [‘192.168.1.100:9093’]
rule_files:
– “rules/*.yml”
scrape_configs:
– job_name: ‘tidb’
static_configs:
– targets: [‘192.168.1.100:10080’]
– job_name: ‘tikv’
static_configs:
– targets: [‘192.168.1.101:20180’, ‘192.168.1.102:20180’, ‘192.168.1.103:20180’]
– job_name: ‘pd’
static_configs:
– targets: [‘192.168.1.100:2379’]

## 3. 部署Grafana
$ tiup cluster deploy fgedu-tidb-cluster 7.5.0 topology.yaml –user root -p

## 4. 配置Grafana
$ tiup cluster edit-config fgedu-tidb-cluster
grafana_servers:
– host: 192.168.1.100
config:
server:
http_port: 3000
security:
admin_user: admin
admin_password: admin
datasources:
datasources.yaml:
apiVersion: 1
datasources:
– name: Prometheus
type: prometheus
url: http://192.168.1.100:9090
access: proxy
isDefault: true

## 5. 重启集群使配置生效
$ tiup cluster reload fgedu-tidb-cluster

## 6. 访问Grafana
# 访问 http://192.168.1.100:3000
# 用户名：admin，密码：admin

## 1. 查看默认监控指标
$ curl http://192.168.1.100:9090/metrics

## 2. 配置自定义监控指标
$ cat > /tidb/app/prometheus/rules/custom.rules.yml << EOF groups: - name: custom_rules rules: - alert: TiDBQueryLatencyHigh expr: sum(rate(tidb_server_handle_query_duration_seconds_sum[5m])) by (instance) / sum(rate(tidb_server_handle_query_duration_seconds_count[5m])) by (instance) > 0.1
for: 5m
labels:
severity: warning
annotations:
summary: “TiDB query latency high”
description: “TiDB instance {{ $labels.instance }} query latency is {{ $value }} seconds”
– alert: TiKVStorageFull
expr: tikv_store_size_bytes / tikv_store_capacity_bytes > 0.8
for: 5m
labels:
severity: warning
annotations:
summary: “TiKV storage full”
description: “TiKV instance {{ $labels.instance }} storage usage is {{ $value | humanizePercentage }}”
EOF

## 3. 重启Prometheus使配置生效
$ tiup cluster reload fgedu-tidb-cluster -R prometheus

## 4. 查看监控指标
$ curl http://192.168.1.100:9090/api/v1/query?query=tidb_server_handle_query_duration_seconds_sum

## 5. 创建Grafana仪表盘
# 访问 http://192.168.1.100:3000
# 创建新的仪表盘，添加面板，选择Prometheus数据源，输入查询语句

## 1. 部署ELK Stack
$ yum install -y docker docker-compose
$ cat > docker-compose.yml << EOF version: '3' services: elasticsearch: image: elasticsearch:7.17.0 ports: - "9200:9200" environment: - discovery.type=single-node - ES_JAVA_OPTS=-Xms1g -Xmx1g volumes: - es_data:/usr/share/elasticsearch/data logstash: image: logstash:7.17.0 ports: - "5044:5044" volumes: - ./logstash/pipeline:/usr/share/logstash/pipeline kibana: image: kibana:7.17.0 ports: - "5601:5601" environment: - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 volumes: es_data: EOF ## 2. 配置Logstash $ mkdir -p logstash/pipeline $ cat > logstash/pipeline/logstash.conf << EOF input { file { path => “/tidb/app/tidb-deploy/tidb-4000/log/tidb.log”
start_position => “beginning”
sincedb_path => “/dev/null”
}
file {
path => “/tidb/app/tidb-deploy/tikv-20160/log/tikv.log”
start_position => “beginning”
sincedb_path => “/dev/null”
}
file {
path => “/tidb/app/tidb-deploy/pd-2379/log/pd.log”
start_position => “beginning”
sincedb_path => “/dev/null”
}
}

filter {
grok {
match => { “message” => “%{TIMESTAMP_ISO8601:timestamp} \[%{DATA:level}\] \[%{DATA:component}\] \[%{DATA:file}\:%{NUMBER:line}\] %{GREEDYDATA:message}” }
}
date {
match => [ “timestamp”, “yyyy-MM-dd HH:mm:ss.SSS” ]
target => “@timestamp”
}
}

output {
elasticsearch {
hosts => [“elasticsearch:9200”]
index => “tidb-logs-%{+YYYY.MM.dd}”
}
}
EOF

## 3. 启动ELK Stack
$ docker-compose up -d

## 4. 访问Kibana
# 访问 http://192.168.1.100:5601
# 创建索引模式 tidb-logs-*，查看日志

## 5. 配置日志告警
# 在Kibana中创建告警，基于日志内容触发告警

## 1. 部署Jaeger
$ docker run -d –name jaeger \
-e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
-p 5775:5775/udp \
-p 6831:6831/udp \
-p 6832:6832/udp \
-p 5778:5778 \
-p 16686:16686 \
-p 14268:14268 \
-p 14250:14250 \
-p 9411:9411 \
jaegertracing/all-in-one:1.35

## 2. 配置TiDB启用追踪
$ tiup cluster edit-config fgedu-tidb-cluster
tidb_servers:
– host: 192.168.1.100
config:
tracing:
enable: true
jaeger:
agent-endpoint: “192.168.1.100:6831”
sampling-rate: 0.1

## 3. 重启TiDB使配置生效
$ tiup cluster reload fgedu-tidb-cluster -R tidb

## 4. 访问Jaeger UI
# 访问 http://192.168.1.100:16686
# 查看分布式追踪数据

## 5. 分析追踪数据
# 在Jaeger UI中查看请求的完整执行路径，分析性能瓶颈

## 1. 配置Alertmanager
$ tiup cluster edit-config fgedu-tidb-cluster
alertmanager_servers:
– host: 192.168.1.100
config:
global:
resolve_timeout: 5m
smtp_smarthost: ‘smtp.fgedu.net.cn:587’
smtp_from: ‘alertmanager@fgedu.net.cn’
smtp_auth_username: ‘alertmanager@fgedu.net.cn’
smtp_auth_password: ‘password’
route:
group_by: [‘alertname’]
group_wait: 30s
group_interval: 5m
repeat_interval: 1h
receiver: ’email’
routes:
– match:
severity: critical
receiver: ’email’
receivers:
– name: ’email’
email_configs:
– to: ‘admin@fgedu.net.cn’
send_resolved: true

## 2. 配置告警规则
$ cat > /tidb/app/prometheus/rules/alert.rules.yml << EOF groups: - name: tidb_alerts rules: - alert: TiDBDown expr: up{job="tidb"} == 0 for: 5m labels: severity: critical annotations: summary: "TiDB instance down" description: "TiDB instance {{ $labels.instance }} has been down for more than 5 minutes" - alert: TiKVDown expr: up{job="tikv"} == 0 for: 5m labels: severity: critical annotations: summary: "TiKV instance down" description: "TiKV instance {{ $labels.instance }} has been down for more than 5 minutes" - alert: PDDown expr: up{job="pd"} == 0 for: 5m labels: severity: critical annotations: summary: "PD instance down" description: "PD instance {{ $labels.instance }} has been down for more than 5 minutes" - alert: TiDBQueryLatencyHigh expr: sum(rate(tidb_server_handle_query_duration_seconds_sum[5m])) by (instance) / sum(rate(tidb_server_handle_query_duration_seconds_count[5m])) by (instance) > 0.1
for: 5m
labels:
severity: warning
annotations:
summary: “TiDB query latency high”
description: “TiDB instance {{ $labels.instance }} query latency is {{ $value }} seconds”
– alert: TiKVStorageFull
expr: tikv_store_size_bytes / tikv_store_capacity_bytes > 0.8
for: 5m
labels:
severity: warning
annotations:
summary: “TiKV storage full”
description: “TiKV instance {{ $labels.instance }} storage usage is {{ $value | humanizePercentage }}”
EOF

## 3. 重启Alertmanager使配置生效
$ tiup cluster reload fgedu-tidb-cluster -R alertmanager

## 4. 查看告警状态
$ curl http://192.168.1.100:9093/api/v2/alerts

## 5. 测试告警
$ curl -X POST -H “Content-Type: application/json” -d ‘{“alerts”:[{“status”:”firing”,”labels”:{“alertname”:”TestAlert”,”severity”:”warning”},”annotations”:{“summary”:”Test alert”,”description”:”This is a test alert”},”generatorURL”:”http://localhost:9090/graph?g0.expr=up\u0026g0.tab=1″}]}’ http://192.168.1.100:9093/api/v2/alerts