# 环境准备
– 操作系统：CentOS 7.9 / RHEL 7.9
– Java版本：JDK 1.8+（Elasticsearch和Logstash需要）学习交流加群风哥QQ113257174
– 内存：8GB+（Elasticsearch建议16GB+）
– 存储：500GB+（根据日志量调整）
– 网络：千兆网络及以上

# 安装Java
$ yum install -y java-1.8.0-openjdk-devel

# 安装Elasticsearch
$ rpm –import https://artifacts.elastic.co/GPG-KEY-elasticsearch
$ cat > /etc/yum.repos.d/elasticsearch.repo << EOF [elasticsearch-7.x] name=Elasticsearch repository for 7.x packages baseurl=https://artifacts.elastic.co/packages/7.x/yum gpgcheck=1 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch enabled=1 autorefresh=1 type=rpm-md EOF $ yum install -y elasticsearch # 安装Kibana $ cat > /etc/yum.repos.d/kibana.repo << EOF [kibana-7.x] name=Kibana repository for 7.x packages baseurl=https://artifacts.elastic.co/packages/7.x/yum gpgcheck=1 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch enabled=1 autorefresh=1 type=rpm-md EOF $ yum install -y kibana # 安装Logstash $ cat > /etc/yum.repos.d/logstash.repo << EOF [logstash-7.x] name=Logstash repository for 7.x packages baseurl=https://artifacts.elastic.co/packages/7.x/yum gpgcheck=1 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch enabled=1 autorefresh=1 type=rpm-md EOF $ yum install -y logstash # 安装Filebeat $ cat > /etc/yum.repos.d/filebeat.repo << EOF [filebeat-7.x] name=Filebeat repository for 7.x packages baseurl=https://artifacts.elastic.co/packages/7.x/yum gpgcheck=1 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch enabled=1 autorefresh=1 type=rpm-md EOF $ yum install -y filebeat

# 配置Elasticsearch
$ vim /etc/elasticsearch/elasticsearch.yml

# 基本配置
cluster.name: fgedu-tidb-logs
node.name: node-1
path.data: /tidb/elasticsearch/data
path.logs: /tidb/elasticsearch/logs
network.host: 0.0.0.0
http.port: 9200

# 集群配置
cluster.initial_master_nodes: [“node-1”]
discovery.seed_hosts: [“192.168.1.100”]

# 内存配置
$ vim /etc/elasticsearch/jvm.options
-Xms8g
-Xmx8g

# 启动Elasticsearch
$ systemctl enable elasticsearch
$ systemctl start elasticsearch

# 验证Elasticsearch
$ curl http://localhost:9200/
{
“name” : “node-1”,
“cluster_name” : “fgedu-tidb-logs”,
“cluster_uuid” : “xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx”,
“version” : {
“number” : “7.17.0”,
“build_flavor” : “default”,
“build_type” : “rpm”,
“build_hash” : “xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx”,
“build_date” : “2022-01-28T08:36:04.379943Z”,
“build_snapshot” : false,
“lucene_version” : “8.11.1”,
“minimum_wire_compatibility_version” : “6.8.0”,
“minimum_index_compatibility_version” : “6.0.0-beta1”
},
“tagline” : “You Know, for Search”
}

# 配置Kibana
$ vim /etc/kibana/kibana.yml

# 基本配置
server.port: 5601
server.host: “0.0.0.0”
elasticsearch.hosts: [“http://localhost:9200”]
kibana.index: “.kibana”

# 启动Kibana
$ systemctl enable kibana
$ systemctl start kibana

# 验证Kibana
# 浏览器访问 http://localhost:5601

# 配置Filebeat
$ vim /etc/filebeat/filebeat.yml

# 输入配置
filebeat.inputs:
– type: log
enabled: true
paths:
– /tidb/app/tidb-deploy/**/log/*.log
scan_frequency: 10s
harvester_limit: 10000
close_inactive: 5m

# 输出配置
output.elasticsearch:
hosts: [“localhost:9200”]
index: “tidb-logs-%{+yyyy.MM.dd}”
pipeline: “tidb-logs”

# 启动Filebeat
$ systemctl enable filebeat
$ systemctl start filebeat

# 验证Filebeat
$ filebeat test output
elasticsearch:
localhost:9200…
parse url… OK
connection…
parse host… OK
dns lookup… OK
addresses: 127.0.0.1
dial up… OK
TLS… WARN secure connection disabled
talk to server… OK
version: 7.17.0

# 创建索引模板
$ curl -X PUT “http://localhost:9200/_index_template/tidb-logs” -H “Content-Type: application/json” -d ‘{
“index_patterns”: [“tidb-logs-*”],
“template”: {
“settings”: {
“number_of_shards”: 3,
“number_of_replicas”: 2,
“index.lifecycle.name”: “tidb-logs-policy”,
“index.lifecycle.rollover_alias”: “tidb-logs”
},
“mappings”: {
“properties”: {
“@timestamp”: {
“type”: “date”
},
“host”: {
“type”: “keyword”
},
“path”: {
“type”: “keyword”
},
“message”: {
“type”: “text”
},
“level”: {
“type”: “keyword”
},
“module”: {
“type”: “keyword”
},
“component”: {
“type”: “keyword”
}
}
}
}
}’

# 创建索引生命周期策略
$ curl -X PUT “http://localhost:9200/_ilm/policy/tidb-logs-policy” -H “Content-Type: application/json” -d ‘{
“policy”: {
“phases”: {
“hot”: {
“actions”: {
“rollover”: {
“max_size”: “30gb”,
“max_age”: “15d”
}
}
},
“delete”: {
“min_age”: “30d”,
“actions”: {
“delete”: {}
}
}
}
}
}’

# 创建初始索引
$ curl -X PUT “http://localhost:9200/tidb-logs-000001” -H “Content-Type: application/json” -d ‘{
“aliases”: {
“tidb-logs”: {
“is_write_index”: true
}
}
}’

# 配置Logstash
$ vim /etc/logstash/conf.d/tidb-logs.conf

input {
beats {
port => 5044
}
}

filter {
grok {
match => {
“message” => “\[%{TIMESTAMP_ISO8601:timestamp}\] \[%{LOGLEVEL:level}\] \[%{DATA:module}\] \[%{DATA:message}\]”
}
}
date {
match => [“timestamp”, “yyyy/MM/dd HH:mm:ss.SSS Z”]
target => “@timestamp”
}
mutate {
remove_field => [“timestamp”]
}
}

output {
elasticsearch {
hosts => [“localhost:9200”]
index => “tidb-logs-%{+yyyy.MM.dd}”
}
}

# 启动Logstash
$ systemctl enable logstash
$ systemctl start logstash

# 验证Logstash
$ curl http://localhost:9600/
{
“host” : “localhost”,
“version” : “7.17.0”,
“http_address” : “127.0.0.1:9600”,
“id” : “xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx”,
“name” : “localhost”,
“build_date” : “2022-01-28T08:36:04.379943Z”,
“build_sha” : “xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx”,
“build_snapshot” : false
}

# 启用Watcher
$ curl -X PUT “http://localhost:9200/_cluster/settings” -H “Content-Type: application/json” -d ‘{
“persistent”: {
“xpack.watcher.enabled”: true
}
}’

# 创建磁盘空间预警
$ curl -X PUT “http://localhost:9200/_watcher/watch/disk-space-warning” -H “Content-Type: application/json” -d ‘{
“trigger”: {
“schedule”: {
“interval”: “10m”
}
},
“input”: {
“search”: {
“request”: {
“indices”: [“tidb-logs-*”],
“body”: {
“query”: {
“bool”: {
“must”: [
{“match”: {“message”: “disk space”}},
{“match”: {“level”: “ERROR”}}
],
“filter”: {
“range”: {
“@timestamp”: {
“gte”: “now-10m”
}
}
}
}
}
}
}
}
},
“condition”: {
“compare”: {
“ctx.payload.hits.total”: {
“gt”: 0
}
}
},
“actions”: {
“send_email”: {
“email”: {
“to”: “admin@fgedu.net.cn”,
“subject”: “TiDB磁盘空间告警”,
“body”: “发现磁盘空间不足的错误日志，请及时处理。”
}
}
}
}’

# 创建连接数预警
$ curl -X PUT “http://localhost:9200/_watcher/watch/connection-warning” -H “Content-Type: application/json” -d ‘{
“trigger”: {
“schedule”: {
“interval”: “5m”
}
},
“input”: {
“search”: {
“request”: {
“indices”: [“tidb-logs-*”],
“body”: {
“query”: {
“bool”: {
“must”: [
{“match”: {“message”: “connection”}},
{“match”: {“level”: “WARN”}}
],
“filter”: {
“range”: {
“@timestamp”: {
“gte”: “now-5m”
}
}
}
}
}
}
}
}
},
“condition”: {
“compare”: {
“ctx.payload.hits.total”: {
“gt”: 10
}
}
},
“actions”: {
“send_email”: {
“email”: {
“to”: “admin@fgedu.net.cn”,
“subject”: “TiDB连接数告警”,
“body”: “连接数过多，可能导致系统性能下降，请及时处理。”
}
}
}
}’

# 配置Kibana告警
# 1. 打开Kibana界面
# 2. 进入Alerts页面
# 3. 点击Create alert
# 4. 选择Alert type: Index threshold
# 5. 配置索引模式: tidb-logs-*
# 6. 配置时间范围: Last 15 minutes
# 7. 配置聚合: Count
# 8. 配置条件: Count is greater than 5
# 9. 配置触发条件: When threshold is met
# 10. 配置通知: Email
# 11. 点击Save