# Elasticsearch配置文件
# cat /etc/elasticsearch/elasticsearch.yml
cluster.name: fgedu-elk-cluster
node.name: node-1
node.roles: [master, data, ingest]

path.data: /data/elasticsearch
path.logs: /var/log/elasticsearch

network.host: 192.168.1.10
http.port: 9200
transport.port: 9300

discovery.seed_hosts: [“192.168.1.10”, “192.168.1.11”, “192.168.1.12”]
cluster.initial_master_nodes: [“node-1”, “node-2”, “node-3”]

gateway.recover_after_nodes: 2
gateway.expected_nodes: 3
gateway.recover_after_time: 5m

action.destructive_requires_name: true

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: elastic-certificates.p12

xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: http-certificates.p12

# JVM配置
# cat /etc/elasticsearch/jvm.options
-Xms16g
-Xmx16g
-XX:+UseG1GC
-XX:G1HeapRegionSize=32m
-XX:InitiatingHeapOccupancyPercent=30
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/var/log/elasticsearch/heapdump.hprof
-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-Xloggc:/var/log/elasticsearch/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=10
8:-XX:GCLogFileSize=100M

# 启动Elasticsearch
# systemctl start elasticsearch
# systemctl enable elasticsearch

# 查看服务状态
# systemctl status elasticsearch
● elasticsearch.service – Elasticsearch
Loaded: loaded (/usr/lib/systemd/system/elasticsearch.service; enabled; vendor preset: disabled)
Active: active (running) since Fri 2026-04-03 10:00:00 CST; 1min ago
Main PID: 12345 (java)
Tasks: 100 (limit: 49143)
Memory: 16.0G
CGroup: /system.slice/elasticsearch.service
└─12345 /usr/share/elasticsearch/jdk/bin/java …

# 查看索引列表
# curl -s http://fgedudb:9200/_cat/indices?v
health status index pri rep docs.count docs.deleted store.size pri.store.size
green open fgedu-app-logs-2026.04.03 5 1 1234567 0 5.2gb 2.6gb
green open fgedu-system-logs-2026.04.03 5 1 567890 0 2.1gb 1.0gb
green open fgedu-security-logs-2026.04.03 5 1 123456 0 0.8gb 0.4gb
green open .kibana_1 1 1 123 0 10.2mb 5.1mb

# 查看分片状态
# curl -s http://fgedudb:9200/_cat/shards?v | head -20
index shard prirep state docs store ip node
fgedu-app-logs-2026.04.03 0 p STARTED 246913 1.0gb 192.168.1.10 node-1
fgedu-app-logs-2026.04.03 0 r STARTED 246913 1.0gb 192.168.1.11 node-2
fgedu-app-logs-2026.04.03 1 p STARTED 246914 1.0gb 192.168.1.11 node-2
fgedu-app-logs-2026.04.03 1 r STARTED 246914 1.0gb 192.168.1.12 node-3

# Logstash配置文件
# cat /etc/logstash/logstash.yml
path.data: /var/lib/logstash
path.config: /etc/logstash/conf.d
path.logs: /var/log/logstash

pipeline.workers: 8
pipeline.batch.size: 125
pipeline.batch.delay: 50

queue.type: persisted
path.queue: /var/lib/logstash/queue
queue.page_capacity: 250mb
queue.max_events: 0
queue.max_bytes: 1024mb

dead_letter_queue.enable: true
path.dead_letter_queue: /var/lib/logstash/dead_letter_queue

xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.hosts: [“https://192.168.1.10:9200”, “https://192.168.1.11:9200”, “https://192.168.1.12:9200”]
xpack.monitoring.elasticsearch.username: logstash_system
xpack.monitoring.elasticsearch.password: Fgedu@Logstash123

# 日志收集管道配置
# cat /etc/logstash/conf.d/fgedu-app-logs.conf
input {
beats {
port => 5044
ssl => true
ssl_certificate => “/etc/logstash/certs/logstash.crt”
ssl_key => “/etc/logstash/certs/logstash.key”
}

kafka {
bootstrap_servers => “192.168.1.20:9092,192.168.1.21:9092,192.168.1.22:9092”
topics => [“fgedu-app-logs”, “fgedu-system-logs”]
group_id => “logstash-consumer”
consumer_threads => 4
decorate_events => true
}
}

filter {
grok {
match => { “message” => “%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:log_level} %{GREEDYDATA:log_message}” }
}

date {
match => [ “timestamp”, “ISO8601” ]
target => “@timestamp”
}

mutate {
remove_field => [ “timestamp”, “host” ]
add_field => { “log_source” => “fgedu-app” }
}

if [log_level] == “ERROR” {
mutate {
add_tag => [ “error” ]
}
}
}

output {
elasticsearch {
hosts => [“https://192.168.1.10:9200”, “https://192.168.1.11:9200”, “https://192.168.1.12:9200”]
index => “fgedu-app-logs-%{+YYYY.MM.dd}”
user => “logstash_writer”
password => “Fgedu@Writer123”
ssl => true
cacert => “/etc/logstash/certs/ca.crt”
}

if “error” in [tags] {
email {
to => “alert@fgedu.net.cn”
from => “logstash@fgedu.net.cn”
subject => “Error Log Alert: %{[log_source]}”
body => “Error detected:\n\n%{message}”
}
}
}

# 启动Logstash
# systemctl start logstash
# systemctl enable logstash

# 查看服务状态
# systemctl status logstash
● logstash.service – Logstash
Loaded: loaded (/usr/lib/systemd/system/logstash.service; enabled; vendor preset: disabled)
Active: active (running) since Fri 2026-04-03 10:00:00 CST; 1min ago
Main PID: 12346 (java)
Tasks: 50 (limit: 49143)
Memory: 4.0G
CGroup: /system.slice/logstash.service
└─12346 /usr/share/logstash/jdk/bin/java …

# 测试配置
# /usr/share/logstash/bin/logstash –config.test_and_exit -f /etc/logstash/conf.d/fgedu-app-logs.conf
Using JAVA_HOME defined java: /usr/share/logstash/jdk
Sending Logstash logs to /var/log/logstash which is now configured via log4j2.properties
[2026-04-03T10:00:00,000][INFO ][logstash.runner ] Starting Logstash {“jvm.options”=>[“-Xms2g”, “-Xmx2g”], “pipeline.workers”=>8, “pipeline.batch.size”=>125}
Configuration OK
[2026-04-03T10:00:05,000][INFO ][logstash.runner ] Logstash shut down.

# 查看管道统计
# curl -s http://fgedudb:9600/_node/stats/pipelines?pretty | head -50
{
“host” : “logstash.fgedu.net.cn”,
“version” : “8.12.0”,
“http_address” : “192.168.1.10:9600”,
“id” : “abc123-def456-ghi789”,
“name” : “logstash”,
“ephemeral_id” : “jkl012-mno345-pqr678”,
“status” : “green”,
“snapshot” : false,
“pipeline” : {
“workers” : 8,
“batch_size” : 125,
“batch_delay” : 50
},
“pipelines” : {
“main” : {
“events” : {
“filtered” : 1234567,
“input” : 1234567,
“output” : 1234567
}
}
}
}

# Kibana配置文件
# cat /etc/kibana/kibana.yml
server.port: 5601
server.host: “192.168.1.10”
server.name: “fgedu-kibana”

elasticsearch.hosts: [“https://192.168.1.10:9200”, “https://192.168.1.11:9200”, “https://192.168.1.12:9200”]
elasticsearch.username: “kibana_system”
elasticsearch.password: “Fgedu@Kibana123”

elasticsearch.ssl.certificateAuthorities: [“/etc/kibana/certs/ca.crt”]
elasticsearch.ssl.verificationMode: certificate

server.ssl.enabled: true
server.ssl.certificate: “/etc/kibana/certs/kibana.crt”
server.ssl.key: “/etc/kibana/certs/kibana.key”

xpack.security.encryptionKey: “fgedu_encryption_key_32_characters_long”
xpack.reporting.encryptionKey: “fgedu_reporting_key_32_characters_long”
xpack.security.session.idleTimeout: “1h”
xpack.security.session.lifespan: “24h”

xpack.encryptedSavedObjects.encryptionKey: “fgedu_saved_objects_key_32_chars”
xpack.reporting.capture.browser.chromium.disableSandbox: true

i18n.locale: “zh-CN”

# 启动Kibana
# systemctl start kibana
# systemctl enable kibana

# 查看服务状态
# systemctl status kibana
● kibana.service – Kibana
Loaded: loaded (/usr/lib/systemd/system/kibana.service; enabled; vendor preset: disabled)
Active: active (running) since Fri 2026-04-03 10:00:00 CST; 1min ago
Main PID: 12347 (node)
Tasks: 20 (limit: 49143)
Memory: 1.5G
CGroup: /system.slice/kibana.service
└─12347 /usr/share/kibana/bin/../node/bin/node …

# 检查Kibana状态
# curl -k https://fgedudb:5601/api/status | jq
{
“name”: “fgedu-kibana”,
“uuid”: “abc123-def456-ghi789”,
“version”: {
“number”: “8.12.0”,
“build_hash”: “jkl012mno345pqr678”
},
“status”: {
“overall”: {
“level”: “available”,
“summary”: “All services are available”
}
}
}

# 创建索引模板
# curl -X PUT “http://fgedudb:9200/_index_template/fgedu-app-logs” -H ‘Content-Type: application/json’ -d’
{
“index_patterns”: [“fgedu-app-logs-*”],
“priority”: 100,
“template”: {
“settings”: {
“number_of_shards”: 5,
“number_of_replicas”: 1,
“refresh_interval”: “30s”,
“index.lifecycle.name”: “fgedu-logs-policy”,
“index.lifecycle.rollover_alias”: “fgedu-app-logs”
},
“mappings”: {
“properties”: {
“@timestamp”: { “type”: “date” },
“log_level”: { “type”: “keyword” },
“log_source”: { “type”: “keyword” },
“message”: { “type”: “text” },
“host”: { “type”: “keyword” },
“application”: { “type”: “keyword” },
“environment”: { “type”: “keyword” },
“trace_id”: { “type”: “keyword” },
“span_id”: { “type”: “keyword” },
“duration_ms”: { “type”: “long” },
“tags”: { “type”: “keyword” }
}
}
}
}’
{“acknowledged”:true}

# 创建索引生命周期策略
# curl -X PUT “http://fgedudb:9200/_ilm/policy/fgedu-logs-policy” -H ‘Content-Type: application/json’ -d’
{
“policy”: {
“phases”: {
“hot”: {
“min_age”: “0ms”,
“actions”: {
“rollover”: {
“max_size”: “50gb”,
“max_age”: “1d”
},
“set_priority”: {
“priority”: 100
}
}
},
“warm”: {
“min_age”: “7d”,
“actions”: {
“shrink”: {
“number_of_shards”: 1
},
“forcemerge”: {
“max_num_segments”: 1
},
“set_priority”: {
“priority”: 50
}
}
},
“cold”: {
“min_age”: “30d”,
“actions”: {
“freeze”: {},
“set_priority”: {
“priority”: 0
}
}
},
“delete”: {
“min_age”: “90d”,
“actions”: {
“delete”: {}
}
}
}
}
}’
{“acknowledged”:true}

# 查看索引模板
# curl -s http://fgedudb:9200/_index_template/fgedu-app-logs | jq
{
“index_templates”: [
{
“name”: “fgedu-app-logs”,
“index_template”: {
“index_patterns”: [
“fgedu-app-logs-*”
],
“priority”: 100,
“template”: {
“settings”: {
“index”: {
“lifecycle”: {
“name”: “fgedu-logs-policy”,
“rollover_alias”: “fgedu-app-logs”
},
“number_of_shards”: “5”,
“number_of_replicas”: “1”
}
}
}
}
}
]
}

# 查看索引统计
# curl -s http://fgedudb:9200/_cat/indices/fgedu-*?v
health status index pri rep docs.count docs.deleted store.size pri.store.size
green open fgedu-app-logs-2026.04.03 5 1 1234567 0 5.2gb 2.6gb
green open fgedu-app-logs-2026.04.02 5 1 2345678 0 8.5gb 4.2gb
green open fgedu-system-logs-2026.04.03 5 1 567890 0 2.1gb 1.0gb

# 手动滚动索引
# curl -X POST “http://fgedudb:9200/fgedu-app-logs-2026.04.03/_rollover”
{
“acknowledged”: true,
“shards_acknowledged”: true,
“old_index”: “fgedu-app-logs-2026.04.03”,
“new_index”: “fgedu-app-logs-2026.04.03-000002”,
“rolled_over”: true,
“dry_run”: false,
“conditions”: {}
}

# Elasticsearch Ingest Pipeline
# curl -X PUT “http://fgedudb:9200/_ingest/pipeline/fgedu-app-pipeline” -H ‘Content-Type: application/json’ -d’
{
“description”: “Pipeline for processing application logs”,
“processors”: [
{
“grok”: {
“field”: “message”,
“patterns”: [
“%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:log_level} \\[%{DATA:thread}\\] %{GREEDYDATA:log_message}”
]
}
},
{
“date”: {
“field”: “timestamp”,
“formats”: [“ISO8601”],
“target_field”: “@timestamp”
}
},
{
“remove”: {
“field”: “timestamp”
}
},
{
“set”: {
“field”: “processed”,
“value”: true
}
},
{
“script”: {
“source”: “if (ctx.log_level == ‘\”ERROR’\”) { ctx.tags = ctx.tags != null ? ctx.tags : []; ctx.tags.add(‘\”error’\”); }”
}
}
],
“on_failure”: [
{
“set”: {
“field”: “error_message”,
“value”: “Pipeline processing failed”
}
}
]
}’
{“acknowledged”:true}

# 测试管道
# curl -X POST “http://fgedudb:9200/_ingest/pipeline/fgedu-app-pipeline/_simulate” -H ‘Content-Type: application/json’ -d’
{
“docs”: [
{
“_source”: {
“message”: “2026-04-03T10:00:00.000Z ERROR [main] Application error occurred”
}
}
]
}’
{
“docs”: [
{
“doc”: {
“_index”: “_index”,
“_id”: “_id”,
“_source”: {
“log_level”: “ERROR”,
“thread”: “main”,
“@timestamp”: “2026-04-03T10:00:00.000Z”,
“log_message”: “Application error occurred”,
“processed”: true,
“tags”: [“error”],
“message”: “2026-04-03T10:00:00.000Z ERROR [main] Application error occurred”
}
}
}
]
}

# 创建Kibana保存对象
# curl -X POST “https://fgedudb:5601/api/saved_objects/visualization” -H ‘kbn-xsrf: true’ -H ‘Content-Type: application/json’ -u elastic:Fgedu@Elastic123 -d’
{
“attributes”: {
“title”: “Application Log Levels”,
“visState”: “{\”title\”:\”Application Log Levels\”,\”type\”:\”pie\”,\”params\”:{\”type\”:\”pie\”},\”aggs\”:[{\”id\”:\”1\”,\”enabled\”:true,\”type\”:\”count\”,\”schema\”:\”metric\”,\”params\”:{}},{\”id\”:\”2\”,\”enabled\”:true,\”type\”:\”terms\”,\”schema\”:\”segment\”,\”params\”:{\”field\”:\”log_level\”,\”size\”:10}}]}”,
“searchSourceJSON”: “{\”index\”:\”fgedu-app-logs-*\”,\”query\”:{\”match_all\”:{}},\”filter\”:[]}”
}
}’

# 创建仪表板
# curl -X POST “https://fgedudb:5601/api/saved_objects/dashboard” -H ‘kbn-xsrf: true’ -H ‘Content-Type: application/json’ -u elastic:Fgedu@Elastic123 -d’
{
“attributes”: {
“title”: “FGedu Application Dashboard”,
“hits”: 0,
“description”: “Dashboard for monitoring application logs”,
“panelsJSON”: “[{\”panelIndex\”:\”1\”,\”gridData\”:{\”x\”:0,\”y\”:0,\”w\”:12,\”h\”:10},\”id\”:\”abc123-def456\”,\”type\”:\”visualization\”}]”,
“optionsJSON”: “{\”darkTheme\”:false}”
}
}’

# 查看仪表板列表
# curl -s “https://fgedudb:5601/api/saved_objects/_find?type=dashboard” -u elastic:Fgedu@Elastic123 | jq
{
“page”: 1,
“per_page”: 20,
“total”: 5,
“saved_objects”: [
{
“id”: “abc123-def456-ghi789”,
“type”: “dashboard”,
“attributes”: {
“title”: “FGedu Application Dashboard”,
“description”: “Dashboard for monitoring application logs”
}
}
]
}

# 创建告警规则
# curl -X POST “https://fgedudb:5601/api/alerting/rules” -H ‘kbn-xsrf: true’ -H ‘Content-Type: application/json’ -u elastic:Fgedu@Elastic123 -d’
{
“name”: “Error Log Alert”,
“rule_type_id”: “.es-query”,
“enabled”: true,
“schedule”: {
“interval”: “5m”
},
“params”: {
“index”: [“fgedu-app-logs-*”],
“esQuery”: “{\”query\”:{\”bool\”:{\”filter\”:[{\”term\”:{\”log_level\”:\”ERROR\”}}],\”must\”:[{\”range\”:{\”@timestamp\”:{\”gte\”:\”now-5m\”}}}]}}}”,
“size”: 100,
“timeField”: “@timestamp”
},
“actions”: [
{
“id”: “email-action”,
“params”: {
“to”: [“alert@fgedu.net.cn”],
“subject”: “Error Log Alert – {{context.title}}”,
“message”: “Found {{context.hits}} error logs in the last 5 minutes”
}
}
]
}’

# 查看告警规则
# curl -s “https://fgedudb:5601/api/alerting/rules/_find” -u elastic:Fgedu@Elastic123 | jq
{
“page”: 1,
“per_page”: 10,
“total”: 3,
“data”: [
{
“id”: “abc123-def456-ghi789”,
“name”: “Error Log Alert”,
“rule_type_id”: “.es-query”,
“enabled”: true,
“schedule”: {
“interval”: “5m”
},
“status”: “OK”,
“last_run”: “2026-04-03T10:00:00.000Z”
}
]
}

# 创建Webhook动作
# curl -X POST “https://fgedudb:5601/api/actions/connector” -H ‘kbn-xsrf: true’ -H ‘Content-Type: application/json’ -u elastic:Fgedu@Elastic123 -d’
{
“name”: “Webhook Alert”,
“connector_type_id”: “.webhook”,
“config”: {
“url”: “https://api.fgedu.net.cn/alerts/webhook”
},
“secrets”: {
“user”: “fgedu_alert”,
“password”: “Fgedu@Alert123”
}
}’

# 查看集群详细状态
# curl -s http://fgedudb:9200/_cluster/stats?pretty | head -50
{
“_nodes” : {
“total” : 3,
“successful” : 3,
“failed” : 0
},
“cluster_name” : “fgedu-elk-cluster”,
“cluster_uuid” : “abc123-def456-ghi789”,
“timestamp” : 1712120400000,
“status” : “green”,
“indices” : {
“count” : 50,
“shards” : {
“total” : 100,
“primaries” : 50,
“replication” : 1.0,
“index” : {
“shards” : {
“min” : 1,
“max” : 5,
“avg” : 2.0
}
}
},
“docs” : {
“count” : 12345678,
“deleted” : 12345
},
“store” : {
“size_in_bytes” : 123456789012,
“reserved_in_bytes” : 0
}
},
“nodes” : {
“count” : {
“total” : 3,
“master” : 3,
“data” : 3
},
“versions” : [
“8.12.0”
],
“os” : {
“available_processors” : 96,
“allocated_processors” : 96
},
“jvm” : {
“max_uptime_in_millis” : 7776000000,
“versions” : [
{
“version” : “17.0.10”,
“vm_name” : “Java HotSpot(TM) 64-Bit Server VM”,
“vm_version” : “17.0.10+9-LTS”
}
]
}
}
}

# 节点维护模式
# curl -X PUT “http://fgedudb:9200/_cluster/settings” -H ‘Content-Type: application/json’ -d’
{
“persistent”: {
“cluster.routing.allocation.exclude._ip”: “192.168.1.12”
}
}’
{“acknowledged”:true}

# 查看分片重分配
# curl -s http://fgedudb:9200/_cat/recovery?v&active_only=true
index shard time type stage source_host target_host repository snapshot files files_recovered files_percent files_total bytes bytes_recovered bytes_percent bytes_total
fgedu-app-logs-2026.04.03 0 1.2s peer done 192.168.1.10 192.168.1.11 n/a n/a 100 100 100.0% 100 1.0gb 1.0gb 100.0% 1.0gb

# Elasticsearch性能调优
# cat >> /etc/elasticsearch/elasticsearch.yml << 'EOF' indices.fielddata.cache.size: 20% indices.fielddata.cache.expire: 6h indices.cache.query.size: 10% index.refresh_interval: 30s index.number_of_replicas: 1 thread_pool: search: size: 20 queue_size: 1000 write: size: 20 queue_size: 1000 bulk: size: 20 queue_size: 1000 bootstrap.memory_lock: true cluster.routing.allocation.node_concurrent_recoveries: 4 cluster.routing.allocation.node_initial_primaries_recoveries: 8 EOF # 系统参数调优 # cat >> /etc/sysctl.conf << 'EOF' vm.max_map_count=262144 vm.swappiness=1 net.core.somaxconn=65535 net.ipv4.tcp_max_syn_backlog=65535 EOF # sysctl -p # 文件描述符限制 # cat >> /etc/security/limits.conf << 'EOF' elasticsearch soft nofile 65536 elasticsearch hard nofile 65536 elasticsearch soft memlock unlimited elasticsearch hard memlock unlimited EOF # 查看集群性能指标 # curl -s http://fgedudb:9200/_nodes/stats/jvm,process,thread_pool?pretty | head -80 { "_nodes" : { "total" : 3, "successful" : 3, "failed" : 0 }, "cluster_name" : "fgedu-elk-cluster", "nodes" : { "node-1" : { "timestamp" : 1712120400000, "name" : "node-1", "jvm" : { "mem" : { "heap_used_in_bytes" : 8589934592, "heap_used_percent" : 50, "heap_max_in_bytes" : 17179869184 }, "gc" : { "collectors" : { "old" : { "collection_count" : 100, "collection_time_in_millis" : 12345 } } } }, "process" : { "cpu" : { "percent" : 25 }, "mem" : { "total_virtual_in_bytes" : 34359738368 } }, "thread_pool" : { "search" : { "threads" : 20, "queue" : 50, "active" : 5 }, "write" : { "threads" : 20, "queue" : 10, "active" : 2 } } } } }