# 查看CloudWatch指标
$ aws cloudwatch get-metric-statistics \
–namespace AWS/EC2 \
–metric-name CPUUtilization \
–dimensions Name=InstanceId,Value=i-12345678 \
–start-time 2026-04-03T00:00:00Z \
–end-time 2026-04-03T12:00:00Z \
–period 3600 \
–statistics Average

# 创建CloudWatch告警
$ aws cloudwatch put-metric-alarm \
–alarm-name HighCPU \
–alarm-description “Alarm when CPU exceeds 70%” \
–metric-name CPUUtilization \
–namespace AWS/EC2 \
–statistic Average \
–period 300 \
–threshold 70 \
–comparison-operator GreaterThanThreshold \
–dimensions Name=InstanceId,Value=i-12345678 \
–evaluation-periods 2 \
–alarm-actions arn:aws:sns:us-west-2:123456789012:MyTopic

# 创建CloudWatch仪表盘
$ aws cloudwatch put-dashboard \
–dashboard-name MyDashboard \
–dashboard-body ‘{“widgets”:[{“type”:”metric”,”x”:0,”y”:0,”width”:12,”height”:6,”properties”:{“metrics”:[[“AWS/EC2″,”CPUUtilization”,”InstanceId”,”i-12345678″]],”period”:300,”stat”:”Average”,”region”:”us-west-2″,”title”:”EC2 CPU Utilization”}},{“type”:”metric”,”x”:0,”y”:6,”width”:12,”height”:6,”properties”:{“metrics”:[[“AWS/S3″,”BucketSizeBytes”,”BucketName”,”my-bucket”,”StorageType”,”StandardStorage”]],”period”:86400,”stat”:”Average”,”region”:”us-west-2″,”title”:”S3 Bucket Size”}}]}’

# 查看CloudWatch告警
$ aws cloudwatch describe-alarms

# 删除CloudWatch告警
$ aws cloudwatch delete-alarms \
–alarm-names HighCPU

# 创建CloudWatch日志组
$ aws logs create-log-group –log-group-name my-log-group

# 创建CloudWatch日志流
$ aws logs create-log-stream \
–log-group-name my-log-group \
–log-stream-name my-log-stream

# 发送日志到CloudWatch
$ aws logs put-log-events \
–log-group-name my-log-group \
–log-stream-name my-log-stream \
–log-events ‘[{“timestamp”:1649090400000,”message”:”Error: Connection failed”},{“timestamp”:1649090401000,”message”:”Info: Service started”}]’ \
–sequence-token 1234567890

# 查看CloudWatch日志
$ aws logs get-log-events \
–log-group-name my-log-group \
–log-stream-name my-log-stream

# 创建CloudWatch日志订阅
$ aws logs put-subscription-filter \
–log-group-name my-log-group \
–filter-name my-filter \
–filter-pattern “Error” \
–destination-arn arn:aws:lambda:us-west-2:123456789012:function:my-function

# 删除CloudWatch日志组
$ aws logs delete-log-group –log-group-name my-log-group

# 创建CloudWatch事件规则
$ aws events put-rule \
–name ec2-state-change \
–event-pattern ‘{“source”:[“aws.ec2″],”detail-type”:[“EC2 Instance State-change Notification”]}’ \
–state ENABLED

# 为规则添加目标
$ aws events put-targets \
–rule ec2-state-change \
–targets “[{\”Id\”:\”1\”,\”Arn\”:\”arn:aws:sns:us-west-2:123456789012:MyTopic\”}]”

# 测试CloudWatch事件规则
$ aws events put-events \
–entries ‘[{“Source”:”test”,”DetailType”:”test”,”Detail”:”{\”key\”:\”value\”}”}]’

# 查看CloudWatch事件规则
$ aws events describe-rule –name ec2-state-change

# 删除CloudWatch事件规则
$ aws events delete-rule –name ec2-state-change

风哥风哥提示：AWS CloudWatch是AWS提供的监控服务，可以监控AWS资源和应用程序的运行状态，及时发现和解决问题。

3. Azure Monitor

3.1 Azure Monitor基础

# 安装Azure CLI
$ curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash

# 登录Azure
$ az login

# 查看Azure Monitor指标
$ az monitor metrics list \
–resource /subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/my-resource-group/providers/Microsoft.Compute/virtualMachines/my-vm \
–metric-names “Percentage CPU” \
–time-grain PT1H \
–start-time 2026-04-03T00:00:00Z \
–end-time 2026-04-03T12:00:00Z

# 创建Azure Monitor告警
$ az monitor metrics alert create \
–name high-cpu \
–resource-group my-resource-group \
–scopes /subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/my-resource-group/providers/Microsoft.Compute/virtualMachines/my-vm \
–condition “avg Percentage CPU > 70” \
–description “Alert when CPU exceeds 70%” \
–action groups “$(az monitor action-group create –name my-action-group –resource-group my-resource-group –short-name MAG –email-receiver name=admin email=admin@fgedu.net.cn –query id -o tsv)”

# 查看Azure Monitor告警
$ az monitor metrics alert list –resource-group my-resource-group

# 删除Azure Monitor告警
$ az monitor metrics alert delete \
–name high-cpu \
–resource-group my-resource-group

# 创建Log Analytics工作区
$ az monitor log-analytics workspace create \
–resource-group my-resource-group \
–workspace-name my-workspace \
–location westus

# 连接VM到Log Analytics工作区
$ az vm extension set \
–resource-group my-resource-group \
–vm-name my-vm \
–name OmsAgentForLinux \
–publisher Microsoft.EnterpriseCloud.Monitoring \
–version 1.13.15 \
–settings ‘{“workspaceId”:”workspace-id”}’ \
–protected-settings ‘{“workspaceKey”:”workspace-key”}’

# 运行日志查询
$ az monitor log-analytics query \
–workspace-name my-workspace \
–analytics-query “Perf | where CounterName == ‘% Processor Time’ | summarize avg(CounterValue) by bin(TimeGenerated, 1h)”

# 创建日志查询告警
$ az monitor scheduled-query create \
–name error-alert \
–resource-group my-resource-group \
–workspace-name my-workspace \
–condition “count ‘where EventLevelName == \”Error\”‘ > 5” \
–description “Alert when there are more than 5 errors” \
–action groups “$(az monitor action-group create –name my-action-group –resource-group my-resource-group –short-name MAG –email-receiver name=admin email=admin@fgedu.net.cn –query id -o tsv)”

# 删除Log Analytics工作区
$ az monitor log-analytics workspace delete \
–resource-group my-resource-group \
–workspace-name my-workspace \
–yes

# 创建Application Insights资源
$ az monitor app-insights component create \
–resource-group my-resource-group \
–app my-application \
–location westus

# 获取Application Insights instrumentation key
$ az monitor app-insights component show \
–resource-group my-resource-group \
–app my-application \
–query instrumentationKey

# 查看Application Insights指标
$ az monitor app-insights metrics get \
–resource-group my-resource-group \
–app my-application \
–metric “requests/count” \
–time-span PT1H

# 创建Application Insights告警
$ az monitor metrics alert create \
–name high-failure-rate \
–resource-group my-resource-group \
–scopes /subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/my-resource-group/providers/microsoft.insights/components/my-application \
–condition “avg requests/failed > 5” \
–description “Alert when failure rate is high” \
–action groups “$(az monitor action-group create –name my-action-group –resource-group my-resource-group –short-name MAG –email-receiver name=admin email=admin@fgedu.net.cn –query id -o tsv)”

# 删除Application Insights资源
$ az monitor app-insights component delete \
–resource-group my-resource-group \
–app my-application

学习交流加群风哥微信: itpux-com

4. GCP Cloud Monitoring

4.1 Cloud Monitoring基础

# 安装gcloud CLI
$ curl https://sdk.cloud.google.com | bash
$ source ~/.bashrc

# 登录GCP
$ gcloud auth login

# 设置项目
$ gcloud config set project my-project

# 查看Cloud Monitoring指标
$ gcloud monitoring metrics list –filter=”metric.type=compute.googleapis.com/instance/cpu/utilization”

# 创建Cloud Monitoring告警策略
$ gcloud alpha monitoring policies create \
–display-name=”High CPU Utilization” \
–condition-display-name=”CPU > 70%” \
–condition-filter=”resource.type=gae_app AND metric.type=compute.googleapis.com/instance/cpu/utilization” \
–condition-aggregator=”mean” \
–condition-threshold-value=0.7 \
–condition-threshold-comparison=COMPARISON_GT \
–condition-duration=”60s” \
–notification-channels=”$(gcloud alpha monitoring channels create –display-name=”Email” –type=email –email-address=admin@fgedu.net.cn –format=value(name))”

# 查看Cloud Monitoring告警策略
$ gcloud alpha monitoring policies list

# 删除Cloud Monitoring告警策略
$ gcloud alpha monitoring policies delete POLICY_ID

# 查看Cloud Logging日志
$ gcloud logging read “resource.type=gae_app AND severity>=ERROR” –limit=10

# 创建Cloud Logging导出
$ gcloud logging sinks create my-sink \
gs://my-bucket \
–log-filter=”resource.type=gae_app AND severity>=ERROR”

# 查看Cloud Logging导出
$ gcloud logging sinks list

# 删除Cloud Logging导出
$ gcloud logging sinks delete my-sink

# 创建Cloud Logging告警
$ gcloud alpha monitoring policies create \
–display-name=”Error Logs” \
–condition-display-name=”Error Count” \
–condition-filter=”resource.type=gae_app AND severity=ERROR” \
–condition-aggregator=”count” \
–condition-threshold-value=5 \
–condition-threshold-comparison=COMPARISON_GT \
–condition-duration=”60s” \
–notification-channels=”$(gcloud alpha monitoring channels create –display-name=”Email” –type=email –email-address=admin@fgedu.net.cn –format=value(name))”

# 查看Cloud Trace数据
$ gcloud beta trace traces list –limit=10

# 查看特定跟踪
$ gcloud beta trace traces describe TRACE_ID

# 创建Cloud Trace导出
$ gcloud beta trace sinks create my-sink \
gs://my-bucket \
–filter=”true”

# 查看Cloud Trace导出
$ gcloud beta trace sinks list

# 删除Cloud Trace导出
$ gcloud beta trace sinks delete my-sink

学习交流加群风哥QQ113257174

5. Prometheus和Grafana

5.1 Prometheus安装与配置

# 安装Prometheus
$ wget https://github.com/prometheus/prometheus/releases/download/v2.33.0/prometheus-2.33.0.linux-amd64.tar.gz
$ tar xvf prometheus-2.33.0.linux-amd64.tar.gz
$ cd prometheus-2.33.0.linux-amd64

# 配置Prometheus
$ cat prometheus.yml
global:
scrape_interval: 15s

scrape_configs:
– job_name: ‘prometheus’
static_configs:
– targets: [‘fgedudb:9090’]
– job_name: ‘node’
static_configs:
– targets: [‘fgedudb:9100’]
– job_name: ‘docker’
static_configs:
– targets: [‘fgedudb:9323’]

# 启动Prometheus
$ ./prometheus –config.file=prometheus.yml

# 访问Prometheus
# 打开浏览器访问 http://fgedudb:9090

# 安装Node Exporter
$ wget https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
$ tar xvf node_exporter-1.3.1.linux-amd64.tar.gz
$ cd node_exporter-1.3.1.linux-amd64

# 启动Node Exporter
$ ./node_exporter

# 安装Docker Exporter
$ docker run -d -p 9323:9323 –name docker-exporter \
-v /var/run/docker.sock:/var/run/docker.sock \
prometheus/docker-exporter

# 安装Grafana
$ wget https://dl.grafana.com/oss/release/grafana_8.3.3_amd64.deb
$ sudo dpkg -i grafana_8.3.3_amd64.deb

# 启动Grafana
$ sudo systemctl start grafana-server
$ sudo systemctl enable grafana-server

# 访问Grafana
# 打开浏览器访问 http://fgedudb:3000
# 默认fgedu和密码: admin/admin

# 配置Prometheus数据源
# 1. 登录Grafana
# 2. 点击”Configuration” -> “Data sources”
# 3. 点击”Add data source”
# 4. 选择”Prometheus”
# 5. 设置URL为 http://fgedudb:9090
# 6. 点击”Save & Test”

# 导入Dashboard
# 1. 点击”+” -> “Import”
# 2. 输入Dashboard ID (例如 1860 用于Node Exporter)
# 3. 选择Prometheus数据源
# 4. 点击”Import”

# 配置Prometheus告警规则
$ cat alerts.yml
groups:
– name: example
rules:
– alert: HighCPU
expr: 100 – (avg by(instance) (irate(node_cpu_seconds_total{mode=”idle”}[5m])) * 100 > 70
for: 5m
labels:
severity: warning
annotations:
summary: “High CPU utilization”
description: “CPU utilization is above 70% for 5 minutes”

# 更新Prometheus配置
$ cat prometheus.yml
global:
scrape_interval: 15s

rule_files:
– alerts.yml

scrape_configs:
– job_name: ‘prometheus’
static_configs:
– targets: [‘fgedudb:9090’]
– job_name: ‘node’
static_configs:
– targets: [‘fgedudb:9100’]

# 重启Prometheus
$ ./prometheus –config.file=prometheus.yml

# 配置Grafana告警通道
# 1. 登录Grafana
# 2. 点击”Alerting” -> “Notification channels”
# 3. 点击”Add channel”
# 4. 设置名称和类型 (例如 Email)
# 5. 配置邮箱地址
# 6. 点击”Save”

# 创建Grafana告警
# 1. 打开Dashboard
# 2. 点击图表标题 -> “Edit”
# 3. 点击”Alert”选项卡
# 4. 配置告警规则
# 5. 选择通知通道
# 6. 点击”Save”

更多学习教程公众号风哥教程itpux_com

6. 日志管理

6.1 集中式日志管理

# 安装ELK Stack
$ docker-compose up -d

$ cat docker-compose.yml
version: ‘3’
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0
environment:
– discovery.type=single-node
ports:
– “9200:9200”
logstash:
image: docker.elastic.co/logstash/logstash:7.14.0
volumes:
– ./logstash.conf:/etc/logstash/conf.d/logstash.conf
ports:
– “5044:5044”
kibana:
image: docker.elastic.co/kibana/kibana:7.14.0
ports:
– “5601:5601”

$ cat logstash.conf
input {
beats {
port => 5044
}
}

filter {
if [type] == “syslog” {
grok {
match => {
“message” => “%{SYSLOGTIMESTAMP:timestamp} %{HOSTNAME:hostname} %{WORD:program}: %{GREEDYDATA:message}”
}
}
}
}

output {
elasticsearch {
hosts => [“elasticsearch:9200”]
index => “%{type}-%{+YYYY.MM.dd}”
}
}

# 安装Filebeat
$ wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.14.0-linux-x86_64.tar.gz
$ tar xvf filebeat-7.14.0-linux-x86_64.tar.gz
$ cd filebeat-7.14.0-linux-x86_64

# 配置Filebeat
$ cat filebeat.yml
filebeat.inputs:
– type: log
enabled: true
paths:
– /var/log/syslog
– /var/log/auth.log

output.logstash:
hosts: [“fgedudb:5044”]

# 启动Filebeat
$ ./filebeat -e -c filebeat.yml

# 访问Kibana
# 打开浏览器访问 http://fgedudb:5601
# 创建索引模式 -> 发现

# 使用Kibana进行日志分析
# 1. 打开Kibana
# 2. 点击”Discover”
# 3. 选择索引模式
# 4. 输入搜索查询

# 示例查询
# 搜索错误日志
error

# 搜索特定时间范围的日志
@timestamp:[now-1h TO now]

# 搜索特定主机的日志
hostname:”server1″

# 搜索特定程序的日志
program:”sshd”

# 创建Kibana仪表盘
# 1. 点击”Dashboard”
# 2. 点击”Create dashboard”
# 3. 点击”Add”
# 4. 选择可视化
# 5. 配置可视化
# 6. 点击”Save”

# 创建Kibana告警
# 1. 点击”Alerting”
# 2. 点击”Create alert”
# 3. 配置告警规则
# 4. 配置通知
# 5. 点击”Save”

# 使用Telegraf收集指标
$ wget https://dl.influxdata.com/telegraf/releases/telegraf-1.21.2_linux_amd64.tar.gz
$ tar xvf telegraf-1.21.2_linux_amd64.tar.gz
$ cd telegraf-1.21.2

# 配置Telegraf
$ cat telegraf.conf
[agent]
interval = “10s”
round_interval = true
metric_batch_size = 1000
metric_buffer_limit = 10000
collection_jitter = “0s”
flush_interval = “10s”
flush_jitter = “0s”
precision = “”
hostname = “”
omit_hostname = false

[[outputs.influxdb]]
urls = [“http://fgedudb:8086”]
database = “telegraf”
retention_policy = “”
write_consistency = “any”
timeout = “5s”

[[inputs.cpu]]
percpu = true
totalcpu = true
collect_cpu_time = false
report_active = false

[[inputs.mem]]

[[inputs.disk]]
ignore_fs = [“tmpfs”, “devtmpfs”, “devfs”]

[[inputs.net]]

# 启动Telegraf
$ ./telegraf –config telegraf.conf

# 安装InfluxDB
$ wget https://dl.influxdata.com/influxdb/releases/influxdb-1.8.10_linux_amd64.tar.gz
$ tar xvf influxdb-1.8.10_linux_amd64.tar.gz
$ cd influxdb-1.8.10

# 启动InfluxDB
$ ./influxd

# 访问InfluxDB
# 打开浏览器访问 http://fgedudb:8086

# 安装Grafana
$ wget https://dl.grafana.com/oss/release/grafana_8.3.3_amd64.deb
$ sudo dpkg -i grafana_8.3.3_amd64.deb

# 启动Grafana
$ sudo systemctl start grafana-server

# 配置InfluxDB数据源
# 1. 登录Grafana
# 2. 点击”Configuration” -> “Data sources”
# 3. 点击”Add data source”
# 4. 选择”InfluxDB”
# 5. 设置URL为 http://fgedudb:8086
# 6. 设置Database为 telegraf
# 7. 点击”Save & Test”

# 使用InfluxDB查询指标
$ ./influx
> USE telegraf
> SELECT mean(“usage_system”) FROM “cpu” WHERE time > now() – 1h GROUP BY time(10m)

# 使用Grafana创建仪表盘
# 1. 登录Grafana
# 2. 点击”+” -> “Dashboard”
# 3. 点击”Add new panel”
# 4. 选择InfluxDB数据源
# 5. 配置查询
# 6. 点击”Apply”

# 示例查询
# CPU使用率
SELECT mean(“usage_system”) FROM “cpu” WHERE $timeFilter GROUP BY time($__interval), “cpu”

# 内存使用率
SELECT mean(“used_percent”) FROM “mem” WHERE $timeFilter GROUP BY time($__interval)

# 磁盘使用率
SELECT mean(“used_percent”) FROM “disk” WHERE $timeFilter GROUP BY time($__interval), “path”

# 网络流量
SELECT mean(“bytes_sent”) AS “sent”, mean(“bytes_recv”) AS “recv” FROM “net” WHERE $timeFilter GROUP BY time($__interval)

author:www.itpux.com

8. 告警管理

8.1 告警策略

# 创建告警策略文档
$ cat alert-policy.md
# 告警策略

## 1. 告警级别
– **Critical**: 严重问题，需要立即处理
– **Warning**: 警告信息，需要关注
– **Info**: 信息性消息，用于通知

## 2. 告警规则
### 2.1 CPU告警
– **Critical**: CPU使用率 > 90% 持续 5 分钟
– **Warning**: CPU使用率 > 70% 持续 10 分钟

### 2.2 内存告警
– **Critical**: 内存使用率 > 90% 持续 5 分钟
– **Warning**: 内存使用率 > 70% 持续 10 分钟

### 2.3 磁盘告警
– **Critical**: 磁盘使用率 > 90% 持续 5 分钟
– **Warning**: 磁盘使用率 > 70% 持续 10 分钟

### 2.4 网络告警
– **Critical**: 网络错误率 > 10% 持续 5 分钟
– **Warning**: 网络错误率 > 5% 持续 10 分钟

### 2.5 应用告警
– **Critical**: 应用错误率 > 5% 持续 5 分钟
– **Warning**: 应用错误率 > 1% 持续 10 分钟

## 3. 告警通知
– **Critical**: 邮件 + 短信 + 电话
– **Warning**: 邮件 + 短信
– **Info**: 邮件

## 4. 告警处理流程
1. 接收告警
2. 确认告警
3. 分析问题
4. 解决问题
5. 关闭告警
6. 记录问题

# 使用AWS Lambda处理告警
$ cat lambda-function.py
import boto3
import json

def lambda_handler(event, context):
# 解析告警事件
alarm_name = event[‘Records’][0][‘Sns’][‘MessageAttributes’][‘AlarmName’][‘Value’]
alarm_description = event[‘Records’][0][‘Sns’][‘MessageAttributes’][‘AlarmDescription’][‘Value’]
alarm_state = event[‘Records’][0][‘Sns’][‘MessageAttributes’][‘NewStateValue’][‘Value’]

# 处理告警
if alarm_state == ‘ALARM’:
if ‘High CPU’ in alarm_name:
# 处理CPU告警
ec2 = boto3.client(‘ec2’)
# 扩展Auto Scaling组
autoscaling = boto3.client(‘autoscaling’)
autoscaling.update_auto_scaling_group(
AutoScalingGroupName=’my-asg’,
MinSize=2,
MaxSize=5,
DesiredCapacity=3
)
print(f’Handled CPU alarm: {alarm_name}’)
elif ‘High Memory’ in alarm_name:
# 处理内存告警
print(f’Handled Memory alarm: {alarm_name}’)

return {
‘statusCode’: 200,
‘body’: json.dumps(‘Alarm handled successfully’)
}

# 创建Lambda函数
$ aws lambda create-function \
–function-name handle-alerts \
–runtime python3.8 \
–role arn:aws:iam::123456789012:role/lambda-role \
–handler lambda-function.lambda_handler \
–zip-file fileb://lambda-function.zip

# 创建SNS主题
$ aws sns create-topic –name alert-topic

# 订阅Lambda函数到SNS主题
$ aws sns subscribe \
–topic-arn arn:aws:sns:us-west-2:123456789012:alert-topic \
–protocol lambda \
–notification-endpoint arn:aws:lambda:us-west-2:123456789012:function:handle-alerts

# 授予SNS权限调用Lambda
$ aws lambda add-permission \
–function-name handle-alerts \
–statement-id sns-topic \
–action “lambda:InvokeFunction” \
–principal sns.amazonaws.com \
–source-arn arn:aws:sns:us-west-2:123456789012:alert-topic

# 创建CloudWatch告警并关联SNS主题
$ aws cloudwatch put-metric-alarm \
–alarm-name HighCPU \
–alarm-description “Alarm when CPU exceeds 70%” \
–metric-name CPUUtilization \
–namespace AWS/EC2 \
–statistic Average \
–period 300 \
–threshold 70 \
–comparison-operator GreaterThanThreshold \
–dimensions Name=InstanceId,Value=i-12345678 \
–evaluation-periods 2 \
–alarm-actions arn:aws:sns:us-west-2:123456789012:alert-topic