NAME: prometheus

LAST DEPLOYED: Mon Apr 3 10:00:00 2026

NAMESPACE: monitoring

STATUS: deployed

from PG视频:www.itpux.com

REVISION: 1

NOTES:…

3.2 自动化脚本开发

创建自动化运维脚本，用于集群日常维护：

#!/bin/bash
# cluster-maintenance.sh
# from:www.itpux.com.qq113257174.wx:itpux-com
# web: http://www.fgedu.net.cn

# 检查集群状态
function check_cluster() {
    echo "检查集群状态..."
    kubectl get nodes
    kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error'
}

# 备份etcd
function backup_etcd() {
    echo "备份etcd..."
    ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-snapshot-$(date +%Y%m%d%H%M%S).db \
        --endpoints=https://192.168.1.10:2379 \
        --cacert=/etc/kubernetes/pki/etcd/ca.crt \
        --cert=/etc/kubernetes/pki/etcd/server.crt \
        --key=/etc/kubernetes/pki/etcd/server.key
}

# 清理未使用的镜像
function cleanup_images() {
    echo "清理未使用的镜像..."
    kubectl get nodes -o name | xargs -I {} ssh {} "docker system prune -f"
}

# 主函数
function main() {
    check_cluster
    backup_etcd
    cleanup_images
    echo "维护完成！"
}

main
        

3.3 告警策略配置

配置Prometheus告警规则：

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: kubernetes-cluster-rules
  namespace: monitoring
spec:
  groups:
  - name: kubernetes-cluster
    rules:
    - alert: HighNodeCPU
      expr: (sum by(instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m])) / sum by(instance) (rate(node_cpu_seconds_total[5m])) * 100 > 80
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "高节点CPU使用率"
        description: "节点 {{ $labels.instance }} CPU使用率超过80%"
    - alert: HighNodeMemory
      expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 85
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "高节点内存使用率"
        description: "节点 {{ $labels.instance }} 内存使用率超过85%"
        

Part04-生产案例与实战讲解

4.1 大规模集群监控部署

实战案例：部署Thanos实现监控数据长期存储和高可用：

---
# k8s-node-maintenance.yml
- name: K8s节点维护
  hosts: k8s_nodes
  become: yes
  tasks:
    - name: 更新系统包
      dnf:
        name: '*'
        state: latest
    
    - name: 重启kubelet服务
      service:
        name: kubelet
        state: restarted
    
    - name: 检查节点状态
      command: kubectl get node {{ inventory_hostname }}
      delegate_to: localhost
      register: node_status
    
    - name: 显示节点状态
      debug:
        msg: "{{ node_status.stdout }}"