services:
  node-exporter:
    image: prom/node-exporter:v1.8.2
    container_name: node-exporter
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
      - '--collector.netclass.ignored-devices=^(veth|docker|br-|flannel|cali)`
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    ports:
      - "9100:9100"
    restart: unless-stopped
    network_mode: host

# 创建目录
ssh <hostname> "sudo mkdir -p <部署路径> && sudo chown $(id -u):$(id -g) <部署路径>"

# 上传 docker-compose.yaml
scp docker-compose.yaml <hostname>:<部署路径>/

# 启动
ssh <hostname> "cd /opt/monitor && docker compose up -d"

# 验证
ssh <hostname> "curl -s localhost:9100/metrics | head -20"

global:
  scrape_interval: 15s
  scrape_timeout: 10s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093

rule_files:
  - /etc/prometheus/alerts.yml

scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets:
          - localhost:9090

  - job_name: robin
    static_configs:
      - targets:
          - node-exporter:9100
      labels:
        hostname: robin

  - job_name: node
    static_configs:
      - targets:
          - <节点IP>:9100
        labels:
          hostname: <hostname>

global:
  resolve_timeout: 5m

route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  receiver: 'openclaw'

receivers:
  - name: 'openclaw'
    webhook_configs:
      - url: 'http://<alert-transformer地址>:9091/alertmanager'
        send_resolved: true

{"HighCPUUsage": "infra-agent", "HighDiskUsage": "infra-agent", "DatabaseDown": "db-agent"}

docker build -t docker-registry:5000/alert-transformer:1.0.0 ./alert-transformer
docker push docker-registry:5000/alert-transformer:1.0.0
ssh rivo "docker pull docker-registry:5000/alert-transformer:1.0.0 && cd /home/ubuntu/openclaw && docker compose up -d alert-transformer"

services:
  openclaw-gateway:
    image: ghcr.nju.edu.cn/openclaw/openclaw:2026.3.28
    environment:
      OPENCLAW_GATEWAY_TOKEN: ${OPENCLAW_GATEWAY_TOKEN}
      HTTP_PROXY: http://<代理服务器IP>:7890
      HTTPS_PROXY: http://<代理服务器IP>:7890
      NO_PROXY: localhost,127.0.0.1,<数据库主机名>,<数据库IP>
    volumes:
      - ./.openclaw:/home/node/.openclaw
      - <用户目录>/.ssh:/home/node/.ssh:ro
    ports:
      - "18789:18789"
      - "18790:18790"
    extra_hosts:
      - "<数据库主机名>:<数据库IP>"
    restart: unless-stopped

groups:
  - name: system_alerts
    rules:
      # CPU 告警
      - alert: HighCPUUsage
        expr: 100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "CPU 使用率超过 80%"

      - alert: CriticalCPUUsage
        expr: 100 - (avg by(hostname) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100 > 90
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "CPU 使用率超过 90%"

      # 内存告警
      - alert: HighMemoryUsage
        expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "内存使用率超过 85%"

      # 磁盘告警
      - alert: HighDiskUsage
        expr: (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs"}) * 100 > 90
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "磁盘使用率超过 90%"

  - name: container_alerts
    rules:
      # 节点宕机
      - alert: ContainerDown
        expr: up{job="node"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "节点宕机"

# 查看容器状态 (<监控服务器>)
ssh <监控服务器> "docker ps --format 'table {{.Names}}\t{{.Status}}'"

# 查看 Prometheus targets
ssh <监控服务器> "curl -s http://localhost:9090/api/v1/targets | python3 -c \"import sys,json; d=json.load(sys.stdin); [print(t['labels']['job'], t['health']) for t in d['data']['activeTargets']]\""

# 查看告警状态
ssh <监控服务器> "curl -s http://localhost:9090/api/v1/alerts | python3 -c 'import sys,json; d=json.load(sys.stdin); [print(a[\"labels\"][\"alertname\"], a[\"state\"]) for a in d[\"data\"]]'"

# 查看 Alertmanager 状态
ssh <监控服务器> "curl -s http://localhost:9093/api/v1/status | python3 -m json.tool"

# 重启 Prometheus
ssh <监控服务器> "cd <部署路径> && sudo docker compose restart prometheus"

# 查看 alert-transformer 日志
ssh <监控服务器> "docker logs -f alert-transformer"

# 查看 OpenClaw 日志
ssh <OpenClaw服务器> "docker logs -f openclaw-gateway"

# 测试节点 exporter
curl -s <节点IP>:9100/metrics | grep node_cpu

# 查看告警规则
ssh <监控服务器> "curl -s http://localhost:9090/api/v1/rules | python3 -c 'import sys,json; d=json.load(sys.stdin); [print(r[\"name\"], r[\"rules\"][0][\"health\"]) for r in d[\"data\"]]'"

- targets:
    - <新节点IP>:9100
  labels:
    hostname: <主机名>

scp monitor/prometheus/prometheus.yml <监控服务器>:/tmp/prometheus.yml
ssh <监控服务器> "sudo cp /tmp/prometheus.yml <部署路径>/prometheus/prometheus.yml"
ssh <监控服务器> "cd <部署路径> && sudo docker compose restart prometheus"

scp monitor/node-exporter/docker-compose.yaml <hostname>:<部署路径>/
ssh <hostname> "cd <部署路径> && docker compose up -d"

ssh <监控服务器> "curl -s http://localhost:9090/api/v1/targets | python3 -c \"import sys,json; d=json.load(sys.stdin); [print(t['labels']['job'], t['labels']['hostname'], t['health']) for t in d['data']['activeTargets']]\""