groups:
  - name: pve_nodes
    interval: 30s
    rules:
      - alert: PVENodeDown
        expr: pve_node_up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "PVE node {{ $labels.node }} is offline"
          description: "Node {{ $labels.node }} has been offline for more than 2 minutes. Check hardware, network, and Proxmox services."

      - alert: PVENodeHighCPU
        expr: >
          pve_node_cpu_usage_ratio{node!="vanlint-ha-2"} > 0.90
        for: 5m
        labels:
          severity: high
        annotations:
          summary: "PVE node {{ $labels.node }} CPU usage > 90%"
          description: "Node {{ $labels.node }} CPU at {{ $value | humanizePercentage }} for more than 5 minutes. Consider migrating guests or investigating runaway processes."

      # vanlint-ha-2 hosts media-vm (iGPU passthrough) — CPU spikes expected during Plex/Tdarr
      - alert: PVEMediaNodeHighCPU
        expr: >
          pve_node_cpu_usage_ratio{node="vanlint-ha-2"} > 0.95
        for: 30m
        labels:
          severity: medium
          media_workload: "true"
        annotations:
          summary: "PVE media node vanlint-ha-2 CPU > 95%"
          description: "vanlint-ha-2 CPU at {{ $value | humanizePercentage }} for 30+ minutes. Expected during heavy Plex transcoding or Tdarr processing."

      - alert: PVENodeHighMemory
        expr: >
          pve_node_memory_usage_ratio{node!="vanlint-ha-2"} > 0.90
        for: 5m
        labels:
          severity: high
        annotations:
          summary: "PVE node {{ $labels.node }} memory usage > 90%"
          description: "Node {{ $labels.node }} memory at {{ $value | humanizePercentage }}. Used: {{ with printf `pve_node_memory_used_bytes{node=\"%s\"}` $labels.node | query }}{{ . | first | value | humanize1024 }}{{ end }} of {{ with printf `pve_node_memory_total_bytes{node=\"%s\"}` $labels.node | query }}{{ . | first | value | humanize1024 }}{{ end }}."

      # vanlint-ha-2 hosts media-vm — memory spikes expected during transcoding
      - alert: PVEMediaNodeHighMemory
        expr: >
          pve_node_memory_usage_ratio{node="vanlint-ha-2"} > 0.95
        for: 30m
        labels:
          severity: medium
          media_workload: "true"
        annotations:
          summary: "PVE media node vanlint-ha-2 memory > 95%"
          description: "vanlint-ha-2 memory at {{ $value | humanizePercentage }} for 30+ minutes. Expected during heavy media processing but risks swapping at this level."

      - alert: PVENodeLikelySwapping
        expr: pve_node_memory_usage_ratio > 0.95
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "PVE node {{ $labels.node }} memory > 95% -- likely swapping"
          description: "Node {{ $labels.node }} memory at {{ $value | humanizePercentage }}. At this level, swap usage is almost certain. Note: direct swap metrics require per-node /status API. Investigate with 'free -m' on the node."

      - alert: PVENodeDiskHigh
        expr: (pve_node_disk_used_bytes / pve_node_disk_total_bytes) > 0.85
        for: 5m
        labels:
          severity: high
        annotations:
          summary: "PVE node {{ $labels.node }} root disk usage > 85%"
          description: "Node {{ $labels.node }} root disk at {{ $value | humanizePercentage }}. Clean up old ISOs, templates, or logs."

      - alert: PVEClusterNotQuorate
        expr: pve_cluster_quorate == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "PVE cluster has lost quorum"
          description: "The cluster is no longer quorate. HA fencing and migrations are disabled. Online: {{ with query `pve_cluster_nodes_online` }}{{ . | first | value }}{{ end }} of {{ with query `pve_cluster_nodes_total` }}{{ . | first | value }}{{ end }} nodes."

      - alert: PVECriticalGuestDown
        expr: pve_guest_up{id=~"209", type!="lxc"} == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Critical PVE guest {{ $labels.name }} ({{ $labels.id }}) is down"
          description: "Critical guest {{ $labels.name }} on node {{ $labels.node }} has been down for 5 minutes. Immediate attention required."

      - alert: PVEGuestDown
        expr: pve_guest_up{id!~"209|220|221|222|900|999", type!="lxc"} == 0
        for: 10m
        labels:
          severity: high
        annotations:
          summary: "PVE guest {{ $labels.name }} ({{ $labels.id }}) is down"
          description: "Guest {{ $labels.name }} ({{ $labels.type }} {{ $labels.id }}) on node {{ $labels.node }} has been down for more than 10 minutes."

