groups:
  - name: docker_containers
    interval: 60s
    rules:
      - alert: DockerContainerNotRunning
        expr: docker_container_running == 0
        for: 5m
        labels:
          severity: high
        annotations:
          summary: "Container {{ $labels.container }} on {{ $labels.host }} is not running"
          description: "Container {{ $labels.container }} (image: {{ $labels.image }}) on {{ $labels.host }} has been in a non-running state for more than 5 minutes."

      - alert: DockerContainerRestarting
        expr: increase(docker_container_restart_count[1h]) > 3
        for: 5m
        labels:
          severity: high
        annotations:
          summary: "Container {{ $labels.container }} on {{ $labels.host }} restarting frequently"
          description: "Container {{ $labels.container }} on {{ $labels.host }} has restarted more than 3 times in the last hour. Check container logs for crash loops."

      - alert: DockerContainerHighMemory
        expr: >
          docker_container_memory_limit_bytes{container!~"plex|tdarr"} > 0
          and (docker_container_memory_usage_bytes{container!~"plex|tdarr"} / docker_container_memory_limit_bytes{container!~"plex|tdarr"}) > 0.90
        for: 5m
        labels:
          severity: high
        annotations:
          summary: "Container {{ $labels.container }} on {{ $labels.host }} memory > 90% of limit"
          description: "Container {{ $labels.container }} on {{ $labels.host }} is using {{ $value | humanizePercentage }} of its memory limit. OOM kill may be imminent."

      - alert: DockerContainerHighCPU
        expr: docker_container_cpu_percent{container!~"plex|tdarr"} > 90
        for: 10m
        labels:
          severity: medium
        annotations:
          summary: "Container {{ $labels.container }} on {{ $labels.host }} CPU > 90%"
          description: "Container {{ $labels.container }} on {{ $labels.host }} CPU at {{ $value }}% for more than 10 minutes."

      # Media containers (plex, tdarr) — higher thresholds for transcoding workloads
      - alert: DockerMediaContainerHighMemory
        expr: >
          docker_container_memory_limit_bytes{container=~"plex|tdarr"} > 0
          and (docker_container_memory_usage_bytes{container=~"plex|tdarr"} / docker_container_memory_limit_bytes{container=~"plex|tdarr"}) > 0.95
        for: 30m
        labels:
          severity: high
          media_workload: "true"
        annotations:
          summary: "Media container {{ $labels.container }} on {{ $labels.host }} memory > 95% of limit"
          description: "{{ $labels.container }} on {{ $labels.host }} is using {{ $value | humanizePercentage }} of its memory limit for 30+ minutes. Sustained near-OOM during transcoding."

      - alert: DockerMediaContainerHighCPU
        expr: docker_container_cpu_percent{container=~"plex|tdarr"} > 95
        for: 30m
        labels:
          severity: info
          media_workload: "true"
        annotations:
          summary: "Media container {{ $labels.container }} on {{ $labels.host }} CPU > 95%"
          description: "{{ $labels.container }} on {{ $labels.host }} CPU at {{ $value }}% for 30+ minutes. Expected during transcoding — informational only."

  - name: docker_hosts
    interval: 120s
    rules:
      - alert: DockerHostDiskHigh
        expr: (docker_host_disk_used_bytes{host!="media-vm"} / docker_host_disk_total_bytes{host!="media-vm"}) > 0.85
        for: 10m
        labels:
          severity: high
        annotations:
          summary: "Docker host {{ $labels.host }} disk usage > 85%"
          description: "Host {{ $labels.host }} disk at {{ $value | humanizePercentage }}. Run docker system prune or expand disk."

      - alert: DockerMediaHostDiskHigh
        expr: (docker_host_disk_used_bytes{host="media-vm"} / docker_host_disk_total_bytes{host="media-vm"}) > 0.85
        for: 10m
        labels:
          severity: high
          media_workload: "true"
        annotations:
          summary: "Media host media-vm disk usage > 85%"
          description: "media-vm disk at {{ $value | humanizePercentage }}. Tdarr output may be accumulating. Run docker system prune or check transcoding output."

      - alert: DockerHostMemoryHigh
        expr: >
          docker_host_memory_total_bytes{host!="media-vm"} > 0
          and (docker_host_memory_used_bytes{host!="media-vm"} / docker_host_memory_total_bytes{host!="media-vm"}) > 0.90
        for: 5m
        labels:
          severity: high
        annotations:
          summary: "Docker host {{ $labels.host }} memory > 90%"
          description: "Host {{ $labels.host }} memory at {{ $value | humanizePercentage }}. Consider stopping unused containers or increasing VM memory."

      # media-vm — relaxed threshold (Plex transcoding causes high memory spikes)
      - alert: DockerMediaHostMemoryHigh
        expr: >
          docker_host_memory_total_bytes{host="media-vm"} > 0
          and (docker_host_memory_used_bytes{host="media-vm"} / docker_host_memory_total_bytes{host="media-vm"}) > 0.95
        for: 30m
        labels:
          severity: high
          media_workload: "true"
        annotations:
          summary: "Media host media-vm memory > 95%"
          description: "media-vm memory at {{ $value | humanizePercentage }} for 30+ minutes. Plex transcoding can cause sustained spikes but this level risks OOM."

      - alert: DockerHostNoRunningContainers
        expr: docker_host_containers_running == 0 and docker_host_containers_total > 0
        for: 10m
        labels:
          severity: high
        annotations:
          summary: "Docker host {{ $labels.host }} has 0 running containers"
          description: "Host {{ $labels.host }} has {{ with printf `docker_host_containers_total{host=\"%s\"}` $labels.host | query }}{{ . | first | value }}{{ end }} containers but none are running. Docker daemon may have restarted or all services are down."
