groups:

  # ──────────────────────────────────────────────
  # 1. Node availability
  # ──────────────────────────────────────────────
  - name: node_availability
    rules:
      - alert: NodeDown
        expr: up{job="node-exporter"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Node {{ $labels.instance }} is down"
          description: "node-exporter has not been reachable for >2 minutes."

      - alert: NodeHighLoad
        expr: >
          node_load15
          / count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})
          > 0.85
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "High 15-min load on {{ $labels.instance }}"
          description: "Load/core ratio is {{ printf \"%.2f\" $value }} (>0.85) for 15 minutes."

      - alert: NodeHighLoadCritical
        expr: >
          node_load15
          / count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})
          > 1.0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Critical load on {{ $labels.instance }}"
          description: "Load/core ratio is {{ printf \"%.2f\" $value }} (>1.0) for 10 minutes."

  # ──────────────────────────────────────────────
  # 2. CPU
  # ──────────────────────────────────────────────
  - name: cpu
    rules:
      - alert: CPUHighWarning
        expr: >
          100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "High CPU on {{ $labels.instance }}"
          description: "CPU usage is {{ printf \"%.1f\" $value }}% for 15 minutes."

      - alert: CPUHighCritical
        expr: >
          100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 98
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Critical CPU on {{ $labels.instance }}"
          description: "CPU usage is {{ printf \"%.1f\" $value }}% for 5 minutes."

  # ──────────────────────────────────────────────
  # 3. Memory
  # ──────────────────────────────────────────────
  - name: memory
    rules:
      - alert: MemoryHighWarning
        expr: >
          (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High memory on {{ $labels.instance }}"
          description: "Memory usage is {{ printf \"%.1f\" $value }}% (>90%)."

      - alert: MemoryHighCritical
        expr: >
          (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Critical memory on {{ $labels.instance }}"
          description: "Memory usage is {{ printf \"%.1f\" $value }}% (>95%)."

      - alert: SwapHighWarning
        expr: >
          (node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes)
          / node_memory_SwapTotal_bytes * 100 > 50
          and node_memory_SwapTotal_bytes > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "High swap usage on {{ $labels.instance }}"
          description: "Swap usage is {{ printf \"%.1f\" $value }}% (>50%)."

  # ──────────────────────────────────────────────
  # 4. Disk / filesystem
  # $value = % used (expr produces percentage directly)
  # ──────────────────────────────────────────────
  - name: disk
    rules:
      - alert: DiskSpaceWarning
        expr: >
          (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"}
          / node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"})
          * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Disk >85% on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "{{ printf \"%.1f\" $value }}% used on {{ $labels.mountpoint }} ({{ $labels.device }})."

      - alert: DiskSpaceCritical
        expr: >
          (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"}
          / node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"})
          * 100 > 95
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Disk >95% on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "{{ printf \"%.1f\" $value }}% used — disk nearly full."

      - alert: DiskFillPredicted24h
        expr: >
          predict_linear(
            node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"}[6h],
            86400
          ) < 0
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Disk fill predicted <24h on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "At current write rate {{ $labels.mountpoint }} will fill within 24 hours."

      - alert: InodesWarning
        expr: >
          (1 - node_filesystem_files_free{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs"}
          / node_filesystem_files{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs"})
          * 100 > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Inodes >90% on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "{{ printf \"%.1f\" $value }}% of inodes used on {{ $labels.mountpoint }}."

      - alert: EFIOrBootPartitionHighWarning
        expr: >
          (1 - node_filesystem_avail_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"}
          / node_filesystem_size_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"})
          * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Boot/EFI partition >80% on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "{{ printf \"%.1f\" $value }}% used on {{ $labels.mountpoint }}. Clean old kernel/EFI entries."

      - alert: EFIOrBootPartitionCritical
        expr: >
          (1 - node_filesystem_avail_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"}
          / node_filesystem_size_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"})
          * 100 > 90
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Boot/EFI partition >90% on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "{{ printf \"%.1f\" $value }}% used. Risk of failed kernel installs or boot failures."

  # ──────────────────────────────────────────────
  # 5. ZFS pool health
  # ──────────────────────────────────────────────
  - name: zfs
    rules:
      - alert: ZFSPoolDegraded
        expr: node_zfs_zpool_state{state="degraded"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "ZFS pool {{ $labels.zpool }} DEGRADED on {{ $labels.instance }}"
          description: "Pool {{ $labels.zpool }} is DEGRADED. A disk has likely failed — check immediately."

      - alert: ZFSPoolFaultedOrUnavail
        expr: node_zfs_zpool_state{state=~"faulted|unavail|suspended"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "ZFS pool {{ $labels.zpool }} is {{ $labels.state }} on {{ $labels.instance }}"
          description: "Pool {{ $labels.zpool }} is {{ $labels.state }}. Immediate attention required."

      - alert: ZFSPoolOfflineOrRemoved
        expr: node_zfs_zpool_state{state=~"offline|removed"} == 1
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "ZFS pool {{ $labels.zpool }} is {{ $labels.state }} on {{ $labels.instance }}"
          description: "Pool {{ $labels.zpool }} reported {{ $labels.state }}. Verify if intentional."

      - alert: ZFSPoolCapacityWarning
        expr: >
          (1 - node_filesystem_avail_bytes{fstype="zfs"}
          / node_filesystem_size_bytes{fstype="zfs"})
          * 100 > 85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "ZFS filesystem >85% on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "{{ printf \"%.1f\" $value }}% used. ZFS performance degrades above 80%."

      - alert: ZFSPoolCapacityCritical
        expr: >
          (1 - node_filesystem_avail_bytes{fstype="zfs"}
          / node_filesystem_size_bytes{fstype="zfs"})
          * 100 > 95
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "ZFS filesystem >95% on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "{{ printf \"%.1f\" $value }}% used. ZFS writes will fail at 100%."

  # ──────────────────────────────────────────────
  # 6. SMART (dormant — requires smartctl-exporter)
  # ──────────────────────────────────────────────
  - name: smart
    rules:
      - alert: SMARTDeviceUnhealthy
        expr: smartctl_device_smart_healthy == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "SMART failure on {{ $labels.instance }} device {{ $labels.device }}"
          description: "SMART overall health check FAILED for {{ $labels.device }}."

      - alert: SMARTReallocatedSectors
        expr: >
          smartctl_device_attribute{attribute_name="Reallocated_Sector_Ct",attribute_value_type="raw"}
          > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Reallocated sectors on {{ $labels.instance }} {{ $labels.device }}"
          description: "{{ $value }} reallocated sectors — early sign of disk failure."

      - alert: SMARTPendingSectors
        expr: >
          smartctl_device_attribute{attribute_name="Current_Pending_Sector",attribute_value_type="raw"}
          > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Pending sectors on {{ $labels.instance }} {{ $labels.device }}"
          description: "{{ $value }} pending (unreadable) sectors. Disk may be failing."

  # ──────────────────────────────────────────────
  # 7. NFS mounts
  # ──────────────────────────────────────────────
  - name: nfs
    rules:
      - alert: NFSMountReadOnly
        expr: node_filesystem_readonly{fstype=~"nfs.*", mountpoint=~"/mnt/.*"} == 1
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "NFS mount read-only on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "NFS mount {{ $labels.mountpoint }} has gone read-only — possible server issue or network error."

      - alert: MediaNFSMountMissing
        expr: absent(node_filesystem_avail_bytes{mountpoint="/mnt/media", instance="192.168.99.183:9100"})
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "Media NFS mount missing on media-server (.183)"
          description: "/mnt/media (TrueNAS Tank) is not mounted. Plex/Immich will lose access to media."

      - alert: SynoNFSMountMissing
        expr: absent(node_filesystem_avail_bytes{mountpoint="/mnt/synology", instance="192.168.99.153:9100"})
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "Synology NFS mount missing on PBS (.153)"
          description: "/mnt/synology is not mounted on PBS. Synology-Remote backup datastore is offline."

  # ──────────────────────────────────────────────
  # 8. UPS / NUT
  # hon95/prometheus-nut-exporter reports 0–1 scale.
  # Expressions multiply to produce % so $value is human-readable.
  # ──────────────────────────────────────────────
  - name: ups
    rules:
      - alert: UPSBatteryLow
        expr: nut_battery_charge * 100 < 30
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "UPS {{ $labels.ups }} battery low ({{ printf \"%.0f\" $value }}%)"
          description: "Battery at {{ printf \"%.0f\" $value }}%. Power loss may be imminent."

      - alert: UPSBatteryWarning
        expr: nut_battery_charge * 100 < 50
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "UPS {{ $labels.ups }} battery below 50%"
          description: "Battery at {{ printf \"%.0f\" $value }}%. May indicate on-battery or slow charging."

      - alert: UPSLowRuntime
        expr: nut_battery_runtime_seconds < 300
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "UPS {{ $labels.ups }} runtime <5 minutes"
          description: "Estimated runtime is {{ printf \"%.0f\" $value }}s. Systems may not shut down cleanly."

      - alert: UPSHighLoad
        expr: nut_load * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "UPS {{ $labels.ups }} load >80%"
          description: "UPS load is {{ printf \"%.0f\" $value }}%. Consider load balancing."

      - alert: NUTExporterDown
        expr: up{job="nut"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "NUT exporter is down"
          description: "UPS monitoring is offline — cannot detect power events or battery state."

  # ──────────────────────────────────────────────
  # 9. TLS certificate expiry
  # (dormant — requires blackbox_exporter or traefik metrics)
  # ──────────────────────────────────────────────
  - name: tls
    rules:
      - alert: TLSCertExpiryWarning
        expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "TLS cert expiring soon for {{ $labels.instance }}"
          description: "Certificate expires in {{ printf \"%.0f\" $value | humanizeDuration }}."

      - alert: TLSCertExpiryCritical
        expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 7
        for: 1h
        labels:
          severity: critical
        annotations:
          summary: "TLS cert expiring in <7 days for {{ $labels.instance }}"
          description: "Certificate expires in {{ printf \"%.0f\" $value | humanizeDuration }}. Renewal may have failed."

      - alert: TraefikCertExpiryWarning
        expr: traefik_tls_certs_not_after - time() < 86400 * 30
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "Traefik TLS cert expiring for {{ $labels.sans }}"
          description: "Certificate expires in {{ printf \"%.0f\" $value | humanizeDuration }}."

  # ──────────────────────────────────────────────
  # 10. Container health
  # ──────────────────────────────────────────────
  - name: containers
    rules:
      - alert: ContainerOOM
        expr: increase(container_oom_events_total{name!=""}[5m]) > 0
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: "Container OOM kill: {{ $labels.name }}"
          description: "Container {{ $labels.name }} was OOM-killed. Memory limit may need to be raised."

      - alert: ContainerRestartLoop
        expr: changes(container_start_time_seconds{name!=""}[1h]) > 3
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container restart loop: {{ $labels.name }}"
          description: "Container {{ $labels.name }} has restarted {{ printf \"%.0f\" $value }} times in the last hour."

      - alert: ContainerHighCPU
        expr: rate(container_cpu_usage_seconds_total{name!="", id!="/"}[5m]) > 0.80
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "High CPU in container {{ $labels.name }}"
          description: "Container {{ $labels.name }} is using {{ printf \"%.2f\" $value }} CPU cores sustained for 15 minutes."

      - alert: ContainerMemoryNearLimit
        expr: >
          (container_memory_working_set_bytes{name!=""}
          / container_spec_memory_limit_bytes{name!=""})
          * 100 > 90
          and container_spec_memory_limit_bytes{name!=""} > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container {{ $labels.name }} memory >90% of limit"
          description: "Using {{ printf \"%.1f\" $value }}% of memory limit. OOM risk."

      - alert: ContainerMemoryAtLimit
        expr: >
          (container_memory_working_set_bytes{name!=""}
          / container_spec_memory_limit_bytes{name!=""})
          * 100 > 98
          and container_spec_memory_limit_bytes{name!=""} > 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Container {{ $labels.name }} memory at limit — OOM imminent"
          description: "Using {{ printf \"%.1f\" $value }}% of memory limit. OOM kill expected shortly."

  # ──────────────────────────────────────────────
  # 11. Prometheus self-monitoring
  # ──────────────────────────────────────────────
  - name: prometheus_self
    rules:
      - alert: PrometheusConfigReloadFailed
        expr: prometheus_config_last_reload_successful == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Prometheus config reload failed"
          description: "Prometheus failed to reload its configuration. Alert rules may be stale."

      - alert: AlertmanagerDown
        expr: up{job="alertmanager"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Alertmanager is down"
          description: "Prometheus cannot reach Alertmanager — no alerts will be delivered."

      - alert: PrometheusTargetDown
        expr: up == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus target down: {{ $labels.job }}/{{ $labels.instance }}"
          description: "Scrape target {{ $labels.instance }} (job={{ $labels.job }}) unreachable for >5 minutes."

      - alert: AlertmanagerNotificationsFailing
        expr: increase(alertmanager_notifications_failed_total[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Alertmanager notifications failing"
          description: "Some alert notifications are failing to deliver. Check bridge logs."

  # ──────────────────────────────────────────────
  # 12. Media / Vault1 storage (TrueNAS Tank NFS)
  # ──────────────────────────────────────────────
  - name: media_storage
    rules:
      - alert: MediaStorageWarning
        expr: >
          (1 - node_filesystem_avail_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"}
          / node_filesystem_size_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"})
          * 100 > 85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Tank/Media share >85% full on {{ $labels.instance }}"
          description: "{{ printf \"%.1f\" $value }}% used on the TrueNAS media share."

      - alert: MediaStorageCritical
        expr: >
          (1 - node_filesystem_avail_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"}
          / node_filesystem_size_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"})
          * 100 > 95
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Tank/Media share >95% full on {{ $labels.instance }}"
          description: "{{ printf \"%.1f\" $value }}% used. Downloads/recordings will fail soon."

  # ──────────────────────────────────────────────
  # 13. UPS battery self-test results
  # Metrics written by /usr/local/bin/ups-quarterly-test.sh via node-exporter textfile collector on Beast.
  # 1=passed, 0=failed/unknown. Cron: first Sunday of Jan/Apr/Jul/Oct at 03:00.
  # ──────────────────────────────────────────────
  - name: ups_battery_test
    rules:
      - alert: UPSBatteryTestFailed
        expr: ups_battery_test_result == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "UPS {{ $labels.ups }} quarterly battery test failed"
          description: "Self-test result was not 'passed'. Run test manually: upscmd -u upsadmin cyberpower1@localhost test.battery.start.quick"

      - alert: UPSBatteryTestStale
        expr: time() - ups_battery_test_timestamp_seconds > 86400 * 100
        for: 6h
        labels:
          severity: warning
        annotations:
          summary: "UPS {{ $labels.ups }} battery test overdue (>100 days)"
          description: "Last self-test was {{ printf \"%.0f\" $value | humanizeDuration }} ago. Quarterly cron may have missed a run on Beast."