diff --git a/observability/prometheus/alert_rules.yml b/observability/prometheus/alert_rules.yml new file mode 100644 index 0000000..f32bbb3 --- /dev/null +++ b/observability/prometheus/alert_rules.yml @@ -0,0 +1,542 @@ +groups: + + # ────────────────────────────────────────────── + # 1. Node availability + # ────────────────────────────────────────────── + - name: node_availability + rules: + - alert: NodeDown + expr: up{job="node-exporter"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Node {{ $labels.instance }} is down" + description: "node-exporter has not been reachable for >2 minutes." + + - alert: NodeHighLoad + expr: > + node_load15 + / count without(cpu, mode) (node_cpu_seconds_total{mode="idle"}) + > 0.85 + for: 15m + labels: + severity: warning + annotations: + summary: "High 15-min load on {{ $labels.instance }}" + description: "Load/core ratio is {{ printf \"%.2f\" $value }} (>0.85) for 15 minutes." + + - alert: NodeHighLoadCritical + expr: > + node_load15 + / count without(cpu, mode) (node_cpu_seconds_total{mode="idle"}) + > 1.0 + for: 10m + labels: + severity: critical + annotations: + summary: "Critical load on {{ $labels.instance }}" + description: "Load/core ratio is {{ printf \"%.2f\" $value }} (>1.0) for 10 minutes." + + # ────────────────────────────────────────────── + # 2. CPU + # ────────────────────────────────────────────── + - name: cpu + rules: + - alert: CPUHighWarning + expr: > + 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 + for: 15m + labels: + severity: warning + annotations: + summary: "High CPU on {{ $labels.instance }}" + description: "CPU usage is {{ printf \"%.1f\" $value }}% for 15 minutes." + + - alert: CPUHighCritical + expr: > + 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 98 + for: 5m + labels: + severity: critical + annotations: + summary: "Critical CPU on {{ $labels.instance }}" + description: "CPU usage is {{ printf \"%.1f\" $value }}% for 5 minutes." + + # ────────────────────────────────────────────── + # 3. Memory + # ────────────────────────────────────────────── + - name: memory + rules: + - alert: MemoryHighWarning + expr: > + (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90 + for: 5m + labels: + severity: warning + annotations: + summary: "High memory on {{ $labels.instance }}" + description: "Memory usage is {{ printf \"%.1f\" $value }}% (>90%)." + + - alert: MemoryHighCritical + expr: > + (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95 + for: 2m + labels: + severity: critical + annotations: + summary: "Critical memory on {{ $labels.instance }}" + description: "Memory usage is {{ printf \"%.1f\" $value }}% (>95%)." + + - alert: SwapHighWarning + expr: > + (node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) + / node_memory_SwapTotal_bytes * 100 > 50 + and node_memory_SwapTotal_bytes > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "High swap usage on {{ $labels.instance }}" + description: "Swap usage is {{ printf \"%.1f\" $value }}% (>50%)." + + # ────────────────────────────────────────────── + # 4. Disk / filesystem + # $value = % used (expr produces percentage directly) + # ────────────────────────────────────────────── + - name: disk + rules: + - alert: DiskSpaceWarning + expr: > + (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"}) + * 100 > 85 + for: 5m + labels: + severity: warning + annotations: + summary: "Disk >85% on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "{{ printf \"%.1f\" $value }}% used on {{ $labels.mountpoint }} ({{ $labels.device }})." + + - alert: DiskSpaceCritical + expr: > + (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"}) + * 100 > 95 + for: 2m + labels: + severity: critical + annotations: + summary: "Disk >95% on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "{{ printf \"%.1f\" $value }}% used — disk nearly full." + + - alert: DiskFillPredicted24h + expr: > + predict_linear( + node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"}[6h], + 86400 + ) < 0 + for: 30m + labels: + severity: warning + annotations: + summary: "Disk fill predicted <24h on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "At current write rate {{ $labels.mountpoint }} will fill within 24 hours." + + - alert: InodesWarning + expr: > + (1 - node_filesystem_files_free{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs"} + / node_filesystem_files{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs"}) + * 100 > 90 + for: 5m + labels: + severity: warning + annotations: + summary: "Inodes >90% on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "{{ printf \"%.1f\" $value }}% of inodes used on {{ $labels.mountpoint }}." + + - alert: EFIOrBootPartitionHighWarning + expr: > + (1 - node_filesystem_avail_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"} + / node_filesystem_size_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"}) + * 100 > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "Boot/EFI partition >80% on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "{{ printf \"%.1f\" $value }}% used on {{ $labels.mountpoint }}. Clean old kernel/EFI entries." + + - alert: EFIOrBootPartitionCritical + expr: > + (1 - node_filesystem_avail_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"} + / node_filesystem_size_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"}) + * 100 > 90 + for: 2m + labels: + severity: critical + annotations: + summary: "Boot/EFI partition >90% on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "{{ printf \"%.1f\" $value }}% used. Risk of failed kernel installs or boot failures." + + # ────────────────────────────────────────────── + # 5. ZFS pool health + # ────────────────────────────────────────────── + - name: zfs + rules: + - alert: ZFSPoolDegraded + expr: node_zfs_zpool_state{state="degraded"} == 1 + for: 1m + labels: + severity: critical + annotations: + summary: "ZFS pool {{ $labels.zpool }} DEGRADED on {{ $labels.instance }}" + description: "Pool {{ $labels.zpool }} is DEGRADED. A disk has likely failed — check immediately." + + - alert: ZFSPoolFaultedOrUnavail + expr: node_zfs_zpool_state{state=~"faulted|unavail|suspended"} == 1 + for: 1m + labels: + severity: critical + annotations: + summary: "ZFS pool {{ $labels.zpool }} is {{ $labels.state }} on {{ $labels.instance }}" + description: "Pool {{ $labels.zpool }} is {{ $labels.state }}. Immediate attention required." + + - alert: ZFSPoolOfflineOrRemoved + expr: node_zfs_zpool_state{state=~"offline|removed"} == 1 + for: 2m + labels: + severity: warning + annotations: + summary: "ZFS pool {{ $labels.zpool }} is {{ $labels.state }} on {{ $labels.instance }}" + description: "Pool {{ $labels.zpool }} reported {{ $labels.state }}. Verify if intentional." + + - alert: ZFSPoolCapacityWarning + expr: > + (1 - node_filesystem_avail_bytes{fstype="zfs"} + / node_filesystem_size_bytes{fstype="zfs"}) + * 100 > 85 + for: 10m + labels: + severity: warning + annotations: + summary: "ZFS filesystem >85% on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "{{ printf \"%.1f\" $value }}% used. ZFS performance degrades above 80%." + + - alert: ZFSPoolCapacityCritical + expr: > + (1 - node_filesystem_avail_bytes{fstype="zfs"} + / node_filesystem_size_bytes{fstype="zfs"}) + * 100 > 95 + for: 5m + labels: + severity: critical + annotations: + summary: "ZFS filesystem >95% on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "{{ printf \"%.1f\" $value }}% used. ZFS writes will fail at 100%." + + # ────────────────────────────────────────────── + # 6. SMART (dormant — requires smartctl-exporter) + # ────────────────────────────────────────────── + - name: smart + rules: + - alert: SMARTDeviceUnhealthy + expr: smartctl_device_smart_healthy == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "SMART failure on {{ $labels.instance }} device {{ $labels.device }}" + description: "SMART overall health check FAILED for {{ $labels.device }}." + + - alert: SMARTReallocatedSectors + expr: > + smartctl_device_attribute{attribute_name="Reallocated_Sector_Ct",attribute_value_type="raw"} + > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Reallocated sectors on {{ $labels.instance }} {{ $labels.device }}" + description: "{{ $value }} reallocated sectors — early sign of disk failure." + + - alert: SMARTPendingSectors + expr: > + smartctl_device_attribute{attribute_name="Current_Pending_Sector",attribute_value_type="raw"} + > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Pending sectors on {{ $labels.instance }} {{ $labels.device }}" + description: "{{ $value }} pending (unreadable) sectors. Disk may be failing." + + # ────────────────────────────────────────────── + # 7. NFS mounts + # ────────────────────────────────────────────── + - name: nfs + rules: + - alert: NFSMountReadOnly + expr: node_filesystem_readonly{fstype=~"nfs.*", mountpoint=~"/mnt/.*"} == 1 + for: 2m + labels: + severity: critical + annotations: + summary: "NFS mount read-only on {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "NFS mount {{ $labels.mountpoint }} has gone read-only — possible server issue or network error." + + - alert: MediaNFSMountMissing + expr: absent(node_filesystem_avail_bytes{mountpoint="/mnt/media", instance="192.168.99.183:9100"}) + for: 3m + labels: + severity: critical + annotations: + summary: "Media NFS mount missing on media-server (.183)" + description: "/mnt/media (TrueNAS Tank) is not mounted. Plex/Immich will lose access to media." + + - alert: SynoNFSMountMissing + expr: absent(node_filesystem_avail_bytes{mountpoint="/mnt/synology", instance="192.168.99.153:9100"}) + for: 3m + labels: + severity: critical + annotations: + summary: "Synology NFS mount missing on PBS (.153)" + description: "/mnt/synology is not mounted on PBS. Synology-Remote backup datastore is offline." + + # ────────────────────────────────────────────── + # 8. UPS / NUT + # hon95/prometheus-nut-exporter reports 0–1 scale. + # Expressions multiply to produce % so $value is human-readable. + # ────────────────────────────────────────────── + - name: ups + rules: + - alert: UPSBatteryLow + expr: nut_battery_charge * 100 < 30 + for: 2m + labels: + severity: critical + annotations: + summary: "UPS {{ $labels.ups }} battery low ({{ printf \"%.0f\" $value }}%)" + description: "Battery at {{ printf \"%.0f\" $value }}%. Power loss may be imminent." + + - alert: UPSBatteryWarning + expr: nut_battery_charge * 100 < 50 + for: 5m + labels: + severity: warning + annotations: + summary: "UPS {{ $labels.ups }} battery below 50%" + description: "Battery at {{ printf \"%.0f\" $value }}%. May indicate on-battery or slow charging." + + - alert: UPSLowRuntime + expr: nut_battery_runtime_seconds < 300 + for: 2m + labels: + severity: critical + annotations: + summary: "UPS {{ $labels.ups }} runtime <5 minutes" + description: "Estimated runtime is {{ printf \"%.0f\" $value }}s. Systems may not shut down cleanly." + + - alert: UPSHighLoad + expr: nut_load * 100 > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "UPS {{ $labels.ups }} load >80%" + description: "UPS load is {{ printf \"%.0f\" $value }}%. Consider load balancing." + + - alert: NUTExporterDown + expr: up{job="nut"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "NUT exporter is down" + description: "UPS monitoring is offline — cannot detect power events or battery state." + + # ────────────────────────────────────────────── + # 9. TLS certificate expiry + # (dormant — requires blackbox_exporter or traefik metrics) + # ────────────────────────────────────────────── + - name: tls + rules: + - alert: TLSCertExpiryWarning + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30 + for: 1h + labels: + severity: warning + annotations: + summary: "TLS cert expiring soon for {{ $labels.instance }}" + description: "Certificate expires in {{ printf \"%.0f\" $value | humanizeDuration }}." + + - alert: TLSCertExpiryCritical + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 7 + for: 1h + labels: + severity: critical + annotations: + summary: "TLS cert expiring in <7 days for {{ $labels.instance }}" + description: "Certificate expires in {{ printf \"%.0f\" $value | humanizeDuration }}. Renewal may have failed." + + - alert: TraefikCertExpiryWarning + expr: traefik_tls_certs_not_after - time() < 86400 * 30 + for: 1h + labels: + severity: warning + annotations: + summary: "Traefik TLS cert expiring for {{ $labels.sans }}" + description: "Certificate expires in {{ printf \"%.0f\" $value | humanizeDuration }}." + + # ────────────────────────────────────────────── + # 10. Container health + # ────────────────────────────────────────────── + - name: containers + rules: + - alert: ContainerOOM + expr: increase(container_oom_events_total{name!=""}[5m]) > 0 + for: 0m + labels: + severity: critical + annotations: + summary: "Container OOM kill: {{ $labels.name }}" + description: "Container {{ $labels.name }} was OOM-killed. Memory limit may need to be raised." + + - alert: ContainerRestartLoop + expr: changes(container_start_time_seconds{name!=""}[1h]) > 3 + for: 5m + labels: + severity: warning + annotations: + summary: "Container restart loop: {{ $labels.name }}" + description: "Container {{ $labels.name }} has restarted {{ printf \"%.0f\" $value }} times in the last hour." + + - alert: ContainerHighCPU + expr: rate(container_cpu_usage_seconds_total{name!="", id!="/"}[5m]) > 0.80 + for: 15m + labels: + severity: warning + annotations: + summary: "High CPU in container {{ $labels.name }}" + description: "Container {{ $labels.name }} is using {{ printf \"%.2f\" $value }} CPU cores sustained for 15 minutes." + + - alert: ContainerMemoryNearLimit + expr: > + (container_memory_working_set_bytes{name!=""} + / container_spec_memory_limit_bytes{name!=""}) + * 100 > 90 + and container_spec_memory_limit_bytes{name!=""} > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Container {{ $labels.name }} memory >90% of limit" + description: "Using {{ printf \"%.1f\" $value }}% of memory limit. OOM risk." + + - alert: ContainerMemoryAtLimit + expr: > + (container_memory_working_set_bytes{name!=""} + / container_spec_memory_limit_bytes{name!=""}) + * 100 > 98 + and container_spec_memory_limit_bytes{name!=""} > 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Container {{ $labels.name }} memory at limit — OOM imminent" + description: "Using {{ printf \"%.1f\" $value }}% of memory limit. OOM kill expected shortly." + + # ────────────────────────────────────────────── + # 11. Prometheus self-monitoring + # ────────────────────────────────────────────── + - name: prometheus_self + rules: + - alert: PrometheusConfigReloadFailed + expr: prometheus_config_last_reload_successful == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Prometheus config reload failed" + description: "Prometheus failed to reload its configuration. Alert rules may be stale." + + - alert: AlertmanagerDown + expr: up{job="alertmanager"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Alertmanager is down" + description: "Prometheus cannot reach Alertmanager — no alerts will be delivered." + + - alert: PrometheusTargetDown + expr: up == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Prometheus target down: {{ $labels.job }}/{{ $labels.instance }}" + description: "Scrape target {{ $labels.instance }} (job={{ $labels.job }}) unreachable for >5 minutes." + + - alert: AlertmanagerNotificationsFailing + expr: increase(alertmanager_notifications_failed_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Alertmanager notifications failing" + description: "Some alert notifications are failing to deliver. Check bridge logs." + + # ────────────────────────────────────────────── + # 12. Media / Vault1 storage (TrueNAS Tank NFS) + # ────────────────────────────────────────────── + - name: media_storage + rules: + - alert: MediaStorageWarning + expr: > + (1 - node_filesystem_avail_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"} + / node_filesystem_size_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"}) + * 100 > 85 + for: 10m + labels: + severity: warning + annotations: + summary: "Tank/Media share >85% full on {{ $labels.instance }}" + description: "{{ printf \"%.1f\" $value }}% used on the TrueNAS media share." + + - alert: MediaStorageCritical + expr: > + (1 - node_filesystem_avail_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"} + / node_filesystem_size_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"}) + * 100 > 95 + for: 5m + labels: + severity: critical + annotations: + summary: "Tank/Media share >95% full on {{ $labels.instance }}" + description: "{{ printf \"%.1f\" $value }}% used. Downloads/recordings will fail soon." + + # ────────────────────────────────────────────── + # 13. UPS battery self-test results + # Metrics written by /usr/local/bin/ups-quarterly-test.sh via node-exporter textfile collector on Beast. + # 1=passed, 0=failed/unknown. Cron: first Sunday of Jan/Apr/Jul/Oct at 03:00. + # ────────────────────────────────────────────── + - name: ups_battery_test + rules: + - alert: UPSBatteryTestFailed + expr: ups_battery_test_result == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "UPS {{ $labels.ups }} quarterly battery test failed" + description: "Self-test result was not 'passed'. Run test manually: upscmd -u upsadmin cyberpower1@localhost test.battery.start.quick" + + - alert: UPSBatteryTestStale + expr: time() - ups_battery_test_timestamp_seconds > 86400 * 100 + for: 6h + labels: + severity: warning + annotations: + summary: "UPS {{ $labels.ups }} battery test overdue (>100 days)" + description: "Last self-test was {{ printf \"%.0f\" $value | humanizeDuration }} ago. Quarterly cron may have missed a run on Beast."