tommy
16489cabe5
ContainerMemoryNearLimit and ContainerMemoryAtLimit divided
container_memory_usage_bytes (which includes reclaimable inactive_file
page cache) by the container limit. On prometheus this read ~99% and
fired AtLimit critical ("OOM kill expected shortly") despite anon
working set being only ~365 MiB and memory.events oom_kill = 0.
Both rules now use container_memory_working_set_bytes, the OOM-relevant
figure (usage minus reclaimable cache). Validated with promtool and
reloaded via SIGHUP; both rules read inactive (~33%).
This versions a fix that was previously live-only at
/home/tommy/observability/prometheus/ and would silently revert on the
next repo-based redeploy.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
543 lines
23 KiB
YAML
543 lines
23 KiB
YAML
groups:
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 1. Node availability
|
||
# ──────────────────────────────────────────────
|
||
- name: node_availability
|
||
rules:
|
||
- alert: NodeDown
|
||
expr: up{job="node-exporter"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Node {{ $labels.instance }} is down"
|
||
description: "node-exporter has not been reachable for >2 minutes."
|
||
|
||
- alert: NodeHighLoad
|
||
expr: >
|
||
node_load15
|
||
/ count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})
|
||
> 0.85
|
||
for: 15m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "High 15-min load on {{ $labels.instance }}"
|
||
description: "Load/core ratio is {{ printf \"%.2f\" $value }} (>0.85) for 15 minutes."
|
||
|
||
- alert: NodeHighLoadCritical
|
||
expr: >
|
||
node_load15
|
||
/ count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})
|
||
> 1.0
|
||
for: 10m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Critical load on {{ $labels.instance }}"
|
||
description: "Load/core ratio is {{ printf \"%.2f\" $value }} (>1.0) for 10 minutes."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 2. CPU
|
||
# ──────────────────────────────────────────────
|
||
- name: cpu
|
||
rules:
|
||
- alert: CPUHighWarning
|
||
expr: >
|
||
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||
for: 15m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "High CPU on {{ $labels.instance }}"
|
||
description: "CPU usage is {{ printf \"%.1f\" $value }}% for 15 minutes."
|
||
|
||
- alert: CPUHighCritical
|
||
expr: >
|
||
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 98
|
||
for: 5m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Critical CPU on {{ $labels.instance }}"
|
||
description: "CPU usage is {{ printf \"%.1f\" $value }}% for 5 minutes."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 3. Memory
|
||
# ──────────────────────────────────────────────
|
||
- name: memory
|
||
rules:
|
||
- alert: MemoryHighWarning
|
||
expr: >
|
||
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "High memory on {{ $labels.instance }}"
|
||
description: "Memory usage is {{ printf \"%.1f\" $value }}% (>90%)."
|
||
|
||
- alert: MemoryHighCritical
|
||
expr: >
|
||
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Critical memory on {{ $labels.instance }}"
|
||
description: "Memory usage is {{ printf \"%.1f\" $value }}% (>95%)."
|
||
|
||
- alert: SwapHighWarning
|
||
expr: >
|
||
(node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes)
|
||
/ node_memory_SwapTotal_bytes * 100 > 50
|
||
and node_memory_SwapTotal_bytes > 0
|
||
for: 15m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "High swap usage on {{ $labels.instance }}"
|
||
description: "Swap usage is {{ printf \"%.1f\" $value }}% (>50%)."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 4. Disk / filesystem
|
||
# $value = % used (expr produces percentage directly)
|
||
# ──────────────────────────────────────────────
|
||
- name: disk
|
||
rules:
|
||
- alert: DiskSpaceWarning
|
||
expr: >
|
||
(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"}
|
||
/ node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"})
|
||
* 100 > 85
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Disk >85% on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||
description: "{{ printf \"%.1f\" $value }}% used on {{ $labels.mountpoint }} ({{ $labels.device }})."
|
||
|
||
- alert: DiskSpaceCritical
|
||
expr: >
|
||
(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"}
|
||
/ node_filesystem_size_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"})
|
||
* 100 > 95
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Disk >95% on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||
description: "{{ printf \"%.1f\" $value }}% used — disk nearly full."
|
||
|
||
- alert: DiskFillPredicted24h
|
||
expr: >
|
||
predict_linear(
|
||
node_filesystem_avail_bytes{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs|efivarfs"}[6h],
|
||
86400
|
||
) < 0
|
||
for: 30m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Disk fill predicted <24h on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||
description: "At current write rate {{ $labels.mountpoint }} will fill within 24 hours."
|
||
|
||
- alert: InodesWarning
|
||
expr: >
|
||
(1 - node_filesystem_files_free{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs"}
|
||
/ node_filesystem_files{fstype!~"tmpfs|ramfs|squashfs|overlay|devtmpfs"})
|
||
* 100 > 90
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Inodes >90% on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||
description: "{{ printf \"%.1f\" $value }}% of inodes used on {{ $labels.mountpoint }}."
|
||
|
||
- alert: EFIOrBootPartitionHighWarning
|
||
expr: >
|
||
(1 - node_filesystem_avail_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"}
|
||
/ node_filesystem_size_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"})
|
||
* 100 > 80
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Boot/EFI partition >80% on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||
description: "{{ printf \"%.1f\" $value }}% used on {{ $labels.mountpoint }}. Clean old kernel/EFI entries."
|
||
|
||
- alert: EFIOrBootPartitionCritical
|
||
expr: >
|
||
(1 - node_filesystem_avail_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"}
|
||
/ node_filesystem_size_bytes{mountpoint=~"/boot.*|/sys/firmware/efi.*"})
|
||
* 100 > 90
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Boot/EFI partition >90% on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||
description: "{{ printf \"%.1f\" $value }}% used. Risk of failed kernel installs or boot failures."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 5. ZFS pool health
|
||
# ──────────────────────────────────────────────
|
||
- name: zfs
|
||
rules:
|
||
- alert: ZFSPoolDegraded
|
||
expr: node_zfs_zpool_state{state="degraded"} == 1
|
||
for: 1m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "ZFS pool {{ $labels.zpool }} DEGRADED on {{ $labels.instance }}"
|
||
description: "Pool {{ $labels.zpool }} is DEGRADED. A disk has likely failed — check immediately."
|
||
|
||
- alert: ZFSPoolFaultedOrUnavail
|
||
expr: node_zfs_zpool_state{state=~"faulted|unavail|suspended"} == 1
|
||
for: 1m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "ZFS pool {{ $labels.zpool }} is {{ $labels.state }} on {{ $labels.instance }}"
|
||
description: "Pool {{ $labels.zpool }} is {{ $labels.state }}. Immediate attention required."
|
||
|
||
- alert: ZFSPoolOfflineOrRemoved
|
||
expr: node_zfs_zpool_state{state=~"offline|removed"} == 1
|
||
for: 2m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "ZFS pool {{ $labels.zpool }} is {{ $labels.state }} on {{ $labels.instance }}"
|
||
description: "Pool {{ $labels.zpool }} reported {{ $labels.state }}. Verify if intentional."
|
||
|
||
- alert: ZFSPoolCapacityWarning
|
||
expr: >
|
||
(1 - node_filesystem_avail_bytes{fstype="zfs"}
|
||
/ node_filesystem_size_bytes{fstype="zfs"})
|
||
* 100 > 85
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "ZFS filesystem >85% on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||
description: "{{ printf \"%.1f\" $value }}% used. ZFS performance degrades above 80%."
|
||
|
||
- alert: ZFSPoolCapacityCritical
|
||
expr: >
|
||
(1 - node_filesystem_avail_bytes{fstype="zfs"}
|
||
/ node_filesystem_size_bytes{fstype="zfs"})
|
||
* 100 > 95
|
||
for: 5m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "ZFS filesystem >95% on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||
description: "{{ printf \"%.1f\" $value }}% used. ZFS writes will fail at 100%."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 6. SMART (dormant — requires smartctl-exporter)
|
||
# ──────────────────────────────────────────────
|
||
- name: smart
|
||
rules:
|
||
- alert: SMARTDeviceUnhealthy
|
||
expr: smartctl_device_smart_healthy == 0
|
||
for: 5m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "SMART failure on {{ $labels.instance }} device {{ $labels.device }}"
|
||
description: "SMART overall health check FAILED for {{ $labels.device }}."
|
||
|
||
- alert: SMARTReallocatedSectors
|
||
expr: >
|
||
smartctl_device_attribute{attribute_name="Reallocated_Sector_Ct",attribute_value_type="raw"}
|
||
> 0
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Reallocated sectors on {{ $labels.instance }} {{ $labels.device }}"
|
||
description: "{{ $value }} reallocated sectors — early sign of disk failure."
|
||
|
||
- alert: SMARTPendingSectors
|
||
expr: >
|
||
smartctl_device_attribute{attribute_name="Current_Pending_Sector",attribute_value_type="raw"}
|
||
> 0
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Pending sectors on {{ $labels.instance }} {{ $labels.device }}"
|
||
description: "{{ $value }} pending (unreadable) sectors. Disk may be failing."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 7. NFS mounts
|
||
# ──────────────────────────────────────────────
|
||
- name: nfs
|
||
rules:
|
||
- alert: NFSMountReadOnly
|
||
expr: node_filesystem_readonly{fstype=~"nfs.*", mountpoint=~"/mnt/.*"} == 1
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "NFS mount read-only on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||
description: "NFS mount {{ $labels.mountpoint }} has gone read-only — possible server issue or network error."
|
||
|
||
- alert: MediaNFSMountMissing
|
||
expr: absent(node_filesystem_avail_bytes{mountpoint="/mnt/media", instance="192.168.99.183:9100"})
|
||
for: 3m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Media NFS mount missing on media-server (.183)"
|
||
description: "/mnt/media (TrueNAS Tank) is not mounted. Plex/Immich will lose access to media."
|
||
|
||
- alert: SynoNFSMountMissing
|
||
expr: absent(node_filesystem_avail_bytes{mountpoint="/mnt/synology", instance="192.168.99.153:9100"})
|
||
for: 3m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Synology NFS mount missing on PBS (.153)"
|
||
description: "/mnt/synology is not mounted on PBS. Synology-Remote backup datastore is offline."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 8. UPS / NUT
|
||
# hon95/prometheus-nut-exporter reports 0–1 scale.
|
||
# Expressions multiply to produce % so $value is human-readable.
|
||
# ──────────────────────────────────────────────
|
||
- name: ups
|
||
rules:
|
||
- alert: UPSBatteryLow
|
||
expr: nut_battery_charge * 100 < 30
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "UPS {{ $labels.ups }} battery low ({{ printf \"%.0f\" $value }}%)"
|
||
description: "Battery at {{ printf \"%.0f\" $value }}%. Power loss may be imminent."
|
||
|
||
- alert: UPSBatteryWarning
|
||
expr: nut_battery_charge * 100 < 50
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "UPS {{ $labels.ups }} battery below 50%"
|
||
description: "Battery at {{ printf \"%.0f\" $value }}%. May indicate on-battery or slow charging."
|
||
|
||
- alert: UPSLowRuntime
|
||
expr: nut_battery_runtime_seconds < 300
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "UPS {{ $labels.ups }} runtime <5 minutes"
|
||
description: "Estimated runtime is {{ printf \"%.0f\" $value }}s. Systems may not shut down cleanly."
|
||
|
||
- alert: UPSHighLoad
|
||
expr: nut_load * 100 > 80
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "UPS {{ $labels.ups }} load >80%"
|
||
description: "UPS load is {{ printf \"%.0f\" $value }}%. Consider load balancing."
|
||
|
||
- alert: NUTExporterDown
|
||
expr: up{job="nut"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "NUT exporter is down"
|
||
description: "UPS monitoring is offline — cannot detect power events or battery state."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 9. TLS certificate expiry
|
||
# (dormant — requires blackbox_exporter or traefik metrics)
|
||
# ──────────────────────────────────────────────
|
||
- name: tls
|
||
rules:
|
||
- alert: TLSCertExpiryWarning
|
||
expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30
|
||
for: 1h
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "TLS cert expiring soon for {{ $labels.instance }}"
|
||
description: "Certificate expires in {{ printf \"%.0f\" $value | humanizeDuration }}."
|
||
|
||
- alert: TLSCertExpiryCritical
|
||
expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 7
|
||
for: 1h
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "TLS cert expiring in <7 days for {{ $labels.instance }}"
|
||
description: "Certificate expires in {{ printf \"%.0f\" $value | humanizeDuration }}. Renewal may have failed."
|
||
|
||
- alert: TraefikCertExpiryWarning
|
||
expr: traefik_tls_certs_not_after - time() < 86400 * 30
|
||
for: 1h
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Traefik TLS cert expiring for {{ $labels.sans }}"
|
||
description: "Certificate expires in {{ printf \"%.0f\" $value | humanizeDuration }}."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 10. Container health
|
||
# ──────────────────────────────────────────────
|
||
- name: containers
|
||
rules:
|
||
- alert: ContainerOOM
|
||
expr: increase(container_oom_events_total{name!=""}[5m]) > 0
|
||
for: 0m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Container OOM kill: {{ $labels.name }}"
|
||
description: "Container {{ $labels.name }} was OOM-killed. Memory limit may need to be raised."
|
||
|
||
- alert: ContainerRestartLoop
|
||
expr: changes(container_start_time_seconds{name!=""}[1h]) > 3
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Container restart loop: {{ $labels.name }}"
|
||
description: "Container {{ $labels.name }} has restarted {{ printf \"%.0f\" $value }} times in the last hour."
|
||
|
||
- alert: ContainerHighCPU
|
||
expr: rate(container_cpu_usage_seconds_total{name!="", id!="/"}[5m]) > 0.80
|
||
for: 15m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "High CPU in container {{ $labels.name }}"
|
||
description: "Container {{ $labels.name }} is using {{ printf \"%.2f\" $value }} CPU cores sustained for 15 minutes."
|
||
|
||
- alert: ContainerMemoryNearLimit
|
||
expr: >
|
||
(container_memory_working_set_bytes{name!=""}
|
||
/ container_spec_memory_limit_bytes{name!=""})
|
||
* 100 > 90
|
||
and container_spec_memory_limit_bytes{name!=""} > 0
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Container {{ $labels.name }} memory >90% of limit"
|
||
description: "Using {{ printf \"%.1f\" $value }}% of memory limit. OOM risk."
|
||
|
||
- alert: ContainerMemoryAtLimit
|
||
expr: >
|
||
(container_memory_working_set_bytes{name!=""}
|
||
/ container_spec_memory_limit_bytes{name!=""})
|
||
* 100 > 98
|
||
and container_spec_memory_limit_bytes{name!=""} > 0
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Container {{ $labels.name }} memory at limit — OOM imminent"
|
||
description: "Using {{ printf \"%.1f\" $value }}% of memory limit. OOM kill expected shortly."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 11. Prometheus self-monitoring
|
||
# ──────────────────────────────────────────────
|
||
- name: prometheus_self
|
||
rules:
|
||
- alert: PrometheusConfigReloadFailed
|
||
expr: prometheus_config_last_reload_successful == 0
|
||
for: 5m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Prometheus config reload failed"
|
||
description: "Prometheus failed to reload its configuration. Alert rules may be stale."
|
||
|
||
- alert: AlertmanagerDown
|
||
expr: up{job="alertmanager"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Alertmanager is down"
|
||
description: "Prometheus cannot reach Alertmanager — no alerts will be delivered."
|
||
|
||
- alert: PrometheusTargetDown
|
||
expr: up == 0
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Prometheus target down: {{ $labels.job }}/{{ $labels.instance }}"
|
||
description: "Scrape target {{ $labels.instance }} (job={{ $labels.job }}) unreachable for >5 minutes."
|
||
|
||
- alert: AlertmanagerNotificationsFailing
|
||
expr: increase(alertmanager_notifications_failed_total[5m]) > 0
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Alertmanager notifications failing"
|
||
description: "Some alert notifications are failing to deliver. Check bridge logs."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 12. Media / Vault1 storage (TrueNAS Tank NFS)
|
||
# ──────────────────────────────────────────────
|
||
- name: media_storage
|
||
rules:
|
||
- alert: MediaStorageWarning
|
||
expr: >
|
||
(1 - node_filesystem_avail_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"}
|
||
/ node_filesystem_size_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"})
|
||
* 100 > 85
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "Tank/Media share >85% full on {{ $labels.instance }}"
|
||
description: "{{ printf \"%.1f\" $value }}% used on the TrueNAS media share."
|
||
|
||
- alert: MediaStorageCritical
|
||
expr: >
|
||
(1 - node_filesystem_avail_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"}
|
||
/ node_filesystem_size_bytes{mountpoint="/mnt/media", fstype=~"nfs.*"})
|
||
* 100 > 95
|
||
for: 5m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: "Tank/Media share >95% full on {{ $labels.instance }}"
|
||
description: "{{ printf \"%.1f\" $value }}% used. Downloads/recordings will fail soon."
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 13. UPS battery self-test results
|
||
# Metrics written by /usr/local/bin/ups-quarterly-test.sh via node-exporter textfile collector on Beast.
|
||
# 1=passed, 0=failed/unknown. Cron: first Sunday of Jan/Apr/Jul/Oct at 03:00.
|
||
# ──────────────────────────────────────────────
|
||
- name: ups_battery_test
|
||
rules:
|
||
- alert: UPSBatteryTestFailed
|
||
expr: ups_battery_test_result == 0
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "UPS {{ $labels.ups }} quarterly battery test failed"
|
||
description: "Self-test result was not 'passed'. Run test manually: upscmd -u upsadmin cyberpower1@localhost test.battery.start.quick"
|
||
|
||
- alert: UPSBatteryTestStale
|
||
expr: time() - ups_battery_test_timestamp_seconds > 86400 * 100
|
||
for: 6h
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: "UPS {{ $labels.ups }} battery test overdue (>100 days)"
|
||
description: "Last self-test was {{ printf \"%.0f\" $value | humanizeDuration }} ago. Quarterly cron may have missed a run on Beast."
|