diff --git a/monitoring/prometheus-offhost-watchdog/README.md b/monitoring/prometheus-offhost-watchdog/README.md new file mode 100644 index 0000000..a01741f --- /dev/null +++ b/monitoring/prometheus-offhost-watchdog/README.md @@ -0,0 +1,36 @@ +# Off-host Prometheus watchdog (dead-man's-switch) + +Independent-hardware watchdog that pages if the media-server (.183) Prometheus +becomes unreachable — the one failure the same-host Uptime-Kuma monitor cannot +detect (media-server itself going dark). + +**Runs on:** Pi / raspberrypi (192.168.99.227), as root via systemd timer. + +## Behaviour +- Timer fires every 60s (`OnBootSec=2min`, `OnUnitActiveSec=60s`). +- Probes `http://192.168.99.183:9090/-/healthy` for the keyword `Healthy`. +- After 3 consecutive failures (~3 min) publishes a **priority-5** "Prometheus DOWN" + page to ntfy topic `prometheus-critical` (compute6, `http://192.168.99.198:8095`); + re-pages every 30 checks while still down. +- On recovery sends a **priority-4** "Prometheus recovered". +- State: `/var/lib/prom-watchdog/state` ("FAILS ALERTED"). + +Reaches the phone because the ntfy server has `upstream-base-url: https://ntfy.sh` +(FCM/APNS relay). + +## Install (on the Pi) +``` +sudo install -m 755 prometheus-watchdog.sh /usr/local/bin/prometheus-watchdog.sh +cp prom-watchdog.env.example prom-watchdog.env # set NTFY_TOKEN only if ntfy auth is enabled +sudo install -m 600 prom-watchdog.env /etc/prom-watchdog.env +sudo install -m 644 prometheus-watchdog.service /etc/systemd/system/prometheus-watchdog.service +sudo install -m 644 prometheus-watchdog.timer /etc/systemd/system/prometheus-watchdog.timer +sudo systemctl daemon-reload +sudo systemctl enable --now prometheus-watchdog.timer +``` + +## Notes +- `prom-watchdog.env` has an **empty** `NTFY_TOKEN` — the `prometheus-critical` + topic is unauthenticated on the LAN ntfy. If auth is ever enabled, put the real + token in the deployed `/etc/prom-watchdog.env` (mode 600) and do **not** commit it. +- Deployed + live-outage tested 2026-06-30 (DOWN + recovered pages confirmed). diff --git a/monitoring/prometheus-offhost-watchdog/prom-watchdog.env.example b/monitoring/prometheus-offhost-watchdog/prom-watchdog.env.example new file mode 100644 index 0000000..1b1f3ac --- /dev/null +++ b/monitoring/prometheus-offhost-watchdog/prom-watchdog.env.example @@ -0,0 +1,5 @@ +# Copy to /etc/prom-watchdog.env (mode 600) on the Pi. +# NTFY_TOKEN stays empty while the prometheus-critical topic is unauthenticated; +# if ntfy auth is ever enabled, set it in the DEPLOYED file only (never commit — *.env is gitignored). +NTFY_URL=http://192.168.99.198:8095/prometheus-critical +NTFY_TOKEN= diff --git a/monitoring/prometheus-offhost-watchdog/prometheus-watchdog.service b/monitoring/prometheus-offhost-watchdog/prometheus-watchdog.service new file mode 100644 index 0000000..dfd5a61 --- /dev/null +++ b/monitoring/prometheus-offhost-watchdog/prometheus-watchdog.service @@ -0,0 +1,8 @@ +[Unit] +Description=Off-host Prometheus dead-man's-switch (probes media-server .183) +After=network-online.target +Wants=network-online.target +[Service] +Type=oneshot +EnvironmentFile=/etc/prom-watchdog.env +ExecStart=/usr/local/bin/prometheus-watchdog.sh diff --git a/monitoring/prometheus-offhost-watchdog/prometheus-watchdog.sh b/monitoring/prometheus-offhost-watchdog/prometheus-watchdog.sh new file mode 100644 index 0000000..24d192f --- /dev/null +++ b/monitoring/prometheus-offhost-watchdog/prometheus-watchdog.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +TARGET_URL="http://192.168.99.183:9090/-/healthy" +EXPECT="Healthy" +NTFY_URL="${NTFY_URL:?}" # from EnvironmentFile +NTFY_TOKEN="${NTFY_TOKEN:-}" +THRESHOLD=3 # consecutive fails before first page (~3 min) +RESEND_EVERY=30 # re-page every N checks while still down +TIMEOUT=10 +STATE_DIR="/var/lib/prom-watchdog"; STATE="$STATE_DIR/state" +mkdir -p "$STATE_DIR" +read -r FAILS ALERTED 2>/dev/null < "$STATE" || { FAILS=0; ALERTED=0; } +notify() { # $1=priority $2=tags $3=title $4=message + local a=(-s -H "Priority: $1" -H "Tags: $2" -H "Title: $3" -d "$4") + [ -n "$NTFY_TOKEN" ] && a+=(-H "Authorization: Bearer $NTFY_TOKEN") + curl "${a[@]}" "$NTFY_URL" >/dev/null 2>&1 || true +} +if curl -fsS --max-time "$TIMEOUT" "$TARGET_URL" 2>/dev/null | grep -q "$EXPECT"; then + [ "$ALERTED" -eq 1 ] && notify 4 white_check_mark "Prometheus recovered" \ + "media-server Prometheus responding again (off-host watchdog)." + echo "0 0" > "$STATE"; exit 0 +fi +FAILS=$((FAILS + 1)) +if [ "$ALERTED" -eq 0 ] && [ "$FAILS" -ge "$THRESHOLD" ]; then + notify 5 "rotating_light,skull" "Prometheus DOWN (off-host watchdog)" \ + "Pi cannot reach Prometheus at $TARGET_URL after $FAILS checks. media-server may be down or prometheus stopped. ALERTING IS BLIND until resolved." + ALERTED=1 +elif [ "$ALERTED" -eq 1 ] && [ $((FAILS % RESEND_EVERY)) -eq 0 ]; then + notify 5 rotating_light "Prometheus STILL down" "Unreachable ~${FAILS} min." +fi +echo "$FAILS $ALERTED" > "$STATE"; exit 0 diff --git a/monitoring/prometheus-offhost-watchdog/prometheus-watchdog.timer b/monitoring/prometheus-offhost-watchdog/prometheus-watchdog.timer new file mode 100644 index 0000000..94eb588 --- /dev/null +++ b/monitoring/prometheus-offhost-watchdog/prometheus-watchdog.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Run Prometheus watchdog every minute +[Timer] +OnBootSec=2min +OnUnitActiveSec=60s +AccuracySec=5s +[Install] +WantedBy=timers.target