tommy
6af09ef9c6
Independent-hardware systemd timer on raspberrypi (.227) probing media-server Prometheus /-/healthy every 60s; pages ntfy prometheus-critical after 3 fails and on recovery. Catches media-server-dark, which the same-host Uptime-Kuma monitor structurally cannot. Deployed + live-outage tested 2026-06-30. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
32 lines
1.5 KiB
Bash
32 lines
1.5 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
TARGET_URL="http://192.168.99.183:9090/-/healthy"
|
|
EXPECT="Healthy"
|
|
NTFY_URL="${NTFY_URL:?}" # from EnvironmentFile
|
|
NTFY_TOKEN="${NTFY_TOKEN:-}"
|
|
THRESHOLD=3 # consecutive fails before first page (~3 min)
|
|
RESEND_EVERY=30 # re-page every N checks while still down
|
|
TIMEOUT=10
|
|
STATE_DIR="/var/lib/prom-watchdog"; STATE="$STATE_DIR/state"
|
|
mkdir -p "$STATE_DIR"
|
|
read -r FAILS ALERTED 2>/dev/null < "$STATE" || { FAILS=0; ALERTED=0; }
|
|
notify() { # $1=priority $2=tags $3=title $4=message
|
|
local a=(-s -H "Priority: $1" -H "Tags: $2" -H "Title: $3" -d "$4")
|
|
[ -n "$NTFY_TOKEN" ] && a+=(-H "Authorization: Bearer $NTFY_TOKEN")
|
|
curl "${a[@]}" "$NTFY_URL" >/dev/null 2>&1 || true
|
|
}
|
|
if curl -fsS --max-time "$TIMEOUT" "$TARGET_URL" 2>/dev/null | grep -q "$EXPECT"; then
|
|
[ "$ALERTED" -eq 1 ] && notify 4 white_check_mark "Prometheus recovered" \
|
|
"media-server Prometheus responding again (off-host watchdog)."
|
|
echo "0 0" > "$STATE"; exit 0
|
|
fi
|
|
FAILS=$((FAILS + 1))
|
|
if [ "$ALERTED" -eq 0 ] && [ "$FAILS" -ge "$THRESHOLD" ]; then
|
|
notify 5 "rotating_light,skull" "Prometheus DOWN (off-host watchdog)" \
|
|
"Pi cannot reach Prometheus at $TARGET_URL after $FAILS checks. media-server may be down or prometheus stopped. ALERTING IS BLIND until resolved."
|
|
ALERTED=1
|
|
elif [ "$ALERTED" -eq 1 ] && [ $((FAILS % RESEND_EVERY)) -eq 0 ]; then
|
|
notify 5 rotating_light "Prometheus STILL down" "Unreachable ~${FAILS} min."
|
|
fi
|
|
echo "$FAILS $ALERTED" > "$STATE"; exit 0
|