monitoring: off-host Prometheus dead-man's-switch watchdog (Pi .227)

Independent-hardware systemd timer on raspberrypi (.227) probing media-server
Prometheus /-/healthy every 60s; pages ntfy prometheus-critical after 3 fails
and on recovery. Catches media-server-dark, which the same-host Uptime-Kuma
monitor structurally cannot. Deployed + live-outage tested 2026-06-30.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

monitoring/prometheus-offhost-watchdog/prometheus-watchdog.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+TARGET_URL="http://192.168.99.183:9090/-/healthy"
+EXPECT="Healthy"
+NTFY_URL="${NTFY_URL:?}"          # from EnvironmentFile
+NTFY_TOKEN="${NTFY_TOKEN:-}"
+THRESHOLD=3                        # consecutive fails before first page (~3 min)
+RESEND_EVERY=30                    # re-page every N checks while still down
+TIMEOUT=10
+STATE_DIR="/var/lib/prom-watchdog"; STATE="$STATE_DIR/state"
+mkdir -p "$STATE_DIR"
+read -r FAILS ALERTED 2>/dev/null < "$STATE" || { FAILS=0; ALERTED=0; }
+notify() {  # $1=priority $2=tags $3=title $4=message
+  local a=(-s -H "Priority: $1" -H "Tags: $2" -H "Title: $3" -d "$4")
+  [ -n "$NTFY_TOKEN" ] && a+=(-H "Authorization: Bearer $NTFY_TOKEN")
+  curl "${a[@]}" "$NTFY_URL" >/dev/null 2>&1 || true
+}
+if curl -fsS --max-time "$TIMEOUT" "$TARGET_URL" 2>/dev/null | grep -q "$EXPECT"; then
+  [ "$ALERTED" -eq 1 ] && notify 4 white_check_mark "Prometheus recovered" \
+    "media-server Prometheus responding again (off-host watchdog)."
+  echo "0 0" > "$STATE"; exit 0
+fi
+FAILS=$((FAILS + 1))
+if [ "$ALERTED" -eq 0 ] && [ "$FAILS" -ge "$THRESHOLD" ]; then
+  notify 5 "rotating_light,skull" "Prometheus DOWN (off-host watchdog)" \
+    "Pi cannot reach Prometheus at $TARGET_URL after $FAILS checks. media-server may be down or prometheus stopped. ALERTING IS BLIND until resolved."
+  ALERTED=1
+elif [ "$ALERTED" -eq 1 ] && [ $((FAILS % RESEND_EVERY)) -eq 0 ]; then
+  notify 5 rotating_light "Prometheus STILL down" "Unreachable ~${FAILS} min."
+fi
+echo "$FAILS $ALERTED" > "$STATE"; exit 0