tommy
7178f6ceea
Reports the speedtest-tracker (on .183) newest result: flags STALE if >14h old (2+ missed 6h cycles → scheduler stalled) or LOW if the best of the last 4 tests falls below the 500/500 Mbps floor (best-of-4 avoids flagging a transient ISP dip). Floors adjustable. Local SQLite read. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
335 lines
15 KiB
Bash
Executable File
335 lines
15 KiB
Bash
Executable File
#!/bin/bash
|
|
# weekly-health-digest.sh — P5-12
|
|
# Lightweight weekly drift-catcher. Read-only. Posts a concise digest to ntfy/health-digest.
|
|
# Does NOT replace the quarterly full health check.
|
|
# Schedule: Monday 07:00 via /etc/cron.d/weekly-health-digest
|
|
# Runs on: media-server (192.168.99.183)
|
|
|
|
set -euo pipefail
|
|
|
|
NTFY_URL="http://192.168.99.198:8095/health-digest"
|
|
PROM_URL="http://localhost:9090"
|
|
GRAFANA_URL="http://localhost:3005"
|
|
TIMEOUT=10 # seconds per HTTP request before giving up on that check
|
|
|
|
# Node-exporter targets (from prometheus.yml)
|
|
NODE_EXPORTER_TARGETS=(
|
|
"192.168.99.200:9100" # beast
|
|
"192.168.99.192:9100" # compute2
|
|
"192.168.99.193:9100" # compute3
|
|
"192.168.99.194:9100" # compute4
|
|
"192.168.99.153:9100" # pbs
|
|
"192.168.99.183:9100" # media-server
|
|
"192.168.99.186:9100" # docker-node01
|
|
"192.168.99.187:9100" # docker-node02
|
|
"192.168.99.190:9100" # ansible-control
|
|
"192.168.99.227:9100" # pi4 (P5-10 — expected down)
|
|
"192.168.99.196:9100" # compute5
|
|
"192.168.99.198:9100" # compute6
|
|
"192.168.99.100:9100" # haos (expected 500)
|
|
)
|
|
|
|
logger -t weekly-health-digest "starting weekly health digest"
|
|
|
|
# ── helpers ─────────────────────────────────────────────────────────────────
|
|
|
|
prom_query() {
|
|
# $1 = PromQL query; returns JSON result array or empty string on failure
|
|
curl -sf --max-time "$TIMEOUT" \
|
|
"${PROM_URL}/api/v1/query?query=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1]))" "$1")" \
|
|
2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(json.dumps(d['data']['result']))" 2>/dev/null || echo "[]"
|
|
}
|
|
|
|
prom_scalar() {
|
|
# Returns first value from a query as a float, or -1 on failure
|
|
local result
|
|
result=$(prom_query "$1")
|
|
python3 -c "
|
|
import json,sys
|
|
try:
|
|
d=json.loads('$result')
|
|
if d: print(float(d[0]['value'][1]))
|
|
else: print(-1)
|
|
except: print(-1)
|
|
" 2>/dev/null || echo "-1"
|
|
}
|
|
|
|
OK="OK"
|
|
FLAG=""
|
|
LINES=()
|
|
ISSUES=0
|
|
|
|
add_line() {
|
|
local label="$1" status="$2" detail="${3:-}"
|
|
if [ "$status" = "OK" ]; then
|
|
LINES+=(" ${label}: OK${detail:+ — $detail}")
|
|
else
|
|
LINES+=(" ${label}: *** ${status}${detail:+ — $detail} ***")
|
|
ISSUES=$((ISSUES + 1))
|
|
fi
|
|
}
|
|
|
|
# ── 1. Proxmox quorum ────────────────────────────────────────────────────────
|
|
# We can't SSH to PVE nodes from here directly, so we infer quorum from node
|
|
# reachability. If beast (the PVE master) is up and answering node-exporter,
|
|
# the cluster is likely healthy. Flag if beast is unreachable.
|
|
beast_up=$(curl -sf --max-time 5 "http://192.168.99.200:9100/metrics" 2>/dev/null | grep -c "node_boot_time_seconds" || true)
|
|
if [ "${beast_up}" -ge 1 ]; then
|
|
add_line "PVE/quorum" "OK" "beast reachable"
|
|
else
|
|
add_line "PVE/quorum" "beast unreachable — quorum check not possible"
|
|
fi
|
|
|
|
# ── 2. Prometheus node-exporter targets ─────────────────────────────────────
|
|
targets_result=$(prom_query 'up{job="node-exporter"}')
|
|
targets_down=$(echo "$targets_result" | python3 -c "
|
|
import json,sys
|
|
try:
|
|
d=json.loads(sys.stdin.read())
|
|
down=[r['metric'].get('instance','?') for r in d if float(r['value'][1])==0]
|
|
# Filter known-persistent issues that don't need weekly noise
|
|
known_down={'192.168.99.227:9100','192.168.99.100:9100'}
|
|
new_down=[h for h in down if h not in known_down]
|
|
print(','.join(new_down) if new_down else '')
|
|
# Always report known-down count for awareness
|
|
known_count=len([h for h in down if h in known_down])
|
|
if known_count: print(f'(+{known_count} known-down: P5-10/HAOS)', end='', file=__import__('sys').stderr)
|
|
except: print('')
|
|
" 2>/tmp/weekly-health-digest-known.txt)
|
|
known_note=$(cat /tmp/weekly-health-digest-known.txt 2>/dev/null || true)
|
|
rm -f /tmp/weekly-health-digest-known.txt
|
|
|
|
if [ -z "$targets_down" ]; then
|
|
add_line "node-exporter targets" "OK" "${known_note:-all up (excl. 2 known-down)}"
|
|
else
|
|
add_line "node-exporter targets" "NEW targets down: ${targets_down}" "${known_note}"
|
|
fi
|
|
|
|
# ── 3. Grafana/Prometheus firing alerts ─────────────────────────────────────
|
|
firing_result=$(curl -sf --max-time "$TIMEOUT" \
|
|
"${PROM_URL}/api/v1/alerts" 2>/dev/null | python3 -c "
|
|
import json,sys
|
|
try:
|
|
d=json.load(sys.stdin)
|
|
firing=[a['labels'].get('alertname','?') for a in d['data']['alerts'] if a['state']=='firing']
|
|
# Suppress known-persistent alerts that are tracked elsewhere
|
|
known_ok={'PrometheusTargetDown','NodeDown'} # covered in target check above
|
|
new_firing=[a for a in firing if a not in known_ok]
|
|
if new_firing:
|
|
print(','.join(sorted(set(new_firing))))
|
|
else:
|
|
suppressed=len([a for a in firing if a in known_ok])
|
|
print(f'none (suppressed {suppressed} known)')
|
|
except: print('query_failed')
|
|
" 2>/dev/null || echo "query_failed")
|
|
|
|
if echo "$firing_result" | grep -qE "none|^$"; then
|
|
add_line "Grafana alerts" "OK" "$firing_result"
|
|
elif [ "$firing_result" = "query_failed" ]; then
|
|
add_line "Grafana alerts" "Prometheus unreachable"
|
|
else
|
|
add_line "Grafana alerts" "FIRING: ${firing_result}"
|
|
fi
|
|
|
|
# ── 4. PBS datastores — backup freshness >48h ────────────────────────────────
|
|
# Check local backup metric: backup_status textfile on media-server node-exporter
|
|
backup_status=$(prom_scalar 'backup_status{instance="192.168.99.183:9100"}')
|
|
if python3 -c "import sys; sys.exit(0 if float('$backup_status') >= 1 else 1)" 2>/dev/null; then
|
|
add_line "PBS/backups" "OK" "backup_status metric healthy"
|
|
else
|
|
add_line "PBS/backups" "backup_status=0 or unavailable — check PBS and local backup jobs"
|
|
fi
|
|
|
|
# Also check PBS ZFS pool presence (covers the INC-001/usb1-zfs and usb2-zfs situation)
|
|
pbs_zfs=$(prom_query 'zfs_pool_present')
|
|
pbs_zfs_missing=$(echo "$pbs_zfs" | python3 -c "
|
|
import json,sys
|
|
try:
|
|
d=json.loads(sys.stdin.read())
|
|
missing=[r['metric'].get('pool','?') for r in d if float(r['value'][1])==0]
|
|
print(','.join(missing) if missing else '')
|
|
except: print('')
|
|
" 2>/dev/null || echo "")
|
|
if [ -z "$pbs_zfs_missing" ]; then
|
|
add_line "PBS ZFS pools" "OK" "usb1-zfs + usb2-zfs imported"
|
|
else
|
|
add_line "PBS ZFS pools" "MISSING: ${pbs_zfs_missing}" "pools not imported on PBS — check enclosure/cables"
|
|
fi
|
|
|
|
# ── 4b. Off-host app backups: LubeLogger + Vaultwarden freshness (<26h) ──────
|
|
# Both push nightly from docker-node01 (.186) to PBS rust-usb: host/vaultwarden
|
|
# (root cron 03:00) and host/lubelogger (systemd timer 03:30). We ask PBS for the
|
|
# newest snapshot time of each group via the lubelogger token, which has
|
|
# datastore-wide audit on rust-usb (covers both groups). Read-only; keyed SSH +
|
|
# passwordless sudo on .186. Quoted heredoc → remote-side var expansion only.
|
|
backup_ages=$(ssh -o ConnectTimeout=8 -o BatchMode=yes tommy@192.168.99.186 'sudo bash -s' 2>/dev/null <<'REMOTE' || true
|
|
source /etc/lubelogger-backup.env 2>/dev/null
|
|
export PBS_PASSWORD PBS_FINGERPRINT
|
|
for grp in lubelogger vaultwarden; do
|
|
t=$(proxmox-backup-client snapshot list "host/$grp" --repository "$PBS_REPOSITORY" --output-format json 2>/dev/null \
|
|
| python3 -c 'import sys,json
|
|
try:
|
|
d=json.load(sys.stdin); print(max(x["backup-time"] for x in d) if d else 0)
|
|
except: print(0)' 2>/dev/null || echo 0)
|
|
echo "$grp $t"
|
|
done
|
|
REMOTE
|
|
)
|
|
|
|
now_epoch=$(date +%s)
|
|
for svc in lubelogger vaultwarden; do
|
|
ts=$(echo "$backup_ages" | awk -v s="$svc" '$1==s{print $2}')
|
|
if [ -z "$ts" ] || [ "$ts" = "0" ]; then
|
|
add_line "Backup ${svc}→PBS" "no snapshot / query failed — check .186 backup job"
|
|
else
|
|
age_h=$(( (now_epoch - ts) / 3600 ))
|
|
if [ "$age_h" -lt 26 ]; then
|
|
add_line "Backup ${svc}→PBS" "OK" "newest snapshot ${age_h}h ago"
|
|
else
|
|
add_line "Backup ${svc}→PBS" "STALE — newest snapshot ${age_h}h ago (>26h)"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# ── 4c. Internet speed (speedtest-tracker on .183) ──────────────────────────
|
|
# Tracker records Ookla tests every 6h (SPEEDTEST_SCHEDULE). Flag if the newest
|
|
# completed result is stale (>14h = 2+ missed cycles → scheduler stalled) or if
|
|
# the best of the last 4 tests is below the floor (using best-of-4, not the
|
|
# single latest, avoids flagging a transient ISP dip). Floors are ADJUSTABLE —
|
|
# set to catch a real regression (e.g. the old 100Mb-uplink cap ≈ 94 Mbps) or
|
|
# sustained ISP degradation, not normal variance. DB is local + tommy-readable.
|
|
ST_DB="/home/tommy/speedtest-tracker/data/database.sqlite"
|
|
ST_FLOOR_DOWN=500 # Mbps
|
|
ST_FLOOR_UP=500 # Mbps
|
|
if [ -r "$ST_DB" ]; then
|
|
st_row=$(sqlite3 "$ST_DB" "
|
|
SELECT
|
|
CAST((julianday('now') - julianday(max(created_at)))*24 AS INT) || '|' ||
|
|
CAST((SELECT download FROM results WHERE status='completed' ORDER BY id DESC LIMIT 1)*8/1e6 AS INT) || '|' ||
|
|
CAST((SELECT upload FROM results WHERE status='completed' ORDER BY id DESC LIMIT 1)*8/1e6 AS INT) || '|' ||
|
|
CAST((SELECT max(download) FROM (SELECT download FROM results WHERE status='completed' ORDER BY id DESC LIMIT 4))*8/1e6 AS INT) || '|' ||
|
|
CAST((SELECT max(upload) FROM (SELECT upload FROM results WHERE status='completed' ORDER BY id DESC LIMIT 4))*8/1e6 AS INT)
|
|
FROM results WHERE status='completed';" 2>/dev/null)
|
|
st_age=$(echo "$st_row" | cut -d'|' -f1)
|
|
st_dl=$(echo "$st_row" | cut -d'|' -f2)
|
|
st_ul=$(echo "$st_row" | cut -d'|' -f3)
|
|
st_mdl=$(echo "$st_row" | cut -d'|' -f4)
|
|
st_mul=$(echo "$st_row" | cut -d'|' -f5)
|
|
if [ -z "$st_age" ]; then
|
|
add_line "Internet speed" "no completed speedtest results found"
|
|
elif [ "$st_age" -gt 14 ]; then
|
|
add_line "Internet speed" "STALE — newest test ${st_age}h ago (expected every 6h; scheduler stalled?)"
|
|
elif [ "$st_mdl" -lt "$ST_FLOOR_DOWN" ] || [ "$st_mul" -lt "$ST_FLOOR_UP" ]; then
|
|
add_line "Internet speed" "LOW — best of last 4: ${st_mdl}/${st_mul} Mbps down/up (floor ${ST_FLOOR_DOWN}/${ST_FLOOR_UP})"
|
|
else
|
|
add_line "Internet speed" "OK" "${st_dl}/${st_ul} Mbps down/up, ${st_age}h ago"
|
|
fi
|
|
else
|
|
add_line "Internet speed" "speedtest DB unreadable ($ST_DB)"
|
|
fi
|
|
|
|
# ── 5. Filesystem >85% ───────────────────────────────────────────────────────
|
|
fs_over=$(prom_query '(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"}) * 100 > 85' | python3 -c "
|
|
import json,sys
|
|
try:
|
|
d=json.loads(sys.stdin.read())
|
|
items=[f'{r[\"metric\"].get(\"instance\",\"?\").split(\":\")[0]}:{r[\"metric\"].get(\"mountpoint\",\"?\")} {float(r[\"value\"][1]):.0f}%' for r in d]
|
|
print(', '.join(items) if items else '')
|
|
except: print('')
|
|
" 2>/dev/null || echo "")
|
|
if [ -z "$fs_over" ]; then
|
|
add_line "Disk usage >85%" "OK" "no filesystem over threshold"
|
|
else
|
|
add_line "Disk usage >85%" "OVER THRESHOLD: ${fs_over}"
|
|
fi
|
|
|
|
# ── 6. ZFS pools ONLINE (cluster-wide) ──────────────────────────────────────
|
|
zfs_bad=$(prom_query 'node_zfs_zpool_state{state=~"degraded|faulted|suspended|unavail|offline|removed"} == 1' | python3 -c "
|
|
import json,sys
|
|
try:
|
|
d=json.loads(sys.stdin.read())
|
|
items=[f'{r[\"metric\"].get(\"instance\",\"?\").split(\":\")[0]}:{r[\"metric\"].get(\"zpool\",\"?\")}({r[\"metric\"].get(\"state\",\"?\")})' for r in d]
|
|
print(', '.join(items) if items else '')
|
|
except: print('')
|
|
" 2>/dev/null || echo "")
|
|
if [ -z "$zfs_bad" ]; then
|
|
add_line "ZFS pools" "OK" "das-mirror, compute3-zfs all ONLINE"
|
|
else
|
|
add_line "ZFS pools" "DEGRADED/FAULTED: ${zfs_bad}"
|
|
fi
|
|
|
|
# ── 7. Docker unhealthy/restarting ──────────────────────────────────────────
|
|
unhealthy=$(docker ps --format "{{.Names}}\t{{.Status}}" 2>/dev/null | \
|
|
awk '/unhealthy|restarting/{print $1}' | \
|
|
grep -v "immich_machine_learning" | \
|
|
tr '\n' ',' | sed 's/,$//' || true)
|
|
# immich_machine_learning is known-unhealthy — suppress
|
|
if [ -z "$unhealthy" ]; then
|
|
add_line "Docker (local)" "OK" "no new unhealthy/restarting containers"
|
|
else
|
|
add_line "Docker (local)" "UNHEALTHY/RESTARTING: ${unhealthy}"
|
|
fi
|
|
|
|
# ── 8. TLS cert expiry <21 days ─────────────────────────────────────────────
|
|
# Check live cert expiry on a representative set of domains
|
|
cert_warn=""
|
|
for domain in traefik.goattw.net auth.goattw.net grafana.goattw.net ha.goattw.net nvr.goattw.net home.goattw.net; do
|
|
expiry=$(echo | timeout 8 openssl s_client -connect "${domain}:443" -servername "${domain}" 2>/dev/null \
|
|
| openssl x509 -noout -enddate 2>/dev/null \
|
|
| cut -d= -f2)
|
|
if [ -n "$expiry" ]; then
|
|
days=$(python3 -c "
|
|
from datetime import datetime
|
|
try:
|
|
exp=datetime.strptime('$expiry','%b %d %H:%M:%S %Y %Z')
|
|
now=datetime.utcnow()
|
|
print(int((exp-now).days))
|
|
except: print(999)
|
|
" 2>/dev/null || echo "999")
|
|
if [ "$days" -lt 21 ] 2>/dev/null; then
|
|
cert_warn="${cert_warn}${domain}(${days}d) "
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ -z "$cert_warn" ]; then
|
|
add_line "TLS cert expiry" "OK" "all checked certs >21 days"
|
|
else
|
|
add_line "TLS cert expiry" "EXPIRING SOON: ${cert_warn}"
|
|
fi
|
|
|
|
# ── Compose and send digest ──────────────────────────────────────────────────
|
|
NOW=$(date '+%Y-%m-%d %H:%M %Z')
|
|
|
|
if [ "$ISSUES" -eq 0 ]; then
|
|
PRIORITY="low"
|
|
SUBJECT="Weekly Health Digest ${NOW} — ALL OK"
|
|
TAGS="white_check_mark"
|
|
else
|
|
PRIORITY="high"
|
|
SUBJECT="Weekly Health Digest ${NOW} — ${ISSUES} ISSUE(S) FLAGGED"
|
|
TAGS="rotating_light"
|
|
fi
|
|
|
|
BODY=$(printf '%s\n' "${LINES[@]}")
|
|
|
|
logger -t weekly-health-digest "digest complete: issues=${ISSUES} priority=${PRIORITY}"
|
|
|
|
# Post to ntfy health-digest topic
|
|
HTTP_STATUS=$(curl -sf --max-time 15 \
|
|
-H "Title: ${SUBJECT}" \
|
|
-H "Priority: ${PRIORITY}" \
|
|
-H "Tags: ${TAGS}" \
|
|
-d "${BODY}" \
|
|
"${NTFY_URL}" \
|
|
-o /dev/null -w "%{http_code}" 2>/dev/null || echo "000")
|
|
|
|
if [ "$HTTP_STATUS" = "200" ] || [ "$HTTP_STATUS" = "201" ]; then
|
|
logger -t weekly-health-digest "ntfy delivery OK (HTTP ${HTTP_STATUS})"
|
|
else
|
|
logger -t weekly-health-digest "WARNING: ntfy delivery failed (HTTP ${HTTP_STATUS})"
|
|
fi
|
|
|
|
# Always exit 0 — cron should never see a failure from a read-only digest
|
|
exit 0
|