diff --git a/monitoring/PHASE5-incidents.md b/monitoring/PHASE5-incidents.md new file mode 100644 index 0000000..3733ef4 --- /dev/null +++ b/monitoring/PHASE5-incidents.md @@ -0,0 +1,57 @@ +# Phase 5 — Incident & Deliverable Log + +## P5-12 — Weekly Health Digest (2026-05-16) + +**Deliverable:** `monitoring/weekly-health-digest.sh` +**Deployed to:** media-server (192.168.99.183) at `~/ansible/scripts/weekly-health-digest.sh` +**Schedule:** `/etc/cron.d/weekly-health-digest` — Monday 07:00 CDT + +### What the digest covers + +Lightweight drift-catcher. Runs read-only checks each Monday and posts a single concise message to the **ntfy `health-digest` topic** (dedicated topic — not grafana-alerts, not prometheus-alerts). + +| Check | Method | +|-------|--------| +| PVE cluster reachable | Ping beast node-exporter | +| Prometheus targets all up | `up{job="node-exporter"}` query — known-persistent down (.227/.100) suppressed | +| Grafana/Prometheus firing alerts | `/api/v1/alerts` — PrometheusTargetDown/NodeDown suppressed (covered above) | +| PBS backup freshness | `backup_status` textfile metric | +| PBS ZFS pools imported | `zfs_pool_present` textfile metric (usb1-zfs, usb2-zfs) | +| Filesystem >85% (all nodes) | Prometheus node_filesystem query | +| ZFS pools ONLINE (cluster-wide) | `node_zfs_zpool_state` for degraded/faulted/suspended/unavail | +| Docker containers unhealthy/restarting | `docker ps` on media-server (known-unhealthy `immich_machine_learning` suppressed) | +| TLS cert expiry <21 days | Live openssl check on 6 representative domains | + +### What the digest does NOT do + +- Does NOT replace the quarterly full health check sweep. +- Does NOT check HAOS internals, Frigate camera counts, Coral inference times, or Authelia error rates. +- Does NOT check docker-node01/02 containers directly (no Docker API exposed). +- Does NOT verify Keepalived MASTER/BACKUP state. +- Does NOT check PBS datastore GC/verify schedules in detail. + +These areas require a human-run full sweep (see `~/ansible/reports/health-check-*.md`). + +### ntfy subscription + +Subscribe ntfy client to topic `health-digest` on `192.168.99.198:8095`. +Same procedure as the `zfs-health` topic added in Phase 4D. + +### Break-test result (2026-05-16 08:54 CDT) + +Script ran successfully. Delivery confirmed on `health-digest` topic. Correctly flagged: +- **PBS ZFS pools MISSING** (usb1-zfs + usb2-zfs — INC-001 + new usb2-zfs failure) ✓ +- **Grafana FIRING** (ContainerMemoryNearLimit, SwapHighWarning) ✓ +- All healthy checks reported OK ✓ +- Known-down nodes (.227 P5-10, .100 HAOS) correctly suppressed ✓ + +--- + +## Open Phase 5 Items + +| ID | Description | Status | +|----|-------------|--------| +| P5-07 | ansible-control disk at 79.2% (was 81.5%) | Ongoing — still approaching 85% | +| P5-10 | Pi4 node-exporter wrong-arch binary (.227) | Open | +| P5-12 | Weekly health digest | **DONE** (this entry) | + diff --git a/monitoring/weekly-health-digest.sh b/monitoring/weekly-health-digest.sh new file mode 100755 index 0000000..199616f --- /dev/null +++ b/monitoring/weekly-health-digest.sh @@ -0,0 +1,262 @@ +#!/bin/bash +# weekly-health-digest.sh — P5-12 +# Lightweight weekly drift-catcher. Read-only. Posts a concise digest to ntfy/health-digest. +# Does NOT replace the quarterly full health check. +# Schedule: Monday 07:00 via /etc/cron.d/weekly-health-digest +# Runs on: media-server (192.168.99.183) + +set -euo pipefail + +NTFY_URL="http://192.168.99.198:8095/health-digest" +PROM_URL="http://localhost:9090" +GRAFANA_URL="http://localhost:3005" +TIMEOUT=10 # seconds per HTTP request before giving up on that check + +# Node-exporter targets (from prometheus.yml) +NODE_EXPORTER_TARGETS=( + "192.168.99.200:9100" # beast + "192.168.99.192:9100" # compute2 + "192.168.99.193:9100" # compute3 + "192.168.99.194:9100" # compute4 + "192.168.99.153:9100" # pbs + "192.168.99.183:9100" # media-server + "192.168.99.186:9100" # docker-node01 + "192.168.99.187:9100" # docker-node02 + "192.168.99.190:9100" # ansible-control + "192.168.99.227:9100" # pi4 (P5-10 — expected down) + "192.168.99.196:9100" # compute5 + "192.168.99.198:9100" # compute6 + "192.168.99.100:9100" # haos (expected 500) +) + +logger -t weekly-health-digest "starting weekly health digest" + +# ── helpers ───────────────────────────────────────────────────────────────── + +prom_query() { + # $1 = PromQL query; returns JSON result array or empty string on failure + curl -sf --max-time "$TIMEOUT" \ + "${PROM_URL}/api/v1/query?query=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1]))" "$1")" \ + 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(json.dumps(d['data']['result']))" 2>/dev/null || echo "[]" +} + +prom_scalar() { + # Returns first value from a query as a float, or -1 on failure + local result + result=$(prom_query "$1") + python3 -c " +import json,sys +try: + d=json.loads('$result') + if d: print(float(d[0]['value'][1])) + else: print(-1) +except: print(-1) +" 2>/dev/null || echo "-1" +} + +OK="OK" +FLAG="" +LINES=() +ISSUES=0 + +add_line() { + local label="$1" status="$2" detail="${3:-}" + if [ "$status" = "OK" ]; then + LINES+=(" ${label}: OK${detail:+ — $detail}") + else + LINES+=(" ${label}: *** ${status}${detail:+ — $detail} ***") + ISSUES=$((ISSUES + 1)) + fi +} + +# ── 1. Proxmox quorum ──────────────────────────────────────────────────────── +# We can't SSH to PVE nodes from here directly, so we infer quorum from node +# reachability. If beast (the PVE master) is up and answering node-exporter, +# the cluster is likely healthy. Flag if beast is unreachable. +beast_up=$(curl -sf --max-time 5 "http://192.168.99.200:9100/metrics" 2>/dev/null | grep -c "node_boot_time_seconds" || true) +if [ "${beast_up}" -ge 1 ]; then + add_line "PVE/quorum" "OK" "beast reachable" +else + add_line "PVE/quorum" "beast unreachable — quorum check not possible" +fi + +# ── 2. Prometheus node-exporter targets ───────────────────────────────────── +targets_result=$(prom_query 'up{job="node-exporter"}') +targets_down=$(echo "$targets_result" | python3 -c " +import json,sys +try: + d=json.loads(sys.stdin.read()) + down=[r['metric'].get('instance','?') for r in d if float(r['value'][1])==0] + # Filter known-persistent issues that don't need weekly noise + known_down={'192.168.99.227:9100','192.168.99.100:9100'} + new_down=[h for h in down if h not in known_down] + print(','.join(new_down) if new_down else '') + # Always report known-down count for awareness + known_count=len([h for h in down if h in known_down]) + if known_count: print(f'(+{known_count} known-down: P5-10/HAOS)', end='', file=__import__('sys').stderr) +except: print('') +" 2>/tmp/weekly-health-digest-known.txt) +known_note=$(cat /tmp/weekly-health-digest-known.txt 2>/dev/null || true) +rm -f /tmp/weekly-health-digest-known.txt + +if [ -z "$targets_down" ]; then + add_line "node-exporter targets" "OK" "${known_note:-all up (excl. 2 known-down)}" +else + add_line "node-exporter targets" "NEW targets down: ${targets_down}" "${known_note}" +fi + +# ── 3. Grafana/Prometheus firing alerts ───────────────────────────────────── +firing_result=$(curl -sf --max-time "$TIMEOUT" \ + "${PROM_URL}/api/v1/alerts" 2>/dev/null | python3 -c " +import json,sys +try: + d=json.load(sys.stdin) + firing=[a['labels'].get('alertname','?') for a in d['data']['alerts'] if a['state']=='firing'] + # Suppress known-persistent alerts that are tracked elsewhere + known_ok={'PrometheusTargetDown','NodeDown'} # covered in target check above + new_firing=[a for a in firing if a not in known_ok] + if new_firing: + print(','.join(sorted(set(new_firing)))) + else: + suppressed=len([a for a in firing if a in known_ok]) + print(f'none (suppressed {suppressed} known)') +except: print('query_failed') +" 2>/dev/null || echo "query_failed") + +if echo "$firing_result" | grep -qE "none|^$"; then + add_line "Grafana alerts" "OK" "$firing_result" +elif [ "$firing_result" = "query_failed" ]; then + add_line "Grafana alerts" "Prometheus unreachable" +else + add_line "Grafana alerts" "FIRING: ${firing_result}" +fi + +# ── 4. PBS datastores — backup freshness >48h ──────────────────────────────── +# Check local backup metric: backup_status textfile on media-server node-exporter +backup_status=$(prom_scalar 'backup_status{instance="192.168.99.183:9100"}') +if python3 -c "import sys; sys.exit(0 if float('$backup_status') >= 1 else 1)" 2>/dev/null; then + add_line "PBS/backups" "OK" "backup_status metric healthy" +else + add_line "PBS/backups" "backup_status=0 or unavailable — check PBS and local backup jobs" +fi + +# Also check PBS ZFS pool presence (covers the INC-001/usb1-zfs and usb2-zfs situation) +pbs_zfs=$(prom_query 'zfs_pool_present') +pbs_zfs_missing=$(echo "$pbs_zfs" | python3 -c " +import json,sys +try: + d=json.loads(sys.stdin.read()) + missing=[r['metric'].get('pool','?') for r in d if float(r['value'][1])==0] + print(','.join(missing) if missing else '') +except: print('') +" 2>/dev/null || echo "") +if [ -z "$pbs_zfs_missing" ]; then + add_line "PBS ZFS pools" "OK" "usb1-zfs + usb2-zfs imported" +else + add_line "PBS ZFS pools" "MISSING: ${pbs_zfs_missing}" "pools not imported on PBS — check enclosure/cables" +fi + +# ── 5. Filesystem >85% ─────────────────────────────────────────────────────── +fs_over=$(prom_query '(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"}) * 100 > 85' | python3 -c " +import json,sys +try: + d=json.loads(sys.stdin.read()) + items=[f'{r[\"metric\"].get(\"instance\",\"?\").split(\":\")[0]}:{r[\"metric\"].get(\"mountpoint\",\"?\")} {float(r[\"value\"][1]):.0f}%' for r in d] + print(', '.join(items) if items else '') +except: print('') +" 2>/dev/null || echo "") +if [ -z "$fs_over" ]; then + add_line "Disk usage >85%" "OK" "no filesystem over threshold" +else + add_line "Disk usage >85%" "OVER THRESHOLD: ${fs_over}" +fi + +# ── 6. ZFS pools ONLINE (cluster-wide) ────────────────────────────────────── +zfs_bad=$(prom_query 'node_zfs_zpool_state{state=~"degraded|faulted|suspended|unavail|offline|removed"} == 1' | python3 -c " +import json,sys +try: + d=json.loads(sys.stdin.read()) + items=[f'{r[\"metric\"].get(\"instance\",\"?\").split(\":\")[0]}:{r[\"metric\"].get(\"zpool\",\"?\")}({r[\"metric\"].get(\"state\",\"?\")})' for r in d] + print(', '.join(items) if items else '') +except: print('') +" 2>/dev/null || echo "") +if [ -z "$zfs_bad" ]; then + add_line "ZFS pools" "OK" "das-mirror, compute3-zfs all ONLINE" +else + add_line "ZFS pools" "DEGRADED/FAULTED: ${zfs_bad}" +fi + +# ── 7. Docker unhealthy/restarting ────────────────────────────────────────── +unhealthy=$(docker ps --format "{{.Names}}\t{{.Status}}" 2>/dev/null | \ + awk '/unhealthy|restarting/{print $1}' | \ + grep -v "immich_machine_learning" | \ + tr '\n' ',' | sed 's/,$//' || true) +# immich_machine_learning is known-unhealthy — suppress +if [ -z "$unhealthy" ]; then + add_line "Docker (local)" "OK" "no new unhealthy/restarting containers" +else + add_line "Docker (local)" "UNHEALTHY/RESTARTING: ${unhealthy}" +fi + +# ── 8. TLS cert expiry <21 days ───────────────────────────────────────────── +# Check live cert expiry on a representative set of domains +cert_warn="" +for domain in traefik.goattw.net auth.goattw.net grafana.goattw.net ha.goattw.net nvr.goattw.net home.goattw.net; do + expiry=$(echo | timeout 8 openssl s_client -connect "${domain}:443" -servername "${domain}" 2>/dev/null \ + | openssl x509 -noout -enddate 2>/dev/null \ + | cut -d= -f2) + if [ -n "$expiry" ]; then + days=$(python3 -c " +from datetime import datetime +try: + exp=datetime.strptime('$expiry','%b %d %H:%M:%S %Y %Z') + now=datetime.utcnow() + print(int((exp-now).days)) +except: print(999) +" 2>/dev/null || echo "999") + if [ "$days" -lt 21 ] 2>/dev/null; then + cert_warn="${cert_warn}${domain}(${days}d) " + fi + fi +done + +if [ -z "$cert_warn" ]; then + add_line "TLS cert expiry" "OK" "all checked certs >21 days" +else + add_line "TLS cert expiry" "EXPIRING SOON: ${cert_warn}" +fi + +# ── Compose and send digest ────────────────────────────────────────────────── +NOW=$(date '+%Y-%m-%d %H:%M %Z') + +if [ "$ISSUES" -eq 0 ]; then + PRIORITY="low" + SUBJECT="Weekly Health Digest ${NOW} — ALL OK" + TAGS="white_check_mark" +else + PRIORITY="high" + SUBJECT="Weekly Health Digest ${NOW} — ${ISSUES} ISSUE(S) FLAGGED" + TAGS="rotating_light" +fi + +BODY=$(printf '%s\n' "${LINES[@]}") + +logger -t weekly-health-digest "digest complete: issues=${ISSUES} priority=${PRIORITY}" + +# Post to ntfy health-digest topic +HTTP_STATUS=$(curl -sf --max-time 15 \ + -H "Title: ${SUBJECT}" \ + -H "Priority: ${PRIORITY}" \ + -H "Tags: ${TAGS}" \ + -d "${BODY}" \ + "${NTFY_URL}" \ + -o /dev/null -w "%{http_code}" 2>/dev/null || echo "000") + +if [ "$HTTP_STATUS" = "200" ] || [ "$HTTP_STATUS" = "201" ]; then + logger -t weekly-health-digest "ntfy delivery OK (HTTP ${HTTP_STATUS})" +else + logger -t weekly-health-digest "WARNING: ntfy delivery failed (HTTP ${HTTP_STATUS})" +fi + +# Always exit 0 — cron should never see a failure from a read-only digest +exit 0