homelab-configs/monitoring/weekly-health-digest.sh

#!/bin/bash
# weekly-health-digest.sh — P5-12
# Lightweight weekly drift-catcher. Read-only. Posts a concise digest to ntfy/health-digest.
# Does NOT replace the quarterly full health check.
# Schedule: Monday 07:00 via /etc/cron.d/weekly-health-digest
# Runs on: media-server (192.168.99.183)

set -euo pipefail

NTFY_URL="http://192.168.99.198:8095/health-digest"
PROM_URL="http://localhost:9090"
GRAFANA_URL="http://localhost:3005"
TIMEOUT=10   # seconds per HTTP request before giving up on that check

# Node-exporter targets (from prometheus.yml)
NODE_EXPORTER_TARGETS=(
    "192.168.99.200:9100"   # beast
    "192.168.99.192:9100"   # compute2
    "192.168.99.193:9100"   # compute3
    "192.168.99.194:9100"   # compute4
    "192.168.99.153:9100"   # pbs
    "192.168.99.183:9100"   # media-server
    "192.168.99.186:9100"   # docker-node01
    "192.168.99.187:9100"   # docker-node02
    "192.168.99.190:9100"   # ansible-control
    "192.168.99.227:9100"   # pi4 (P5-10 — expected down)
    "192.168.99.196:9100"   # compute5
    "192.168.99.198:9100"   # compute6
    "192.168.99.100:9100"   # haos (expected 500)
)

logger -t weekly-health-digest "starting weekly health digest"

# ── helpers ─────────────────────────────────────────────────────────────────

prom_query() {
    # $1 = PromQL query; returns JSON result array or empty string on failure
    curl -sf --max-time "$TIMEOUT" \
        "${PROM_URL}/api/v1/query?query=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1]))" "$1")" \
        2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(json.dumps(d['data']['result']))" 2>/dev/null || echo "[]"
}

prom_scalar() {
    # Returns first value from a query as a float, or -1 on failure
    local result
    result=$(prom_query "$1")
    python3 -c "
import json,sys
try:
    d=json.loads('$result')
    if d: print(float(d[0]['value'][1]))
    else: print(-1)
except: print(-1)
" 2>/dev/null || echo "-1"
}

OK="OK"
FLAG=""
LINES=()
ISSUES=0

add_line() {
    local label="$1" status="$2" detail="${3:-}"
    if [ "$status" = "OK" ]; then
        LINES+=("  ${label}: OK${detail:+ — $detail}")
    else
        LINES+=("  ${label}: *** ${status}${detail:+ — $detail} ***")
        ISSUES=$((ISSUES + 1))
    fi
}

# ── 1. Proxmox quorum ────────────────────────────────────────────────────────
# We can't SSH to PVE nodes from here directly, so we infer quorum from node
# reachability. If beast (the PVE master) is up and answering node-exporter,
# the cluster is likely healthy. Flag if beast is unreachable.
beast_up=$(curl -sf --max-time 5 "http://192.168.99.200:9100/metrics" 2>/dev/null | grep -c "node_boot_time_seconds" || true)
if [ "${beast_up}" -ge 1 ]; then
    add_line "PVE/quorum" "OK" "beast reachable"
else
    add_line "PVE/quorum" "beast unreachable — quorum check not possible"
fi

# ── 2. Prometheus node-exporter targets ─────────────────────────────────────
targets_result=$(prom_query 'up{job="node-exporter"}')
targets_down=$(echo "$targets_result" | python3 -c "
import json,sys
try:
    d=json.loads(sys.stdin.read())
    down=[r['metric'].get('instance','?') for r in d if float(r['value'][1])==0]
    # Filter known-persistent issues that don't need weekly noise
    known_down={'192.168.99.227:9100','192.168.99.100:9100'}
    new_down=[h for h in down if h not in known_down]
    print(','.join(new_down) if new_down else '')
    # Always report known-down count for awareness
    known_count=len([h for h in down if h in known_down])
    if known_count: print(f'(+{known_count} known-down: P5-10/HAOS)', end='', file=__import__('sys').stderr)
except: print('')
" 2>/tmp/weekly-health-digest-known.txt)
known_note=$(cat /tmp/weekly-health-digest-known.txt 2>/dev/null || true)
rm -f /tmp/weekly-health-digest-known.txt

if [ -z "$targets_down" ]; then
    add_line "node-exporter targets" "OK" "${known_note:-all up (excl. 2 known-down)}"
else
    add_line "node-exporter targets" "NEW targets down: ${targets_down}" "${known_note}"
fi

# ── 3. Grafana/Prometheus firing alerts ─────────────────────────────────────
firing_result=$(curl -sf --max-time "$TIMEOUT" \
    "${PROM_URL}/api/v1/alerts" 2>/dev/null | python3 -c "
import json,sys
try:
    d=json.load(sys.stdin)
    firing=[a['labels'].get('alertname','?') for a in d['data']['alerts'] if a['state']=='firing']
    # Suppress known-persistent alerts that are tracked elsewhere
    known_ok={'PrometheusTargetDown','NodeDown'}  # covered in target check above
    new_firing=[a for a in firing if a not in known_ok]
    if new_firing:
        print(','.join(sorted(set(new_firing))))
    else:
        suppressed=len([a for a in firing if a in known_ok])
        print(f'none (suppressed {suppressed} known)')
except: print('query_failed')
" 2>/dev/null || echo "query_failed")

if echo "$firing_result" | grep -qE "none|^$"; then
    add_line "Grafana alerts" "OK" "$firing_result"
elif [ "$firing_result" = "query_failed" ]; then
    add_line "Grafana alerts" "Prometheus unreachable"
else
    add_line "Grafana alerts" "FIRING: ${firing_result}"
fi

# ── 4. PBS datastores — backup freshness >48h ────────────────────────────────
# Check local backup metric: backup_status textfile on media-server node-exporter
backup_status=$(prom_scalar 'backup_status{instance="192.168.99.183:9100"}')
if python3 -c "import sys; sys.exit(0 if float('$backup_status') >= 1 else 1)" 2>/dev/null; then
    add_line "PBS/backups" "OK" "backup_status metric healthy"
else
    add_line "PBS/backups" "backup_status=0 or unavailable — check PBS and local backup jobs"
fi

# Also check PBS ZFS pool presence (covers the INC-001/usb1-zfs and usb2-zfs situation)
pbs_zfs=$(prom_query 'zfs_pool_present')
pbs_zfs_missing=$(echo "$pbs_zfs" | python3 -c "
import json,sys
try:
    d=json.loads(sys.stdin.read())
    missing=[r['metric'].get('pool','?') for r in d if float(r['value'][1])==0]
    print(','.join(missing) if missing else '')
except: print('')
" 2>/dev/null || echo "")
if [ -z "$pbs_zfs_missing" ]; then
    add_line "PBS ZFS pools" "OK" "usb1-zfs + usb2-zfs imported"
else
    add_line "PBS ZFS pools" "MISSING: ${pbs_zfs_missing}" "pools not imported on PBS — check enclosure/cables"
fi

# ── 4b. Off-host app backups: LubeLogger + Vaultwarden freshness (<26h) ──────
# Both push nightly from docker-node01 (.186) to PBS rust-usb: host/vaultwarden
# (root cron 03:00) and host/lubelogger (systemd timer 03:30). We ask PBS for the
# newest snapshot time of each group via the lubelogger token, which has
# datastore-wide audit on rust-usb (covers both groups). Read-only; keyed SSH +
# passwordless sudo on .186. Quoted heredoc → remote-side var expansion only.
backup_ages=$(ssh -o ConnectTimeout=8 -o BatchMode=yes tommy@192.168.99.186 'sudo bash -s' 2>/dev/null <<'REMOTE' || true
source /etc/lubelogger-backup.env 2>/dev/null
export PBS_PASSWORD PBS_FINGERPRINT
for grp in lubelogger vaultwarden; do
  t=$(proxmox-backup-client snapshot list "host/$grp" --repository "$PBS_REPOSITORY" --output-format json 2>/dev/null \
      | python3 -c 'import sys,json
try:
    d=json.load(sys.stdin); print(max(x["backup-time"] for x in d) if d else 0)
except: print(0)' 2>/dev/null || echo 0)
  echo "$grp $t"
done
REMOTE
)

now_epoch=$(date +%s)
for svc in lubelogger vaultwarden; do
    ts=$(echo "$backup_ages" | awk -v s="$svc" '$1==s{print $2}')
    if [ -z "$ts" ] || [ "$ts" = "0" ]; then
        add_line "Backup ${svc}→PBS" "no snapshot / query failed — check .186 backup job"
    else
        age_h=$(( (now_epoch - ts) / 3600 ))
        if [ "$age_h" -lt 26 ]; then
            add_line "Backup ${svc}→PBS" "OK" "newest snapshot ${age_h}h ago"
        else
            add_line "Backup ${svc}→PBS" "STALE — newest snapshot ${age_h}h ago (>26h)"
        fi
    fi
done

# ── 5. Filesystem >85% ───────────────────────────────────────────────────────
fs_over=$(prom_query '(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"}) * 100 > 85' | python3 -c "
import json,sys
try:
    d=json.loads(sys.stdin.read())
    items=[f'{r[\"metric\"].get(\"instance\",\"?\").split(\":\")[0]}:{r[\"metric\"].get(\"mountpoint\",\"?\")} {float(r[\"value\"][1]):.0f}%' for r in d]
    print(', '.join(items) if items else '')
except: print('')
" 2>/dev/null || echo "")
if [ -z "$fs_over" ]; then
    add_line "Disk usage >85%" "OK" "no filesystem over threshold"
else
    add_line "Disk usage >85%" "OVER THRESHOLD: ${fs_over}"
fi

# ── 6. ZFS pools ONLINE (cluster-wide) ──────────────────────────────────────
zfs_bad=$(prom_query 'node_zfs_zpool_state{state=~"degraded|faulted|suspended|unavail|offline|removed"} == 1' | python3 -c "
import json,sys
try:
    d=json.loads(sys.stdin.read())
    items=[f'{r[\"metric\"].get(\"instance\",\"?\").split(\":\")[0]}:{r[\"metric\"].get(\"zpool\",\"?\")}({r[\"metric\"].get(\"state\",\"?\")})' for r in d]
    print(', '.join(items) if items else '')
except: print('')
" 2>/dev/null || echo "")
if [ -z "$zfs_bad" ]; then
    add_line "ZFS pools" "OK" "das-mirror, compute3-zfs all ONLINE"
else
    add_line "ZFS pools" "DEGRADED/FAULTED: ${zfs_bad}"
fi

# ── 7. Docker unhealthy/restarting ──────────────────────────────────────────
unhealthy=$(docker ps --format "{{.Names}}\t{{.Status}}" 2>/dev/null | \
    awk '/unhealthy|restarting/{print $1}' | \
    grep -v "immich_machine_learning" | \
    tr '\n' ',' | sed 's/,$//' || true)
# immich_machine_learning is known-unhealthy — suppress
if [ -z "$unhealthy" ]; then
    add_line "Docker (local)" "OK" "no new unhealthy/restarting containers"
else
    add_line "Docker (local)" "UNHEALTHY/RESTARTING: ${unhealthy}"
fi

# ── 8. TLS cert expiry <21 days ─────────────────────────────────────────────
# Check live cert expiry on a representative set of domains
cert_warn=""
for domain in traefik.goattw.net auth.goattw.net grafana.goattw.net ha.goattw.net nvr.goattw.net home.goattw.net; do
    expiry=$(echo | timeout 8 openssl s_client -connect "${domain}:443" -servername "${domain}" 2>/dev/null \
        | openssl x509 -noout -enddate 2>/dev/null \
        | cut -d= -f2)
    if [ -n "$expiry" ]; then
        days=$(python3 -c "
from datetime import datetime
try:
    exp=datetime.strptime('$expiry','%b %d %H:%M:%S %Y %Z')
    now=datetime.utcnow()
    print(int((exp-now).days))
except: print(999)
" 2>/dev/null || echo "999")
        if [ "$days" -lt 21 ] 2>/dev/null; then
            cert_warn="${cert_warn}${domain}(${days}d) "
        fi
    fi
done

if [ -z "$cert_warn" ]; then
    add_line "TLS cert expiry" "OK" "all checked certs >21 days"
else
    add_line "TLS cert expiry" "EXPIRING SOON: ${cert_warn}"
fi

# ── Compose and send digest ──────────────────────────────────────────────────
NOW=$(date '+%Y-%m-%d %H:%M %Z')

if [ "$ISSUES" -eq 0 ]; then
    PRIORITY="low"
    SUBJECT="Weekly Health Digest ${NOW} — ALL OK"
    TAGS="white_check_mark"
else
    PRIORITY="high"
    SUBJECT="Weekly Health Digest ${NOW} — ${ISSUES} ISSUE(S) FLAGGED"
    TAGS="rotating_light"
fi

BODY=$(printf '%s\n' "${LINES[@]}")

logger -t weekly-health-digest "digest complete: issues=${ISSUES} priority=${PRIORITY}"

# Post to ntfy health-digest topic
HTTP_STATUS=$(curl -sf --max-time 15 \
    -H "Title: ${SUBJECT}" \
    -H "Priority: ${PRIORITY}" \
    -H "Tags: ${TAGS}" \
    -d "${BODY}" \
    "${NTFY_URL}" \
    -o /dev/null -w "%{http_code}" 2>/dev/null || echo "000")

if [ "$HTTP_STATUS" = "200" ] || [ "$HTTP_STATUS" = "201" ]; then
    logger -t weekly-health-digest "ntfy delivery OK (HTTP ${HTTP_STATUS})"
else
    logger -t weekly-health-digest "WARNING: ntfy delivery failed (HTTP ${HTTP_STATUS})"
fi

# Always exit 0 — cron should never see a failure from a read-only digest
exit 0