From a567f64508cad767d02105f2a8d56d0f35126906 Mon Sep 17 00:00:00 2001
From: tommy <tommy@goattw.net>
Date: Tue, 7 Jul 2026 17:39:24 -0500
Subject: [PATCH] weekly-health-digest: add LubeLogger + Vaultwarden PBS backup
 freshness

New section 4b flags either app backup if its newest PBS snapshot in
rust-usb is >26h old (catches a missed nightly run). Queries via the
lubelogger token (datastore-wide audit covers both groups) over keyed
SSH to docker-node01; read-only and failure-guarded.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 monitoring/weekly-health-digest.sh | 35 ++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/monitoring/weekly-health-digest.sh b/monitoring/weekly-health-digest.sh
index 199616f..e348641 100755
--- a/monitoring/weekly-health-digest.sh
+++ b/monitoring/weekly-health-digest.sh
@@ -156,6 +156,41 @@ else
     add_line "PBS ZFS pools" "MISSING: ${pbs_zfs_missing}" "pools not imported on PBS — check enclosure/cables"
 fi
 
+# ── 4b. Off-host app backups: LubeLogger + Vaultwarden freshness (<26h) ──────
+# Both push nightly from docker-node01 (.186) to PBS rust-usb: host/vaultwarden
+# (root cron 03:00) and host/lubelogger (systemd timer 03:30). We ask PBS for the
+# newest snapshot time of each group via the lubelogger token, which has
+# datastore-wide audit on rust-usb (covers both groups). Read-only; keyed SSH +
+# passwordless sudo on .186. Quoted heredoc → remote-side var expansion only.
+backup_ages=$(ssh -o ConnectTimeout=8 -o BatchMode=yes tommy@192.168.99.186 'sudo bash -s' 2>/dev/null <<'REMOTE' || true
+source /etc/lubelogger-backup.env 2>/dev/null
+export PBS_PASSWORD PBS_FINGERPRINT
+for grp in lubelogger vaultwarden; do
+  t=$(proxmox-backup-client snapshot list "host/$grp" --repository "$PBS_REPOSITORY" --output-format json 2>/dev/null \
+      | python3 -c 'import sys,json
+try:
+    d=json.load(sys.stdin); print(max(x["backup-time"] for x in d) if d else 0)
+except: print(0)' 2>/dev/null || echo 0)
+  echo "$grp $t"
+done
+REMOTE
+)
+
+now_epoch=$(date +%s)
+for svc in lubelogger vaultwarden; do
+    ts=$(echo "$backup_ages" | awk -v s="$svc" '$1==s{print $2}')
+    if [ -z "$ts" ] || [ "$ts" = "0" ]; then
+        add_line "Backup ${svc}→PBS" "no snapshot / query failed — check .186 backup job"
+    else
+        age_h=$(( (now_epoch - ts) / 3600 ))
+        if [ "$age_h" -lt 26 ]; then
+            add_line "Backup ${svc}→PBS" "OK" "newest snapshot ${age_h}h ago"
+        else
+            add_line "Backup ${svc}→PBS" "STALE — newest snapshot ${age_h}h ago (>26h)"
+        fi
+    fi
+done
+
 # ── 5. Filesystem >85% ───────────────────────────────────────────────────────
 fs_over=$(prom_query '(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"}) * 100 > 85' | python3 -c "
 import json,sys