tommy/homelab-configs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

weekly-health-digest: add LubeLogger + Vaultwarden PBS backup freshness

Browse Source 
New section 4b flags either app backup if its newest PBS snapshot in
rust-usb is >26h old (catches a missed nightly run). Queries via the
lubelogger token (datastore-wide audit covers both groups) over keyed
SSH to docker-node01; read-only and failure-guarded.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
tommy
2026-07-07 17:39:24 -05:00
parent b43007c9a9
commit a567f64508
1 changed files with 35 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
monitoring/weekly-health-digest.sh
+35
View File
@@ -156,6 +156,41 @@ else
add_line "PBS ZFS pools" "MISSING: ${pbs_zfs_missing}" "pools not imported on PBS — check enclosure/cables"
fi
# ── 4b. Off-host app backups: LubeLogger + Vaultwarden freshness (<26h) ──────
# Both push nightly from docker-node01 (.186) to PBS rust-usb: host/vaultwarden
# (root cron 03:00) and host/lubelogger (systemd timer 03:30). We ask PBS for the
# newest snapshot time of each group via the lubelogger token, which has
# datastore-wide audit on rust-usb (covers both groups). Read-only; keyed SSH +
# passwordless sudo on .186. Quoted heredoc → remote-side var expansion only.
backup_ages=$(ssh -o ConnectTimeout=8 -o BatchMode=yes tommy@192.168.99.186 'sudo bash -s' 2>/dev/null <<'REMOTE' || true
source /etc/lubelogger-backup.env 2>/dev/null
export PBS_PASSWORD PBS_FINGERPRINT
for grp in lubelogger vaultwarden; do
t=$(proxmox-backup-client snapshot list "host/$grp" --repository "$PBS_REPOSITORY" --output-format json 2>/dev/null \
| python3 -c 'import sys,json
try:
d=json.load(sys.stdin); print(max(x["backup-time"] for x in d) if d else 0)
except: print(0)' 2>/dev/null || echo 0)
echo "$grp $t"
done
REMOTE
)
now_epoch=$(date +%s)
for svc in lubelogger vaultwarden; do
ts=$(echo "$backup_ages" | awk -v s="$svc" '$1==s{print $2}')
if [ -z "$ts" ] || [ "$ts" = "0" ]; then
add_line "Backup ${svc}→PBS" "no snapshot / query failed — check .186 backup job"
else
age_h=$(( (now_epoch - ts) / 3600 ))
if [ "$age_h" -lt 26 ]; then
add_line "Backup ${svc}→PBS" "OK" "newest snapshot ${age_h}h ago"
else
add_line "Backup ${svc}→PBS" "STALE — newest snapshot ${age_h}h ago (>26h)"
fi
fi
done
# ── 5. Filesystem >85% ───────────────────────────────────────────────────────
fs_over=$(prom_query '(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"}) * 100 > 85' | python3 -c "
import json,sys