From 9e6afe27c565ba5feda7db42f37d45a54847ee39 Mon Sep 17 00:00:00 2001 From: tommy Date: Thu, 2 Jul 2026 11:21:53 -0500 Subject: [PATCH] runbooks: INC-001 PBS USB-ZFS recovery + import/alert hardening (P5-14) Sync pbs/zfs-health-check.sh to the deployed version and log the full INC-001 resolution arc: - recovery by importing pools on stable /dev/disk/by-id paths - by-id cachefile pin so the fix survives USB re-enumeration - wait-for-zfs-usb.sh + zfs-import-cache drop-in for the boot race - alert split: devices-present (import) vs devices-absent (hardware), new zfs_pool_members_present metric - cron ownership fix: root-managed single entry, tommy evicted, unmarked root line removed (double-fire root cause) Marks the acute import/alert half of INC-001 resolved; USB-hub reliability history retained as a separate hardware concern. Co-Authored-By: Claude Opus 4.8 --- pbs/zfs-health-check.sh | 54 +++++++++++++++++++++++++++------ runbooks/phase5-incident-log.md | 50 +++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 11 deletions(-) diff --git a/pbs/zfs-health-check.sh b/pbs/zfs-health-check.sh index f1d566e..19de554 100755 --- a/pbs/zfs-health-check.sh +++ b/pbs/zfs-health-check.sh @@ -1,8 +1,11 @@ #!/bin/bash # ZFS pool health check — /usr/local/bin/zfs-health-check.sh # Runs every 5 min via root crontab on PBS. -# Three states: ONLINE (silent), DEGRADED/FAULTED (ntfy high), MISSING (ntfy urgent). +# States: ONLINE (silent), DEGRADED/FAULTED (ntfy high), NOT-IMPORTED w/ members +# present (ntfy high — failed import, hardware fine), MISSING members (ntfy urgent). # Writes Prometheus textfile metrics after full loop completes only. +# NOTE: deployed by Ansible role `pbs` (roles/pbs/files/zfs-health-check.sh). +# Edit the role file, not the live copy — the playbook overwrites it. NTFY_URL="https://ntfy.goattw.net/zfs-health" EXPECTED_POOLS=(usb1-zfs usb2-zfs) @@ -18,6 +21,8 @@ logger -t zfs-health-check "starting pool health check" METRICS="" METRICS+="# HELP zfs_pool_present 1 if pool imported and healthy, 0 if missing\n" METRICS+="# TYPE zfs_pool_present gauge\n" +METRICS+="# HELP zfs_pool_members_present count of expected by-id members visible when a pool is not imported\n" +METRICS+="# TYPE zfs_pool_members_present gauge\n" METRICS+="# HELP zfs_health_last_run_seconds Unix timestamp of last successful check\n" METRICS+="# TYPE zfs_health_last_run_seconds gauge\n" @@ -40,16 +45,45 @@ for pool in "${EXPECTED_POOLS[@]}"; do || logger -t zfs-health-check "WARNING: ntfy post failed for $pool" else - # Empty or error — pool not imported / missing - logger -t zfs-health-check "pool $pool: MISSING — alerting" + # Pool not imported. Distinguish two very different causes before alerting: + # devices ABSENT -> real hardware (cable/hub/enclosure/power) + # devices PRESENT -> pool merely not imported (stale cache, boot race) -> import fixes it + case "$pool" in + usb1-zfs) members="/dev/disk/by-id/usb-ASMT_2115_ACAAEBBB2E5F-0:0-part1 /dev/disk/by-id/wwn-0x5000cca234ca7110-part1" ;; + usb2-zfs) members="/dev/disk/by-id/wwn-0x50014ee2075b3126-part1 /dev/disk/by-id/wwn-0x50014ee211ed75f6-part1" ;; + *) members="" ;; + esac + + present=0; total=0 + for m in $members; do + total=$((total+1)) + [ -e "$m" ] && present=$((present+1)) + done + METRICS+="zfs_pool_present{pool=\"$pool\"} 0\n" - curl -sf \ - -H "Priority: urgent" \ - -H "Tags: rotating_light" \ - -H "Title: PBS ZFS pool MISSING: $pool" \ - -d "Pool $pool not imported on PBS. Check cables. Fix: zpool import $pool" \ - "$NTFY_URL" \ - || logger -t zfs-health-check "WARNING: ntfy post failed for $pool" + METRICS+="zfs_pool_members_present{pool=\"$pool\"} $present\n" + + if [ "$total" -gt 0 ] && [ "$present" -eq "$total" ]; then + # All members enumerated — this is a failed import, NOT a hardware fault. + logger -t zfs-health-check "pool $pool: NOT IMPORTED (all $total members present) — alerting" + curl -sf \ + -H "Priority: high" \ + -H "Tags: warning" \ + -H "Title: PBS ZFS pool not imported: $pool" \ + -d "Pool $pool not imported, but all $total members are present. Hardware is fine — likely failed auto-import. Fix: zpool import -d /dev/disk/by-id $pool" \ + "$NTFY_URL" \ + || logger -t zfs-health-check "WARNING: ntfy post failed for $pool" + else + # Members missing (or unknown pool) — genuine hardware suspicion. + logger -t zfs-health-check "pool $pool: MISSING ($present/$total members present) — alerting" + curl -sf \ + -H "Priority: urgent" \ + -H "Tags: rotating_light" \ + -H "Title: PBS ZFS pool MISSING: $pool" \ + -d "Pool $pool not imported; only $present of $total members visible. Check cables/enclosure/power. Then: zpool import -d /dev/disk/by-id $pool" \ + "$NTFY_URL" \ + || logger -t zfs-health-check "WARNING: ntfy post failed for $pool" + fi fi done diff --git a/runbooks/phase5-incident-log.md b/runbooks/phase5-incident-log.md index c9bee30..19ec919 100644 --- a/runbooks/phase5-incident-log.md +++ b/runbooks/phase5-incident-log.md @@ -80,7 +80,7 @@ DEADTIME 15 → DEADTIME 300 --- -### INC-001 — PBS USB Hub Failure (ACTIVE — hardware fix in progress) +### INC-001 — PBS USB Hub Failure (import/alerting RESOLVED + hardened 2026-07-02 — see P5-14; hardware-hub history retained below) **First event:** March 12, 2026 (earliest confirmed crash in journal) **Crash count:** 34+ unclean reboots since April 21; ~50+ total since March 12 @@ -330,6 +330,54 @@ Result: fresh `us_denver` peer (endpoint `181.41.206.159:1337`), gluetun Healthy --- +### P5-14 — INC-001 PBS USB-ZFS Recovery + Import/Alert Hardening (DONE 2026-07-02) + +**Host:** PBS (192.168.99.153). Closes the acute import/alert half of INC-001 (see full section above). + +**Context — this instance was NOT a hardware drop.** Both `usb1-zfs` and `usb2-zfs` again showed MISSING (`zpool list` = "no pools available"; `zfs-import-cache.service` + `zfs-import@usb{1,2}-zfs` units failed). But all four ASM1153E bridges were enumerated on the hub and all four mirror members had intact `zfs_member` labels. Root cause was a **failed auto-import**: stale cache-import + `zfs-import-scan` disabled (no fallback) + non-deterministic `/dev/sdX` USB enumeration order through the hub, so cached device paths no longer matched. + +**1. Resolution — import by stable by-id paths** (no `-f` needed): +```bash +zpool import -d /dev/disk/by-id usb1-zfs +zpool import -d /dev/disk/by-id usb2-zfs +zpool status -P # confirmed paths now read /dev/disk/by-id/... not /dev/sdX +``` +Both ONLINE, 0 errors. `usb1-zfs` 613G data intact (mounted `/mnt/datastore/usb1-zfs`); `usb2-zfs` ~empty. Scrubs clean. + +**2. by-id cachefile pin** (persists the fix across USB re-enumeration): +```bash +zpool set cachefile=/etc/zfs/zpool.cache usb1-zfs +zpool set cachefile=/etc/zfs/zpool.cache usb2-zfs +``` +Also codified as a task in the `pbs` Ansible role. + +**3. Boot-race guard — `wait-for-zfs-usb.sh` + systemd drop-in.** `systemd-udev-settle` drains the current udev queue but does not wait for slow USB hub/enclosure enumeration, so `zfs-import-cache.service` can fire before the by-id member symlinks exist. Added: +- `/usr/local/sbin/wait-for-zfs-usb.sh` — polls for the 4 by-id members, max 60s, **always exits 0** (never wedges boot) +- `/etc/systemd/system/zfs-import-cache.service.d/wait-for-usb.conf` — `ExecStartPre=/usr/local/sbin/wait-for-zfs-usb.sh` + +Real validation is a PBS reboot (deferred to a maintenance window). + +**4. Alert split — devices-present vs devices-absent** (fixes the "check cables" misfire). The `zfs-health-check.sh` MISSING branch previously always cried *"Check cables. Fix: zpool import"* — it could not tell a real hardware drop from a pool merely unimported. Now it tests member presence: +- all by-id members present → ntfy **high**: "not imported, hardware fine — Fix: `zpool import -d /dev/disk/by-id`" +- members missing → ntfy **urgent**: "only N of M visible — check cables/enclosure/power" + +New metric `zfs_pool_members_present` gives Prometheus a scrapable signal (a fully-unimported pool emits no `node_zfs_zpool_state` series, so Prometheus was previously blind to this exact case). + +**5. Cron ownership fix (double-fire root cause).** The alert script was running **twice** every 5 min — once from an unmarked root crontab entry (the original) and once from the Ansible-managed `user: tommy` entry (which also carried a duplicated `#Ansible:` marker). This is what produced the **paired ~9:55/10:00 ntfy notifications**: two independent runs firing seconds apart. Worse, the tommy run **silently failed to write** the node_exporter textfile (dir is `root:root`), so `zfs_pool_members_present` would never have landed under it. Converged to ONE root-managed entry: +- Ansible cron task repointed `user: tommy` → `user: root`, `state: present` (mirrors the working `docker-image-prune` pattern) +- added eviction task (`name: zfs-health-check`, `user: tommy`, `state: absent`) to drop the tommy job + duplicate marker +- one-time **manual** delete of the unmarked original root line (Ansible can't dedupe an entry lacking its marker — done first, before the converge, to avoid a momentary double-root window) + +**Deployment.** Script + cron ownership + cachefile now live in the `pbs` Ansible role (`tommy/ansible` commit `c9a028b`); deploy via `ansible-playbook pbs.yml --tags zfs --ask-become-pass`. The role had been shipping an inline **stub** that drifted far behind the live script — running it would have clobbered the good 3-state/metrics script; now reconciled to `roles/pbs/files/zfs-health-check.sh` (this repo's `pbs/zfs-health-check.sh` is the synced copy). + +**Scope note.** The acute import failure and the alert/cron bugs are resolved and hardened. The long-running USB-**hub reliability** history above (34+ crash-reboots) is a separate hardware concern — these changes harden import + alerting, they do not replace the hub. Current state: all members enumerated, both pools ONLINE and stable. + +**Deferred / passive follow-ups (nothing to actively revisit):** +- **Reboot verification** — `wait-for-zfs-usb.sh` (ExecStartPre) is only exercised on a cold boot. On the next PBS reboot, confirm it ran: `journalctl -b -u zfs-import-cache.service | grep wait-for-zfs-usb` (expect "all N members present after Ns", then a clean cache import with both pools auto-imported). +- **Orphaned cron marker** — root's crontab still carries a stray `#Ansible: Prune PBS task logs` marker with **no matching task** in the `pbs` role (unmanaged leftover from a removed task). Harmless — no job is re-created by any playbook. Delete via `crontab -e` (as root) whenever convenient. + +--- + ## Known Config Drift (not incidents — track for cleanup) | Item | Location | Drift |