tommy
9e6afe27c5
Sync pbs/zfs-health-check.sh to the deployed version and log the full INC-001 resolution arc: - recovery by importing pools on stable /dev/disk/by-id paths - by-id cachefile pin so the fix survives USB re-enumeration - wait-for-zfs-usb.sh + zfs-import-cache drop-in for the boot race - alert split: devices-present (import) vs devices-absent (hardware), new zfs_pool_members_present metric - cron ownership fix: root-managed single entry, tommy evicted, unmarked root line removed (double-fire root cause) Marks the acute import/alert half of INC-001 resolved; USB-hub reliability history retained as a separate hardware concern. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
102 lines
4.7 KiB
Bash
Executable File
102 lines
4.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# ZFS pool health check — /usr/local/bin/zfs-health-check.sh
|
|
# Runs every 5 min via root crontab on PBS.
|
|
# States: ONLINE (silent), DEGRADED/FAULTED (ntfy high), NOT-IMPORTED w/ members
|
|
# present (ntfy high — failed import, hardware fine), MISSING members (ntfy urgent).
|
|
# Writes Prometheus textfile metrics after full loop completes only.
|
|
# NOTE: deployed by Ansible role `pbs` (roles/pbs/files/zfs-health-check.sh).
|
|
# Edit the role file, not the live copy — the playbook overwrites it.
|
|
|
|
NTFY_URL="https://ntfy.goattw.net/zfs-health"
|
|
EXPECTED_POOLS=(usb1-zfs usb2-zfs)
|
|
TEXTFILE="/var/lib/node_exporter/textfile/zfs_health.prom"
|
|
|
|
TMPFILE=$(mktemp /tmp/zfs_health.XXXXXX.prom)
|
|
# On crash, clean up tmpfile — do NOT write partial results to final path
|
|
trap 'rm -f "$TMPFILE"' EXIT
|
|
|
|
logger -t zfs-health-check "starting pool health check"
|
|
|
|
# Accumulate metric lines; written atomically only after full loop
|
|
METRICS=""
|
|
METRICS+="# HELP zfs_pool_present 1 if pool imported and healthy, 0 if missing\n"
|
|
METRICS+="# TYPE zfs_pool_present gauge\n"
|
|
METRICS+="# HELP zfs_pool_members_present count of expected by-id members visible when a pool is not imported\n"
|
|
METRICS+="# TYPE zfs_pool_members_present gauge\n"
|
|
METRICS+="# HELP zfs_health_last_run_seconds Unix timestamp of last successful check\n"
|
|
METRICS+="# TYPE zfs_health_last_run_seconds gauge\n"
|
|
|
|
for pool in "${EXPECTED_POOLS[@]}"; do
|
|
status=$(zpool list -H -o health "$pool" 2>/dev/null)
|
|
|
|
if [[ "$status" == "ONLINE" ]]; then
|
|
logger -t zfs-health-check "pool $pool: ONLINE"
|
|
METRICS+="zfs_pool_present{pool=\"$pool\"} 1\n"
|
|
|
|
elif [[ "$status" =~ ^(DEGRADED|FAULTED|REMOVED|UNAVAIL)$ ]]; then
|
|
logger -t zfs-health-check "pool $pool: $status — alerting"
|
|
METRICS+="zfs_pool_present{pool=\"$pool\"} 1\n"
|
|
curl -sf \
|
|
-H "Priority: high" \
|
|
-H "Tags: warning" \
|
|
-H "Title: PBS ZFS pool degraded: $pool" \
|
|
-d "Pool $pool is $status on PBS — check immediately" \
|
|
"$NTFY_URL" \
|
|
|| logger -t zfs-health-check "WARNING: ntfy post failed for $pool"
|
|
|
|
else
|
|
# Pool not imported. Distinguish two very different causes before alerting:
|
|
# devices ABSENT -> real hardware (cable/hub/enclosure/power)
|
|
# devices PRESENT -> pool merely not imported (stale cache, boot race) -> import fixes it
|
|
case "$pool" in
|
|
usb1-zfs) members="/dev/disk/by-id/usb-ASMT_2115_ACAAEBBB2E5F-0:0-part1 /dev/disk/by-id/wwn-0x5000cca234ca7110-part1" ;;
|
|
usb2-zfs) members="/dev/disk/by-id/wwn-0x50014ee2075b3126-part1 /dev/disk/by-id/wwn-0x50014ee211ed75f6-part1" ;;
|
|
*) members="" ;;
|
|
esac
|
|
|
|
present=0; total=0
|
|
for m in $members; do
|
|
total=$((total+1))
|
|
[ -e "$m" ] && present=$((present+1))
|
|
done
|
|
|
|
METRICS+="zfs_pool_present{pool=\"$pool\"} 0\n"
|
|
METRICS+="zfs_pool_members_present{pool=\"$pool\"} $present\n"
|
|
|
|
if [ "$total" -gt 0 ] && [ "$present" -eq "$total" ]; then
|
|
# All members enumerated — this is a failed import, NOT a hardware fault.
|
|
logger -t zfs-health-check "pool $pool: NOT IMPORTED (all $total members present) — alerting"
|
|
curl -sf \
|
|
-H "Priority: high" \
|
|
-H "Tags: warning" \
|
|
-H "Title: PBS ZFS pool not imported: $pool" \
|
|
-d "Pool $pool not imported, but all $total members are present. Hardware is fine — likely failed auto-import. Fix: zpool import -d /dev/disk/by-id $pool" \
|
|
"$NTFY_URL" \
|
|
|| logger -t zfs-health-check "WARNING: ntfy post failed for $pool"
|
|
else
|
|
# Members missing (or unknown pool) — genuine hardware suspicion.
|
|
logger -t zfs-health-check "pool $pool: MISSING ($present/$total members present) — alerting"
|
|
curl -sf \
|
|
-H "Priority: urgent" \
|
|
-H "Tags: rotating_light" \
|
|
-H "Title: PBS ZFS pool MISSING: $pool" \
|
|
-d "Pool $pool not imported; only $present of $total members visible. Check cables/enclosure/power. Then: zpool import -d /dev/disk/by-id $pool" \
|
|
"$NTFY_URL" \
|
|
|| logger -t zfs-health-check "WARNING: ntfy post failed for $pool"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Loop completed — write timestamp and atomically replace textfile
|
|
METRICS+="zfs_health_last_run_seconds $(date +%s)\n"
|
|
printf "%b" "$METRICS" > "$TMPFILE"
|
|
mkdir -p "$(dirname "$TEXTFILE")"
|
|
mv "$TMPFILE" "$TEXTFILE"
|
|
chmod 644 "$TEXTFILE"
|
|
|
|
# Disarm cleanup trap — mv succeeded
|
|
trap - EXIT
|
|
|
|
logger -t zfs-health-check "completed — metrics written to $TEXTFILE"
|
|
exit 0
|