#!/bin/bash
# ZFS pool health check — /usr/local/bin/zfs-health-check.sh
# Runs every 5 min via root crontab on PBS.
# States: ONLINE (silent), DEGRADED/FAULTED (ntfy high), NOT-IMPORTED w/ members
#   present (ntfy high — failed import, hardware fine), MISSING members (ntfy urgent).
# Writes Prometheus textfile metrics after full loop completes only.
# NOTE: deployed by Ansible role `pbs` (roles/pbs/files/zfs-health-check.sh).
#       Edit the role file, not the live copy — the playbook overwrites it.

NTFY_URL="https://ntfy.goattw.net/zfs-health"
EXPECTED_POOLS=(usb1-zfs usb2-zfs)
TEXTFILE="/var/lib/node_exporter/textfile/zfs_health.prom"

TMPFILE=$(mktemp /tmp/zfs_health.XXXXXX.prom)
# On crash, clean up tmpfile — do NOT write partial results to final path
trap 'rm -f "$TMPFILE"' EXIT

logger -t zfs-health-check "starting pool health check"

# Accumulate metric lines; written atomically only after full loop
METRICS=""
METRICS+="# HELP zfs_pool_present 1 if pool imported and healthy, 0 if missing\n"
METRICS+="# TYPE zfs_pool_present gauge\n"
METRICS+="# HELP zfs_pool_members_present count of expected by-id members visible when a pool is not imported\n"
METRICS+="# TYPE zfs_pool_members_present gauge\n"
METRICS+="# HELP zfs_health_last_run_seconds Unix timestamp of last successful check\n"
METRICS+="# TYPE zfs_health_last_run_seconds gauge\n"

for pool in "${EXPECTED_POOLS[@]}"; do
    status=$(zpool list -H -o health "$pool" 2>/dev/null)

    if [[ "$status" == "ONLINE" ]]; then
        logger -t zfs-health-check "pool $pool: ONLINE"
        METRICS+="zfs_pool_present{pool=\"$pool\"} 1\n"

    elif [[ "$status" =~ ^(DEGRADED|FAULTED|REMOVED|UNAVAIL)$ ]]; then
        logger -t zfs-health-check "pool $pool: $status — alerting"
        METRICS+="zfs_pool_present{pool=\"$pool\"} 1\n"
        curl -sf \
            -H "Priority: high" \
            -H "Tags: warning" \
            -H "Title: PBS ZFS pool degraded: $pool" \
            -d "Pool $pool is $status on PBS — check immediately" \
            "$NTFY_URL" \
            || logger -t zfs-health-check "WARNING: ntfy post failed for $pool"

    else
        # Pool not imported. Distinguish two very different causes before alerting:
        #   devices ABSENT  -> real hardware (cable/hub/enclosure/power)
        #   devices PRESENT -> pool merely not imported (stale cache, boot race) -> import fixes it
        case "$pool" in
            usb1-zfs) members="/dev/disk/by-id/usb-ASMT_2115_ACAAEBBB2E5F-0:0-part1 /dev/disk/by-id/wwn-0x5000cca234ca7110-part1" ;;
            usb2-zfs) members="/dev/disk/by-id/wwn-0x50014ee2075b3126-part1 /dev/disk/by-id/wwn-0x50014ee211ed75f6-part1" ;;
            *)        members="" ;;
        esac

        present=0; total=0
        for m in $members; do
            total=$((total+1))
            [ -e "$m" ] && present=$((present+1))
        done

        METRICS+="zfs_pool_present{pool=\"$pool\"} 0\n"
        METRICS+="zfs_pool_members_present{pool=\"$pool\"} $present\n"

        if [ "$total" -gt 0 ] && [ "$present" -eq "$total" ]; then
            # All members enumerated — this is a failed import, NOT a hardware fault.
            logger -t zfs-health-check "pool $pool: NOT IMPORTED (all $total members present) — alerting"
            curl -sf \
                -H "Priority: high" \
                -H "Tags: warning" \
                -H "Title: PBS ZFS pool not imported: $pool" \
                -d "Pool $pool not imported, but all $total members are present. Hardware is fine — likely failed auto-import. Fix: zpool import -d /dev/disk/by-id $pool" \
                "$NTFY_URL" \
                || logger -t zfs-health-check "WARNING: ntfy post failed for $pool"
        else
            # Members missing (or unknown pool) — genuine hardware suspicion.
            logger -t zfs-health-check "pool $pool: MISSING ($present/$total members present) — alerting"
            curl -sf \
                -H "Priority: urgent" \
                -H "Tags: rotating_light" \
                -H "Title: PBS ZFS pool MISSING: $pool" \
                -d "Pool $pool not imported; only $present of $total members visible. Check cables/enclosure/power. Then: zpool import -d /dev/disk/by-id $pool" \
                "$NTFY_URL" \
                || logger -t zfs-health-check "WARNING: ntfy post failed for $pool"
        fi
    fi
done

# Loop completed — write timestamp and atomically replace textfile
METRICS+="zfs_health_last_run_seconds $(date +%s)\n"
printf "%b" "$METRICS" > "$TMPFILE"
mkdir -p "$(dirname "$TEXTFILE")"
mv "$TMPFILE" "$TEXTFILE"
chmod 644 "$TEXTFILE"

# Disarm cleanup trap — mv succeeded
trap - EXIT

logger -t zfs-health-check "completed — metrics written to $TEXTFILE"
exit 0