homelab-configs/monitoring/ups-quarterly-test.sh

#!/bin/bash
# Quarterly UPS battery self-test — first Sunday of Jan/Apr/Jul/Oct at 03:00
# Writes results to Prometheus textfile collector for alerting.
#
# Safety: tests are SEQUENTIAL — second UPS not tested until first is confirmed
# back on line power. Abort if first UPS doesn't return OL within timeout.
# This prevents both UPS being on battery simultaneously, which would trigger
# the upssched earlyshutdown FSD timer.

# NUT_PASS is sourced from a root-only file kept OUT of git (see CRED_FILE below).
NUT_USER='upsadmin'
NUT_HOST='localhost'
TEXTFILE_DIR='/var/lib/node_exporter/textfile_collector'
OUTFILE="${TEXTFILE_DIR}/ups_test.prom"
OL_TIMEOUT=120    # seconds to wait for OL restoration before aborting
OL_BUFFER=35      # seconds to wait after OL confirmed before starting next test
                  # (>30s earlyshutdown timer to ensure no residual timer from prior state)
RESULT_TIMEOUT=90 # seconds to wait for ups.test.result to reach a terminal state

log() { logger -t ups-quarterly-test "$*"; }
ts()  { date '+%Y-%m-%dT%H:%M:%S'; }

# Load NUT credentials from a root-only file (chmod 600), kept OUT of git so the
# password never lands in a repo. Created by ups-fix.sh.
CRED_FILE='/etc/ups-quarterly-test.env'
if [[ -r "$CRED_FILE" ]]; then
    # shellcheck source=/dev/null
    source "$CRED_FILE"
else
    log "$(ts) ERROR: missing ${CRED_FILE} (defines NUT_PASS) — aborting"
    exit 1
fi

# FIX: cron cannot express "first Sunday" (setting both DOM and DOW makes cron
# OR them). Cron now runs every Sunday of the quarter months; this guard limits
# execution to the FIRST Sunday (day-of-month 1-7).
if (( $(date +%-d) > 7 )); then
    log "$(ts) not the first Sunday of the month (day $(date +%-d)) — skipping"
    exit 0
fi

log "$(ts) starting quarterly battery self-test"

# Wait for a given UPS to return to OL after a test.
# Returns 0 if OL confirmed within timeout, 1 if timeout exceeded.
wait_for_online() {
    local ups="$1"
    local deadline=$(( $(date +%s) + OL_TIMEOUT ))
    while (( $(date +%s) < deadline )); do
        local status
        status=$(upsc "${ups}@${NUT_HOST}" ups.status 2>/dev/null)
        log "$(ts) ${ups} status: ${status}"
        if [[ "$status" == OL* ]]; then
            log "$(ts) ${ups} confirmed OL"
            return 0
        fi
        sleep 5
    done
    log "$(ts) ERROR: ${ups} did not return OL within ${OL_TIMEOUT}s — ABORTING"
    return 1
}

# Run test on one UPS and return result string.
# Returns "Done and passed", "Done and failed", or "TIMEOUT"
run_test() {
    local ups="$1"
    local label="$2"
    local result

    log "$(ts) submitting test: ${ups} (${label})"
    upscmd -u "$NUT_USER" -p "$NUT_PASS" "${ups}@${NUT_HOST}" test.battery.start.quick 2>&1 \
        | logger -t ups-quarterly-test

    log "$(ts) ${ups} test submitted — polling for OB then OL"

    # Wait for test to complete (UPS goes OB then returns OL)
    if ! wait_for_online "$ups"; then
        echo "TIMEOUT"
        return
    fi

    log "$(ts) ${ups} OL confirmed — waiting ${OL_BUFFER}s buffer before reading result"
    sleep "$OL_BUFFER"

    # FIX: CyberPower updates ups.test.result slower than the OL transition, so a
    # single read races and can catch a non-terminal value (e.g. "In progress")
    # → false 0 despite a passing test. Poll until the result is terminal.
    local rdeadline=$(( $(date +%s) + RESULT_TIMEOUT ))
    while (( $(date +%s) < rdeadline )); do
        result=$(upsc "${ups}@${NUT_HOST}" ups.test.result 2>/dev/null)
        if [[ "$result" == *[Pp]assed* || "$result" == *[Ff]ailed* ]]; then
            break
        fi
        log "$(ts) ${ups} result not terminal yet: '${result}' — waiting"
        sleep 5
    done

    log "$(ts) ${ups} test result: ${result}"
    echo "$result"
}

# FIX: match on "passed" substring (robust to formatting) rather than an exact
# "Done and passed" string.
to_gauge() {
    [[ "$1" == *[Pp]assed* ]] && echo 1 || echo 0
}

ntfy_alert() {
    local msg="$1"
    log "$(ts) ntfy: ${msg}"
    curl -s -X POST \
        -H "Title: UPS Test Warning" \
        -H "Priority: high" \
        -d "$msg" \
        http://192.168.99.198:8081/homelab 2>/dev/null || true
}

# --- Test cyberpower1 ---
result1=$(run_test cyberpower1 "Tower")

if [[ "$result1" == "TIMEOUT" ]]; then
    ntfy_alert "cyberpower1 quarterly test: did not return OL within ${OL_TIMEOUT}s — cyberpower2 NOT tested. Manual inspection required."
    log "$(ts) ABORT: cyberpower1 timeout — cyberpower2 test skipped"
    g1=0
    result2="skipped"
    g2=0
else
    log "$(ts) cyberpower1 complete. Waiting ${OL_BUFFER}s before starting cyberpower2 test"
    sleep "$OL_BUFFER"

    # --- Test cyberpower2 ---
    result2=$(run_test cyberpower2 "Rackmount")

    if [[ "$result2" == "TIMEOUT" ]]; then
        ntfy_alert "cyberpower2 quarterly test: did not return OL within ${OL_TIMEOUT}s. Manual inspection required."
        log "$(ts) ABORT: cyberpower2 timeout"
        g2=0
    else
        g2=$(to_gauge "$result2")
    fi
    g1=$(to_gauge "$result1")
fi

now=$(date +%s)

{
    printf '# HELP ups_battery_test_result Last quarterly UPS battery self-test: 1=passed, 0=failed/unknown.\n'
    printf '# TYPE ups_battery_test_result gauge\n'
    printf 'ups_battery_test_result{ups="cyberpower1"} %s\n' "$g1"
    printf 'ups_battery_test_result{ups="cyberpower2"} %s\n' "$g2"
    printf '# HELP ups_battery_test_timestamp_seconds Unix timestamp of last UPS battery self-test run.\n'
    printf '# TYPE ups_battery_test_timestamp_seconds gauge\n'
    printf 'ups_battery_test_timestamp_seconds{ups="cyberpower1"} %s\n' "$now"
    printf 'ups_battery_test_timestamp_seconds{ups="cyberpower2"} %s\n' "$now"
} > "${OUTFILE}.tmp" && mv "${OUTFILE}.tmp" "$OUTFILE"

if [[ "$g1" == "1" && "$g2" == "1" ]]; then
    log "$(ts) PASS: both UPS batteries healthy"
else
    log "$(ts) FAIL: cyberpower1=${result1} cyberpower2=${result2}"
    ntfy_alert "UPS quarterly test FAILED: cyberpower1=${result1} cyberpower2=${result2}"
fi