ups-quarterly-test: sync tracked copy to live + fix false-alarm bugs

The tracked copy was a stale pre-refactor version (no sequential-test
safety, no textfile metric). Replace with the current script plus fixes
for the UPSBatteryTestFailed false alarm:

- Poll ups.test.result until terminal ("passed"/"failed") instead of a
  single racy read — CyberPower updates the string slower than the OL
  transition, which recorded 0 despite a passing test.
- to_gauge matches *passed* substring, not exact "Done and passed".
- Fix cron OR-logic: setting both DOM (1-7) and DOW (0) made cron run on
  EITHER (daily Jul 1-7 + Sundays). Script now self-guards to day<=7;
  cron fires every Sunday of the quarter months.
- Externalize NUT_PASS to /etc/ups-quarterly-test.env (root:600) so the
  password is no longer embedded in the repo. Deploy via ups-fix.sh.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

monitoring/ups-quarterly-test.sh

@@ -1,29 +1,162 @@
 #!/bin/bash
-# Quarterly UPS battery self-test — runs via cron first Sunday of Jan/Apr/Jul/Oct
+# Quarterly UPS battery self-test — first Sunday of Jan/Apr/Jul/Oct at 03:00
-NUT_PASS='Sparky$100'
+# Writes results to Prometheus textfile collector for alerting.
+#
+# Safety: tests are SEQUENTIAL — second UPS not tested until first is confirmed
+# back on line power. Abort if first UPS doesn't return OL within timeout.
+# This prevents both UPS being on battery simultaneously, which would trigger
+# the upssched earlyshutdown FSD timer.
+
+# NUT_PASS is sourced from a root-only file kept OUT of git (see CRED_FILE below).
 NUT_USER='upsadmin'
 NUT_HOST='localhost'
+TEXTFILE_DIR='/var/lib/node_exporter/textfile_collector'
+OUTFILE="${TEXTFILE_DIR}/ups_test.prom"
+OL_TIMEOUT=120    # seconds to wait for OL restoration before aborting
+OL_BUFFER=35      # seconds to wait after OL confirmed before starting next test
+                  # (>30s earlyshutdown timer to ensure no residual timer from prior state)
+RESULT_TIMEOUT=90 # seconds to wait for ups.test.result to reach a terminal state
 
-logger -t ups-quarterly-test 'starting quarterly battery self-test'
+log() { logger -t ups-quarterly-test "$*"; }
+ts()  { date '+%Y-%m-%dT%H:%M:%S'; }
 
-logger -t ups-quarterly-test 'testing cyberpower1 (Tower)'
+# Load NUT credentials from a root-only file (chmod 600), kept OUT of git so the
-upscmd -u "$NUT_USER" -p "$NUT_PASS" cyberpower1@$NUT_HOST test.battery.start.quick 2>&1 |   logger -t ups-quarterly-test
+# password never lands in a repo. Created by ups-fix.sh.
-
+CRED_FILE='/etc/ups-quarterly-test.env'
-sleep 120
+if [[ -r "$CRED_FILE" ]]; then
-
+    # shellcheck source=/dev/null
-result1=$(upsc cyberpower1@$NUT_HOST ups.test.result 2>/dev/null)
+    source "$CRED_FILE"
-logger -t ups-quarterly-test "cyberpower1 result: $result1"
-
-logger -t ups-quarterly-test 'testing cyberpower2 (Rackmount)'
-upscmd -u "$NUT_USER" -p "$NUT_PASS" cyberpower2@$NUT_HOST test.battery.start.quick 2>&1 |   logger -t ups-quarterly-test
-
-sleep 120
-
-result2=$(upsc cyberpower2@$NUT_HOST ups.test.result 2>/dev/null)
-logger -t ups-quarterly-test "cyberpower2 result: $result2"
-
-if [[ "$result1" == 'Done and passed' && "$result2" == 'Done and passed' ]]; then
-    logger -t ups-quarterly-test 'PASS: both UPS batteries healthy'
 else
-    logger -t ups-quarterly-test 'FAIL: one or more UPS batteries need attention'
+    log "$(ts) ERROR: missing ${CRED_FILE} (defines NUT_PASS) — aborting"
+    exit 1
+fi
+
+# FIX: cron cannot express "first Sunday" (setting both DOM and DOW makes cron
+# OR them). Cron now runs every Sunday of the quarter months; this guard limits
+# execution to the FIRST Sunday (day-of-month 1-7).
+if (( $(date +%-d) > 7 )); then
+    log "$(ts) not the first Sunday of the month (day $(date +%-d)) — skipping"
+    exit 0
+fi
+
+log "$(ts) starting quarterly battery self-test"
+
+# Wait for a given UPS to return to OL after a test.
+# Returns 0 if OL confirmed within timeout, 1 if timeout exceeded.
+wait_for_online() {
+    local ups="$1"
+    local deadline=$(( $(date +%s) + OL_TIMEOUT ))
+    while (( $(date +%s) < deadline )); do
+        local status
+        status=$(upsc "${ups}@${NUT_HOST}" ups.status 2>/dev/null)
+        log "$(ts) ${ups} status: ${status}"
+        if [[ "$status" == OL* ]]; then
+            log "$(ts) ${ups} confirmed OL"
+            return 0
+        fi
+        sleep 5
+    done
+    log "$(ts) ERROR: ${ups} did not return OL within ${OL_TIMEOUT}s — ABORTING"
+    return 1
+}
+
+# Run test on one UPS and return result string.
+# Returns "Done and passed", "Done and failed", or "TIMEOUT"
+run_test() {
+    local ups="$1"
+    local label="$2"
+    local result
+
+    log "$(ts) submitting test: ${ups} (${label})"
+    upscmd -u "$NUT_USER" -p "$NUT_PASS" "${ups}@${NUT_HOST}" test.battery.start.quick 2>&1 \
+        | logger -t ups-quarterly-test
+
+    log "$(ts) ${ups} test submitted — polling for OB then OL"
+
+    # Wait for test to complete (UPS goes OB then returns OL)
+    if ! wait_for_online "$ups"; then
+        echo "TIMEOUT"
+        return
+    fi
+
+    log "$(ts) ${ups} OL confirmed — waiting ${OL_BUFFER}s buffer before reading result"
+    sleep "$OL_BUFFER"
+
+    # FIX: CyberPower updates ups.test.result slower than the OL transition, so a
+    # single read races and can catch a non-terminal value (e.g. "In progress")
+    # → false 0 despite a passing test. Poll until the result is terminal.
+    local rdeadline=$(( $(date +%s) + RESULT_TIMEOUT ))
+    while (( $(date +%s) < rdeadline )); do
+        result=$(upsc "${ups}@${NUT_HOST}" ups.test.result 2>/dev/null)
+        if [[ "$result" == *[Pp]assed* || "$result" == *[Ff]ailed* ]]; then
+            break
+        fi
+        log "$(ts) ${ups} result not terminal yet: '${result}' — waiting"
+        sleep 5
+    done
+
+    log "$(ts) ${ups} test result: ${result}"
+    echo "$result"
+}
+
+# FIX: match on "passed" substring (robust to formatting) rather than an exact
+# "Done and passed" string.
+to_gauge() {
+    [[ "$1" == *[Pp]assed* ]] && echo 1 || echo 0
+}
+
+ntfy_alert() {
+    local msg="$1"
+    log "$(ts) ntfy: ${msg}"
+    curl -s -X POST \
+        -H "Title: UPS Test Warning" \
+        -H "Priority: high" \
+        -d "$msg" \
+        http://192.168.99.198:8081/homelab 2>/dev/null || true
+}
+
+# --- Test cyberpower1 ---
+result1=$(run_test cyberpower1 "Tower")
+
+if [[ "$result1" == "TIMEOUT" ]]; then
+    ntfy_alert "cyberpower1 quarterly test: did not return OL within ${OL_TIMEOUT}s — cyberpower2 NOT tested. Manual inspection required."
+    log "$(ts) ABORT: cyberpower1 timeout — cyberpower2 test skipped"
+    g1=0
+    result2="skipped"
+    g2=0
+else
+    log "$(ts) cyberpower1 complete. Waiting ${OL_BUFFER}s before starting cyberpower2 test"
+    sleep "$OL_BUFFER"
+
+    # --- Test cyberpower2 ---
+    result2=$(run_test cyberpower2 "Rackmount")
+
+    if [[ "$result2" == "TIMEOUT" ]]; then
+        ntfy_alert "cyberpower2 quarterly test: did not return OL within ${OL_TIMEOUT}s. Manual inspection required."
+        log "$(ts) ABORT: cyberpower2 timeout"
+        g2=0
+    else
+        g2=$(to_gauge "$result2")
+    fi
+    g1=$(to_gauge "$result1")
+fi
+
+now=$(date +%s)
+
+{
+    printf '# HELP ups_battery_test_result Last quarterly UPS battery self-test: 1=passed, 0=failed/unknown.\n'
+    printf '# TYPE ups_battery_test_result gauge\n'
+    printf 'ups_battery_test_result{ups="cyberpower1"} %s\n' "$g1"
+    printf 'ups_battery_test_result{ups="cyberpower2"} %s\n' "$g2"
+    printf '# HELP ups_battery_test_timestamp_seconds Unix timestamp of last UPS battery self-test run.\n'
+    printf '# TYPE ups_battery_test_timestamp_seconds gauge\n'
+    printf 'ups_battery_test_timestamp_seconds{ups="cyberpower1"} %s\n' "$now"
+    printf 'ups_battery_test_timestamp_seconds{ups="cyberpower2"} %s\n' "$now"
+} > "${OUTFILE}.tmp" && mv "${OUTFILE}.tmp" "$OUTFILE"
+
+if [[ "$g1" == "1" && "$g2" == "1" ]]; then
+    log "$(ts) PASS: both UPS batteries healthy"
+else
+    log "$(ts) FAIL: cyberpower1=${result1} cyberpower2=${result2}"
+    ntfy_alert "UPS quarterly test FAILED: cyberpower1=${result1} cyberpower2=${result2}"
 fi