From 250fea8679d94716fbcc76f58cb50bdfbc629d22 Mon Sep 17 00:00:00 2001 From: tommy Date: Tue, 7 Jul 2026 18:31:46 -0500 Subject: [PATCH] ups-quarterly-test: sync tracked copy to live + fix false-alarm bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tracked copy was a stale pre-refactor version (no sequential-test safety, no textfile metric). Replace with the current script plus fixes for the UPSBatteryTestFailed false alarm: - Poll ups.test.result until terminal ("passed"/"failed") instead of a single racy read — CyberPower updates the string slower than the OL transition, which recorded 0 despite a passing test. - to_gauge matches *passed* substring, not exact "Done and passed". - Fix cron OR-logic: setting both DOM (1-7) and DOW (0) made cron run on EITHER (daily Jul 1-7 + Sundays). Script now self-guards to day<=7; cron fires every Sunday of the quarter months. - Externalize NUT_PASS to /etc/ups-quarterly-test.env (root:600) so the password is no longer embedded in the repo. Deploy via ups-fix.sh. Co-Authored-By: Claude Opus 4.8 --- monitoring/ups-quarterly-test.sh | 177 +++++++++++++++++++++++++++---- 1 file changed, 155 insertions(+), 22 deletions(-) diff --git a/monitoring/ups-quarterly-test.sh b/monitoring/ups-quarterly-test.sh index 38b1aed..9f8f4b7 100755 --- a/monitoring/ups-quarterly-test.sh +++ b/monitoring/ups-quarterly-test.sh @@ -1,29 +1,162 @@ #!/bin/bash -# Quarterly UPS battery self-test — runs via cron first Sunday of Jan/Apr/Jul/Oct -NUT_PASS='Sparky$100' +# Quarterly UPS battery self-test — first Sunday of Jan/Apr/Jul/Oct at 03:00 +# Writes results to Prometheus textfile collector for alerting. +# +# Safety: tests are SEQUENTIAL — second UPS not tested until first is confirmed +# back on line power. Abort if first UPS doesn't return OL within timeout. +# This prevents both UPS being on battery simultaneously, which would trigger +# the upssched earlyshutdown FSD timer. + +# NUT_PASS is sourced from a root-only file kept OUT of git (see CRED_FILE below). NUT_USER='upsadmin' NUT_HOST='localhost' +TEXTFILE_DIR='/var/lib/node_exporter/textfile_collector' +OUTFILE="${TEXTFILE_DIR}/ups_test.prom" +OL_TIMEOUT=120 # seconds to wait for OL restoration before aborting +OL_BUFFER=35 # seconds to wait after OL confirmed before starting next test + # (>30s earlyshutdown timer to ensure no residual timer from prior state) +RESULT_TIMEOUT=90 # seconds to wait for ups.test.result to reach a terminal state -logger -t ups-quarterly-test 'starting quarterly battery self-test' +log() { logger -t ups-quarterly-test "$*"; } +ts() { date '+%Y-%m-%dT%H:%M:%S'; } -logger -t ups-quarterly-test 'testing cyberpower1 (Tower)' -upscmd -u "$NUT_USER" -p "$NUT_PASS" cyberpower1@$NUT_HOST test.battery.start.quick 2>&1 | logger -t ups-quarterly-test - -sleep 120 - -result1=$(upsc cyberpower1@$NUT_HOST ups.test.result 2>/dev/null) -logger -t ups-quarterly-test "cyberpower1 result: $result1" - -logger -t ups-quarterly-test 'testing cyberpower2 (Rackmount)' -upscmd -u "$NUT_USER" -p "$NUT_PASS" cyberpower2@$NUT_HOST test.battery.start.quick 2>&1 | logger -t ups-quarterly-test - -sleep 120 - -result2=$(upsc cyberpower2@$NUT_HOST ups.test.result 2>/dev/null) -logger -t ups-quarterly-test "cyberpower2 result: $result2" - -if [[ "$result1" == 'Done and passed' && "$result2" == 'Done and passed' ]]; then - logger -t ups-quarterly-test 'PASS: both UPS batteries healthy' +# Load NUT credentials from a root-only file (chmod 600), kept OUT of git so the +# password never lands in a repo. Created by ups-fix.sh. +CRED_FILE='/etc/ups-quarterly-test.env' +if [[ -r "$CRED_FILE" ]]; then + # shellcheck source=/dev/null + source "$CRED_FILE" else - logger -t ups-quarterly-test 'FAIL: one or more UPS batteries need attention' + log "$(ts) ERROR: missing ${CRED_FILE} (defines NUT_PASS) — aborting" + exit 1 +fi + +# FIX: cron cannot express "first Sunday" (setting both DOM and DOW makes cron +# OR them). Cron now runs every Sunday of the quarter months; this guard limits +# execution to the FIRST Sunday (day-of-month 1-7). +if (( $(date +%-d) > 7 )); then + log "$(ts) not the first Sunday of the month (day $(date +%-d)) — skipping" + exit 0 +fi + +log "$(ts) starting quarterly battery self-test" + +# Wait for a given UPS to return to OL after a test. +# Returns 0 if OL confirmed within timeout, 1 if timeout exceeded. +wait_for_online() { + local ups="$1" + local deadline=$(( $(date +%s) + OL_TIMEOUT )) + while (( $(date +%s) < deadline )); do + local status + status=$(upsc "${ups}@${NUT_HOST}" ups.status 2>/dev/null) + log "$(ts) ${ups} status: ${status}" + if [[ "$status" == OL* ]]; then + log "$(ts) ${ups} confirmed OL" + return 0 + fi + sleep 5 + done + log "$(ts) ERROR: ${ups} did not return OL within ${OL_TIMEOUT}s — ABORTING" + return 1 +} + +# Run test on one UPS and return result string. +# Returns "Done and passed", "Done and failed", or "TIMEOUT" +run_test() { + local ups="$1" + local label="$2" + local result + + log "$(ts) submitting test: ${ups} (${label})" + upscmd -u "$NUT_USER" -p "$NUT_PASS" "${ups}@${NUT_HOST}" test.battery.start.quick 2>&1 \ + | logger -t ups-quarterly-test + + log "$(ts) ${ups} test submitted — polling for OB then OL" + + # Wait for test to complete (UPS goes OB then returns OL) + if ! wait_for_online "$ups"; then + echo "TIMEOUT" + return + fi + + log "$(ts) ${ups} OL confirmed — waiting ${OL_BUFFER}s buffer before reading result" + sleep "$OL_BUFFER" + + # FIX: CyberPower updates ups.test.result slower than the OL transition, so a + # single read races and can catch a non-terminal value (e.g. "In progress") + # → false 0 despite a passing test. Poll until the result is terminal. + local rdeadline=$(( $(date +%s) + RESULT_TIMEOUT )) + while (( $(date +%s) < rdeadline )); do + result=$(upsc "${ups}@${NUT_HOST}" ups.test.result 2>/dev/null) + if [[ "$result" == *[Pp]assed* || "$result" == *[Ff]ailed* ]]; then + break + fi + log "$(ts) ${ups} result not terminal yet: '${result}' — waiting" + sleep 5 + done + + log "$(ts) ${ups} test result: ${result}" + echo "$result" +} + +# FIX: match on "passed" substring (robust to formatting) rather than an exact +# "Done and passed" string. +to_gauge() { + [[ "$1" == *[Pp]assed* ]] && echo 1 || echo 0 +} + +ntfy_alert() { + local msg="$1" + log "$(ts) ntfy: ${msg}" + curl -s -X POST \ + -H "Title: UPS Test Warning" \ + -H "Priority: high" \ + -d "$msg" \ + http://192.168.99.198:8081/homelab 2>/dev/null || true +} + +# --- Test cyberpower1 --- +result1=$(run_test cyberpower1 "Tower") + +if [[ "$result1" == "TIMEOUT" ]]; then + ntfy_alert "cyberpower1 quarterly test: did not return OL within ${OL_TIMEOUT}s — cyberpower2 NOT tested. Manual inspection required." + log "$(ts) ABORT: cyberpower1 timeout — cyberpower2 test skipped" + g1=0 + result2="skipped" + g2=0 +else + log "$(ts) cyberpower1 complete. Waiting ${OL_BUFFER}s before starting cyberpower2 test" + sleep "$OL_BUFFER" + + # --- Test cyberpower2 --- + result2=$(run_test cyberpower2 "Rackmount") + + if [[ "$result2" == "TIMEOUT" ]]; then + ntfy_alert "cyberpower2 quarterly test: did not return OL within ${OL_TIMEOUT}s. Manual inspection required." + log "$(ts) ABORT: cyberpower2 timeout" + g2=0 + else + g2=$(to_gauge "$result2") + fi + g1=$(to_gauge "$result1") +fi + +now=$(date +%s) + +{ + printf '# HELP ups_battery_test_result Last quarterly UPS battery self-test: 1=passed, 0=failed/unknown.\n' + printf '# TYPE ups_battery_test_result gauge\n' + printf 'ups_battery_test_result{ups="cyberpower1"} %s\n' "$g1" + printf 'ups_battery_test_result{ups="cyberpower2"} %s\n' "$g2" + printf '# HELP ups_battery_test_timestamp_seconds Unix timestamp of last UPS battery self-test run.\n' + printf '# TYPE ups_battery_test_timestamp_seconds gauge\n' + printf 'ups_battery_test_timestamp_seconds{ups="cyberpower1"} %s\n' "$now" + printf 'ups_battery_test_timestamp_seconds{ups="cyberpower2"} %s\n' "$now" +} > "${OUTFILE}.tmp" && mv "${OUTFILE}.tmp" "$OUTFILE" + +if [[ "$g1" == "1" && "$g2" == "1" ]]; then + log "$(ts) PASS: both UPS batteries healthy" +else + log "$(ts) FAIL: cyberpower1=${result1} cyberpower2=${result2}" + ntfy_alert "UPS quarterly test FAILED: cyberpower1=${result1} cyberpower2=${result2}" fi