From 7178f6ceea75925298412776c704bb55376315ac Mon Sep 17 00:00:00 2001 From: tommy Date: Tue, 7 Jul 2026 18:13:16 -0500 Subject: [PATCH] weekly-health-digest: add internet speed check (section 4c) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reports the speedtest-tracker (on .183) newest result: flags STALE if >14h old (2+ missed 6h cycles → scheduler stalled) or LOW if the best of the last 4 tests falls below the 500/500 Mbps floor (best-of-4 avoids flagging a transient ISP dip). Floors adjustable. Local SQLite read. Co-Authored-By: Claude Opus 4.8 --- monitoring/weekly-health-digest.sh | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/monitoring/weekly-health-digest.sh b/monitoring/weekly-health-digest.sh index e348641..ab6a41d 100755 --- a/monitoring/weekly-health-digest.sh +++ b/monitoring/weekly-health-digest.sh @@ -191,6 +191,43 @@ for svc in lubelogger vaultwarden; do fi done +# ── 4c. Internet speed (speedtest-tracker on .183) ────────────────────────── +# Tracker records Ookla tests every 6h (SPEEDTEST_SCHEDULE). Flag if the newest +# completed result is stale (>14h = 2+ missed cycles → scheduler stalled) or if +# the best of the last 4 tests is below the floor (using best-of-4, not the +# single latest, avoids flagging a transient ISP dip). Floors are ADJUSTABLE — +# set to catch a real regression (e.g. the old 100Mb-uplink cap ≈ 94 Mbps) or +# sustained ISP degradation, not normal variance. DB is local + tommy-readable. +ST_DB="/home/tommy/speedtest-tracker/data/database.sqlite" +ST_FLOOR_DOWN=500 # Mbps +ST_FLOOR_UP=500 # Mbps +if [ -r "$ST_DB" ]; then + st_row=$(sqlite3 "$ST_DB" " + SELECT + CAST((julianday('now') - julianday(max(created_at)))*24 AS INT) || '|' || + CAST((SELECT download FROM results WHERE status='completed' ORDER BY id DESC LIMIT 1)*8/1e6 AS INT) || '|' || + CAST((SELECT upload FROM results WHERE status='completed' ORDER BY id DESC LIMIT 1)*8/1e6 AS INT) || '|' || + CAST((SELECT max(download) FROM (SELECT download FROM results WHERE status='completed' ORDER BY id DESC LIMIT 4))*8/1e6 AS INT) || '|' || + CAST((SELECT max(upload) FROM (SELECT upload FROM results WHERE status='completed' ORDER BY id DESC LIMIT 4))*8/1e6 AS INT) + FROM results WHERE status='completed';" 2>/dev/null) + st_age=$(echo "$st_row" | cut -d'|' -f1) + st_dl=$(echo "$st_row" | cut -d'|' -f2) + st_ul=$(echo "$st_row" | cut -d'|' -f3) + st_mdl=$(echo "$st_row" | cut -d'|' -f4) + st_mul=$(echo "$st_row" | cut -d'|' -f5) + if [ -z "$st_age" ]; then + add_line "Internet speed" "no completed speedtest results found" + elif [ "$st_age" -gt 14 ]; then + add_line "Internet speed" "STALE — newest test ${st_age}h ago (expected every 6h; scheduler stalled?)" + elif [ "$st_mdl" -lt "$ST_FLOOR_DOWN" ] || [ "$st_mul" -lt "$ST_FLOOR_UP" ]; then + add_line "Internet speed" "LOW — best of last 4: ${st_mdl}/${st_mul} Mbps down/up (floor ${ST_FLOOR_DOWN}/${ST_FLOOR_UP})" + else + add_line "Internet speed" "OK" "${st_dl}/${st_ul} Mbps down/up, ${st_age}h ago" + fi +else + add_line "Internet speed" "speedtest DB unreadable ($ST_DB)" +fi + # ── 5. Filesystem >85% ─────────────────────────────────────────────────────── fs_over=$(prom_query '(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs"}) * 100 > 85' | python3 -c " import json,sys