From 25bae3b4a2957501f1702a7060a626971f585978 Mon Sep 17 00:00:00 2001 From: c4ch3c4d3 Date: Sat, 22 Aug 2026 15:51:05 -0600 Subject: [PATCH] test(smoke): on-device reliability suite --- scripts/smoke.sh | 421 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100755 scripts/smoke.sh diff --git a/scripts/smoke.sh b/scripts/smoke.sh new file mode 100755 index 0000000..6524239 --- /dev/null +++ b/scripts/smoke.sh @@ -0,0 +1,421 @@ +#!/bin/sh +# Mark VIII on-device reliability smoke suite (POSIX sh, BusyBox-safe). +# +# Usage: smoke.sh [--write] +# +# Read-only checks (default): +# 1. Web UI answers GET / +# 2. mk8-guard installed and enabled (S49 boot symlink) +# 3. Safe-UCI invariants + SSID pool size <= 20 +# 4. Event journal: last line of /mmc/mk8/events.log parses as JSON +# with a 'kind' field (via python3) +# 5. Monitor interfaces wlan0mon + wlan1mon exist +# 6. Deployed server.py SERVER_VERSION == payload.sh Version header +# 7. Authenticated API path: POST /api/login -> GET /api/health +# (requires webui password in $PASS; skipped when unset) +# +# Destructive drills (--write only; values auto-restored): +# 8. Bad-value drill: feeds --reconcile a wrong bands value and an +# oversized SSID pool (25 dummy entries), verifies both are +# repaired, restores originals. +# 9. Rollback watchdog drill: runs mk8-watchdog.sh against a config +# profile while the web UI is up (expects clean promote exit), +# then STOPS the pagerwebui service and expects the watchdog to +# roll back and journal a 'rollback' event, then restarts webui. +# Preceded by a 5-second warning countdown; brief web outage. +# +# Environment: +# PASS webui password used for POST /api/login (check 7). +# +# Exit status: 0 when every executed check passes, 1 otherwise. + +set -u + +BASE=/mmc/mk8 +REL="$BASE/releases/current" +LEGACY=/root/payloads/user/remote_access/pager-webui +URL=http://127.0.0.1:8080 +GUARD_INIT=/etc/init.d/mk8-guard +GUARD_LINK=/etc/rc.d/S49mk8-guard +WEBUI_INIT=/etc/init.d/pagerwebui +JAR=/tmp/mk8-smoke-cookies.$$ + +WRITE=0 +PASS="${PASS:-}" +WEB_STOPPED=0 +FAILED=0 +WAIT_RC=0 + +PY="$(command -v python3 2>/dev/null || true)" +[ -n "$PY" ] || PY=/usr/bin/python3 +[ -x "$PY" ] || PY="" + +usage() { + printf 'Usage: smoke.sh [--write]\n' + printf '\n' + printf 'Read-only checks run by default. --write adds two destructive\n' + printf 'drills that briefly toggle UCI config and stop/start the\n' + printf 'pagerwebui service; original values are restored automatically.\n' + printf 'Set PASS= to enable the authenticated API check.\n' +} + +on_exit() { + rm -f "$JAR" 2>/dev/null + if [ "$WEB_STOPPED" = "1" ]; then + info 'restoring pagerwebui service' + "$WEBUI_INIT" start >/dev/null 2>&1 + fi +} + +pass() { printf 'PASS %s\n' "$1"; } +fail() { printf 'FAIL %s\n' "$1"; FAILED=$((FAILED + 1)); } +skip() { printf 'SKIP %s\n' "$1"; } +info() { printf ' %s\n' "$1"; } + +uci_get() { uci -q get "$1" 2>/dev/null | tr -d '\r'; } + +rollback_count() { + if [ -z "$PY" ] || [ ! -f "$BASE/events.log" ]; then + printf 0 + return + fi + tail -n 400 "$BASE/events.log" 2>/dev/null | "$PY" -c ' +import json, sys +n = 0 +for line in sys.stdin: + try: + d = json.loads(line) + except Exception: + continue + if isinstance(d, dict) and d.get("kind") == "rollback": + n += 1 +print(n)' 2>/dev/null || printf 0 +} + +# wait_exit : poll for background job exit; sets WAIT_RC +# and returns 0 once reaped, 1 on timeout (job left running). +wait_exit() { + _pid="$1"; _t="$2"; _n=0 + while [ "$_n" -lt "$_t" ]; do + if ! kill -0 "$_pid" 2>/dev/null; then + wait "$_pid" + WAIT_RC=$? + return 0 + fi + sleep 1 + _n=$((_n + 1)) + done + return 1 +} + +kill_bg() { + kill "$1" 2>/dev/null + sleep 1 + kill -9 "$1" 2>/dev/null + wait "$1" 2>/dev/null +} + +check_web_up() { + if curl -fsS -m 5 "$URL/" >/dev/null 2>&1; then + pass 'web: GET / answered' + else + fail 'web: GET / failed' + fi +} + +check_guard() { + if [ -x "$GUARD_INIT" ] && [ -e "$GUARD_LINK" ]; then + pass 'guard: init script executable and enabled (S49)' + else + fail "guard: missing executable/init or S49 link ($GUARD_INIT $GUARD_LINK)" + fi +} + +check_invariants() { + inv_fail=0 + while IFS= read -r kv; do + [ -n "$kv" ] || continue + key="${kv%%=*}" + want="${kv#*=}" + got="$(uci_get "$key")" + if [ "$got" != "$want" ]; then + inv_fail=$((inv_fail + 1)) + info "uci $key=$got (want $want)" + fi + done </dev/null)" + out="$(printf '%s' "$last" | "$PY" -c ' +import json, sys +raw = sys.stdin.read().strip() +if not raw: + raise SystemExit("events.log empty or missing") +try: + d = json.loads(raw) +except Exception as exc: + raise SystemExit("not JSON: %s" % exc) +kind = d.get("kind") if isinstance(d, dict) else None +if not kind: + raise SystemExit("last entry has no kind field") +print(kind)' 2>&1)" + rc=$? + if [ "$rc" -eq 0 ]; then + pass "journal: last entry ok (kind=$out)" + else + fail "journal: $out" + fi +} + +check_monitors() { + miss="" + for m in wlan0mon wlan1mon; do + ip link show "$m" >/dev/null 2>&1 || miss="$miss $m" + done + if [ -z "$miss" ]; then + pass 'monitors: wlan0mon + wlan1mon present' + else + fail "monitors: down:$miss" + fi +} + +check_versions() { + sv="$(grep '^SERVER_VERSION' "$REL/server.py" 2>/dev/null | head -n 1 \ + | sed -e 's/^SERVER_VERSION = //' -e "s/'//g" | tr -d '\r')" + pv="$(grep '^#[ ]*[Vv]ersion:' "$REL/payload.sh" 2>/dev/null | head -n 1 \ + | sed 's/^#[ ]*[Vv]ersion:[ ]*//' | tr -d '\r')" + if [ -n "$sv" ] && [ "$sv" = "$pv" ]; then + pass "versions: release server.py and payload.sh agree ($sv)" + else + fail "versions: server.py='$sv' payload.sh='$pv'" + fi +} + +check_authed_api() { + if [ -z "$PASS" ]; then + skip 'authed API: PASS not set' + return + fi + if [ -z "$PY" ]; then + fail 'authed API: python3 not found for response check' + return + fi + body='{"password":"'"$PASS"'"}' + code="$(curl -fsS -m 10 -o /dev/null -w '%{http_code}' \ + -c "$JAR" -H 'Content-Type: application/json' \ + -d "$body" "$URL/api/login" 2>/dev/null)" + if [ "$code" != "200" ]; then + fail "authed API: login failed (http=$code)" + return + fi + shape="$(curl -fsS -m 10 -b "$JAR" "$URL/api/health" 2>/dev/null \ + | "$PY" -c ' +import json, sys +try: + d = json.loads(sys.stdin.read()) +except Exception: + raise SystemExit("unparseable") +print("dict" if isinstance(d, dict) else "other" )' 2>/dev/null)" + if [ "$shape" = "dict" ]; then + pass 'authed API: login + cookie-authenticated /api/health ok' + else + fail 'authed API: /api/health did not return a JSON object' + fi +} + +drill_bad_values() { + info 'bad-value drill: bands=2,5 then 25-entry SSID pool' + orig_bands="$(uci_get pineapd.wlan1mon.bands)" + [ -n "$orig_bands" ] || orig_bands=5 + orig_pool="$(uci_get 'pineapd.@ssidpool[0].ssid')" + + uci set pineapd.wlan1mon.bands='2,5' + uci commit pineapd + "$PY" "$REL/server.py" --reconcile >/dev/null 2>&1 + got="$(uci_get pineapd.wlan1mon.bands)" + if [ "$got" = "5" ]; then + pass 'drill: reconcile repaired bands 2,5 -> 5' + else + fail "drill: bands not repaired (got '$got')" + fi + uci set pineapd.wlan1mon.bands="$orig_bands" + + list="" + i=0 + while [ "$i" -lt 25 ]; do + list="$list smoke$i" + i=$((i + 1)) + done + uci set "pineapd.@ssidpool[0].ssid=${list# }" + uci commit pineapd + "$PY" "$REL/server.py" --reconcile >/dev/null 2>&1 + got="$(uci_get 'pineapd.@ssidpool[0].ssid')" + if [ -z "$got" ]; then + pass 'drill: reconcile cleared oversized SSID pool' + else + fail "drill: oversized pool survived ($(printf '%s' "$got" \ + | awk '{n += NF} END {print n + 0}') entries)" + fi + if [ -n "$orig_pool" ]; then + uci set "pineapd.@ssidpool[0].ssid=$orig_pool" + else + uci -q delete 'pineapd.@ssidpool[0].ssid' + fi + uci commit pineapd + "$PY" "$REL/server.py" --reconcile >/dev/null 2>&1 +} + +drill_watchdog() { + WDOG="$REL/mk8-watchdog.sh" + [ -f "$WDOG" ] || WDOG="$LEGACY/mk8-watchdog.sh" + if [ ! -f "$WDOG" ]; then + fail 'drill: mk8-watchdog.sh not found in release or legacy dir' + return + fi + prof=lastknown-good + [ -d "$BASE/profiles/$prof" ] \ + || prof="$(ls "$BASE/profiles" 2>/dev/null | head -n 1)" + if [ -z "$prof" ]; then + fail 'drill: no profiles under /mmc/mk8/profiles to exercise watchdog' + return + fi + + printf ' WARNING: watchdog drill stops/starts the webui service.\n' + n=5 + while [ "$n" -gt 0 ]; do + printf ' starting in %ds (ctrl-c to abort)\n' "$n" + sleep 1 + n=$((n - 1)) + done + + if ! curl -fsS -m 5 "$URL/" >/dev/null 2>&1; then + fail 'drill: web must be up before watchdog promote phase' + return + fi + + # Phase 1: healthy system -> watchdog promotes snapshot, exits 0. + "$WDOG" "$prof" 1 2 2 30 & + wp=$! + if wait_exit "$wp" 40; then + if [ "$WAIT_RC" -eq 0 ]; then + pass "drill: watchdog promote path exited 0 (profile=$prof)" + else + fail "drill: watchdog promote exit=$WAIT_RC" + fi + else + kill_bg "$wp" + fail 'drill: watchdog promote did not exit within 40s' + fi + + # Phase 2: web stopped -> watchdog rolls back and journals it. + before="$(rollback_count)" + WEB_STOPPED=1 + "$WEBUI_INIT" stop >/dev/null 2>&1 + "$WDOG" "$prof" 1 2 2 60 & + wp=$! + rolled=0 + t=0 + while [ "$t" -lt 150 ]; do + now="$(rollback_count)" + [ "$now" -gt "$before" ] && { rolled=1; break; } + kill -0 "$wp" 2>/dev/null || break + sleep 2 + t=$((t + 2)) + done + kill_bg "$wp" + WEB_STOPPED=0 + "$WEBUI_INIT" start >/dev/null 2>&1 + up=0 + t=0 + while [ "$t" -lt 45 ]; do + curl -fsS -m 3 "$URL/" >/dev/null 2>&1 && { up=1; break; } + sleep 2 + t=$((t + 2)) + done + if [ "$rolled" -eq 1 ]; then + pass 'drill: watchdog rollback journaled' + else + fail 'drill: no new rollback entry in events.log' + fi + if [ "$up" -eq 1 ]; then + pass 'drill: webui restored after rollback drill' + else + fail 'drill: webui did not come back within 45s' + fi +} + +main() { + for arg in "$@"; do + case "$arg" in + --write) WRITE=1 ;; + -h|--help) usage; exit 0 ;; + *) printf 'unknown argument: %s\n' "$arg" >&2; usage >&2; exit 2 ;; + esac + done + + if [ "$(id -u 2>/dev/null)" != "0" ]; then + printf 'FAIL smoke: must run as root on the device\n' + exit 1 + fi + + printf 'Mark VIII smoke suite (%s%s)\n' \ + "$(date '+%Y-%m-%d %H:%M:%S')" \ + "$([ "$WRITE" = "1" ] && printf ' --write')" + + check_web_up + check_guard + check_invariants + check_journal + check_monitors + check_versions + check_authed_api + + if [ "$WRITE" = "1" ]; then + if [ -z "$PY" ]; then + fail 'drills: python3 required but not found' + else + printf -- '--- --write drills ---\n' + drill_bad_values + drill_watchdog + fi + fi + + printf -- '---\n' + if [ "$FAILED" -eq 0 ]; then + printf 'SMOKE OK\n' + exit 0 + fi + printf 'SMOKE FAILED (%d check(s))\n' "$FAILED" + exit 1 +} + +trap on_exit EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +main "$@"