501 lines
14 KiB
Bash
Executable File
501 lines
14 KiB
Bash
Executable File
#!/bin/sh
|
|
# Mark VIII on-device reliability smoke suite (POSIX sh, BusyBox-safe).
|
|
#
|
|
# Usage: smoke.sh [--write]
|
|
#
|
|
# Read-only checks (default):
|
|
# 1. Web UI answers GET /
|
|
# 2. mk8-guard installed and enabled (S49 boot symlink)
|
|
# 3. Safe-UCI invariants + SSID pool size <= 20
|
|
# 4. Event journal: last line of /mmc/mk8/events.log parses as JSON
|
|
# with a 'kind' field (via python3)
|
|
# 5. Monitor interfaces wlan0mon + wlan1mon exist
|
|
# 6. Deployed server.py SERVER_VERSION == payload.sh Version header
|
|
# 7. Authenticated API path: POST /api/login -> GET /api/health
|
|
# (requires webui password in $PASS; skipped when unset)
|
|
#
|
|
# Destructive drills (--write only; values auto-restored):
|
|
# 8. Bad-value drill: feeds --reconcile a wrong bands value and an
|
|
# oversized SSID pool (25 dummy entries), verifies both are
|
|
# repaired, restores originals.
|
|
# 9. RF role drill (only when SMOKE_UPLINK_SSID is set): switches
|
|
# radio1 to the uplink role against the named lab AP, expects an
|
|
# association, then back to attack with hopping resumed.
|
|
# 10. Rollback watchdog drill: runs mk8-watchdog.sh against a config
|
|
# profile while the web UI is up (expects clean promote exit),
|
|
# then STOPS the pagerwebui service and expects the watchdog to
|
|
# roll back and journal a 'rollback' event, then restarts webui.
|
|
# Preceded by a 5-second warning countdown; brief web outage.
|
|
# Drills run only when every read-only check has passed.
|
|
#
|
|
# Environment:
|
|
# PASS webui password used for POST /api/login (check 7).
|
|
# SMOKE_UPLINK_SSID lab AP SSID; enables the --write RF role drill.
|
|
# SMOKE_UPLINK_PSK optional PSK for the lab AP.
|
|
#
|
|
# Exit status: 0 when every executed check passes, 1 otherwise.
|
|
|
|
set -u
|
|
|
|
BASE=/mmc/mk8
|
|
REL="$BASE/releases/current"
|
|
LEGACY=/root/payloads/user/remote_access/pager-webui
|
|
URL=http://127.0.0.1:8080
|
|
GUARD_INIT=/etc/init.d/mk8-guard
|
|
GUARD_LINK=/etc/rc.d/S49mk8-guard
|
|
WEBUI_INIT=/etc/init.d/pagerwebui
|
|
JAR=/tmp/mk8-smoke-cookies.$$
|
|
|
|
WRITE=0
|
|
PASS="${PASS:-}"
|
|
UPLINK_SSID="${SMOKE_UPLINK_SSID:-}"
|
|
UPLINK_PSK="${SMOKE_UPLINK_PSK:-}"
|
|
WEB_STOPPED=0
|
|
FAILED=0
|
|
WAIT_RC=0
|
|
wp=""
|
|
|
|
PY="$(command -v python3 2>/dev/null || true)"
|
|
[ -n "$PY" ] || PY=/usr/bin/python3
|
|
[ -x "$PY" ] || PY=""
|
|
|
|
usage() {
|
|
printf 'Usage: smoke.sh [--write]\n'
|
|
printf '\n'
|
|
printf 'Read-only checks run by default. --write adds destructive drills\n'
|
|
printf '(run only if every read-only check passed) that briefly toggle\n'
|
|
printf 'UCI config and stop/start the pagerwebui service; original values\n'
|
|
printf 'are restored automatically.\n'
|
|
printf 'Set PASS=<webui password> to enable the authenticated API check.\n'
|
|
printf 'Set SMOKE_UPLINK_SSID=[<PSK via SMOKE_UPLINK_PSK>] to enable the\n'
|
|
printf '--write RF role drill against a lab AP.\n'
|
|
}
|
|
|
|
on_exit() {
|
|
rm -f "$JAR" 2>/dev/null
|
|
if [ -n "$wp" ]; then
|
|
kill "$wp" 2>/dev/null
|
|
fi
|
|
if [ "$WEB_STOPPED" = "1" ]; then
|
|
info 'restoring pagerwebui service'
|
|
"$WEBUI_INIT" start >/dev/null 2>&1
|
|
fi
|
|
}
|
|
|
|
pass() { printf 'PASS %s\n' "$1"; }
|
|
fail() { printf 'FAIL %s\n' "$1"; FAILED=$((FAILED + 1)); }
|
|
skip() { printf 'SKIP %s\n' "$1"; }
|
|
info() { printf ' %s\n' "$1"; }
|
|
|
|
uci_get() { uci -q get "$1" 2>/dev/null | tr -d '\r'; }
|
|
|
|
rollback_count() {
|
|
if [ -z "$PY" ] || [ ! -f "$BASE/events.log" ]; then
|
|
printf 0
|
|
return
|
|
fi
|
|
tail -n 400 "$BASE/events.log" 2>/dev/null | "$PY" -c '
|
|
import json, sys
|
|
n = 0
|
|
for line in sys.stdin:
|
|
try:
|
|
d = json.loads(line)
|
|
except Exception:
|
|
continue
|
|
if isinstance(d, dict) and d.get("kind") == "rollback":
|
|
n += 1
|
|
print(n)' 2>/dev/null || printf 0
|
|
}
|
|
|
|
# wait_exit <pid> <seconds>: poll for background job exit; sets WAIT_RC
|
|
# and returns 0 once reaped, 1 on timeout (job left running).
|
|
wait_exit() {
|
|
_pid="$1"; _t="$2"; _n=0
|
|
while [ "$_n" -lt "$_t" ]; do
|
|
if ! kill -0 "$_pid" 2>/dev/null; then
|
|
wait "$_pid"
|
|
WAIT_RC=$?
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
_n=$((_n + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
kill_bg() {
|
|
kill "$1" 2>/dev/null
|
|
sleep 1
|
|
kill -9 "$1" 2>/dev/null
|
|
wait "$1" 2>/dev/null
|
|
}
|
|
|
|
check_web_up() {
|
|
if curl -fsS -m 5 "$URL/" >/dev/null 2>&1; then
|
|
pass 'web: GET / answered'
|
|
else
|
|
fail 'web: GET / failed'
|
|
fi
|
|
}
|
|
|
|
check_guard() {
|
|
if [ -x "$GUARD_INIT" ] && [ -e "$GUARD_LINK" ]; then
|
|
pass 'guard: init script executable and enabled (S49)'
|
|
else
|
|
fail "guard: missing executable/init or S49 link ($GUARD_INIT $GUARD_LINK)"
|
|
fi
|
|
}
|
|
|
|
check_invariants() {
|
|
inv_fail=0
|
|
while IFS= read -r kv; do
|
|
[ -n "$kv" ] || continue
|
|
key="${kv%%=*}"
|
|
want="${kv#*=}"
|
|
got="$(uci_get "$key")"
|
|
if [ "$got" != "$want" ]; then
|
|
inv_fail=$((inv_fail + 1))
|
|
info "uci $key=$got (want $want)"
|
|
fi
|
|
done <<EOF
|
|
pineapd.@ssidpool[0].disable=1
|
|
pineapd.wlan2mon.disable=1
|
|
pineapd.wlan2mon.hop=0
|
|
pineapd.wlan1mon.bands=5
|
|
pineapd.wlan0mon.bands=2
|
|
pineapd.wlan1mon.hop=0
|
|
pineapd.@pineapd[0].autossidpool=0
|
|
EOF
|
|
pool="$(uci_get 'pineapd.@ssidpool[0].ssid')"
|
|
pool_n=$(printf '%s' "$pool" | awk '{n += NF} END {print n + 0}')
|
|
if [ "$pool_n" -le 20 ]; then
|
|
info "ssid pool size: $pool_n (max 20)"
|
|
else
|
|
inv_fail=$((inv_fail + 1))
|
|
info "ssid pool size: $pool_n (max 20)"
|
|
fi
|
|
if [ "$inv_fail" -eq 0 ]; then
|
|
pass 'invariants: safe UCI values + pool <= 20'
|
|
else
|
|
fail "invariants: $inv_fail violation(s)"
|
|
fi
|
|
}
|
|
|
|
check_journal() {
|
|
if [ -z "$PY" ]; then
|
|
fail 'journal: python3 not found for JSON check'
|
|
return
|
|
fi
|
|
last="$(tail -n 1 "$BASE/events.log" 2>/dev/null)"
|
|
out="$(printf '%s' "$last" | "$PY" -c '
|
|
import json, sys
|
|
raw = sys.stdin.read().strip()
|
|
if not raw:
|
|
raise SystemExit("events.log empty or missing")
|
|
try:
|
|
d = json.loads(raw)
|
|
except Exception as exc:
|
|
raise SystemExit("not JSON: %s" % exc)
|
|
kind = d.get("kind") if isinstance(d, dict) else None
|
|
if not kind:
|
|
raise SystemExit("last entry has no kind field")
|
|
print(kind)' 2>&1)"
|
|
rc=$?
|
|
if [ "$rc" -eq 0 ]; then
|
|
pass "journal: last entry ok (kind=$out)"
|
|
else
|
|
fail "journal: $out"
|
|
fi
|
|
}
|
|
|
|
check_monitors() {
|
|
miss=""
|
|
for m in wlan0mon wlan1mon; do
|
|
ip link show "$m" >/dev/null 2>&1 || miss="$miss $m"
|
|
done
|
|
if [ -z "$miss" ]; then
|
|
pass 'monitors: wlan0mon + wlan1mon present'
|
|
else
|
|
fail "monitors: down:$miss"
|
|
fi
|
|
}
|
|
|
|
check_versions() {
|
|
sv="$(grep '^SERVER_VERSION' "$REL/server.py" 2>/dev/null | head -n 1 \
|
|
| sed -e 's/^SERVER_VERSION = //' -e "s/'//g" | tr -d '\r')"
|
|
pv="$(grep '^#[ ]*[Vv]ersion:' "$REL/payload.sh" 2>/dev/null | head -n 1 \
|
|
| sed 's/^#[ ]*[Vv]ersion:[ ]*//' | tr -d '\r')"
|
|
if [ -n "$sv" ] && [ "$sv" = "$pv" ]; then
|
|
pass "versions: release server.py and payload.sh agree ($sv)"
|
|
else
|
|
fail "versions: server.py='$sv' payload.sh='$pv'"
|
|
fi
|
|
}
|
|
|
|
check_authed_api() {
|
|
if [ -z "$PASS" ]; then
|
|
skip 'authed API: PASS not set'
|
|
return
|
|
fi
|
|
if [ -z "$PY" ]; then
|
|
fail 'authed API: python3 not found for response check'
|
|
return
|
|
fi
|
|
body='{"password":"'"$PASS"'"}'
|
|
code="$(curl -fsS -m 10 -o /dev/null -w '%{http_code}' \
|
|
-c "$JAR" -H 'Content-Type: application/json' \
|
|
-d "$body" "$URL/api/login" 2>/dev/null)"
|
|
if [ "$code" != "200" ]; then
|
|
fail "authed API: login failed (http=$code)"
|
|
return
|
|
fi
|
|
shape="$(curl -fsS -m 10 -b "$JAR" "$URL/api/health" 2>/dev/null \
|
|
| "$PY" -c '
|
|
import json, sys
|
|
try:
|
|
d = json.loads(sys.stdin.read())
|
|
except Exception:
|
|
raise SystemExit("unparseable")
|
|
print("dict" if isinstance(d, dict) else "other" )' 2>/dev/null)"
|
|
if [ "$shape" = "dict" ]; then
|
|
pass 'authed API: login + cookie-authenticated /api/health ok'
|
|
else
|
|
fail 'authed API: /api/health did not return a JSON object'
|
|
fi
|
|
}
|
|
|
|
drill_bad_values() {
|
|
info 'bad-value drill: bands=2,5 then 25-entry SSID pool'
|
|
orig_bands="$(uci_get pineapd.wlan1mon.bands)"
|
|
[ -n "$orig_bands" ] || orig_bands=5
|
|
orig_pool="$(uci_get 'pineapd.@ssidpool[0].ssid')"
|
|
|
|
uci set pineapd.wlan1mon.bands='2,5'
|
|
uci commit pineapd
|
|
"$PY" "$REL/server.py" --reconcile >/dev/null 2>&1
|
|
got="$(uci_get pineapd.wlan1mon.bands)"
|
|
if [ "$got" = "5" ]; then
|
|
pass 'drill: reconcile repaired bands 2,5 -> 5'
|
|
else
|
|
fail "drill: bands not repaired (got '$got')"
|
|
fi
|
|
uci set pineapd.wlan1mon.bands="$orig_bands"
|
|
|
|
list=""
|
|
i=0
|
|
while [ "$i" -lt 25 ]; do
|
|
list="$list smoke$i"
|
|
i=$((i + 1))
|
|
done
|
|
uci set "pineapd.@ssidpool[0].ssid=${list# }"
|
|
uci commit pineapd
|
|
"$PY" "$REL/server.py" --reconcile >/dev/null 2>&1
|
|
got="$(uci_get 'pineapd.@ssidpool[0].ssid')"
|
|
if [ -z "$got" ]; then
|
|
pass 'drill: reconcile cleared oversized SSID pool'
|
|
else
|
|
fail "drill: oversized pool survived ($(printf '%s' "$got" \
|
|
| awk '{n += NF} END {print n + 0}') entries)"
|
|
fi
|
|
if [ -n "$orig_pool" ]; then
|
|
uci set "pineapd.@ssidpool[0].ssid=$orig_pool"
|
|
else
|
|
uci -q delete 'pineapd.@ssidpool[0].ssid'
|
|
fi
|
|
uci commit pineapd
|
|
"$PY" "$REL/server.py" --reconcile >/dev/null 2>&1
|
|
}
|
|
|
|
drill_role() {
|
|
if [ -z "$UPLINK_SSID" ]; then
|
|
skip 'role drill: set SMOKE_UPLINK_SSID to enable'
|
|
return
|
|
fi
|
|
info "role drill: uplink '$UPLINK_SSID' then attack"
|
|
out="$("$PY" - "$REL" "$LEGACY" "$UPLINK_SSID" "$UPLINK_PSK" <<'PYEOF' 2>&1
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
rel, legacy, ssid, psk = sys.argv[1:5]
|
|
mk8_rfplan = None
|
|
for d in (rel, legacy):
|
|
if os.path.isfile(os.path.join(d, 'mk8_rfplan.py')):
|
|
sys.path.insert(0, d)
|
|
try:
|
|
import mk8_rfplan
|
|
break
|
|
except Exception:
|
|
sys.path.remove(d)
|
|
if mk8_rfplan is None:
|
|
print('FAIL mk8_rfplan not importable from release or legacy dir')
|
|
raise SystemExit(1)
|
|
|
|
r1 = mk8_rfplan.set_role(
|
|
'uplink', ssid=ssid or None, psk=psk or None)
|
|
if not isinstance(r1, dict) or not r1.get('ok'):
|
|
print('FAIL uplink set_role failed: %s' % json.dumps(r1))
|
|
raise SystemExit(1)
|
|
|
|
assoc = mk8_rfplan.associated()
|
|
if not assoc:
|
|
print('FAIL uplink associated() returned nothing after set_role')
|
|
else:
|
|
print('associated as %s' % assoc)
|
|
|
|
try:
|
|
r2 = mk8_rfplan.set_role('attack')
|
|
except Exception as exc:
|
|
r2 = {'ok': False, 'error': str(exc)}
|
|
if not isinstance(r2, dict) or not r2.get('ok'):
|
|
print('FAIL attack set_role failed: %s' % json.dumps(r2))
|
|
raise SystemExit(1)
|
|
raise SystemExit(0 if assoc else 1)
|
|
PYEOF
|
|
)"
|
|
rc=$?
|
|
printf '%s\n' "$out" | sed 's/^/ /'
|
|
if [ "$rc" -ne 0 ]; then
|
|
fail 'role drill: uplink association failed (see detail above)'
|
|
return
|
|
fi
|
|
hop="$(uci_get pineapd.wlan1mon.hop)"
|
|
if [ "$hop" = "1" ]; then
|
|
pass 'role drill: uplink assoc + attack role + hopping resumed'
|
|
else
|
|
fail "role drill: pineapd.wlan1mon.hop=$hop after attack role (want 1)"
|
|
fi
|
|
}
|
|
|
|
drill_watchdog() {
|
|
WDOG="$REL/mk8-watchdog.sh"
|
|
[ -f "$WDOG" ] || WDOG="$LEGACY/mk8-watchdog.sh"
|
|
if [ ! -f "$WDOG" ]; then
|
|
fail 'drill: mk8-watchdog.sh not found in release or legacy dir'
|
|
return
|
|
fi
|
|
prof=lastknown-good
|
|
[ -d "$BASE/profiles/$prof" ] \
|
|
|| prof="$(ls "$BASE/profiles" 2>/dev/null | head -n 1)"
|
|
if [ -z "$prof" ]; then
|
|
fail 'drill: no profiles under /mmc/mk8/profiles to exercise watchdog'
|
|
return
|
|
fi
|
|
|
|
printf ' WARNING: watchdog drill stops/starts the webui service.\n'
|
|
n=5
|
|
while [ "$n" -gt 0 ]; do
|
|
printf ' starting in %ds (ctrl-c to abort)\n' "$n"
|
|
sleep 1
|
|
n=$((n - 1))
|
|
done
|
|
|
|
if ! curl -fsS -m 5 "$URL/" >/dev/null 2>&1; then
|
|
fail 'drill: web must be up before watchdog promote phase'
|
|
return
|
|
fi
|
|
|
|
# Phase 1: healthy system -> watchdog promotes snapshot, exits 0.
|
|
"$WDOG" "$prof" 1 2 2 30 &
|
|
wp=$!
|
|
if wait_exit "$wp" 40; then
|
|
if [ "$WAIT_RC" -eq 0 ]; then
|
|
pass "drill: watchdog promote path exited 0 (profile=$prof)"
|
|
else
|
|
fail "drill: watchdog promote exit=$WAIT_RC"
|
|
fi
|
|
else
|
|
kill_bg "$wp"
|
|
fail 'drill: watchdog promote did not exit within 40s'
|
|
fi
|
|
|
|
# Phase 2: web stopped -> watchdog rolls back and journals it.
|
|
before="$(rollback_count)"
|
|
WEB_STOPPED=1
|
|
"$WEBUI_INIT" stop >/dev/null 2>&1
|
|
"$WDOG" "$prof" 1 2 2 60 &
|
|
wp=$!
|
|
rolled=0
|
|
t=0
|
|
while [ "$t" -lt 150 ]; do
|
|
now="$(rollback_count)"
|
|
[ "$now" -gt "$before" ] && { rolled=1; break; }
|
|
kill -0 "$wp" 2>/dev/null || break
|
|
sleep 2
|
|
t=$((t + 2))
|
|
done
|
|
kill_bg "$wp"
|
|
WEB_STOPPED=0
|
|
"$WEBUI_INIT" start >/dev/null 2>&1
|
|
up=0
|
|
t=0
|
|
while [ "$t" -lt 45 ]; do
|
|
curl -fsS -m 3 "$URL/" >/dev/null 2>&1 && { up=1; break; }
|
|
sleep 2
|
|
t=$((t + 2))
|
|
done
|
|
if [ "$rolled" -eq 1 ]; then
|
|
pass 'drill: watchdog rollback journaled'
|
|
else
|
|
fail 'drill: no new rollback entry in events.log'
|
|
fi
|
|
if [ "$up" -eq 1 ]; then
|
|
pass 'drill: webui restored after rollback drill'
|
|
else
|
|
fail 'drill: webui did not come back within 45s'
|
|
fi
|
|
}
|
|
|
|
main() {
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--write) WRITE=1 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) printf 'unknown argument: %s\n' "$arg" >&2; usage >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
if [ "$(id -u 2>/dev/null)" != "0" ]; then
|
|
printf 'FAIL smoke: must run as root on the device\n'
|
|
exit 1
|
|
fi
|
|
|
|
printf 'Mark VIII smoke suite (%s%s)\n' \
|
|
"$(date '+%Y-%m-%d %H:%M:%S')" \
|
|
"$([ "$WRITE" = "1" ] && printf ' --write')"
|
|
|
|
check_web_up
|
|
check_guard
|
|
check_invariants
|
|
check_journal
|
|
check_monitors
|
|
check_versions
|
|
check_authed_api
|
|
|
|
if [ "$WRITE" = "1" ]; then
|
|
if [ -z "$PY" ]; then
|
|
fail 'drills: python3 required but not found'
|
|
elif [ "$FAILED" -ne 0 ]; then
|
|
skip 'drills: read-only checks failed; refusing drills'
|
|
else
|
|
printf -- '--- --write drills ---\n'
|
|
drill_bad_values
|
|
drill_role
|
|
drill_watchdog
|
|
fi
|
|
fi
|
|
|
|
printf -- '---\n'
|
|
if [ "$FAILED" -eq 0 ]; then
|
|
printf 'SMOKE OK\n'
|
|
exit 0
|
|
fi
|
|
printf 'SMOKE FAILED (%d check(s))\n' "$FAILED"
|
|
exit 1
|
|
}
|
|
|
|
trap on_exit EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
main "$@"
|