Mark-VIII/scripts/smoke.sh

#!/bin/sh
# Mark VIII on-device reliability smoke suite (POSIX sh, BusyBox-safe).
#
# Usage: smoke.sh [--write]
#
# Read-only checks (default):
#   1. Web UI answers GET /
#   2. mk8-guard installed and enabled (S49 boot symlink)
#   3. Safe-UCI invariants + SSID pool size <= 20
#   4. Event journal: last line of /mmc/mk8/events.log parses as JSON
#      with a 'kind' field (via python3)
#   5. Monitor interfaces wlan0mon + wlan1mon exist
#   6. Deployed server.py SERVER_VERSION == payload.sh Version header
#   7. Authenticated API path: POST /api/login -> GET /api/health
#      (requires webui password in $PASS; skipped when unset)
#
# Destructive drills (--write only; values auto-restored):
#   8. Bad-value drill: feeds --reconcile a wrong bands value and an
#      oversized SSID pool (25 dummy entries), verifies both are
#      repaired, restores originals.
#   9. Rollback watchdog drill: runs mk8-watchdog.sh against a config
#      profile while the web UI is up (expects clean promote exit),
#      then STOPS the pagerwebui service and expects the watchdog to
#      roll back and journal a 'rollback' event, then restarts webui.
#      Preceded by a 5-second warning countdown; brief web outage.
#
# Environment:
#   PASS   webui password used for POST /api/login (check 7).
#
# Exit status: 0 when every executed check passes, 1 otherwise.

set -u

BASE=/mmc/mk8
REL="$BASE/releases/current"
LEGACY=/root/payloads/user/remote_access/pager-webui
URL=http://127.0.0.1:8080
GUARD_INIT=/etc/init.d/mk8-guard
GUARD_LINK=/etc/rc.d/S49mk8-guard
WEBUI_INIT=/etc/init.d/pagerwebui
JAR=/tmp/mk8-smoke-cookies.$$

WRITE=0
PASS="${PASS:-}"
WEB_STOPPED=0
FAILED=0
WAIT_RC=0

PY="$(command -v python3 2>/dev/null || true)"
[ -n "$PY" ] || PY=/usr/bin/python3
[ -x "$PY" ] || PY=""

usage() {
    printf 'Usage: smoke.sh [--write]\n'
    printf '\n'
    printf 'Read-only checks run by default. --write adds two destructive\n'
    printf 'drills that briefly toggle UCI config and stop/start the\n'
    printf 'pagerwebui service; original values are restored automatically.\n'
    printf 'Set PASS=<webui password> to enable the authenticated API check.\n'
}

on_exit() {
    rm -f "$JAR" 2>/dev/null
    if [ "$WEB_STOPPED" = "1" ]; then
        info 'restoring pagerwebui service'
        "$WEBUI_INIT" start >/dev/null 2>&1
    fi
}

pass() { printf 'PASS %s\n' "$1"; }
fail() { printf 'FAIL %s\n' "$1"; FAILED=$((FAILED + 1)); }
skip() { printf 'SKIP %s\n' "$1"; }
info() { printf '     %s\n' "$1"; }

uci_get() { uci -q get "$1" 2>/dev/null | tr -d '\r'; }

rollback_count() {
    if [ -z "$PY" ] || [ ! -f "$BASE/events.log" ]; then
        printf 0
        return
    fi
    tail -n 400 "$BASE/events.log" 2>/dev/null | "$PY" -c '
import json, sys
n = 0
for line in sys.stdin:
    try:
        d = json.loads(line)
    except Exception:
        continue
    if isinstance(d, dict) and d.get("kind") == "rollback":
        n += 1
print(n)' 2>/dev/null || printf 0
}

# wait_exit <pid> <seconds>: poll for background job exit; sets WAIT_RC
# and returns 0 once reaped, 1 on timeout (job left running).
wait_exit() {
    _pid="$1"; _t="$2"; _n=0
    while [ "$_n" -lt "$_t" ]; do
        if ! kill -0 "$_pid" 2>/dev/null; then
            wait "$_pid"
            WAIT_RC=$?
            return 0
        fi
        sleep 1
        _n=$((_n + 1))
    done
    return 1
}

kill_bg() {
    kill "$1" 2>/dev/null
    sleep 1
    kill -9 "$1" 2>/dev/null
    wait "$1" 2>/dev/null
}

check_web_up() {
    if curl -fsS -m 5 "$URL/" >/dev/null 2>&1; then
        pass 'web: GET / answered'
    else
        fail 'web: GET / failed'
    fi
}

check_guard() {
    if [ -x "$GUARD_INIT" ] && [ -e "$GUARD_LINK" ]; then
        pass 'guard: init script executable and enabled (S49)'
    else
        fail "guard: missing executable/init or S49 link ($GUARD_INIT $GUARD_LINK)"
    fi
}

check_invariants() {
    inv_fail=0
    while IFS= read -r kv; do
        [ -n "$kv" ] || continue
        key="${kv%%=*}"
        want="${kv#*=}"
        got="$(uci_get "$key")"
        if [ "$got" != "$want" ]; then
            inv_fail=$((inv_fail + 1))
            info "uci $key=$got (want $want)"
        fi
    done <<EOF
pineapd.@ssidpool[0].disable=1
pineapd.wlan2mon.disable=1
pineapd.wlan2mon.hop=0
pineapd.wlan1mon.bands=5
pineapd.wlan0mon.bands=2
pineapd.wlan1mon.hop=0
pineapd.@pineapd[0].autossidpool=0
EOF
    pool="$(uci_get 'pineapd.@ssidpool[0].ssid')"
    pool_n=$(printf '%s' "$pool" | awk '{n += NF} END {print n + 0}')
    if [ "$pool_n" -le 20 ]; then
        info "ssid pool size: $pool_n (max 20)"
    else
        inv_fail=$((inv_fail + 1))
        info "ssid pool size: $pool_n (max 20)"
    fi
    if [ "$inv_fail" -eq 0 ]; then
        pass 'invariants: safe UCI values + pool <= 20'
    else
        fail "invariants: $inv_fail violation(s)"
    fi
}

check_journal() {
    if [ -z "$PY" ]; then
        fail 'journal: python3 not found for JSON check'
        return
    fi
    last="$(tail -n 1 "$BASE/events.log" 2>/dev/null)"
    out="$(printf '%s' "$last" | "$PY" -c '
import json, sys
raw = sys.stdin.read().strip()
if not raw:
    raise SystemExit("events.log empty or missing")
try:
    d = json.loads(raw)
except Exception as exc:
    raise SystemExit("not JSON: %s" % exc)
kind = d.get("kind") if isinstance(d, dict) else None
if not kind:
    raise SystemExit("last entry has no kind field")
print(kind)' 2>&1)"
    rc=$?
    if [ "$rc" -eq 0 ]; then
        pass "journal: last entry ok (kind=$out)"
    else
        fail "journal: $out"
    fi
}

check_monitors() {
    miss=""
    for m in wlan0mon wlan1mon; do
        ip link show "$m" >/dev/null 2>&1 || miss="$miss $m"
    done
    if [ -z "$miss" ]; then
        pass 'monitors: wlan0mon + wlan1mon present'
    else
        fail "monitors: down:$miss"
    fi
}

check_versions() {
    sv="$(grep '^SERVER_VERSION' "$REL/server.py" 2>/dev/null | head -n 1 \
        | sed -e 's/^SERVER_VERSION = //' -e "s/'//g" | tr -d '\r')"
    pv="$(grep '^#[ ]*[Vv]ersion:' "$REL/payload.sh" 2>/dev/null | head -n 1 \
        | sed 's/^#[ ]*[Vv]ersion:[ ]*//' | tr -d '\r')"
    if [ -n "$sv" ] && [ "$sv" = "$pv" ]; then
        pass "versions: release server.py and payload.sh agree ($sv)"
    else
        fail "versions: server.py='$sv' payload.sh='$pv'"
    fi
}

check_authed_api() {
    if [ -z "$PASS" ]; then
        skip 'authed API: PASS not set'
        return
    fi
    if [ -z "$PY" ]; then
        fail 'authed API: python3 not found for response check'
        return
    fi
    body='{"password":"'"$PASS"'"}'
    code="$(curl -fsS -m 10 -o /dev/null -w '%{http_code}' \
        -c "$JAR" -H 'Content-Type: application/json' \
        -d "$body" "$URL/api/login" 2>/dev/null)"
    if [ "$code" != "200" ]; then
        fail "authed API: login failed (http=$code)"
        return
    fi
    shape="$(curl -fsS -m 10 -b "$JAR" "$URL/api/health" 2>/dev/null \
        | "$PY" -c '
import json, sys
try:
    d = json.loads(sys.stdin.read())
except Exception:
    raise SystemExit("unparseable")
print("dict" if isinstance(d, dict) else "other" )' 2>/dev/null)"
    if [ "$shape" = "dict" ]; then
        pass 'authed API: login + cookie-authenticated /api/health ok'
    else
        fail 'authed API: /api/health did not return a JSON object'
    fi
}

drill_bad_values() {
    info 'bad-value drill: bands=2,5 then 25-entry SSID pool'
    orig_bands="$(uci_get pineapd.wlan1mon.bands)"
    [ -n "$orig_bands" ] || orig_bands=5
    orig_pool="$(uci_get 'pineapd.@ssidpool[0].ssid')"

    uci set pineapd.wlan1mon.bands='2,5'
    uci commit pineapd
    "$PY" "$REL/server.py" --reconcile >/dev/null 2>&1
    got="$(uci_get pineapd.wlan1mon.bands)"
    if [ "$got" = "5" ]; then
        pass 'drill: reconcile repaired bands 2,5 -> 5'
    else
        fail "drill: bands not repaired (got '$got')"
    fi
    uci set pineapd.wlan1mon.bands="$orig_bands"

    list=""
    i=0
    while [ "$i" -lt 25 ]; do
        list="$list smoke$i"
        i=$((i + 1))
    done
    uci set "pineapd.@ssidpool[0].ssid=${list# }"
    uci commit pineapd
    "$PY" "$REL/server.py" --reconcile >/dev/null 2>&1
    got="$(uci_get 'pineapd.@ssidpool[0].ssid')"
    if [ -z "$got" ]; then
        pass 'drill: reconcile cleared oversized SSID pool'
    else
        fail "drill: oversized pool survived ($(printf '%s' "$got" \
            | awk '{n += NF} END {print n + 0}') entries)"
    fi
    if [ -n "$orig_pool" ]; then
        uci set "pineapd.@ssidpool[0].ssid=$orig_pool"
    else
        uci -q delete 'pineapd.@ssidpool[0].ssid'
    fi
    uci commit pineapd
    "$PY" "$REL/server.py" --reconcile >/dev/null 2>&1
}

drill_watchdog() {
    WDOG="$REL/mk8-watchdog.sh"
    [ -f "$WDOG" ] || WDOG="$LEGACY/mk8-watchdog.sh"
    if [ ! -f "$WDOG" ]; then
        fail 'drill: mk8-watchdog.sh not found in release or legacy dir'
        return
    fi
    prof=lastknown-good
    [ -d "$BASE/profiles/$prof" ] \
        || prof="$(ls "$BASE/profiles" 2>/dev/null | head -n 1)"
    if [ -z "$prof" ]; then
        fail 'drill: no profiles under /mmc/mk8/profiles to exercise watchdog'
        return
    fi

    printf '     WARNING: watchdog drill stops/starts the webui service.\n'
    n=5
    while [ "$n" -gt 0 ]; do
        printf '     starting in %ds (ctrl-c to abort)\n' "$n"
        sleep 1
        n=$((n - 1))
    done

    if ! curl -fsS -m 5 "$URL/" >/dev/null 2>&1; then
        fail 'drill: web must be up before watchdog promote phase'
        return
    fi

    # Phase 1: healthy system -> watchdog promotes snapshot, exits 0.
    "$WDOG" "$prof" 1 2 2 30 &
    wp=$!
    if wait_exit "$wp" 40; then
        if [ "$WAIT_RC" -eq 0 ]; then
            pass "drill: watchdog promote path exited 0 (profile=$prof)"
        else
            fail "drill: watchdog promote exit=$WAIT_RC"
        fi
    else
        kill_bg "$wp"
        fail 'drill: watchdog promote did not exit within 40s'
    fi

    # Phase 2: web stopped -> watchdog rolls back and journals it.
    before="$(rollback_count)"
    WEB_STOPPED=1
    "$WEBUI_INIT" stop >/dev/null 2>&1
    "$WDOG" "$prof" 1 2 2 60 &
    wp=$!
    rolled=0
    t=0
    while [ "$t" -lt 150 ]; do
        now="$(rollback_count)"
        [ "$now" -gt "$before" ] && { rolled=1; break; }
        kill -0 "$wp" 2>/dev/null || break
        sleep 2
        t=$((t + 2))
    done
    kill_bg "$wp"
    WEB_STOPPED=0
    "$WEBUI_INIT" start >/dev/null 2>&1
    up=0
    t=0
    while [ "$t" -lt 45 ]; do
        curl -fsS -m 3 "$URL/" >/dev/null 2>&1 && { up=1; break; }
        sleep 2
        t=$((t + 2))
    done
    if [ "$rolled" -eq 1 ]; then
        pass 'drill: watchdog rollback journaled'
    else
        fail 'drill: no new rollback entry in events.log'
    fi
    if [ "$up" -eq 1 ]; then
        pass 'drill: webui restored after rollback drill'
    else
        fail 'drill: webui did not come back within 45s'
    fi
}

main() {
    for arg in "$@"; do
        case "$arg" in
            --write) WRITE=1 ;;
            -h|--help) usage; exit 0 ;;
            *) printf 'unknown argument: %s\n' "$arg" >&2; usage >&2; exit 2 ;;
        esac
    done

    if [ "$(id -u 2>/dev/null)" != "0" ]; then
        printf 'FAIL smoke: must run as root on the device\n'
        exit 1
    fi

    printf 'Mark VIII smoke suite (%s%s)\n' \
        "$(date '+%Y-%m-%d %H:%M:%S')" \
        "$([ "$WRITE" = "1" ] && printf ' --write')"

    check_web_up
    check_guard
    check_invariants
    check_journal
    check_monitors
    check_versions
    check_authed_api

    if [ "$WRITE" = "1" ]; then
        if [ -z "$PY" ]; then
            fail 'drills: python3 required but not found'
        else
            printf -- '--- --write drills ---\n'
            drill_bad_values
            drill_watchdog
        fi
    fi

    printf -- '---\n'
    if [ "$FAILED" -eq 0 ]; then
        printf 'SMOKE OK\n'
        exit 0
    fi
    printf 'SMOKE FAILED (%d check(s))\n' "$FAILED"
    exit 1
}

trap on_exit EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
main "$@"