feat(reliability): risky-op preflight snapshots + detached rollback watchdog

payload/user/remote_access/pager-webui/mk8-watchdog.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+# Usage: mk8-watchdog.sh <profile> <interval> <fail_after> <healthy_after>
+PROFILE="$1"; IV="${2:-5}"; FA="${3:-6}"; HA="${4:-6}"
+DIR="/root/payloads/user/remote_access/pager-webui"
+[ -f "$DIR/server.py" ] || DIR="/mmc/mk8/releases/current"
+fails=0; oks=0; tripped=0
+probe() {
+    curl -fsS -m 3 http://127.0.0.1:8080/ >/dev/null 2>&1 &&
+    { ip link show wlan0mon >/dev/null 2>&1 ||
+      ip link show wlan1mon >/dev/null 2>&1; }
+}
+while true; do
+    if probe; then
+        fails=0
+        if [ "$tripped" = "1" ]; then
+            oks=$((oks + 1))
+            if [ "$oks" -ge "$HA" ]; then
+                /usr/bin/python3 "$DIR/server.py" --promote-snapshot "$PROFILE" >/dev/null 2>&1
+                exit 0
+            fi
+        fi
+    else
+        fails=$((fails + 1)); oks=0
+        if [ "$tripped" = "0" ] && [ "$fails" -ge "$FA" ]; then
+            tripped=1
+            /usr/bin/python3 "$DIR/server.py" --rollback-snapshot "$PROFILE" >/dev/null 2>&1
+        fi
+    fi
+    sleep "$IV"
+done

payload/user/remote_access/pager-webui/mk8_gate.py

@@ -0,0 +1,49 @@
+"""Risky-operation gate: preflight config snapshot + detached rollback watchdog."""
+import shlex
+import subprocess
+
+WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh'
+FAIL_AFTER = 6          # consecutive local-liveness failures -> rollback
+HEALTHY_AFTER = 6       # consecutive successes after failure -> promote
+INTERVAL = 5            # seconds between probes
+
+# Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing
+# this module never snapshots or spawns anything.
+ENABLED = False
+
+
+def watchdog_decision(state):
+    """state: {'fails': int, 'oks': int, 'tripped': bool,
+               'fail_after': 6, 'healthy_after': 6}
+    Returns (action, new_state): action in {'rollback','promote',None}."""
+    s = dict(state)
+    fa = s.get('fail_after', FAIL_AFTER)
+    ha = s.get('healthy_after', HEALTHY_AFTER)
+    if not s['tripped'] and s['fails'] >= fa:
+        return 'rollback', dict(s, tripped=True, oks=0)
+    if s['tripped'] and s['oks'] >= ha:
+        return 'promote', s
+    return None, s
+
+
+def _spawn_watchdog(name):
+    cmd = ('setsid sh %s %s %d %d %d >/dev/null 2>&1 &'
+           % (shlex.quote(WATCHDOG), shlex.quote(name),
+              INTERVAL, FAIL_AFTER, HEALTHY_AFTER))
+    return subprocess.Popen(cmd, shell=True, start_new_session=True)
+
+
+def enter(op):
+    """Snapshot + spawn watchdog. Returns profile name or None when disabled."""
+    if not ENABLED:
+        return None
+    import mk8_profiles
+    name = mk8_profiles.auto_name(op)
+    mk8_profiles.snapshot(name)
+    _spawn_watchdog(name)
+    try:
+        import mk8_events
+        mk8_events.log_event('gate', msg='preflight snapshot %s' % name)
+    except Exception:
+        pass
+    return name

payload/user/remote_access/pager-webui/server.py

@@ -3526,6 +3526,8 @@ def _apply_radio1_ap(openap, wpa):
 
 
 def h_pineap_wifi_set_ap(ctx):
+    import mk8_gate
+    mk8_gate.enter('ap_change')
     body = ctx.body or {}
     wpa = body.get('wpa') or {}
     openap = body.get('open') or {}
@@ -4350,6 +4352,8 @@ def _enterprise_boot_recover():
 
 
 def h_attacks_deploy(ctx):
+    import mk8_gate
+    mk8_gate.enter('attack_deploy')
     body = ctx.body or {}
     kind = (body.get('kind') or '').strip().lower()
     if kind not in ('wpa', 'open', 'enterprise'):
@@ -4439,6 +4443,8 @@ def _radio1_ap_active():
 
 
 def h_attacks_stop(ctx):
+    import mk8_gate
+    mk8_gate.enter('attack_stop')
     kind = ((ctx.body or {}).get('kind') or '')
     stopped = []
     if kind in ('wpa', 'open'):
@@ -4890,6 +4896,8 @@ def _disable_sta_uplink():
     keeps it off across reboots/wifi reloads; taking wlan0 down immediately
     frees phy0's channel for wlan0mon. Never runs `wifi reload` here — that
     tears down live APs and drops the monitors mid-assessment."""
+    import mk8_gate
+    mk8_gate.enter('uplink_disable')
     device_run(['uci', 'set', 'wireless.dummy_radio0.disabled=1'])
     device_run(['uci', 'commit', 'wireless'])
     for iface in ('wlan0',):
@@ -7189,6 +7197,8 @@ ROUTER.add('POST', r'/api/mode/release', h_mode_release)
 
 
 def serve():
+    import mk8_gate
+    mk8_gate.ENABLED = True
     LIVE_STOP.clear()
     HEALTH_STOP.clear()
     _recon_hopper_stop.set()
@@ -7255,6 +7265,26 @@ if __name__ == '__main__':
         except Exception as exc:  # boot must never fail here
             print(json.dumps({'error': str(exc)}))
         sys.exit(0)
+    if '--rollback-snapshot' in sys.argv:
+        import mk8_gate
+        mk8_gate.ENABLED = True
+        name = sys.argv[sys.argv.index('--rollback-snapshot') + 1]
+        import mk8_profiles
+        result = mk8_profiles.restore(name)
+        device_run(['wifi', 'reload'], timeout=90)
+        import mk8_events
+        mk8_events.log_event('rollback', sev='warn',
+                             msg='watchdog restored %s' % name,
+                             meta=result)
+        print(json.dumps(result))
+        sys.exit(0)
+    if '--promote-snapshot' in sys.argv:
+        import mk8_gate
+        mk8_gate.ENABLED = True
+        name = sys.argv[sys.argv.index('--promote-snapshot') + 1]
+        import mk8_profiles
+        print(json.dumps({'promoted': mk8_profiles.promote_lastknown_good()}))
+        sys.exit(0)
     signal.signal(signal.SIGTERM, _request_shutdown)
     signal.signal(signal.SIGINT, _request_shutdown)
     serve()