feat(reliability): risky-op preflight snapshots + detached rollback watchdog

payload/user/remote_access/pager-webui/mk8-watchdog.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+# Usage: mk8-watchdog.sh <profile> <interval> <fail_after> <healthy_after>
+PROFILE="$1"; IV="${2:-5}"; FA="${3:-6}"; HA="${4:-6}"
+DIR="/root/payloads/user/remote_access/pager-webui"
+[ -f "$DIR/server.py" ] || DIR="/mmc/mk8/releases/current"
+fails=0; oks=0; tripped=0
+probe() {
+    curl -fsS -m 3 http://127.0.0.1:8080/ >/dev/null 2>&1 &&
+    { ip link show wlan0mon >/dev/null 2>&1 ||
+      ip link show wlan1mon >/dev/null 2>&1; }
+}
+while true; do
+    if probe; then
+        fails=0
+        if [ "$tripped" = "1" ]; then
+            oks=$((oks + 1))
+            if [ "$oks" -ge "$HA" ]; then
+                /usr/bin/python3 "$DIR/server.py" --promote-snapshot "$PROFILE" >/dev/null 2>&1
+                exit 0
+            fi
+        fi
+    else
+        fails=$((fails + 1)); oks=0
+        if [ "$tripped" = "0" ] && [ "$fails" -ge "$FA" ]; then
+            tripped=1
+            /usr/bin/python3 "$DIR/server.py" --rollback-snapshot "$PROFILE" >/dev/null 2>&1
+        fi
+    fi
+    sleep "$IV"
+done

payload/user/remote_access/pager-webui/mk8_gate.py

@@ -0,0 +1,49 @@
+"""Risky-operation gate: preflight config snapshot + detached rollback watchdog."""
+import shlex
+import subprocess
+
+WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh'
+FAIL_AFTER = 6          # consecutive local-liveness failures -> rollback
+HEALTHY_AFTER = 6       # consecutive successes after failure -> promote
+INTERVAL = 5            # seconds between probes
+
+# Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing
+# this module never snapshots or spawns anything.
+ENABLED = False
+
+
+def watchdog_decision(state):
+    """state: {'fails': int, 'oks': int, 'tripped': bool,
+               'fail_after': 6, 'healthy_after': 6}
+    Returns (action, new_state): action in {'rollback','promote',None}."""
+    s = dict(state)
+    fa = s.get('fail_after', FAIL_AFTER)
+    ha = s.get('healthy_after', HEALTHY_AFTER)
+    if not s['tripped'] and s['fails'] >= fa:
+        return 'rollback', dict(s, tripped=True, oks=0)
+    if s['tripped'] and s['oks'] >= ha:
+        return 'promote', s
+    return None, s
+
+
+def _spawn_watchdog(name):
+    cmd = ('setsid sh %s %s %d %d %d >/dev/null 2>&1 &'
+           % (shlex.quote(WATCHDOG), shlex.quote(name),
+              INTERVAL, FAIL_AFTER, HEALTHY_AFTER))
+    return subprocess.Popen(cmd, shell=True, start_new_session=True)
+
+
+def enter(op):
+    """Snapshot + spawn watchdog. Returns profile name or None when disabled."""
+    if not ENABLED:
+        return None
+    import mk8_profiles
+    name = mk8_profiles.auto_name(op)
+    mk8_profiles.snapshot(name)
+    _spawn_watchdog(name)
+    try:
+        import mk8_events
+        mk8_events.log_event('gate', msg='preflight snapshot %s' % name)
+    except Exception:
+        pass
+    return name

payload/user/remote_access/pager-webui/server.py

@@ -3526,6 +3526,8 @@ def _apply_radio1_ap(openap, wpa):
 
 
 def h_pineap_wifi_set_ap(ctx):
+    import mk8_gate
+    mk8_gate.enter('ap_change')
     body = ctx.body or {}
     wpa = body.get('wpa') or {}
     openap = body.get('open') or {}
@@ -4350,6 +4352,8 @@ def _enterprise_boot_recover():
 
 
 def h_attacks_deploy(ctx):
+    import mk8_gate
+    mk8_gate.enter('attack_deploy')
     body = ctx.body or {}
     kind = (body.get('kind') or '').strip().lower()
     if kind not in ('wpa', 'open', 'enterprise'):
@@ -4439,6 +4443,8 @@ def _radio1_ap_active():
 
 
 def h_attacks_stop(ctx):
+    import mk8_gate
+    mk8_gate.enter('attack_stop')
     kind = ((ctx.body or {}).get('kind') or '')
     stopped = []
     if kind in ('wpa', 'open'):
@@ -4890,6 +4896,8 @@ def _disable_sta_uplink():
     keeps it off across reboots/wifi reloads; taking wlan0 down immediately
     frees phy0's channel for wlan0mon. Never runs `wifi reload` here — that
     tears down live APs and drops the monitors mid-assessment."""
+    import mk8_gate
+    mk8_gate.enter('uplink_disable')
     device_run(['uci', 'set', 'wireless.dummy_radio0.disabled=1'])
     device_run(['uci', 'commit', 'wireless'])
     for iface in ('wlan0',):
@@ -7189,6 +7197,8 @@ ROUTER.add('POST', r'/api/mode/release', h_mode_release)
 
 
 def serve():
+    import mk8_gate
+    mk8_gate.ENABLED = True
     LIVE_STOP.clear()
     HEALTH_STOP.clear()
     _recon_hopper_stop.set()
@@ -7255,6 +7265,26 @@ if __name__ == '__main__':
         except Exception as exc:  # boot must never fail here
             print(json.dumps({'error': str(exc)}))
         sys.exit(0)
+    if '--rollback-snapshot' in sys.argv:
+        import mk8_gate
+        mk8_gate.ENABLED = True
+        name = sys.argv[sys.argv.index('--rollback-snapshot') + 1]
+        import mk8_profiles
+        result = mk8_profiles.restore(name)
+        device_run(['wifi', 'reload'], timeout=90)
+        import mk8_events
+        mk8_events.log_event('rollback', sev='warn',
+                             msg='watchdog restored %s' % name,
+                             meta=result)
+        print(json.dumps(result))
+        sys.exit(0)
+    if '--promote-snapshot' in sys.argv:
+        import mk8_gate
+        mk8_gate.ENABLED = True
+        name = sys.argv[sys.argv.index('--promote-snapshot') + 1]
+        import mk8_profiles
+        print(json.dumps({'promoted': mk8_profiles.promote_lastknown_good()}))
+        sys.exit(0)
     signal.signal(signal.SIGTERM, _request_shutdown)
     signal.signal(signal.SIGINT, _request_shutdown)
     serve()

tests/test_mk8_gate.py

@@ -0,0 +1,73 @@
+import os, sys, types, unittest
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'payload', 'user', 'remote_access', 'pager-webui'))
+import mk8_gate
+
+
+class DecisionTest(unittest.TestCase):
+    def tick(self, state):
+        action, new = mk8_gate.watchdog_decision(state)
+        return action, new
+
+    def test_no_action_below_fail_threshold(self):
+        action, s = self.tick({'fails': 5, 'oks': 0, 'tripped': False})
+        self.assertIsNone(action)
+        self.assertFalse(s['tripped'])
+
+    def test_rollback_at_threshold(self):
+        action, s = self.tick({'fails': 6, 'oks': 0, 'tripped': False})
+        self.assertEqual(action, 'rollback')
+        self.assertTrue(s['tripped'])
+        self.assertEqual(s['oks'], 0)
+
+    def test_promote_after_recovery(self):
+        action, s = self.tick({'fails': 6, 'oks': 6, 'tripped': True})
+        self.assertEqual(action, 'promote')
+
+    def test_no_promote_before_recovery_threshold(self):
+        action, s = self.tick({'fails': 6, 'oks': 5, 'tripped': True})
+        self.assertIsNone(action)
+
+
+class EnterTest(unittest.TestCase):
+    def setUp(self):
+        self.addCleanup(setattr, mk8_gate, 'ENABLED', mk8_gate.ENABLED)
+        self.spawned = []
+
+    def _install_fake_profiles(self):
+        snaps = []
+        fake = types.ModuleType('mk8_profiles')
+        fake.auto_name = lambda op: 'pre-%s-42' % op
+        fake.snapshot = lambda name: (snaps.append(name), True)[1]
+        old = sys.modules.get('mk8_profiles')
+        sys.modules['mk8_profiles'] = fake
+        self.addCleanup(sys.modules.__setitem__, 'mk8_profiles', old)
+        return snaps
+
+    def _capture_popen(self):
+        cmds = []
+        old = mk8_gate.subprocess.Popen
+        mk8_gate.subprocess.Popen = lambda cmd, **kw: cmds.append(cmd)
+        self.addCleanup(setattr, mk8_gate.subprocess, 'Popen', old)
+        return cmds
+
+    def test_enter_disabled_is_noop(self):
+        mk8_gate.ENABLED = False
+        cmds = self._capture_popen()
+        snaps = self._install_fake_profiles()
+        self.assertIsNone(mk8_gate.enter('ap_change'))
+        self.assertEqual(snaps, [])
+        self.assertEqual(cmds, [])
+
+    def test_enter_enabled_snapshots_and_spawns(self):
+        mk8_gate.ENABLED = True
+        cmds = self._capture_popen()
+        snaps = self._install_fake_profiles()
+        name = mk8_gate.enter('attack_deploy')
+        self.assertEqual(name, 'pre-attack_deploy-42')
+        self.assertEqual(snaps, [name])
+        self.assertEqual(len(cmds), 1)
+        self.assertIn('setsid sh', cmds[0])
+        self.assertIn(name, cmds[0])
+
+if __name__ == '__main__':
+    unittest.main()