From 60db235f0b07608c5354d5dee5debbbd06f044b6 Mon Sep 17 00:00:00 2001 From: c4ch3c4d3 Date: Sat, 22 Aug 2026 16:52:29 -0600 Subject: [PATCH] fix(reliability): converge runtime on uplink revert; bound health endpoint cost (I3,I4) --- .../remote_access/pager-webui/mk8_events.py | 38 ++++++++++++++++ .../remote_access/pager-webui/mk8_rfplan.py | 5 +++ .../user/remote_access/pager-webui/server.py | 5 ++- tests/test_mk8_events.py | 45 +++++++++++++++++++ tests/test_mk8_rfplan.py | 13 ++++++ 5 files changed, 104 insertions(+), 2 deletions(-) diff --git a/payload/user/remote_access/pager-webui/mk8_events.py b/payload/user/remote_access/pager-webui/mk8_events.py index 9f939da..a758477 100644 --- a/payload/user/remote_access/pager-webui/mk8_events.py +++ b/payload/user/remote_access/pager-webui/mk8_events.py @@ -1,5 +1,6 @@ """Mark VIII reliability event journal. JSONL on /mmc, rotated.""" import json, os, threading, time +from collections import deque MK8_DIR = '/mmc/mk8' EVENTS_PATH = os.path.join(MK8_DIR, 'events.log') @@ -87,6 +88,43 @@ def counters(): return counts +def snapshot(event_limit=20, scan=2000): + """Newest-first events (up to event_limit) plus kind counters computed + over at most `scan` most-recent entries, in ONE parse pass. Bounded so a + large rotated journal cannot spike memory/CPU on every health poll.""" + events = [] + counts = {v: 0 for v in _COUNTER_KEYS.values()} + scanned = 0 + paths = [EVENTS_PATH] + paths.extend(EVENTS_PATH + '.%d' % i for i in range(KEEP, 0, -1)) + tail_len = max(scan, event_limit) + for path in paths: + if len(events) >= event_limit and scanned >= scan: + break + try: + with open(path) as f: + tail = deque((l for l in f if l.strip()), maxlen=tail_len) + except OSError: + continue + for line in reversed(tail): + try: + row = json.loads(line) + except ValueError: + continue + if not isinstance(row, dict): + continue + scanned += 1 + kind = row.get('kind') + if kind in _COUNTER_KEYS: + counts[_COUNTER_KEYS[kind]] += 1 + if len(events) < event_limit: + events.append(row) + if len(events) >= event_limit and scanned >= scan: + break + events.sort(key=lambda r: r.get('ts', 0), reverse=True) + return {'events': events[:event_limit], 'reliability': counts} + + def mark_boot(unexpected=False): log_event('unexpected_boot' if unexpected else 'boot', sev='warn' if unexpected else 'info', diff --git a/payload/user/remote_access/pager-webui/mk8_rfplan.py b/payload/user/remote_access/pager-webui/mk8_rfplan.py index 96d4018..1028436 100644 --- a/payload/user/remote_access/pager-webui/mk8_rfplan.py +++ b/payload/user/remote_access/pager-webui/mk8_rfplan.py @@ -94,6 +94,11 @@ def set_role(role, ssid=None, psk=None): break if not assoc: disable_uplink() + # UCI alone does not converge runtime: without a reload wlan1up + # keeps scanning/authenticating and pins phy1 until some unrelated + # future reload, while current_role() already reports idle. + # Converge now like the idle branch, then reapply the hop policy. + device_run(['wifi', 'reload'], timeout=60) _resume_hop() return {'ok': False, 'error': 'association failed; reverted'} return {'ok': True, 'role': 'uplink', 'assoc': assoc} diff --git a/payload/user/remote_access/pager-webui/server.py b/payload/user/remote_access/pager-webui/server.py index 9fbd110..f6173f1 100644 --- a/payload/user/remote_access/pager-webui/server.py +++ b/payload/user/remote_access/pager-webui/server.py @@ -4808,8 +4808,9 @@ def h_health(ctx): for k in ('pass', 'fixed', 'warn', 'fail')}, 'steps': ENV_CHECK_STATE['report'], } - h['reliability'] = mk8_events.counters() - h['events'] = mk8_events.read_events(limit=20) + snap = mk8_events.snapshot() + h['reliability'] = snap['reliability'] + h['events'] = snap['events'] h['guard'] = mk8_guard.guard_report() return 200, h diff --git a/tests/test_mk8_events.py b/tests/test_mk8_events.py index 1087b4a..bd9b058 100644 --- a/tests/test_mk8_events.py +++ b/tests/test_mk8_events.py @@ -31,6 +31,51 @@ class EventsTest(unittest.TestCase): self.assertEqual(c['restarts'], 1) self.assertEqual(c['guard_fixes'], 1) + def test_snapshot_bounded_and_counts(self): + # Deterministic window: 3 restarts (oldest, outside the scan window), + # then 40 ticks, then 1 boot LAST so the newest-first window of 30 + # includes it. + for _ in range(3): + mk8_events.log_event('restart', msg='old') + for i in range(40): + mk8_events.log_event('tick', msg='t%d' % i) + mk8_events.log_event('boot', msg='final boot') + snap = mk8_events.snapshot(event_limit=5, scan=30) + self.assertLessEqual(len(snap['events']), 5) + ev = snap['events'] + tss = [r.get('ts', 0) for r in ev] + self.assertEqual(tss, sorted(tss, reverse=True), 'must be newest-first') + self.assertEqual(ev[0]['kind'], 'boot') + rel = snap['reliability'] + self.assertEqual(rel['boots'], 1, + 'boot is inside the scanned window') + self.assertEqual(rel['restarts'], 0, + 'counters must cover only the bounded window') + self.assertEqual(rel['unexpected_boots'], 0) + # Full-journal counters() stays available and unbounded for callers + # that want it. + self.assertEqual(mk8_events.counters()['restarts'], 3) + + def test_snapshot_spans_rotation(self): + mk8_events.MAX_BYTES = 200 + for i in range(8): + mk8_events.log_event('tick', msg='x' * 20) + if i == 3: + mk8_events.log_event('rollback', sev='warn', msg='mid') + rotated = [i for i in range(1, mk8_events.KEEP + 1) + if os.path.exists(mk8_events.EVENTS_PATH + '.%d' % i)] + self.assertGreaterEqual(len(rotated) + 1, 2, + 'journal must actually rotate here') + snap = mk8_events.snapshot(event_limit=100, scan=1000) + # Snapshot must span every rotated generation read_events sees. + self.assertEqual(len(snap['events']), + len(mk8_events.read_events(limit=1000))) + tss = [r.get('ts', 0) for r in snap['events']] + self.assertEqual(tss, sorted(tss, reverse=True)) + kinds = {r['kind'] for r in snap['events']} + self.assertEqual(kinds, {'tick', 'rollback'}) + self.assertEqual(snap['reliability']['rollbacks'], 1) + def test_rotation_keeps_recent(self): mk8_events.MAX_BYTES = 200 for i in range(20): diff --git a/tests/test_mk8_rfplan.py b/tests/test_mk8_rfplan.py index 86f2c2d..6738790 100644 --- a/tests/test_mk8_rfplan.py +++ b/tests/test_mk8_rfplan.py @@ -157,6 +157,19 @@ class RfPlanTest(unittest.TestCase): self.assertEqual(len(self.sleeps), 5) self.assertIn('uci set wireless.wlan1up.disabled=1', self.cmds) self.assertEqual(self.resumed, 1) + # Revert must converge runtime like the idle branch: a wifi reload + # after the STA disable, with hop resumed only after the reload. + cmds = self.cmds + reloads = [i for i, c in enumerate(cmds) if c == 'wifi reload'] + self.assertEqual(len(reloads), 2, cmds) + disable_set = cmds.index('uci set wireless.wlan1up.disabled=1') + last_commit = len(cmds) - 1 - cmds[::-1].index('uci commit wireless') + resume_idx = self.seq.index('resume') + self.assertLess(disable_set, last_commit) + self.assertLess(last_commit, reloads[-1]) + self.assertLess(reloads[-1], resume_idx) + # cmds[i] maps 1:1 onto self.runs[i]; revert reload must be gated. + self.assertEqual(self.runs[reloads[-1]], (['wifi', 'reload'], 60)) def test_set_role_uplink_requires_ssid(self): result = mk8_rfplan.set_role('uplink')