fix(reliability): converge runtime on uplink revert; bound health endpoint cost (I3,I4)

This commit is contained in:
2026-08-22 16:52:29 -06:00
parent b25c98b7c7
commit 60db235f0b
5 changed files with 104 additions and 2 deletions
@@ -1,5 +1,6 @@
"""Mark VIII reliability event journal. JSONL on /mmc, rotated."""
import json, os, threading, time
from collections import deque
MK8_DIR = '/mmc/mk8'
EVENTS_PATH = os.path.join(MK8_DIR, 'events.log')
@@ -87,6 +88,43 @@ def counters():
return counts
def snapshot(event_limit=20, scan=2000):
"""Newest-first events (up to event_limit) plus kind counters computed
over at most `scan` most-recent entries, in ONE parse pass. Bounded so a
large rotated journal cannot spike memory/CPU on every health poll."""
events = []
counts = {v: 0 for v in _COUNTER_KEYS.values()}
scanned = 0
paths = [EVENTS_PATH]
paths.extend(EVENTS_PATH + '.%d' % i for i in range(KEEP, 0, -1))
tail_len = max(scan, event_limit)
for path in paths:
if len(events) >= event_limit and scanned >= scan:
break
try:
with open(path) as f:
tail = deque((l for l in f if l.strip()), maxlen=tail_len)
except OSError:
continue
for line in reversed(tail):
try:
row = json.loads(line)
except ValueError:
continue
if not isinstance(row, dict):
continue
scanned += 1
kind = row.get('kind')
if kind in _COUNTER_KEYS:
counts[_COUNTER_KEYS[kind]] += 1
if len(events) < event_limit:
events.append(row)
if len(events) >= event_limit and scanned >= scan:
break
events.sort(key=lambda r: r.get('ts', 0), reverse=True)
return {'events': events[:event_limit], 'reliability': counts}
def mark_boot(unexpected=False):
log_event('unexpected_boot' if unexpected else 'boot', sev='warn'
if unexpected else 'info',
@@ -94,6 +94,11 @@ def set_role(role, ssid=None, psk=None):
break
if not assoc:
disable_uplink()
# UCI alone does not converge runtime: without a reload wlan1up
# keeps scanning/authenticating and pins phy1 until some unrelated
# future reload, while current_role() already reports idle.
# Converge now like the idle branch, then reapply the hop policy.
device_run(['wifi', 'reload'], timeout=60)
_resume_hop()
return {'ok': False, 'error': 'association failed; reverted'}
return {'ok': True, 'role': 'uplink', 'assoc': assoc}
@@ -4808,8 +4808,9 @@ def h_health(ctx):
for k in ('pass', 'fixed', 'warn', 'fail')},
'steps': ENV_CHECK_STATE['report'],
}
h['reliability'] = mk8_events.counters()
h['events'] = mk8_events.read_events(limit=20)
snap = mk8_events.snapshot()
h['reliability'] = snap['reliability']
h['events'] = snap['events']
h['guard'] = mk8_guard.guard_report()
return 200, h
+45
View File
@@ -31,6 +31,51 @@ class EventsTest(unittest.TestCase):
self.assertEqual(c['restarts'], 1)
self.assertEqual(c['guard_fixes'], 1)
def test_snapshot_bounded_and_counts(self):
# Deterministic window: 3 restarts (oldest, outside the scan window),
# then 40 ticks, then 1 boot LAST so the newest-first window of 30
# includes it.
for _ in range(3):
mk8_events.log_event('restart', msg='old')
for i in range(40):
mk8_events.log_event('tick', msg='t%d' % i)
mk8_events.log_event('boot', msg='final boot')
snap = mk8_events.snapshot(event_limit=5, scan=30)
self.assertLessEqual(len(snap['events']), 5)
ev = snap['events']
tss = [r.get('ts', 0) for r in ev]
self.assertEqual(tss, sorted(tss, reverse=True), 'must be newest-first')
self.assertEqual(ev[0]['kind'], 'boot')
rel = snap['reliability']
self.assertEqual(rel['boots'], 1,
'boot is inside the scanned window')
self.assertEqual(rel['restarts'], 0,
'counters must cover only the bounded window')
self.assertEqual(rel['unexpected_boots'], 0)
# Full-journal counters() stays available and unbounded for callers
# that want it.
self.assertEqual(mk8_events.counters()['restarts'], 3)
def test_snapshot_spans_rotation(self):
mk8_events.MAX_BYTES = 200
for i in range(8):
mk8_events.log_event('tick', msg='x' * 20)
if i == 3:
mk8_events.log_event('rollback', sev='warn', msg='mid')
rotated = [i for i in range(1, mk8_events.KEEP + 1)
if os.path.exists(mk8_events.EVENTS_PATH + '.%d' % i)]
self.assertGreaterEqual(len(rotated) + 1, 2,
'journal must actually rotate here')
snap = mk8_events.snapshot(event_limit=100, scan=1000)
# Snapshot must span every rotated generation read_events sees.
self.assertEqual(len(snap['events']),
len(mk8_events.read_events(limit=1000)))
tss = [r.get('ts', 0) for r in snap['events']]
self.assertEqual(tss, sorted(tss, reverse=True))
kinds = {r['kind'] for r in snap['events']}
self.assertEqual(kinds, {'tick', 'rollback'})
self.assertEqual(snap['reliability']['rollbacks'], 1)
def test_rotation_keeps_recent(self):
mk8_events.MAX_BYTES = 200
for i in range(20):
+13
View File
@@ -157,6 +157,19 @@ class RfPlanTest(unittest.TestCase):
self.assertEqual(len(self.sleeps), 5)
self.assertIn('uci set wireless.wlan1up.disabled=1', self.cmds)
self.assertEqual(self.resumed, 1)
# Revert must converge runtime like the idle branch: a wifi reload
# after the STA disable, with hop resumed only after the reload.
cmds = self.cmds
reloads = [i for i, c in enumerate(cmds) if c == 'wifi reload']
self.assertEqual(len(reloads), 2, cmds)
disable_set = cmds.index('uci set wireless.wlan1up.disabled=1')
last_commit = len(cmds) - 1 - cmds[::-1].index('uci commit wireless')
resume_idx = self.seq.index('resume')
self.assertLess(disable_set, last_commit)
self.assertLess(last_commit, reloads[-1])
self.assertLess(reloads[-1], resume_idx)
# cmds[i] maps 1:1 onto self.runs[i]; revert reload must be gated.
self.assertEqual(self.runs[reloads[-1]], (['wifi', 'reload'], 60))
def test_set_role_uplink_requires_ssid(self):
result = mk8_rfplan.set_role('uplink')