diff --git a/payload/user/remote_access/pager-webui/server.py b/payload/user/remote_access/pager-webui/server.py index baa21aa..fd1f3ff 100644 --- a/payload/user/remote_access/pager-webui/server.py +++ b/payload/user/remote_access/pager-webui/server.py @@ -4682,6 +4682,24 @@ def _monitor_down(name): return not _iface_up(name) +def _mem_percent(path='/proc/meminfo'): + try: + vals = {} + with open(path) as f: + for line in f: + k, v = line.split(':') + vals[k] = int(v.strip().split()[0]) + total = vals.get('MemTotal', 0) + avail = vals.get('MemAvailable', vals.get('MemFree', 0)) + return round(100.0 * (total - avail) / total) if total else 0 + except (OSError, ValueError): + return 0 + + +MEM_WARN_PERCENT = 85 +MEM_WARN_STREAK = 5 + + def health_check(): """One health pass. Returns the health dict. Fix actions are rate-limited by HEALTH_FIX_COOLDOWN. @@ -4690,6 +4708,8 @@ def health_check(): socket every 15s collides with the stock daemon's own socket writes ('[PineAp] Error writing' -> daemon watchdog SIGTERMs pineapd). """ + import mk8_events + h = _health rc, out, err = device_run(['pidof', 'pineapd'], timeout=10) h['pineap_up'] = rc == 0 and bool((out or '').strip()) @@ -4708,11 +4728,20 @@ def health_check(): h['sigsegv_last'] = _sigsegv_count() h['last_fix'] = now h['fixes'] += 1 + mk8_events.log_event('restart', msg='pineapd restarted by health monitor') return dict(h) # pineapd is healthy, but wifi reloads still drop the monitors (pineapd # does not bring secondary monitors back). Repair them without cooldown. if _monitor_down('wlan1mon') or _monitor_down('wlan0mon'): _bring_monitors_up(h) + h['mem_percent'] = _mem_percent() + if h['mem_percent'] >= MEM_WARN_PERCENT: + h['mem_streak'] = h.get('mem_streak', 0) + 1 + else: + h['mem_streak'] = 0 + if h['mem_streak'] == MEM_WARN_STREAK: + mk8_events.log_event('mem_warn', sev='warn', + msg='memory above %d%% sustained' % MEM_WARN_PERCENT) return dict(h) @@ -4741,6 +4770,9 @@ def _bring_monitors_up(h): def h_health(ctx): + import mk8_events + import mk8_guard + h = dict(_health) h['sigsegv_count'] = h.pop('sigsegv_last') h['pool_disabled'] = _uci_section('pineapd.@ssidpool[0]').get('disable') == '1' @@ -4755,6 +4787,9 @@ def h_health(ctx): for k in ('pass', 'fixed', 'warn', 'fail')}, 'steps': ENV_CHECK_STATE['report'], } + h['reliability'] = mk8_events.counters() + h['events'] = mk8_events.read_events(limit=20) + h['guard'] = mk8_guard.guard_report() return 200, h @@ -4939,6 +4974,21 @@ def env_check(): return report +BOOT_MARKER = '/mmc/mk8/boot.marker' + + +def check_boot_marker(): + import os, mk8_events + try: + unexpected = os.path.exists(BOOT_MARKER) + mk8_events.mark_boot(unexpected=unexpected) + with open(BOOT_MARKER, 'w') as f: + f.write(str(int(time.time()))) + return unexpected + except OSError: + return False + + def startup_env_check(attempts=STARTUP_CHECK_ATTEMPTS, delay=STARTUP_CHECK_DELAY): """Run and print the startup contract, allowing boot dependencies time. @@ -4959,6 +5009,10 @@ def startup_env_check(attempts=STARTUP_CHECK_ATTEMPTS, if ENV_CHECK_STATE.get('overall') != 'fail': print('ENVIRONMENT CHECK: %s' % ENV_CHECK_STATE['overall'].upper(), flush=True) + try: + check_boot_marker() + except Exception: + pass return last_report if attempt < attempts: print('ENVIRONMENT CHECK: FAIL; retry %d/%d in %ds' % diff --git a/tests/test_health.py b/tests/test_health.py index 0be24cb..181721d 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -139,5 +139,38 @@ class HealthCheckTest(unittest.TestCase): self.assertIn('pool_disabled', payload) +class SupervisorExtrasTest(unittest.TestCase): + def runTestWith(self): # helper: reuse existing setUp fake_run + pass + + def test_mem_percent_math(self): + import tempfile + content = 'MemTotal: 250000 kB\nMemAvailable: 100000 kB\n' + path = tempfile.mktemp() + open(path, 'w').write(content) + self.assertEqual(server._mem_percent(path), 60) + + def test_health_reports_events_and_counters(self): + import mk8_events + mk8_events.log_event('restart', msg='x') + status, h = server.h_health(None) + self.assertEqual(status, 200) + self.assertIn('events', h) + self.assertIn('boots', h['reliability']) + + def test_boot_marker_detects_unexpected(self): + import mk8_events, tempfile, os + marker = tempfile.mktemp() + old = server.BOOT_MARKER + server.BOOT_MARKER = marker + try: + open(marker, 'w').write('0') + self.assertTrue(server.check_boot_marker()) + os.unlink(marker) + self.assertFalse(server.check_boot_marker()) + finally: + server.BOOT_MARKER = old + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_reliability_api.py b/tests/test_reliability_api.py new file mode 100644 index 0000000..cf53d30 --- /dev/null +++ b/tests/test_reliability_api.py @@ -0,0 +1,63 @@ +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'payload', + 'user', 'remote_access', 'pager-webui')) +import server + + +def setUpModule(): + __import__('importlib').reload(server) + + +class ReliabilityApiTest(unittest.TestCase): + def setUp(self): + self.runs = [] + self.old_device_run = server.device_run + server._health.update({ + 'sigsegv_last': None, 'last_fix': 0.0, 'fixes': 0, + 'last_action': None, 'pineap_up': False, 'monitor_fixes': 0}) + + def fake_run(args, timeout=20, input_data=None): + self.runs.append((list(args), timeout)) + return (0, '', '') + + server.device_run = fake_run + + def tearDown(self): + server.device_run = self.old_device_run + + def test_h_health_exposes_reliability_feed(self): + status, h = server.h_health(None) + self.assertEqual(status, 200) + for key in ('reliability', 'events', 'guard'): + self.assertIn(key, h) + for counter in ('boots', 'unexpected_boots', 'rollbacks', + 'restarts', 'guard_fixes'): + self.assertIn(counter, h['reliability']) + self.assertIsInstance(h['events'], list) + self.assertIn('in_sync', h['guard']) + self.assertIn('pool_size', h['guard']) + + def test_check_boot_marker_uses_module_marker_path(self): + import tempfile + marker = tempfile.mktemp() + old = server.BOOT_MARKER + server.BOOT_MARKER = marker + try: + if os.path.exists(marker): + os.unlink(marker) + self.assertFalse(server.check_boot_marker()) + self.assertTrue(os.path.exists(marker), + 'check_boot_marker must use server.BOOT_MARKER') + open(marker, 'w').write('0') + self.assertTrue(server.check_boot_marker()) + finally: + if os.path.exists(marker): + os.unlink(marker) + server.BOOT_MARKER = old + + +if __name__ == '__main__': + unittest.main()