feat(reliability): supervisor sampling, event feed, boot marker

payload/user/remote_access/pager-webui/server.py

@@ -4682,6 +4682,24 @@ def _monitor_down(name):
     return not _iface_up(name)
 
 
+def _mem_percent(path='/proc/meminfo'):
+    try:
+        vals = {}
+        with open(path) as f:
+            for line in f:
+                k, v = line.split(':')
+                vals[k] = int(v.strip().split()[0])
+        total = vals.get('MemTotal', 0)
+        avail = vals.get('MemAvailable', vals.get('MemFree', 0))
+        return round(100.0 * (total - avail) / total) if total else 0
+    except (OSError, ValueError):
+        return 0
+
+
+MEM_WARN_PERCENT = 85
+MEM_WARN_STREAK = 5
+
+
 def health_check():
     """One health pass. Returns the health dict. Fix actions are
     rate-limited by HEALTH_FIX_COOLDOWN.
@@ -4690,6 +4708,8 @@ def health_check():
     socket every 15s collides with the stock daemon's own socket writes
     ('[PineAp] Error writing' -> daemon watchdog SIGTERMs pineapd).
     """
+    import mk8_events
+
     h = _health
     rc, out, err = device_run(['pidof', 'pineapd'], timeout=10)
     h['pineap_up'] = rc == 0 and bool((out or '').strip())
@@ -4708,11 +4728,20 @@ def health_check():
         h['sigsegv_last'] = _sigsegv_count()
         h['last_fix'] = now
         h['fixes'] += 1
+        mk8_events.log_event('restart', msg='pineapd restarted by health monitor')
         return dict(h)
     # pineapd is healthy, but wifi reloads still drop the monitors (pineapd
     # does not bring secondary monitors back). Repair them without cooldown.
     if _monitor_down('wlan1mon') or _monitor_down('wlan0mon'):
         _bring_monitors_up(h)
+    h['mem_percent'] = _mem_percent()
+    if h['mem_percent'] >= MEM_WARN_PERCENT:
+        h['mem_streak'] = h.get('mem_streak', 0) + 1
+    else:
+        h['mem_streak'] = 0
+    if h['mem_streak'] == MEM_WARN_STREAK:
+        mk8_events.log_event('mem_warn', sev='warn',
+                             msg='memory above %d%% sustained' % MEM_WARN_PERCENT)
     return dict(h)
 
 
@@ -4741,6 +4770,9 @@ def _bring_monitors_up(h):
 
 
 def h_health(ctx):
+    import mk8_events
+    import mk8_guard
+
     h = dict(_health)
     h['sigsegv_count'] = h.pop('sigsegv_last')
     h['pool_disabled'] = _uci_section('pineapd.@ssidpool[0]').get('disable') == '1'
@@ -4755,6 +4787,9 @@ def h_health(ctx):
                        for k in ('pass', 'fixed', 'warn', 'fail')},
             'steps': ENV_CHECK_STATE['report'],
         }
+    h['reliability'] = mk8_events.counters()
+    h['events'] = mk8_events.read_events(limit=20)
+    h['guard'] = mk8_guard.guard_report()
     return 200, h
 
 
@@ -4939,6 +4974,21 @@ def env_check():
     return report
 
 
+BOOT_MARKER = '/mmc/mk8/boot.marker'
+
+
+def check_boot_marker():
+    import os, mk8_events
+    try:
+        unexpected = os.path.exists(BOOT_MARKER)
+        mk8_events.mark_boot(unexpected=unexpected)
+        with open(BOOT_MARKER, 'w') as f:
+            f.write(str(int(time.time())))
+        return unexpected
+    except OSError:
+        return False
+
+
 def startup_env_check(attempts=STARTUP_CHECK_ATTEMPTS,
                       delay=STARTUP_CHECK_DELAY):
     """Run and print the startup contract, allowing boot dependencies time.
@@ -4959,6 +5009,10 @@ def startup_env_check(attempts=STARTUP_CHECK_ATTEMPTS,
         if ENV_CHECK_STATE.get('overall') != 'fail':
             print('ENVIRONMENT CHECK: %s' %
                   ENV_CHECK_STATE['overall'].upper(), flush=True)
+            try:
+                check_boot_marker()
+            except Exception:
+                pass
             return last_report
         if attempt < attempts:
             print('ENVIRONMENT CHECK: FAIL; retry %d/%d in %ds' %

tests/test_health.py

@@ -139,5 +139,38 @@ class HealthCheckTest(unittest.TestCase):
         self.assertIn('pool_disabled', payload)
 
 
+class SupervisorExtrasTest(unittest.TestCase):
+    def runTestWith(self):  # helper: reuse existing setUp fake_run
+        pass
+
+    def test_mem_percent_math(self):
+        import tempfile
+        content = 'MemTotal:  250000 kB\nMemAvailable: 100000 kB\n'
+        path = tempfile.mktemp()
+        open(path, 'w').write(content)
+        self.assertEqual(server._mem_percent(path), 60)
+
+    def test_health_reports_events_and_counters(self):
+        import mk8_events
+        mk8_events.log_event('restart', msg='x')
+        status, h = server.h_health(None)
+        self.assertEqual(status, 200)
+        self.assertIn('events', h)
+        self.assertIn('boots', h['reliability'])
+
+    def test_boot_marker_detects_unexpected(self):
+        import mk8_events, tempfile, os
+        marker = tempfile.mktemp()
+        old = server.BOOT_MARKER
+        server.BOOT_MARKER = marker
+        try:
+            open(marker, 'w').write('0')
+            self.assertTrue(server.check_boot_marker())
+            os.unlink(marker)
+            self.assertFalse(server.check_boot_marker())
+        finally:
+            server.BOOT_MARKER = old
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_reliability_api.py

@@ -0,0 +1,63 @@
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'payload',
+                                'user', 'remote_access', 'pager-webui'))
+import server
+
+
+def setUpModule():
+    __import__('importlib').reload(server)
+
+
+class ReliabilityApiTest(unittest.TestCase):
+    def setUp(self):
+        self.runs = []
+        self.old_device_run = server.device_run
+        server._health.update({
+            'sigsegv_last': None, 'last_fix': 0.0, 'fixes': 0,
+            'last_action': None, 'pineap_up': False, 'monitor_fixes': 0})
+
+        def fake_run(args, timeout=20, input_data=None):
+            self.runs.append((list(args), timeout))
+            return (0, '', '')
+
+        server.device_run = fake_run
+
+    def tearDown(self):
+        server.device_run = self.old_device_run
+
+    def test_h_health_exposes_reliability_feed(self):
+        status, h = server.h_health(None)
+        self.assertEqual(status, 200)
+        for key in ('reliability', 'events', 'guard'):
+            self.assertIn(key, h)
+        for counter in ('boots', 'unexpected_boots', 'rollbacks',
+                        'restarts', 'guard_fixes'):
+            self.assertIn(counter, h['reliability'])
+        self.assertIsInstance(h['events'], list)
+        self.assertIn('in_sync', h['guard'])
+        self.assertIn('pool_size', h['guard'])
+
+    def test_check_boot_marker_uses_module_marker_path(self):
+        import tempfile
+        marker = tempfile.mktemp()
+        old = server.BOOT_MARKER
+        server.BOOT_MARKER = marker
+        try:
+            if os.path.exists(marker):
+                os.unlink(marker)
+            self.assertFalse(server.check_boot_marker())
+            self.assertTrue(os.path.exists(marker),
+                            'check_boot_marker must use server.BOOT_MARKER')
+            open(marker, 'w').write('0')
+            self.assertTrue(server.check_boot_marker())
+        finally:
+            if os.path.exists(marker):
+                os.unlink(marker)
+            server.BOOT_MARKER = old
+
+
+if __name__ == '__main__':
+    unittest.main()