feat(reliability): supervisor sampling, event feed, boot marker

payload/user/remote_access/pager-webui/server.py

@@ -4682,6 +4682,24 @@ def _monitor_down(name):
     return not _iface_up(name)
 
 
+def _mem_percent(path='/proc/meminfo'):
+    try:
+        vals = {}
+        with open(path) as f:
+            for line in f:
+                k, v = line.split(':')
+                vals[k] = int(v.strip().split()[0])
+        total = vals.get('MemTotal', 0)
+        avail = vals.get('MemAvailable', vals.get('MemFree', 0))
+        return round(100.0 * (total - avail) / total) if total else 0
+    except (OSError, ValueError):
+        return 0
+
+
+MEM_WARN_PERCENT = 85
+MEM_WARN_STREAK = 5
+
+
 def health_check():
     """One health pass. Returns the health dict. Fix actions are
     rate-limited by HEALTH_FIX_COOLDOWN.
@@ -4690,6 +4708,8 @@ def health_check():
     socket every 15s collides with the stock daemon's own socket writes
     ('[PineAp] Error writing' -> daemon watchdog SIGTERMs pineapd).
     """
+    import mk8_events
+
     h = _health
     rc, out, err = device_run(['pidof', 'pineapd'], timeout=10)
     h['pineap_up'] = rc == 0 and bool((out or '').strip())
@@ -4708,11 +4728,20 @@ def health_check():
         h['sigsegv_last'] = _sigsegv_count()
         h['last_fix'] = now
         h['fixes'] += 1
+        mk8_events.log_event('restart', msg='pineapd restarted by health monitor')
         return dict(h)
     # pineapd is healthy, but wifi reloads still drop the monitors (pineapd
     # does not bring secondary monitors back). Repair them without cooldown.
     if _monitor_down('wlan1mon') or _monitor_down('wlan0mon'):
         _bring_monitors_up(h)
+    h['mem_percent'] = _mem_percent()
+    if h['mem_percent'] >= MEM_WARN_PERCENT:
+        h['mem_streak'] = h.get('mem_streak', 0) + 1
+    else:
+        h['mem_streak'] = 0
+    if h['mem_streak'] == MEM_WARN_STREAK:
+        mk8_events.log_event('mem_warn', sev='warn',
+                             msg='memory above %d%% sustained' % MEM_WARN_PERCENT)
     return dict(h)
 
 
@@ -4741,6 +4770,9 @@ def _bring_monitors_up(h):
 
 
 def h_health(ctx):
+    import mk8_events
+    import mk8_guard
+
     h = dict(_health)
     h['sigsegv_count'] = h.pop('sigsegv_last')
     h['pool_disabled'] = _uci_section('pineapd.@ssidpool[0]').get('disable') == '1'
@@ -4755,6 +4787,9 @@ def h_health(ctx):
                        for k in ('pass', 'fixed', 'warn', 'fail')},
             'steps': ENV_CHECK_STATE['report'],
         }
+    h['reliability'] = mk8_events.counters()
+    h['events'] = mk8_events.read_events(limit=20)
+    h['guard'] = mk8_guard.guard_report()
     return 200, h
 
 
@@ -4939,6 +4974,21 @@ def env_check():
     return report
 
 
+BOOT_MARKER = '/mmc/mk8/boot.marker'
+
+
+def check_boot_marker():
+    import os, mk8_events
+    try:
+        unexpected = os.path.exists(BOOT_MARKER)
+        mk8_events.mark_boot(unexpected=unexpected)
+        with open(BOOT_MARKER, 'w') as f:
+            f.write(str(int(time.time())))
+        return unexpected
+    except OSError:
+        return False
+
+
 def startup_env_check(attempts=STARTUP_CHECK_ATTEMPTS,
                       delay=STARTUP_CHECK_DELAY):
     """Run and print the startup contract, allowing boot dependencies time.
@@ -4959,6 +5009,10 @@ def startup_env_check(attempts=STARTUP_CHECK_ATTEMPTS,
         if ENV_CHECK_STATE.get('overall') != 'fail':
             print('ENVIRONMENT CHECK: %s' %
                   ENV_CHECK_STATE['overall'].upper(), flush=True)
+            try:
+                check_boot_marker()
+            except Exception:
+                pass
             return last_report
         if attempt < attempts:
             print('ENVIRONMENT CHECK: FAIL; retry %d/%d in %ds' %