feat(reliability): JSONL event journal with rotation and counters

payload/user/remote_access/pager-webui/mk8_events.py

@@ -0,0 +1,74 @@
+"""Mark VIII reliability event journal. JSONL on /mmc, rotated."""
+import json, os, threading
+
+MK8_DIR = '/mmc/mk8'
+EVENTS_PATH = os.path.join(MK8_DIR, 'events.log')
+MAX_BYTES = 5 * 1024 * 1024
+KEEP = 4
+_LOCK = threading.Lock()
+
+COUNTER_KINDS = ('boot', 'unexpected_boot', 'rollback', 'restart',
+                 'guard_fix')
+_COUNTER_KEYS = {'boot': 'boots', 'unexpected_boot': 'unexpected_boots',
+                 'rollback': 'rollbacks', 'restart': 'restarts',
+                 'guard_fix': 'guard_fixes'}
+
+
+def _ensure_dir():
+    try:
+        os.makedirs(MK8_DIR, exist_ok=True)
+    except OSError:
+        pass
+
+
+def log_event(kind, sev='info', msg='', meta=None):
+    entry = {'ts': int(__import__('time').time()), 'kind': str(kind),
+             'sev': sev, 'msg': msg[:500]}
+    if meta:
+        entry['meta'] = meta
+    line = json.dumps(entry) + '\n'
+    with _LOCK:
+        _ensure_dir()
+        try:
+            if os.path.exists(EVENTS_PATH) and \
+                    os.path.getsize(EVENTS_PATH) > MAX_BYTES:
+                for i in range(KEEP - 1, 0, -1):
+                    src = '%s.%d' % (EVENTS_PATH, i)
+                    dst = '%s.%d' % (EVENTS_PATH, i + 1)
+                    if os.path.exists(src):
+                        os.replace(src, dst)
+                if os.path.exists(EVENTS_PATH):
+                    os.replace(EVENTS_PATH, EVENTS_PATH + '.1')
+            with open(EVENTS_PATH, 'a') as f:
+                f.write(line)
+        except OSError:
+            pass
+
+
+def read_events(limit=100):
+    out = []
+    paths = [EVENTS_PATH + '.%d' % i for i in range(KEEP, 0, -1)]
+    paths.append(EVENTS_PATH)
+    for path in paths:
+        try:
+            with open(path) as f:
+                out.extend(json.loads(l) for l in f if l.strip())
+        except (OSError, ValueError):
+            continue
+    out.sort(key=lambda r: r.get('ts', 0))
+    return out[-limit:][::-1]
+
+
+def counters():
+    counts = {v: 0 for v in _COUNTER_KEYS.values()}
+    for row in read_events(limit=5000):
+        k = row.get('kind')
+        if k in _COUNTER_KEYS:
+            counts[_COUNTER_KEYS[k]] += 1
+    return counts
+
+
+def mark_boot(unexpected=False):
+    log_event('unexpected_boot' if unexpected else 'boot', sev='warn'
+              if unexpected else 'info',
+              msg='service started' + ('' if unexpected else ' cleanly'))

tests/test_mk8_events.py

@@ -0,0 +1,43 @@
+import json, os, sys, tempfile, unittest
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'payload', 'user', 'remote_access', 'pager-webui'))
+import mk8_events
+
+class EventsTest(unittest.TestCase):
+    def setUp(self):
+        self.dir = tempfile.mkdtemp()
+        self.old = mk8_events.MK8_DIR
+        mk8_events.MK8_DIR = self.dir
+        mk8_events.EVENTS_PATH = os.path.join(self.dir, 'events.log')
+
+    def tearDown(self):
+        mk8_events.MK8_DIR = self.old
+
+    def test_log_and_read_newest_first(self):
+        mk8_events.log_event('boot', msg='first')
+        mk8_events.log_event('rollback', sev='warn', msg='second', meta={'op': 'wifi'})
+        rows = mk8_events.read_events()
+        self.assertEqual(rows[0]['kind'], 'rollback')
+        self.assertEqual(rows[1]['kind'], 'boot')
+        self.assertEqual(rows[0]['meta'], {'op': 'wifi'})
+
+    def test_counters(self):
+        mk8_events.log_event('boot'); mk8_events.log_event('rollback')
+        mk8_events.log_event('restart'); mk8_events.log_event('guard_fix')
+        c = mk8_events.counters()
+        self.assertEqual(c['boots'], 1)
+        self.assertEqual(c['rollbacks'], 1)
+        self.assertEqual(c['restarts'], 1)
+        self.assertEqual(c['guard_fixes'], 1)
+
+    def test_rotation_keeps_recent(self):
+        mk8_events.MAX_BYTES = 200
+        for i in range(20):
+            mk8_events.log_event('tick', msg='x' * 30)
+        rows = mk8_events.read_events()
+        self.assertGreater(len(rows), 0)
+        self.assertLessEqual(len(rows), 20)
+        self.assertTrue(os.path.exists(mk8_events.EVENTS_PATH + '.1'))
+        self.assertFalse(os.path.exists(mk8_events.EVENTS_PATH + '.5'))
+
+if __name__ == '__main__':
+    unittest.main()