feat(reliability): supervisor sampling, event feed, boot marker
This commit is contained in:
@@ -4682,6 +4682,24 @@ def _monitor_down(name):
|
||||
return not _iface_up(name)
|
||||
|
||||
|
||||
def _mem_percent(path='/proc/meminfo'):
|
||||
try:
|
||||
vals = {}
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
k, v = line.split(':')
|
||||
vals[k] = int(v.strip().split()[0])
|
||||
total = vals.get('MemTotal', 0)
|
||||
avail = vals.get('MemAvailable', vals.get('MemFree', 0))
|
||||
return round(100.0 * (total - avail) / total) if total else 0
|
||||
except (OSError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
MEM_WARN_PERCENT = 85
|
||||
MEM_WARN_STREAK = 5
|
||||
|
||||
|
||||
def health_check():
|
||||
"""One health pass. Returns the health dict. Fix actions are
|
||||
rate-limited by HEALTH_FIX_COOLDOWN.
|
||||
@@ -4690,6 +4708,8 @@ def health_check():
|
||||
socket every 15s collides with the stock daemon's own socket writes
|
||||
('[PineAp] Error writing' -> daemon watchdog SIGTERMs pineapd).
|
||||
"""
|
||||
import mk8_events
|
||||
|
||||
h = _health
|
||||
rc, out, err = device_run(['pidof', 'pineapd'], timeout=10)
|
||||
h['pineap_up'] = rc == 0 and bool((out or '').strip())
|
||||
@@ -4708,11 +4728,20 @@ def health_check():
|
||||
h['sigsegv_last'] = _sigsegv_count()
|
||||
h['last_fix'] = now
|
||||
h['fixes'] += 1
|
||||
mk8_events.log_event('restart', msg='pineapd restarted by health monitor')
|
||||
return dict(h)
|
||||
# pineapd is healthy, but wifi reloads still drop the monitors (pineapd
|
||||
# does not bring secondary monitors back). Repair them without cooldown.
|
||||
if _monitor_down('wlan1mon') or _monitor_down('wlan0mon'):
|
||||
_bring_monitors_up(h)
|
||||
h['mem_percent'] = _mem_percent()
|
||||
if h['mem_percent'] >= MEM_WARN_PERCENT:
|
||||
h['mem_streak'] = h.get('mem_streak', 0) + 1
|
||||
else:
|
||||
h['mem_streak'] = 0
|
||||
if h['mem_streak'] == MEM_WARN_STREAK:
|
||||
mk8_events.log_event('mem_warn', sev='warn',
|
||||
msg='memory above %d%% sustained' % MEM_WARN_PERCENT)
|
||||
return dict(h)
|
||||
|
||||
|
||||
@@ -4741,6 +4770,9 @@ def _bring_monitors_up(h):
|
||||
|
||||
|
||||
def h_health(ctx):
|
||||
import mk8_events
|
||||
import mk8_guard
|
||||
|
||||
h = dict(_health)
|
||||
h['sigsegv_count'] = h.pop('sigsegv_last')
|
||||
h['pool_disabled'] = _uci_section('pineapd.@ssidpool[0]').get('disable') == '1'
|
||||
@@ -4755,6 +4787,9 @@ def h_health(ctx):
|
||||
for k in ('pass', 'fixed', 'warn', 'fail')},
|
||||
'steps': ENV_CHECK_STATE['report'],
|
||||
}
|
||||
h['reliability'] = mk8_events.counters()
|
||||
h['events'] = mk8_events.read_events(limit=20)
|
||||
h['guard'] = mk8_guard.guard_report()
|
||||
return 200, h
|
||||
|
||||
|
||||
@@ -4939,6 +4974,21 @@ def env_check():
|
||||
return report
|
||||
|
||||
|
||||
BOOT_MARKER = '/mmc/mk8/boot.marker'
|
||||
|
||||
|
||||
def check_boot_marker():
|
||||
import os, mk8_events
|
||||
try:
|
||||
unexpected = os.path.exists(BOOT_MARKER)
|
||||
mk8_events.mark_boot(unexpected=unexpected)
|
||||
with open(BOOT_MARKER, 'w') as f:
|
||||
f.write(str(int(time.time())))
|
||||
return unexpected
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def startup_env_check(attempts=STARTUP_CHECK_ATTEMPTS,
|
||||
delay=STARTUP_CHECK_DELAY):
|
||||
"""Run and print the startup contract, allowing boot dependencies time.
|
||||
@@ -4959,6 +5009,10 @@ def startup_env_check(attempts=STARTUP_CHECK_ATTEMPTS,
|
||||
if ENV_CHECK_STATE.get('overall') != 'fail':
|
||||
print('ENVIRONMENT CHECK: %s' %
|
||||
ENV_CHECK_STATE['overall'].upper(), flush=True)
|
||||
try:
|
||||
check_boot_marker()
|
||||
except Exception:
|
||||
pass
|
||||
return last_report
|
||||
if attempt < attempts:
|
||||
print('ENVIRONMENT CHECK: FAIL; retry %d/%d in %ds' %
|
||||
|
||||
@@ -139,5 +139,38 @@ class HealthCheckTest(unittest.TestCase):
|
||||
self.assertIn('pool_disabled', payload)
|
||||
|
||||
|
||||
class SupervisorExtrasTest(unittest.TestCase):
|
||||
def runTestWith(self): # helper: reuse existing setUp fake_run
|
||||
pass
|
||||
|
||||
def test_mem_percent_math(self):
|
||||
import tempfile
|
||||
content = 'MemTotal: 250000 kB\nMemAvailable: 100000 kB\n'
|
||||
path = tempfile.mktemp()
|
||||
open(path, 'w').write(content)
|
||||
self.assertEqual(server._mem_percent(path), 60)
|
||||
|
||||
def test_health_reports_events_and_counters(self):
|
||||
import mk8_events
|
||||
mk8_events.log_event('restart', msg='x')
|
||||
status, h = server.h_health(None)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIn('events', h)
|
||||
self.assertIn('boots', h['reliability'])
|
||||
|
||||
def test_boot_marker_detects_unexpected(self):
|
||||
import mk8_events, tempfile, os
|
||||
marker = tempfile.mktemp()
|
||||
old = server.BOOT_MARKER
|
||||
server.BOOT_MARKER = marker
|
||||
try:
|
||||
open(marker, 'w').write('0')
|
||||
self.assertTrue(server.check_boot_marker())
|
||||
os.unlink(marker)
|
||||
self.assertFalse(server.check_boot_marker())
|
||||
finally:
|
||||
server.BOOT_MARKER = old
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'payload',
|
||||
'user', 'remote_access', 'pager-webui'))
|
||||
import server
|
||||
|
||||
|
||||
def setUpModule():
|
||||
__import__('importlib').reload(server)
|
||||
|
||||
|
||||
class ReliabilityApiTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.runs = []
|
||||
self.old_device_run = server.device_run
|
||||
server._health.update({
|
||||
'sigsegv_last': None, 'last_fix': 0.0, 'fixes': 0,
|
||||
'last_action': None, 'pineap_up': False, 'monitor_fixes': 0})
|
||||
|
||||
def fake_run(args, timeout=20, input_data=None):
|
||||
self.runs.append((list(args), timeout))
|
||||
return (0, '', '')
|
||||
|
||||
server.device_run = fake_run
|
||||
|
||||
def tearDown(self):
|
||||
server.device_run = self.old_device_run
|
||||
|
||||
def test_h_health_exposes_reliability_feed(self):
|
||||
status, h = server.h_health(None)
|
||||
self.assertEqual(status, 200)
|
||||
for key in ('reliability', 'events', 'guard'):
|
||||
self.assertIn(key, h)
|
||||
for counter in ('boots', 'unexpected_boots', 'rollbacks',
|
||||
'restarts', 'guard_fixes'):
|
||||
self.assertIn(counter, h['reliability'])
|
||||
self.assertIsInstance(h['events'], list)
|
||||
self.assertIn('in_sync', h['guard'])
|
||||
self.assertIn('pool_size', h['guard'])
|
||||
|
||||
def test_check_boot_marker_uses_module_marker_path(self):
|
||||
import tempfile
|
||||
marker = tempfile.mktemp()
|
||||
old = server.BOOT_MARKER
|
||||
server.BOOT_MARKER = marker
|
||||
try:
|
||||
if os.path.exists(marker):
|
||||
os.unlink(marker)
|
||||
self.assertFalse(server.check_boot_marker())
|
||||
self.assertTrue(os.path.exists(marker),
|
||||
'check_boot_marker must use server.BOOT_MARKER')
|
||||
open(marker, 'w').write('0')
|
||||
self.assertTrue(server.check_boot_marker())
|
||||
finally:
|
||||
if os.path.exists(marker):
|
||||
os.unlink(marker)
|
||||
server.BOOT_MARKER = old
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user