feat: pineapd health monitor with crash-loop auto-fix

15s poll; on PING failure, detects SIGSEGV growth (SSID-pool crash-loop),
disables pool broadcast, restarts pineapd, brings wlan1mon up. Rate-limited
fix actions, /api/health endpoint.
This commit is contained in:
2026-08-18 19:28:32 -05:00
parent 63fa5ae94a
commit 01c84caaaa
2 changed files with 173 additions and 0 deletions
@@ -3163,6 +3163,86 @@ def _pineap(*args, timeout=30):
return rc, out, err
# --------------------------------------------------------------------------
# Health monitor: keep pineapd alive and the monitor radios up. The stock
# SSID-pool broadcast segfaults pineapd on this firmware; when a crash-loop
# is detected the pool broadcast is disabled and pineapd restarted.
# --------------------------------------------------------------------------
HEALTH_POLL_SECONDS = 15
HEALTH_FIX_COOLDOWN = 20.0
HEALTH_STOP = threading.Event()
_health = {
'sigsegv_last': None,
'last_fix': 0.0,
'fixes': 0,
'last_action': None,
'pineap_up': False,
}
def _sigsegv_count():
rc, out, err = device_run(['logread'], timeout=15)
return out.count('SIGSEGV')
def _monitor_down(name):
rc, out, err = device_run(['iw', 'dev'], timeout=10)
return name not in out
def health_check():
"""One health pass. Returns the health dict. Fix actions are
rate-limited by HEALTH_FIX_COOLDOWN."""
h = _health
rc, out, err = device_run([HAK5CMD, 'PING'], timeout=10)
h['pineap_up'] = rc == 0 and 'PONG' in (out or '')
if h['pineap_up']:
return dict(h)
now = time.time()
if now - h['last_fix'] < HEALTH_FIX_COOLDOWN:
return dict(h)
count = _sigsegv_count()
if h['sigsegv_last'] is not None and count > h['sigsegv_last']:
# Crash-loop signature: disable the SSID pool broadcast and restart.
device_run(['uci', 'set', 'pineapd.@ssidpool[0].disable=1'])
device_run(['uci', 'commit', 'pineapd'])
device_run(['/etc/init.d/pineapd', 'restart'], timeout=30)
h['last_action'] = 'pool-disabled + pineapd restart (SIGSEGV crash-loop)'
elif _monitor_down('wlan1mon'):
device_run(['ip', 'link', 'set', 'wlan1mon', 'up'], timeout=10)
h['last_action'] = 'wlan1mon brought up'
else:
device_run(['/etc/init.d/pineapd', 'restart'], timeout=30)
h['last_action'] = 'pineapd restart'
h['sigsegv_last'] = count
h['last_fix'] = now
h['fixes'] += 1
return dict(h)
def h_health(ctx):
h = dict(_health)
h['sigsegv_count'] = h.pop('sigsegv_last')
h['pool_disabled'] = _uci_section('pineapd.@ssidpool[0]').get('disable') == '1'
h['wlan1mon_up'] = not _monitor_down('wlan1mon')
h['wlan0mon_up'] = not _monitor_down('wlan0mon')
return 200, h
def _health_loop():
while not HEALTH_STOP.is_set():
try:
health_check()
except Exception:
pass
HEALTH_STOP.wait(HEALTH_POLL_SECONDS)
def start_health_monitor():
threading.Thread(target=_health_loop, daemon=True).start()
def h_attacks_clients(ctx):
"""Clients (recon devices) plus the APs matching an SSID, for targeting."""
ssid = ((ctx.query or {}).get('ssid') or '').strip()
@@ -4199,6 +4279,7 @@ ROUTER.add('POST', r'/api/attacks/capture', h_attacks_capture)
ROUTER.add('GET', r'/api/attacks/export/hc22000', h_attacks_export_hc22000)
ROUTER.add('POST', r'/api/attacks/deauth', h_attacks_deauth)
ROUTER.add('GET', r'/api/attacks/clients', h_attacks_clients)
ROUTER.add('GET', r'/api/health', h_health)
ROUTER.add('POST', r'/api/recon/start', h_recon_start)
ROUTER.add('POST', r'/api/recon/stop', h_recon_stop)
ROUTER.add('GET', r'/api/recon/status', h_recon_status)
@@ -4324,6 +4405,7 @@ def _recon_watchdog_loop():
def serve():
threading.Thread(target=live_loop, daemon=True).start()
threading.Thread(target=_recon_watchdog_loop, daemon=True).start()
start_health_monitor()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((HOST, PORT))