56 lines
1.9 KiB
Python
56 lines
1.9 KiB
Python
"""Risky-operation gate: preflight config snapshot + detached rollback watchdog."""
|
|
import shlex
|
|
import subprocess
|
|
import threading
|
|
|
|
WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh'
|
|
FAIL_AFTER = 6 # consecutive local-liveness failures -> rollback
|
|
HEALTHY_AFTER = 6 # consecutive successes after failure -> promote
|
|
INTERVAL = 5 # seconds between probes
|
|
MAX_TICKS = 120 # watchdog self-exits after this many quiet ticks
|
|
_ENTER_LOCK = threading.Lock()
|
|
|
|
# Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing
|
|
# this module never snapshots or spawns anything.
|
|
ENABLED = False
|
|
|
|
|
|
def watchdog_decision(state):
|
|
"""state: {'fails': int, 'oks': int, 'tripped': bool,
|
|
'fail_after': 6, 'healthy_after': 6}
|
|
Returns (action, new_state): action in {'rollback','promote',None}."""
|
|
s = dict(state)
|
|
fa = s.get('fail_after', FAIL_AFTER)
|
|
ha = s.get('healthy_after', HEALTHY_AFTER)
|
|
if not s['tripped'] and s['fails'] >= fa:
|
|
return 'rollback', dict(s, tripped=True, oks=0)
|
|
if s['tripped'] and s['oks'] >= ha:
|
|
return 'promote', s
|
|
return None, s
|
|
|
|
|
|
def _spawn_watchdog(name):
|
|
cmd = ('setsid sh %s %s %d %d %d %d >/dev/null 2>&1 &'
|
|
% (shlex.quote(WATCHDOG), shlex.quote(name),
|
|
INTERVAL, FAIL_AFTER, HEALTHY_AFTER, MAX_TICKS))
|
|
return subprocess.Popen(cmd, shell=True, start_new_session=True)
|
|
|
|
|
|
def enter(op):
|
|
"""Snapshot + spawn watchdog. Returns profile name or None when disabled.
|
|
Serialized so concurrent gated ops cannot interleave snapshots or spawn
|
|
racing watchdogs."""
|
|
if not ENABLED:
|
|
return None
|
|
with _ENTER_LOCK:
|
|
import mk8_profiles
|
|
name = mk8_profiles.auto_name(op)
|
|
mk8_profiles.snapshot(name)
|
|
_spawn_watchdog(name)
|
|
try:
|
|
import mk8_events
|
|
mk8_events.log_event('gate', msg='preflight snapshot %s' % name)
|
|
except Exception:
|
|
pass
|
|
return name
|