fix(reliability): watchdog max lifetime + serialized gate entry (review fixes)

This commit is contained in:
2026-08-22 14:08:25 -06:00
parent a008bb9167
commit 78aab64af0
2 changed files with 21 additions and 10 deletions
@@ -1,11 +1,14 @@
"""Risky-operation gate: preflight config snapshot + detached rollback watchdog."""
import shlex
import subprocess
import threading
WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh'
FAIL_AFTER = 6 # consecutive local-liveness failures -> rollback
HEALTHY_AFTER = 6 # consecutive successes after failure -> promote
INTERVAL = 5 # seconds between probes
MAX_TICKS = 120 # watchdog self-exits after this many quiet ticks
_ENTER_LOCK = threading.Lock()
# Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing
# this module never snapshots or spawns anything.
@@ -27,20 +30,23 @@ def watchdog_decision(state):
def _spawn_watchdog(name):
cmd = ('setsid sh %s %s %d %d %d >/dev/null 2>&1 &'
cmd = ('setsid sh %s %s %d %d %d %d >/dev/null 2>&1 &'
% (shlex.quote(WATCHDOG), shlex.quote(name),
INTERVAL, FAIL_AFTER, HEALTHY_AFTER))
INTERVAL, FAIL_AFTER, HEALTHY_AFTER, MAX_TICKS))
return subprocess.Popen(cmd, shell=True, start_new_session=True)
def enter(op):
"""Snapshot + spawn watchdog. Returns profile name or None when disabled."""
"""Snapshot + spawn watchdog. Returns profile name or None when disabled.
Serialized so concurrent gated ops cannot interleave snapshots or spawn
racing watchdogs."""
if not ENABLED:
return None
import mk8_profiles
name = mk8_profiles.auto_name(op)
mk8_profiles.snapshot(name)
_spawn_watchdog(name)
with _ENTER_LOCK:
import mk8_profiles
name = mk8_profiles.auto_name(op)
mk8_profiles.snapshot(name)
_spawn_watchdog(name)
try:
import mk8_events
mk8_events.log_event('gate', msg='preflight snapshot %s' % name)