fix(reliability): watchdog max lifetime + serialized gate entry (review fixes)
This commit is contained in:
@@ -1,11 +1,14 @@
|
||||
"""Risky-operation gate: preflight config snapshot + detached rollback watchdog."""
|
||||
import shlex
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh'
|
||||
FAIL_AFTER = 6 # consecutive local-liveness failures -> rollback
|
||||
HEALTHY_AFTER = 6 # consecutive successes after failure -> promote
|
||||
INTERVAL = 5 # seconds between probes
|
||||
MAX_TICKS = 120 # watchdog self-exits after this many quiet ticks
|
||||
_ENTER_LOCK = threading.Lock()
|
||||
|
||||
# Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing
|
||||
# this module never snapshots or spawns anything.
|
||||
@@ -27,20 +30,23 @@ def watchdog_decision(state):
|
||||
|
||||
|
||||
def _spawn_watchdog(name):
|
||||
cmd = ('setsid sh %s %s %d %d %d >/dev/null 2>&1 &'
|
||||
cmd = ('setsid sh %s %s %d %d %d %d >/dev/null 2>&1 &'
|
||||
% (shlex.quote(WATCHDOG), shlex.quote(name),
|
||||
INTERVAL, FAIL_AFTER, HEALTHY_AFTER))
|
||||
INTERVAL, FAIL_AFTER, HEALTHY_AFTER, MAX_TICKS))
|
||||
return subprocess.Popen(cmd, shell=True, start_new_session=True)
|
||||
|
||||
|
||||
def enter(op):
|
||||
"""Snapshot + spawn watchdog. Returns profile name or None when disabled."""
|
||||
"""Snapshot + spawn watchdog. Returns profile name or None when disabled.
|
||||
Serialized so concurrent gated ops cannot interleave snapshots or spawn
|
||||
racing watchdogs."""
|
||||
if not ENABLED:
|
||||
return None
|
||||
import mk8_profiles
|
||||
name = mk8_profiles.auto_name(op)
|
||||
mk8_profiles.snapshot(name)
|
||||
_spawn_watchdog(name)
|
||||
with _ENTER_LOCK:
|
||||
import mk8_profiles
|
||||
name = mk8_profiles.auto_name(op)
|
||||
mk8_profiles.snapshot(name)
|
||||
_spawn_watchdog(name)
|
||||
try:
|
||||
import mk8_events
|
||||
mk8_events.log_event('gate', msg='preflight snapshot %s' % name)
|
||||
|
||||
Reference in New Issue
Block a user