fix(reliability): watchdog max lifetime + serialized gate entry (review fixes)

This commit is contained in:
2026-08-22 14:08:25 -06:00
parent a008bb9167
commit 78aab64af0
2 changed files with 21 additions and 10 deletions
@@ -1,9 +1,10 @@
#!/bin/sh
# Usage: mk8-watchdog.sh <profile> <interval> <fail_after> <healthy_after>
PROFILE="$1"; IV="${2:-5}"; FA="${3:-6}"; HA="${4:-6}"
# Usage: mk8-watchdog.sh <profile> <interval> <fail_after> <healthy_after> [max_ticks]
# Exits quietly after max_ticks healthy ticks so sentinels cannot accumulate.
PROFILE="$1"; IV="${2:-5}"; FA="${3:-6}"; HA="${4:-6}"; MT="${5:-120}"
DIR="/root/payloads/user/remote_access/pager-webui"
[ -f "$DIR/server.py" ] || DIR="/mmc/mk8/releases/current"
fails=0; oks=0; tripped=0
fails=0; oks=0; tripped=0; ticks=0
probe() {
curl -fsS -m 3 http://127.0.0.1:8080/ >/dev/null 2>&1 &&
{ ip link show wlan0mon >/dev/null 2>&1 ||
@@ -12,6 +13,10 @@ probe() {
while true; do
if probe; then
fails=0
ticks=$((ticks + 1))
if [ "$tripped" = "0" ] && [ "$ticks" -ge "$MT" ]; then
exit 0
fi
if [ "$tripped" = "1" ]; then
oks=$((oks + 1))
if [ "$oks" -ge "$HA" ]; then
@@ -1,11 +1,14 @@
"""Risky-operation gate: preflight config snapshot + detached rollback watchdog."""
import shlex
import subprocess
import threading
WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh'
FAIL_AFTER = 6 # consecutive local-liveness failures -> rollback
HEALTHY_AFTER = 6 # consecutive successes after failure -> promote
INTERVAL = 5 # seconds between probes
MAX_TICKS = 120 # watchdog self-exits after this many quiet ticks
_ENTER_LOCK = threading.Lock()
# Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing
# this module never snapshots or spawns anything.
@@ -27,16 +30,19 @@ def watchdog_decision(state):
def _spawn_watchdog(name):
cmd = ('setsid sh %s %s %d %d %d >/dev/null 2>&1 &'
cmd = ('setsid sh %s %s %d %d %d %d >/dev/null 2>&1 &'
% (shlex.quote(WATCHDOG), shlex.quote(name),
INTERVAL, FAIL_AFTER, HEALTHY_AFTER))
INTERVAL, FAIL_AFTER, HEALTHY_AFTER, MAX_TICKS))
return subprocess.Popen(cmd, shell=True, start_new_session=True)
def enter(op):
"""Snapshot + spawn watchdog. Returns profile name or None when disabled."""
"""Snapshot + spawn watchdog. Returns profile name or None when disabled.
Serialized so concurrent gated ops cannot interleave snapshots or spawn
racing watchdogs."""
if not ENABLED:
return None
with _ENTER_LOCK:
import mk8_profiles
name = mk8_profiles.auto_name(op)
mk8_profiles.snapshot(name)