fix(reliability): watchdog max lifetime + serialized gate entry (review fixes)
This commit is contained in:
@@ -1,9 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Usage: mk8-watchdog.sh <profile> <interval> <fail_after> <healthy_after>
|
||||
PROFILE="$1"; IV="${2:-5}"; FA="${3:-6}"; HA="${4:-6}"
|
||||
# Usage: mk8-watchdog.sh <profile> <interval> <fail_after> <healthy_after> [max_ticks]
|
||||
# Exits quietly after max_ticks healthy ticks so sentinels cannot accumulate.
|
||||
PROFILE="$1"; IV="${2:-5}"; FA="${3:-6}"; HA="${4:-6}"; MT="${5:-120}"
|
||||
DIR="/root/payloads/user/remote_access/pager-webui"
|
||||
[ -f "$DIR/server.py" ] || DIR="/mmc/mk8/releases/current"
|
||||
fails=0; oks=0; tripped=0
|
||||
fails=0; oks=0; tripped=0; ticks=0
|
||||
probe() {
|
||||
curl -fsS -m 3 http://127.0.0.1:8080/ >/dev/null 2>&1 &&
|
||||
{ ip link show wlan0mon >/dev/null 2>&1 ||
|
||||
@@ -12,6 +13,10 @@ probe() {
|
||||
while true; do
|
||||
if probe; then
|
||||
fails=0
|
||||
ticks=$((ticks + 1))
|
||||
if [ "$tripped" = "0" ] && [ "$ticks" -ge "$MT" ]; then
|
||||
exit 0
|
||||
fi
|
||||
if [ "$tripped" = "1" ]; then
|
||||
oks=$((oks + 1))
|
||||
if [ "$oks" -ge "$HA" ]; then
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
"""Risky-operation gate: preflight config snapshot + detached rollback watchdog."""
|
||||
import shlex
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh'
|
||||
FAIL_AFTER = 6 # consecutive local-liveness failures -> rollback
|
||||
HEALTHY_AFTER = 6 # consecutive successes after failure -> promote
|
||||
INTERVAL = 5 # seconds between probes
|
||||
MAX_TICKS = 120 # watchdog self-exits after this many quiet ticks
|
||||
_ENTER_LOCK = threading.Lock()
|
||||
|
||||
# Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing
|
||||
# this module never snapshots or spawns anything.
|
||||
@@ -27,16 +30,19 @@ def watchdog_decision(state):
|
||||
|
||||
|
||||
def _spawn_watchdog(name):
|
||||
cmd = ('setsid sh %s %s %d %d %d >/dev/null 2>&1 &'
|
||||
cmd = ('setsid sh %s %s %d %d %d %d >/dev/null 2>&1 &'
|
||||
% (shlex.quote(WATCHDOG), shlex.quote(name),
|
||||
INTERVAL, FAIL_AFTER, HEALTHY_AFTER))
|
||||
INTERVAL, FAIL_AFTER, HEALTHY_AFTER, MAX_TICKS))
|
||||
return subprocess.Popen(cmd, shell=True, start_new_session=True)
|
||||
|
||||
|
||||
def enter(op):
|
||||
"""Snapshot + spawn watchdog. Returns profile name or None when disabled."""
|
||||
"""Snapshot + spawn watchdog. Returns profile name or None when disabled.
|
||||
Serialized so concurrent gated ops cannot interleave snapshots or spawn
|
||||
racing watchdogs."""
|
||||
if not ENABLED:
|
||||
return None
|
||||
with _ENTER_LOCK:
|
||||
import mk8_profiles
|
||||
name = mk8_profiles.auto_name(op)
|
||||
mk8_profiles.snapshot(name)
|
||||
|
||||
Reference in New Issue
Block a user