bzuccaro/Mark-VIII
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(reliability): watchdog max lifetime + serialized gate entry (review fixes)

Browse Source
This commit is contained in:
bzuccaro
2026-08-22 14:08:25 -06:00
parent a008bb9167
commit 78aab64af0
2 changed files with 21 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
payload/user/remote_access/pager-webui/mk8-watchdog.sh
+8 -3
View File
@@ -1,9 +1,10 @@
#!/bin/sh #!/bin/sh
# Usage: mk8-watchdog.sh <profile> <interval> <fail_after> <healthy_after> # Usage: mk8-watchdog.sh <profile> <interval> <fail_after> <healthy_after> [max_ticks]
PROFILE="$1"; IV="${2:-5}"; FA="${3:-6}"; HA="${4:-6}" # Exits quietly after max_ticks healthy ticks so sentinels cannot accumulate.
PROFILE="$1"; IV="${2:-5}"; FA="${3:-6}"; HA="${4:-6}"; MT="${5:-120}"
DIR="/root/payloads/user/remote_access/pager-webui" DIR="/root/payloads/user/remote_access/pager-webui"
[ -f "$DIR/server.py" ] || DIR="/mmc/mk8/releases/current" [ -f "$DIR/server.py" ] || DIR="/mmc/mk8/releases/current"
fails=0; oks=0; tripped=0 fails=0; oks=0; tripped=0; ticks=0
probe() { probe() {
curl -fsS -m 3 http://127.0.0.1:8080/ >/dev/null 2>&1 && curl -fsS -m 3 http://127.0.0.1:8080/ >/dev/null 2>&1 &&
{ ip link show wlan0mon >/dev/null 2>&1 || { ip link show wlan0mon >/dev/null 2>&1 ||
@@ -12,6 +13,10 @@ probe() {
while true; do while true; do
if probe; then if probe; then
fails=0 fails=0
ticks=$((ticks + 1))
if [ "$tripped" = "0" ] && [ "$ticks" -ge "$MT" ]; then
exit 0
fi
if [ "$tripped" = "1" ]; then if [ "$tripped" = "1" ]; then
oks=$((oks + 1)) oks=$((oks + 1))
if [ "$oks" -ge "$HA" ]; then if [ "$oks" -ge "$HA" ]; then
payload/user/remote_access/pager-webui/mk8_gate.py
+13 -7
View File
@@ -1,11 +1,14 @@
"""Risky-operation gate: preflight config snapshot + detached rollback watchdog.""" """Risky-operation gate: preflight config snapshot + detached rollback watchdog."""
import shlex import shlex
import subprocess import subprocess
import threading
WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh' WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh'
FAIL_AFTER = 6 # consecutive local-liveness failures -> rollback FAIL_AFTER = 6 # consecutive local-liveness failures -> rollback
HEALTHY_AFTER = 6 # consecutive successes after failure -> promote HEALTHY_AFTER = 6 # consecutive successes after failure -> promote
INTERVAL = 5 # seconds between probes INTERVAL = 5 # seconds between probes
MAX_TICKS = 120 # watchdog self-exits after this many quiet ticks
_ENTER_LOCK = threading.Lock()
# Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing # Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing
# this module never snapshots or spawns anything. # this module never snapshots or spawns anything.
@@ -27,20 +30,23 @@ def watchdog_decision(state):
def _spawn_watchdog(name): def _spawn_watchdog(name):
cmd = ('setsid sh %s %s %d %d %d >/dev/null 2>&1 &' cmd = ('setsid sh %s %s %d %d %d %d >/dev/null 2>&1 &'
% (shlex.quote(WATCHDOG), shlex.quote(name), % (shlex.quote(WATCHDOG), shlex.quote(name),
INTERVAL, FAIL_AFTER, HEALTHY_AFTER)) INTERVAL, FAIL_AFTER, HEALTHY_AFTER, MAX_TICKS))
return subprocess.Popen(cmd, shell=True, start_new_session=True) return subprocess.Popen(cmd, shell=True, start_new_session=True)
def enter(op): def enter(op):
"""Snapshot + spawn watchdog. Returns profile name or None when disabled.""" """Snapshot + spawn watchdog. Returns profile name or None when disabled.
Serialized so concurrent gated ops cannot interleave snapshots or spawn
racing watchdogs."""
if not ENABLED: if not ENABLED:
return None return None
import mk8_profiles with _ENTER_LOCK:
name = mk8_profiles.auto_name(op) import mk8_profiles
mk8_profiles.snapshot(name) name = mk8_profiles.auto_name(op)
_spawn_watchdog(name) mk8_profiles.snapshot(name)
_spawn_watchdog(name)
try: try:
import mk8_events import mk8_events
mk8_events.log_event('gate', msg='preflight snapshot %s' % name) mk8_events.log_event('gate', msg='preflight snapshot %s' % name)