feat(reliability): risky-op preflight snapshots + detached rollback watchdog

payload/user/remote_access/pager-webui/mk8_gate.py

@@ -0,0 +1,49 @@
+"""Risky-operation gate: preflight config snapshot + detached rollback watchdog."""
+import shlex
+import subprocess
+
+WATCHDOG = '/root/payloads/user/remote_access/pager-webui/mk8-watchdog.sh'
+FAIL_AFTER = 6          # consecutive local-liveness failures -> rollback
+HEALTHY_AFTER = 6       # consecutive successes after failure -> promote
+INTERVAL = 5            # seconds between probes
+
+# Dormant until an entrypoint (serve() / CLI ops) flips it on, so importing
+# this module never snapshots or spawns anything.
+ENABLED = False
+
+
+def watchdog_decision(state):
+    """state: {'fails': int, 'oks': int, 'tripped': bool,
+               'fail_after': 6, 'healthy_after': 6}
+    Returns (action, new_state): action in {'rollback','promote',None}."""
+    s = dict(state)
+    fa = s.get('fail_after', FAIL_AFTER)
+    ha = s.get('healthy_after', HEALTHY_AFTER)
+    if not s['tripped'] and s['fails'] >= fa:
+        return 'rollback', dict(s, tripped=True, oks=0)
+    if s['tripped'] and s['oks'] >= ha:
+        return 'promote', s
+    return None, s
+
+
+def _spawn_watchdog(name):
+    cmd = ('setsid sh %s %s %d %d %d >/dev/null 2>&1 &'
+           % (shlex.quote(WATCHDOG), shlex.quote(name),
+              INTERVAL, FAIL_AFTER, HEALTHY_AFTER))
+    return subprocess.Popen(cmd, shell=True, start_new_session=True)
+
+
+def enter(op):
+    """Snapshot + spawn watchdog. Returns profile name or None when disabled."""
+    if not ENABLED:
+        return None
+    import mk8_profiles
+    name = mk8_profiles.auto_name(op)
+    mk8_profiles.snapshot(name)
+    _spawn_watchdog(name)
+    try:
+        import mk8_events
+        mk8_events.log_event('gate', msg='preflight snapshot %s' % name)
+    except Exception:
+        pass
+    return name