Capture hardening: deterministic env shim, extended lazy sweep, nav budget with bot-wall fast-fail

Salvaged and re-implemented from PR #15:
- Deterministic environment shim (seeded Math.random, pinned-but-advancing
  Date consistent across constructor/now/getTime/valueOf/toISOString) so
  time/random-dependent pages capture reproducibly; epoch and seed are
  recorded run metadata, performance.now stays real for motion sampling
- Post-snapshot lazy-asset discovery sweep for channels the walker misses:
  noscript contents (DOMParser), inline style url(), data-background(-image),
  img[loading=lazy] currentSrc
- Navigation gets a total 90s budget, an early bot-wall probe that aborts
  with a pollution-style error instead of capturing garbage (WALL_RE now
  shared with the validator gate), and one fresh-context retry scoped to
  the initial nav on retryable failure classes only

505 tests pass (29 new), typecheck clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

compiler/test/captureFailure.test.ts

@@ -0,0 +1,117 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import {
+  WALL_RE,
+  isWallText,
+  isBotWall,
+  WALL_MAX_NODES,
+  classifyNavFailure,
+  isRetryableNavFailure,
+} from "../src/util/captureFailure.js";
+
+// ---------------------------------------------------------------------------
+// Item 3: capture-side fast-fail predicates (pure — no browser).
+// ---------------------------------------------------------------------------
+
+describe("captureFailure: wall-text detection", () => {
+  it("matches common bot/auth-wall fingerprints", () => {
+    for (const t of [
+      "Just a moment...",
+      "Checking your browser before accessing",
+      "Please enable JavaScript to continue",
+      "Access to this page has been denied",
+      "Are you a human?",
+      "Verify you are human",
+      "Attention Required! | Cloudflare",
+      "Please complete the CAPTCHA",
+      "DDoS protection by Cloudflare",
+    ]) {
+      assert.ok(isWallText(t), `expected wall text to match: ${t}`);
+    }
+  });
+
+  it("does not match ordinary marketing/body copy", () => {
+    for (const t of [
+      "Welcome to our store — shop the new collection",
+      "Our team of humans is here to help you 24/7",
+      "Enable notifications to get the latest deals",
+      "About us · Careers · Contact",
+    ]) {
+      assert.equal(isWallText(t), false, `expected NOT to match: ${t}`);
+    }
+  });
+
+  it("WALL_RE is the same instance shared with the pollution gate (single source of truth)", () => {
+    // gates.ts imports WALL_RE from this module; assert the export is a real RegExp
+    // so a refactor that turns it into a function would fail here.
+    assert.ok(WALL_RE instanceof RegExp);
+  });
+});
+
+describe("captureFailure: isBotWall (matches the pollution gate's small+wall rule)", () => {
+  it("flags a small page with wall text", () => {
+    assert.ok(isBotWall({ text: "Just a moment...", nodes: 8 }));
+    assert.ok(isBotWall({ text: "checking your browser", nodes: WALL_MAX_NODES - 1 }));
+  });
+
+  it("does NOT flag a large page that merely mentions a wall phrase", () => {
+    // A real page can say "captcha" in its help docs — node count is the discriminator.
+    assert.equal(isBotWall({ text: "how our captcha works", nodes: WALL_MAX_NODES }), false);
+    assert.equal(isBotWall({ text: "how our captcha works", nodes: 5000 }), false);
+  });
+
+  it("does NOT flag a small page with no wall text", () => {
+    assert.equal(isBotWall({ text: "Home · About · Contact", nodes: 10 }), false);
+  });
+
+  it("is null-safe", () => {
+    assert.equal(isBotWall(null), false);
+    assert.equal(isBotWall(undefined), false);
+  });
+});
+
+describe("captureFailure: classifyNavFailure", () => {
+  it("classifies session-death / crash / transient network as retryable", () => {
+    for (const msg of [
+      "Target page, context or browser has been closed",
+      "Target closed",
+      "browser has been closed",
+      "page has crashed",
+      "net::ERR_CONNECTION_RESET at https://example.com",
+      "net::ERR_TIMED_OUT",
+      "Timeout 45000ms exceeded",
+      "Navigation timeout of 30000 ms exceeded",
+    ]) {
+      assert.equal(classifyNavFailure(new Error(msg)), "retryable", msg);
+      assert.equal(isRetryableNavFailure(new Error(msg)), true, msg);
+    }
+  });
+
+  it("classifies wall fingerprints as wall (not retryable — a fresh context can't help)", () => {
+    const e = new Error("navigation failed: just a moment... checking your browser");
+    assert.equal(classifyNavFailure(e), "wall");
+    assert.equal(isRetryableNavFailure(e), false);
+  });
+
+  it("classifies hard failures as terminal (not retryable)", () => {
+    for (const msg of [
+      "net::ERR_NAME_NOT_RESOLVED",
+      "net::ERR_CERT_AUTHORITY_INVALID",
+      "net::ERR_CONNECTION_REFUSED",
+      "something totally unexpected",
+    ]) {
+      assert.equal(classifyNavFailure(new Error(msg)), "terminal", msg);
+      assert.equal(isRetryableNavFailure(new Error(msg)), false, msg);
+    }
+  });
+
+  it("wall wins over a transient signature (a wall served over a reset is still a wall)", () => {
+    const e = new Error("net::ERR_CONNECTION_RESET — access denied");
+    assert.equal(classifyNavFailure(e), "wall");
+  });
+
+  it("accepts a bare string or non-Error value", () => {
+    assert.equal(classifyNavFailure("Target closed"), "retryable");
+    assert.equal(classifyNavFailure(undefined), "terminal");
+  });
+});

compiler/test/envShim.test.ts

@@ -0,0 +1,162 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { chromium, type Browser, type Page } from "playwright";
+import {
+  buildDeterministicEnvShim,
+  captureEpochMs,
+  DEFAULT_CAPTURE_EPOCH_MS,
+  DEFAULT_PRNG_SEED,
+} from "../src/util/envShim.js";
+
+const ESBUILD_SHIM =
+  "globalThis.__name = globalThis.__name || ((fn) => fn);" +
+  "globalThis.__defProp = globalThis.__defProp || Object.defineProperty;";
+
+// ---------------------------------------------------------------------------
+// Item 1: deterministic-env shim — pure epoch resolution + in-browser behavior.
+// ---------------------------------------------------------------------------
+
+describe("envShim: captureEpochMs resolution", () => {
+  it("defaults to the fixed epoch when no override is given", () => {
+    assert.equal(captureEpochMs(), DEFAULT_CAPTURE_EPOCH_MS);
+    assert.equal(captureEpochMs(undefined), DEFAULT_CAPTURE_EPOCH_MS);
+  });
+
+  it("passes a finite numeric override through unchanged", () => {
+    assert.equal(captureEpochMs(1234567890), 1234567890);
+  });
+
+  it("parses a numeric string and an ISO date string", () => {
+    assert.equal(captureEpochMs("1234567890"), 1234567890);
+    assert.equal(captureEpochMs("2020-01-01T00:00:00Z"), Date.parse("2020-01-01T00:00:00Z"));
+  });
+
+  it("falls back to the default for garbage input", () => {
+    assert.equal(captureEpochMs("not-a-date"), DEFAULT_CAPTURE_EPOCH_MS);
+    assert.equal(captureEpochMs(NaN), DEFAULT_CAPTURE_EPOCH_MS);
+  });
+});
+
+describe("envShim: buildDeterministicEnvShim source safety", () => {
+  it("interpolates only numeric literals (no injection surface)", () => {
+    const src = buildDeterministicEnvShim({ seed: 42, epochMs: 1000 });
+    assert.match(src, /let __s = 42 \| 0;/);
+    assert.match(src, /const delta = 1000 - realNow\(\);/);
+    // performance.now must remain untouched — the shim only patches Math.random + Date,
+    // never assigns to performance.now (motion.ts velocity sampling depends on the real one).
+    assert.equal(/performance\s*\.\s*now\s*=/.test(src), false);
+    assert.equal(/defineProperty\s*\(\s*performance/.test(src), false);
+  });
+
+  it("truncates non-integer seed/epoch and uses defaults for non-finite", () => {
+    const src = buildDeterministicEnvShim({ seed: 3.9, epochMs: 5.9 });
+    assert.match(src, /let __s = 3 \| 0;/);
+    assert.match(src, /const delta = 5 - realNow\(\);/);
+    const dflt = buildDeterministicEnvShim({ seed: NaN, epochMs: Infinity });
+    assert.match(dflt, new RegExp(`let __s = ${DEFAULT_PRNG_SEED} \\| 0;`));
+    assert.match(dflt, new RegExp(`const delta = ${DEFAULT_CAPTURE_EPOCH_MS} - realNow\\(\\);`));
+  });
+});
+
+const VW = 1280, VH = 800;
+
+describe("envShim: in-browser behavior", () => {
+  let browser: Browser;
+  before(async () => { browser = await chromium.launch(); });
+  after(async () => { await browser.close(); });
+
+  // Fresh page with the shim installed BEFORE any page script (via addInitScript),
+  // exactly as capture.ts wires it. We navigate to a data: URL rather than setContent
+  // so the document is CREATED via navigation — addInitScript fires on document
+  // creation, which setContent (a content swap on the existing about:blank) skips.
+  // The inline <script> then runs after the init scripts and samples the pinned env.
+  const loadWithShim = async (opts: { seed?: number; epochMs?: number }, bodyScript: string): Promise<Page> => {
+    const page = await browser.newPage();
+    await page.setViewportSize({ width: VW, height: VH });
+    await page.addInitScript(ESBUILD_SHIM);
+    await page.addInitScript(buildDeterministicEnvShim(opts));
+    const html = `<!doctype html><html><body><script>${bodyScript}</script></body></html>`;
+    await page.goto("data:text/html," + encodeURIComponent(html));
+    return page;
+  };
+
+  it("seeds Math.random reproducibly across two independent loads (same seed → same sequence)", async () => {
+    const script = "window.__draws = Array.from({length: 10}, () => Math.random());";
+    const p1 = await loadWithShim({ seed: 12345, epochMs: DEFAULT_CAPTURE_EPOCH_MS }, script);
+    const p2 = await loadWithShim({ seed: 12345, epochMs: DEFAULT_CAPTURE_EPOCH_MS }, script);
+    const [a, b] = await Promise.all([
+      p1.evaluate(() => (window as unknown as { __draws: number[] }).__draws),
+      p2.evaluate(() => (window as unknown as { __draws: number[] }).__draws),
+    ]);
+    assert.deepEqual(a, b, "two loads with the same seed produce identical Math.random sequences");
+    assert.ok(a.every((n) => n >= 0 && n < 1), "draws are in [0,1)");
+    // Not a constant stream — a broken PRNG returning the same value would also be
+    // "stable"; assert genuine variation so the test can't pass trivially.
+    assert.ok(new Set(a).size > 1, "the sequence varies");
+    await p1.close(); await p2.close();
+  });
+
+  it("a different seed yields a different sequence", async () => {
+    const script = "window.__draws = Array.from({length: 10}, () => Math.random());";
+    const p1 = await loadWithShim({ seed: 1, epochMs: DEFAULT_CAPTURE_EPOCH_MS }, script);
+    const p2 = await loadWithShim({ seed: 2, epochMs: DEFAULT_CAPTURE_EPOCH_MS }, script);
+    const [a, b] = await Promise.all([
+      p1.evaluate(() => (window as unknown as { __draws: number[] }).__draws),
+      p2.evaluate(() => (window as unknown as { __draws: number[] }).__draws),
+    ]);
+    assert.notDeepEqual(a, b);
+    await p1.close(); await p2.close();
+  });
+
+  it("pins the Date epoch consistently across constructor / now() / getTime() / valueOf() / toISOString()", async () => {
+    const epoch = Date.parse("2021-06-15T12:00:00Z");
+    const page = await loadWithShim(
+      { seed: 1, epochMs: epoch },
+      `window.__t = {
+         now: Date.now(),
+         ctorGetTime: new Date().getTime(),
+         ctorValueOf: new Date().valueOf(),
+         iso: new Date().toISOString(),
+       };`,
+    );
+    const t = await page.evaluate(() => (window as unknown as { __t: { now: number; ctorGetTime: number; ctorValueOf: number; iso: string } }).__t);
+    // All reads land at the pinned epoch (within a few ms of real elapsed time between them).
+    assert.ok(Math.abs(t.now - epoch) < 5000, `Date.now near epoch: ${t.now} vs ${epoch}`);
+    assert.ok(Math.abs(t.ctorGetTime - epoch) < 5000, "new Date().getTime() near epoch");
+    assert.ok(Math.abs(t.ctorValueOf - epoch) < 5000, "new Date().valueOf() near epoch");
+    assert.match(t.iso, /^2021-06-15T12:00:0/, `toISOString pinned: ${t.iso}`);
+    // now() and getTime() must agree (both read the same shifted instant).
+    assert.ok(Math.abs(t.now - t.ctorGetTime) < 5000, "now() and getTime() agree");
+    await page.close();
+  });
+
+  it("keeps the clock ADVANCING from the pinned epoch (timers still behave)", async () => {
+    const epoch = DEFAULT_CAPTURE_EPOCH_MS;
+    const page = await loadWithShim(
+      { seed: 1, epochMs: epoch },
+      `window.__adv = new Promise((res) => {
+         const t0 = Date.now();
+         setTimeout(() => res({ t0, t1: Date.now() }), 60);
+       });`,
+    );
+    const { t0, t1 } = await page.evaluate(() => (window as unknown as { __adv: Promise<{ t0: number; t1: number }> }).__adv);
+    assert.ok(t0 >= epoch - 5000 && t0 <= epoch + 5000, "start is pinned near epoch");
+    assert.ok(t1 > t0, "time advances across a real setTimeout delay");
+    await page.close();
+  });
+
+  it("leaves performance.now() REAL (motion sampling depends on it) and advancing", async () => {
+    const page = await loadWithShim(
+      { seed: 1, epochMs: DEFAULT_CAPTURE_EPOCH_MS },
+      `window.__perf = new Promise((res) => {
+         const p0 = performance.now();
+         setTimeout(() => res({ p0, p1: performance.now() }), 40);
+       });`,
+    );
+    const { p0, p1 } = await page.evaluate(() => (window as unknown as { __perf: Promise<{ p0: number; p1: number }> }).__perf);
+    // performance.now() is a real monotonic clock from navigation start — small, not the epoch.
+    assert.ok(p0 >= 0 && p0 < 60_000, `performance.now() is real (not pinned to epoch): ${p0}`);
+    assert.ok(p1 > p0, "performance.now() advances");
+    await page.close();
+  });
+});

compiler/test/lazySweep.test.ts

@@ -0,0 +1,85 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { chromium, type Browser, type Page } from "playwright";
+import { discoverLazyAssetsInPage } from "../src/capture/capture.js";
+
+const ESBUILD_SHIM =
+  "globalThis.__name = globalThis.__name || ((fn) => fn);" +
+  "globalThis.__defProp = globalThis.__defProp || Object.defineProperty;";
+
+const VW = 1280, VH = 800;
+const BASE = "https://example.com/page/";
+
+// ---------------------------------------------------------------------------
+// Item 2: extended lazy-asset discovery sweep (in-page discovery function).
+// ---------------------------------------------------------------------------
+describe("lazySweep: discoverLazyAssetsInPage", () => {
+  let browser: Browser;
+  before(async () => { browser = await chromium.launch(); });
+  after(async () => { await browser.close(); });
+
+  const discover = async (bodyHtml: string): Promise<string[]> => {
+    const page: Page = await browser.newPage();
+    await page.setViewportSize({ width: VW, height: VH });
+    await page.addInitScript(ESBUILD_SHIM);
+    // A <base> gives relative refs a stable absolute form to assert against.
+    await page.setContent(`<!doctype html><html><head><base href="${BASE}"></head><body>${bodyHtml}</body></html>`);
+    await page.evaluate(ESBUILD_SHIM);
+    const refs = await page.evaluate(discoverLazyAssetsInPage);
+    await page.close();
+    return refs;
+  };
+
+  it("discovers img/source src + srcset inside <noscript> fallback markup", async () => {
+    const refs = await discover(`
+      <noscript>
+        <img src="hero.jpg" srcset="hero-2x.jpg 2x, hero-3x.jpg 3x">
+        <picture><source srcset="alt.webp 1x, alt-2x.webp 2x"></picture>
+      </noscript>
+    `);
+    assert.ok(refs.includes(BASE + "hero.jpg"), "noscript img src");
+    assert.ok(refs.includes(BASE + "hero-2x.jpg"), "first srcset variant from noscript img");
+    assert.ok(refs.includes(BASE + "alt.webp"), "first srcset variant from noscript source");
+  });
+
+  it("discovers url(...) values from an element's inline style attribute", async () => {
+    const refs = await discover(`
+      <div style="background-image: url('bg.png'); color: red"></div>
+      <span style='background: url("sprite.svg") no-repeat'></span>
+    `);
+    assert.ok(refs.includes(BASE + "bg.png"), "inline background-image url()");
+    assert.ok(refs.includes(BASE + "sprite.svg"), "inline background shorthand url()");
+  });
+
+  it("discovers data-background / data-background-image (raw URL and url(...) forms)", async () => {
+    const refs = await discover(`
+      <div data-background="lazybg.jpg"></div>
+      <div data-background-image="url('lazybg2.png')"></div>
+    `);
+    assert.ok(refs.includes(BASE + "lazybg.jpg"), "raw data-background URL");
+    assert.ok(refs.includes(BASE + "lazybg2.png"), "url()-wrapped data-background-image");
+  });
+
+  it("discovers the resolved src of an img[loading=lazy]", async () => {
+    const refs = await discover(`<img loading="lazy" src="belowfold.jpg">`);
+    assert.ok(refs.includes(BASE + "belowfold.jpg"));
+  });
+
+  it("drops data: URIs and dedupes + sorts", async () => {
+    const refs = await discover(`
+      <div style="background: url('dup.png')"></div>
+      <div data-background="dup.png"></div>
+      <img loading="lazy" src="data:image/gif;base64,R0lGOD">
+    `);
+    assert.equal(refs.filter((u) => u.endsWith("dup.png")).length, 1, "deduped across channels");
+    assert.equal(refs.some((u) => u.startsWith("data:")), false, "data: URIs dropped");
+    assert.deepEqual(refs, [...refs].sort(), "sorted");
+  });
+
+  it("returns nothing for a page with no lazy references", async () => {
+    const refs = await discover(`<div><p>hello</p><img src="eager.jpg"></div>`);
+    // The eager <img src> is the walker's job, not this sweep — the sweep only harvests
+    // the channels the walker misses.
+    assert.deepEqual(refs, []);
+  });
+});