Capture hardening: deterministic env shim, extended lazy sweep, nav budget with bot-wall fast-fail
Salvaged and re-implemented from PR #15: - Deterministic environment shim (seeded Math.random, pinned-but-advancing Date consistent across constructor/now/getTime/valueOf/toISOString) so time/random-dependent pages capture reproducibly; epoch and seed are recorded run metadata, performance.now stays real for motion sampling - Post-snapshot lazy-asset discovery sweep for channels the walker misses: noscript contents (DOMParser), inline style url(), data-background(-image), img[loading=lazy] currentSrc - Navigation gets a total 90s budget, an early bot-wall probe that aborts with a pollution-style error instead of capturing garbage (WALL_RE now shared with the validator gate), and one fresh-context retry scoped to the initial nav on retryable failure classes only 505 tests pass (29 new), typecheck clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Samraaj Bath
co-authored by
Claude Fable 5
Claude Fable 5
parent
3faf1ef043
commit
7e8863fc7e
+238
-53
@@ -15,6 +15,8 @@ import { discoverBreakpoints } from "./breakpoints.js";
|
|||||||
import { writeJSON, writeJSONCompact, writeBytes, ensureDir } from "../util/fsx.js";
|
import { writeJSON, writeJSONCompact, writeBytes, ensureDir } from "../util/fsx.js";
|
||||||
import { sha1_12, round } from "../util/canonical.js";
|
import { sha1_12, round } from "../util/canonical.js";
|
||||||
import { isZipArchive, extractDotLottieJson } from "./dotlottie.js";
|
import { isZipArchive, extractDotLottieJson } from "./dotlottie.js";
|
||||||
|
import { buildDeterministicEnvShim, captureEpochMs, DEFAULT_PRNG_SEED } from "../util/envShim.js";
|
||||||
|
import { isBotWall, classifyNavFailure } from "../util/captureFailure.js";
|
||||||
|
|
||||||
export const REQUIRED_VIEWPORTS = [375, 768, 1280, 1920] as const;
|
export const REQUIRED_VIEWPORTS = [375, 768, 1280, 1920] as const;
|
||||||
// The dense width set captured for SIZE INFERENCE: a node sampled at 9 widths reveals its sizing
|
// The dense width set captured for SIZE INFERENCE: a node sampled at 9 widths reveals its sizing
|
||||||
@@ -65,6 +67,10 @@ export type SeoResource = {
|
|||||||
export type CaptureResult = {
|
export type CaptureResult = {
|
||||||
sourceUrl: string;
|
sourceUrl: string;
|
||||||
capturedAt: string;
|
capturedAt: string;
|
||||||
|
// Deterministic-env shim parameters actually used for this run (Item 1). Recorded
|
||||||
|
// so a recapture can reuse the exact {seed, epochMs} — the pinned wall clock is a
|
||||||
|
// run-metadata value, not a hardcoded constant. Absent when the shim was disabled.
|
||||||
|
deterministicEnv?: { seed: number; epochMs: number };
|
||||||
viewports: number[];
|
viewports: number[];
|
||||||
// Browser-as-oracle: the widths where the SOURCE layout actually restructures (display /
|
// Browser-as-oracle: the widths where the SOURCE layout actually restructures (display /
|
||||||
// flex-direction / wrap / grid-track-count / position / visibility flips), found by sweeping the
|
// flex-direction / wrap / grid-track-count / position / visibility flips), found by sweeping the
|
||||||
@@ -797,6 +803,66 @@ async function captureCanvasStills(page: import("playwright").Page): Promise<Can
|
|||||||
// instead of returning a truncated image on pathologically tall pages.
|
// instead of returning a truncated image on pathologically tall pages.
|
||||||
const CDP_MAX_SHOT_DIMENSION = 16_384;
|
const CDP_MAX_SHOT_DIMENSION = 16_384;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Item 2: extended lazy-asset DISCOVERY. Returns the deduped, sorted set of asset
|
||||||
|
* URLs referenced through channels the walker + promoteLazyMedia miss:
|
||||||
|
* (1) <noscript> fallback markup (the walker skips <noscript> entirely) — parsed as
|
||||||
|
* HTML so real img/source src + srcset are read, not regex-scraped;
|
||||||
|
* (2) inline element style="…url(…)…" (the walker harvests url() from STYLESHEET
|
||||||
|
* rules only, not from an element's own style attribute);
|
||||||
|
* (3) data-background / data-background-image lazy-bg aliases (beyond promoteLazyMedia's
|
||||||
|
* single data-bg);
|
||||||
|
* (4) img[loading=lazy] currentSrc — the variant a below-fold image resolved to.
|
||||||
|
* data: URLs and unresolvable refs are dropped. Discovery-only: the caller RECORDS these
|
||||||
|
* for the fallback downloader; nothing here mutates the DOM (snapshots are already taken).
|
||||||
|
* Exported for the fixture tests (page.evaluate'd directly).
|
||||||
|
*/
|
||||||
|
export function discoverLazyAssetsInPage(): string[] {
|
||||||
|
const urls = new Set<string>();
|
||||||
|
const push = (v: string | null | undefined): void => {
|
||||||
|
if (!v) return;
|
||||||
|
const s = v.trim();
|
||||||
|
if (!s || s.startsWith("data:")) return;
|
||||||
|
try { urls.add(new URL(s, document.baseURI).href); } catch { /* not a URL */ }
|
||||||
|
};
|
||||||
|
const firstOfSrcset = (ss: string): void => {
|
||||||
|
for (const part of ss.split(",")) push(part.trim().split(/\s+/)[0]);
|
||||||
|
};
|
||||||
|
// (1) <noscript> fallback markup.
|
||||||
|
for (const ns of Array.from(document.querySelectorAll("noscript"))) {
|
||||||
|
const html = ns.textContent ?? "";
|
||||||
|
if (!html.includes("<")) continue;
|
||||||
|
let frag: Document;
|
||||||
|
try { frag = new DOMParser().parseFromString(html, "text/html"); } catch { continue; }
|
||||||
|
for (const el of Array.from(frag.querySelectorAll("img,source"))) {
|
||||||
|
push(el.getAttribute("src"));
|
||||||
|
const ss = el.getAttribute("srcset");
|
||||||
|
if (ss) firstOfSrcset(ss);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// (2) inline style url(...) on any element (background-image / mask / etc.).
|
||||||
|
for (const el of Array.from(document.querySelectorAll("[style]"))) {
|
||||||
|
const st = el.getAttribute("style") ?? "";
|
||||||
|
if (!st.includes("url(")) continue;
|
||||||
|
for (const m of st.matchAll(/url\(\s*(['"]?)([^'")]+)\1\s*\)/gi)) push(m[2]);
|
||||||
|
}
|
||||||
|
// (3) lazy-background data-* aliases the promoter doesn't cover.
|
||||||
|
const BG_ATTRS = ["data-background", "data-background-image"];
|
||||||
|
for (const el of Array.from(document.querySelectorAll(BG_ATTRS.map((a) => `[${a}]`).join(",")))) {
|
||||||
|
for (const a of BG_ATTRS) {
|
||||||
|
const raw = el.getAttribute(a);
|
||||||
|
if (!raw) continue;
|
||||||
|
const m = /url\(\s*(['"]?)([^'")]+)\1\s*\)/i.exec(raw);
|
||||||
|
push(m ? m[2] : raw);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// (4) the variant a lazy image actually resolved to.
|
||||||
|
for (const img of Array.from(document.querySelectorAll<HTMLImageElement>("img[loading=lazy]"))) {
|
||||||
|
push(img.currentSrc || img.src);
|
||||||
|
}
|
||||||
|
return [...urls].sort();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Full-page screenshot via CDP `Page.captureScreenshot` with `captureBeyondViewport:true`.
|
* Full-page screenshot via CDP `Page.captureScreenshot` with `captureBeyondViewport:true`.
|
||||||
* Unlike Playwright's `fullPage:true` (which scroll-stitches — scrolling the page to render
|
* Unlike Playwright's `fullPage:true` (which scroll-stitches — scrolling the page to render
|
||||||
@@ -1004,6 +1070,14 @@ export async function captureSite(opts: {
|
|||||||
interactions?: boolean; // Stage 4: opt-in interaction capture (hover/focus + patterns)
|
interactions?: boolean; // Stage 4: opt-in interaction capture (hover/focus + patterns)
|
||||||
motion?: boolean; // Stage 5: opt-in motion capture (WAAPI + rotating text)
|
motion?: boolean; // Stage 5: opt-in motion capture (WAAPI + rotating text)
|
||||||
breakpoints?: boolean; // discover the source's real responsive band edges (default on; read-only sweep)
|
breakpoints?: boolean; // discover the source's real responsive band edges (default on; read-only sweep)
|
||||||
|
deterministicEnv?: boolean; // Item 1: seed Math.random + pin the Date epoch BEFORE page JS runs
|
||||||
|
// (default on), so shuffled carousels / random ids / relative-time text
|
||||||
|
// ("posted N minutes ago") render the same across captures. Relative time
|
||||||
|
// still advances (timers behave) and performance.now() stays real.
|
||||||
|
captureEpochMs?: number | string; // pinned wall-clock origin for the shim (ms since epoch, or a
|
||||||
|
// date string). Recorded in the result; a recapture passes the
|
||||||
|
// previously-recorded value to reproduce time-dependent content.
|
||||||
|
prngSeed?: number; // mulberry32 seed for the pinned Math.random (recorded in the result).
|
||||||
screenshots?: boolean; // write per-viewport full-page PNGs (default on). ONLY the validator reads these
|
screenshots?: boolean; // write per-viewport full-page PNGs (default on). ONLY the validator reads these
|
||||||
// (generation never touches pixels), and full-page shots of tall pages are the
|
// (generation never touches pixels), and full-page shots of tall pages are the
|
||||||
// dominant capture cost — so a production clone that won't be perceptually graded
|
// dominant capture cost — so a production clone that won't be perceptually graded
|
||||||
@@ -1013,6 +1087,13 @@ export async function captureSite(opts: {
|
|||||||
}): Promise<CaptureResult> {
|
}): Promise<CaptureResult> {
|
||||||
const viewports = opts.viewports ?? [...REQUIRED_VIEWPORTS];
|
const viewports = opts.viewports ?? [...REQUIRED_VIEWPORTS];
|
||||||
const log = opts.log ?? (() => {});
|
const log = opts.log ?? (() => {});
|
||||||
|
// Item 1: resolve the deterministic-env parameters once, at run start, and record
|
||||||
|
// them in the result. The epoch is a run-metadata value (recorded, reproducible),
|
||||||
|
// not a hardcoded constant; performance.now() is left real so motion capture is
|
||||||
|
// unaffected. Defaults on; opts.deterministicEnv === false disables the shim.
|
||||||
|
const deterministicEnvOn = opts.deterministicEnv !== false;
|
||||||
|
const prngSeed = Number.isFinite(opts.prngSeed) ? Math.trunc(opts.prngSeed!) : DEFAULT_PRNG_SEED;
|
||||||
|
const epochMs = captureEpochMs(opts.captureEpochMs);
|
||||||
const captureDir = join(opts.outDir, "capture");
|
const captureDir = join(opts.outDir, "capture");
|
||||||
const screenshotsDir = join(opts.outDir, "screenshots");
|
const screenshotsDir = join(opts.outDir, "screenshots");
|
||||||
const cssDir = join(captureDir, "css");
|
const cssDir = join(captureDir, "css");
|
||||||
@@ -1178,64 +1259,136 @@ export async function captureSite(opts: {
|
|||||||
// hero on each fresh load) and session-reuse degeneration (warbyparker returned
|
// hero on each fresh load) and session-reuse degeneration (warbyparker returned
|
||||||
// a 13-node shell when reloaded with a carried session). The IR's cross-viewport
|
// a 13-node shell when reloaded with a carried session). The IR's cross-viewport
|
||||||
// alignment then operates on one logical DOM that CSS merely reflows.
|
// alignment then operates on one logical DOM that CSS merely reflows.
|
||||||
const context: BrowserContext = await browser.newContext({
|
|
||||||
ignoreHTTPSErrors: true,
|
|
||||||
viewport: { width: canonical, height: viewportHeight(canonical) },
|
|
||||||
deviceScaleFactor: 1,
|
|
||||||
userAgent: DESKTOP_UA,
|
|
||||||
javaScriptEnabled: true,
|
|
||||||
});
|
|
||||||
const page = await context.newPage();
|
|
||||||
// tsx/esbuild wraps functions with a __name() helper for stack traces; that
|
|
||||||
// helper does not exist in the browser when we serialize page.evaluate
|
|
||||||
// callbacks. Shim it (as a raw string so it isn't itself transformed).
|
|
||||||
await page.addInitScript(ESBUILD_SHIM);
|
|
||||||
const bodyPromises: Promise<void>[] = [];
|
const bodyPromises: Promise<void>[] = [];
|
||||||
|
|
||||||
page.on("response", (resp) => {
|
// Response listener + init scripts are attached the same way to the initial page
|
||||||
try {
|
// and to any fresh recovery page (Item 3c), so factor the wiring into one helper.
|
||||||
const url = resp.url();
|
const attachResponseListener = (pg: import("playwright").Page): void => {
|
||||||
if (url.startsWith("data:") || url.startsWith("blob:")) return;
|
pg.on("response", (resp) => {
|
||||||
const ct = resp.headers()["content-type"] || null;
|
try {
|
||||||
const type = classifyAsset(url, ct);
|
const url = resp.url();
|
||||||
if (!type) return;
|
if (url.startsWith("data:") || url.startsWith("blob:")) return;
|
||||||
const status = resp.status();
|
const ct = resp.headers()["content-type"] || null;
|
||||||
recordAsset(url, type, ct, status, "network");
|
const type = classifyAsset(url, ct);
|
||||||
const existing = assetMap.get(url);
|
if (!type) return;
|
||||||
if (existing?.storedAs) return;
|
const status = resp.status();
|
||||||
if (status >= 400) return;
|
recordAsset(url, type, ct, status, "network");
|
||||||
// A 206 body is a RANGE FRAGMENT (media seek), not the asset — storing it would
|
const existing = assetMap.get(url);
|
||||||
// ship a corrupt file under first-stored-wins and the full-download fallback
|
if (existing?.storedAs) return;
|
||||||
// would then skip the asset. Record only; the fallback pass fetches the 200 body.
|
if (status >= 400) return;
|
||||||
if (status === 206) return;
|
// A 206 body is a RANGE FRAGMENT (media seek), not the asset — storing it would
|
||||||
bodyPromises.push(
|
// ship a corrupt file under first-stored-wins and the full-download fallback
|
||||||
(async () => {
|
// would then skip the asset. Record only; the fallback pass fetches the 200 body.
|
||||||
try {
|
if (status === 206) return;
|
||||||
const buf = await resp.body();
|
bodyPromises.push(
|
||||||
storeBytes(url, type, buf);
|
(async () => {
|
||||||
if (type === "css") cssTextsForParsing.push({ baseUrl: url, text: buf.toString("utf8") });
|
try {
|
||||||
if (type === "manifest") parseManifestForAssets(buf.toString("utf8"), url);
|
const buf = await resp.body();
|
||||||
} catch { /* body unavailable */ }
|
storeBytes(url, type, buf);
|
||||||
})(),
|
if (type === "css") cssTextsForParsing.push({ baseUrl: url, text: buf.toString("utf8") });
|
||||||
);
|
if (type === "manifest") parseManifestForAssets(buf.toString("utf8"), url);
|
||||||
} catch { /* ignore */ }
|
} catch { /* body unavailable */ }
|
||||||
});
|
})(),
|
||||||
|
);
|
||||||
|
} catch { /* ignore */ }
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
// Single navigation at the canonical width; every viewport below is a resize.
|
const newSession = async (): Promise<{ context: BrowserContext; page: import("playwright").Page }> => {
|
||||||
log({ event: "goto", url: opts.url });
|
const ctx: BrowserContext = await browser.newContext({
|
||||||
let navigated = false;
|
ignoreHTTPSErrors: true,
|
||||||
let navErr: unknown = null;
|
viewport: { width: canonical, height: viewportHeight(canonical) },
|
||||||
for (let attempt = 0; attempt < 3 && !navigated; attempt++) {
|
deviceScaleFactor: 1,
|
||||||
try {
|
userAgent: DESKTOP_UA,
|
||||||
await page.goto(opts.url, { waitUntil: attempt === 0 ? "load" : "domcontentloaded", timeout: 45000 });
|
javaScriptEnabled: true,
|
||||||
navigated = true;
|
});
|
||||||
} catch (e) {
|
const pg = await ctx.newPage();
|
||||||
navErr = e;
|
attachResponseListener(pg);
|
||||||
await page.waitForTimeout(1000);
|
// tsx/esbuild wraps functions with a __name() helper for stack traces; that
|
||||||
|
// helper does not exist in the browser when we serialize page.evaluate
|
||||||
|
// callbacks. Shim it (as a raw string so it isn't itself transformed).
|
||||||
|
await pg.addInitScript(ESBUILD_SHIM);
|
||||||
|
// Item 1: deterministic-env shim — seed Math.random + pin the Date epoch BEFORE
|
||||||
|
// any page script runs. Raw string so tsx/esbuild leaves it byte-exact; the
|
||||||
|
// {seed, epochMs} are the recorded run-metadata values. performance.now() stays
|
||||||
|
// real, so motion.ts marquee/rotator velocity sampling is unaffected.
|
||||||
|
if (deterministicEnvOn) {
|
||||||
|
await pg.addInitScript(buildDeterministicEnvShim({ seed: prngSeed, epochMs }));
|
||||||
}
|
}
|
||||||
|
return { context: ctx, page: pg };
|
||||||
|
};
|
||||||
|
|
||||||
|
// Single context + single navigation; every viewport is captured by RESIZING the
|
||||||
|
// same loaded page (see the block comment above). Item 3c: session recovery is
|
||||||
|
// deliberately scoped to the INITIAL navigation only — carrying one session across
|
||||||
|
// viewports is load-bearing (a reload can degenerate: warbyparker's 13-node shell),
|
||||||
|
// so we never recover mid-loop; we only retry the first nav with a fresh context
|
||||||
|
// when the page/browser died before any content was captured.
|
||||||
|
let { context, page } = await newSession();
|
||||||
|
|
||||||
|
// Total nav budget (Item 3a): a hung origin used to stall for up to ~3×45s + retries
|
||||||
|
// with no ceiling. Bound the WHOLE navigation phase so a dead host fails fast with a
|
||||||
|
// structured error instead of tying up the pipeline. Attempt 0 waits for `load`;
|
||||||
|
// later attempts fall back to `domcontentloaded` (a heavy page may never fire `load`).
|
||||||
|
const NAV_BUDGET_MS = 90_000;
|
||||||
|
const navigateLoaded = async (pg: import("playwright").Page): Promise<void> => {
|
||||||
|
log({ event: "goto", url: opts.url });
|
||||||
|
const navStart = Date.now();
|
||||||
|
let navigated = false;
|
||||||
|
let navErr: unknown = null;
|
||||||
|
for (let attempt = 0; attempt < 3 && !navigated; attempt++) {
|
||||||
|
const remaining = NAV_BUDGET_MS - (Date.now() - navStart);
|
||||||
|
if (remaining < 5_000) break; // not enough budget left for a meaningful attempt
|
||||||
|
try {
|
||||||
|
await pg.goto(opts.url, {
|
||||||
|
waitUntil: attempt === 0 ? "load" : "domcontentloaded",
|
||||||
|
timeout: Math.min(attempt === 0 ? 45_000 : 20_000, remaining),
|
||||||
|
});
|
||||||
|
navigated = true;
|
||||||
|
} catch (e) {
|
||||||
|
navErr = e;
|
||||||
|
await pg.waitForTimeout(1000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!navigated) {
|
||||||
|
throw new Error(
|
||||||
|
`navigation failed for ${opts.url} within ${Math.round((Date.now() - navStart) / 1000)}s: ${String((navErr as { message?: string })?.message ?? navErr).slice(0, 300)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await settle(pg);
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
await navigateLoaded(page);
|
||||||
|
} catch (navErr) {
|
||||||
|
// Item 3c: one fresh-context retry, but ONLY for a session-death class of failure
|
||||||
|
// (browser/page/context closed, crash, transient reset/timeout). A wall or a hard
|
||||||
|
// terminal error (DNS/cert/refused) is not retried — a fresh context can't fix it.
|
||||||
|
const cls = classifyNavFailure(navErr);
|
||||||
|
if (cls !== "retryable") throw navErr;
|
||||||
|
log({ event: "capture_recover", reason: "session_death_on_initial_nav" });
|
||||||
|
try { if (!page.isClosed()) await page.close(); } catch { /* ignore */ }
|
||||||
|
try { await context.close(); } catch { /* ignore */ }
|
||||||
|
({ context, page } = await newSession());
|
||||||
|
await navigateLoaded(page); // second failure propagates (no further retry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Item 3b: bot/auth-wall fast-fail. A wall page would otherwise burn the full
|
||||||
|
// multi-viewport capture and only get flagged by the pollution gate afterward.
|
||||||
|
// Uses the SAME signatures + node-count threshold as the gate (util/captureFailure.ts)
|
||||||
|
// so the abort and the grade agree; the pollution gate stays the shipped-capture
|
||||||
|
// authority, this only saves the wasted work of capturing an obvious wall.
|
||||||
|
const wallProbe = await page
|
||||||
|
.evaluate(() => ({
|
||||||
|
text: (document.body?.innerText ?? "").slice(0, 20_000),
|
||||||
|
nodes: document.querySelectorAll("*").length,
|
||||||
|
}))
|
||||||
|
.catch(() => null);
|
||||||
|
if (isBotWall(wallProbe)) {
|
||||||
|
throw new Error(
|
||||||
|
`auth/bot wall detected at ${opts.url} (${wallProbe!.nodes} nodes, wall text matched): capture aborted early`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if (!navigated) throw navErr;
|
|
||||||
await settle(page);
|
|
||||||
|
|
||||||
// Stage 2: lazy-loader promotion. WP Rocket/lazysizes keep a 0-size placeholder in
|
// Stage 2: lazy-loader promotion. WP Rocket/lazysizes keep a 0-size placeholder in
|
||||||
// `src` with the real URL in data attrs; autoScroll outruns their IntersectionObserver
|
// `src` with the real URL in data attrs; autoScroll outruns their IntersectionObserver
|
||||||
@@ -1641,6 +1794,37 @@ export async function captureSite(opts: {
|
|||||||
log({ event: "captured", viewport: vw, nodes: snapshot.doc.nodeCount, scrollHeight: snapshot.doc.scrollHeight });
|
log({ event: "captured", viewport: vw, nodes: snapshot.doc.nodeCount, scrollHeight: snapshot.doc.scrollHeight });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Item 2: extended lazy-asset discovery sweep. The walker + promoteLazyMedia
|
||||||
|
// already harvest img/source src/srcset and the common data-* lazy attrs
|
||||||
|
// (data-src / data-lazy-src / data-original / data-ll-src / data-(lazy-)srcset)
|
||||||
|
// and picture <source> variants — this sweep ONLY adds the references those miss:
|
||||||
|
// • <noscript> contents — the walker skips <noscript> entirely, but the fallback
|
||||||
|
// markup real browsers render with JS off often carries the true <img>/<source> URL;
|
||||||
|
// • inline element style="…url(…)…" — the walker harvests url() from STYLESHEET
|
||||||
|
// rules, not from an element's own style attribute (hero background-image inlined);
|
||||||
|
// • data-background / data-background-image — lazy-bg aliases beyond promoteLazyMedia's
|
||||||
|
// single data-bg;
|
||||||
|
// • img[loading=lazy] currentSrc — the resolved variant a below-fold image settled on.
|
||||||
|
// Discovery-ONLY: URLs are recorded through recordAsset so the fallback downloader
|
||||||
|
// fetches them; nothing is promoted into the DOM (the snapshots are already taken, so
|
||||||
|
// this can never shift a captured layout). Bounded so a pathological page can't explode
|
||||||
|
// the fallback fetch phase.
|
||||||
|
try {
|
||||||
|
const sweptRefs: string[] = await page.evaluate(discoverLazyAssetsInPage);
|
||||||
|
let swept = 0;
|
||||||
|
for (const url of sweptRefs) {
|
||||||
|
if (swept >= 120) break; // bound pathological pages (fallback fetch is 30s/asset)
|
||||||
|
if (!/^https?:\/\//i.test(url) || assetMap.has(url)) continue;
|
||||||
|
const t = classifyAsset(url, null);
|
||||||
|
if (!t || !["image", "svg", "video", "font", "lottie"].includes(t)) continue;
|
||||||
|
recordAsset(url, t, null, null, "lazy-sweep");
|
||||||
|
swept++;
|
||||||
|
}
|
||||||
|
if (swept) log({ event: "lazy_sweep", recorded: swept, seen: sweptRefs.length });
|
||||||
|
} catch (e) {
|
||||||
|
log({ event: "lazy_sweep_error", error: String((e as { message?: string })?.message ?? e).slice(0, 160) });
|
||||||
|
}
|
||||||
|
|
||||||
// Browser-as-oracle: discover the source's real responsive band edges by sweeping the viewport
|
// Browser-as-oracle: discover the source's real responsive band edges by sweeping the viewport
|
||||||
// and binary-searching each width where the discrete (media-query-toggled) layout signature
|
// and binary-searching each width where the discrete (media-query-toggled) layout signature
|
||||||
// changes. Read-only and bounded; runs once here — overlays are dismissed and the DOM settled, so
|
// changes. Read-only and bounded; runs once here — overlays are dismissed and the DOM settled, so
|
||||||
@@ -1754,6 +1938,7 @@ export async function captureSite(opts: {
|
|||||||
const result: CaptureResult = {
|
const result: CaptureResult = {
|
||||||
sourceUrl: opts.url,
|
sourceUrl: opts.url,
|
||||||
capturedAt: new Date().toISOString(),
|
capturedAt: new Date().toISOString(),
|
||||||
|
...(deterministicEnvOn ? { deterministicEnv: { seed: prngSeed, epochMs } } : {}),
|
||||||
viewports,
|
viewports,
|
||||||
breakpoints: discoveredBreakpoints,
|
breakpoints: discoveredBreakpoints,
|
||||||
perViewport,
|
perViewport,
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
/** Capture-side fast-fail helpers (Item 3).
|
||||||
|
*
|
||||||
|
* Two concerns share this module because both are pure, testable predicates the
|
||||||
|
* capture flow keys off:
|
||||||
|
*
|
||||||
|
* 1. Wall-text detection — the SAME signature set the validator's pollution gate
|
||||||
|
* uses (validate/gates.ts imports WALL_RE from here). Extracted so the two
|
||||||
|
* judgments — capture-side abort and validator-side grade — can never drift.
|
||||||
|
* The pollution gate remains the AUTHORITY on whether a shipped capture is
|
||||||
|
* polluted; this module only lets capture bail early on an obvious wall
|
||||||
|
* instead of burning the full multi-viewport pass on garbage.
|
||||||
|
*
|
||||||
|
* 2. Nav-failure classification — decides whether a navigation/session error is
|
||||||
|
* worth ONE fresh-context retry (transient: the browser/page died, a socket
|
||||||
|
* reset, a nav timeout) versus terminal (a wall, a hard DNS/cert failure that
|
||||||
|
* a retry cannot fix). Pure string classification so it unit-tests without a
|
||||||
|
* browser.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Bot/egress/auth-wall text signatures. One regex, shared with the pollution gate
|
||||||
|
* (validate/gates.ts) so capture-abort and validator-grade use identical fingerprints. */
|
||||||
|
export const WALL_RE =
|
||||||
|
/blocked by egress|access denied|access to this page has been denied|are you a (human|robot)|verify you are human|enable javascript to|please enable javascript|checking your browser|just a moment|attention required|request blocked|why have i been blocked|captcha|cf-browser-verification|ddos protection by/i;
|
||||||
|
|
||||||
|
export function isWallText(text: string): boolean {
|
||||||
|
return WALL_RE.test(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Node-count ceiling under which wall text is treated as a genuine interstitial
|
||||||
|
* rather than an incidental mention on a real page. Matches the pollution gate's
|
||||||
|
* `wall && nodeCount < 220` threshold so the two agree on the same boundary. */
|
||||||
|
export const WALL_MAX_NODES = 220;
|
||||||
|
|
||||||
|
export type WallProbe = { text: string; nodes: number };
|
||||||
|
|
||||||
|
/** Capture-side wall verdict: the page is a bot/auth wall worth aborting on iff it
|
||||||
|
* is BOTH small (few nodes) AND carries wall text — identical to the gate's rule.
|
||||||
|
* A large page that merely mentions "captcha" in body copy is NOT a wall. */
|
||||||
|
export function isBotWall(probe: WallProbe | null | undefined): boolean {
|
||||||
|
if (!probe) return false;
|
||||||
|
return probe.nodes < WALL_MAX_NODES && isWallText(probe.text);
|
||||||
|
}
|
||||||
|
|
||||||
|
export type NavFailureClass = "retryable" | "wall" | "terminal";
|
||||||
|
|
||||||
|
/** Classify a navigation/session failure by its error text.
|
||||||
|
*
|
||||||
|
* - "wall" → a bot/auth interstitial; a retry with a fresh context won't help
|
||||||
|
* and the caller should surface it as a pollution-style abort.
|
||||||
|
* - "retryable" → the browser/page/context died, a transient socket/timeout/reset;
|
||||||
|
* ONE fresh-context retry is worthwhile.
|
||||||
|
* - "terminal" → a hard failure a retry cannot change (DNS, cert, refused, unknown).
|
||||||
|
*
|
||||||
|
* Ordering matters: a wall signature wins over a transient one (a wall served over
|
||||||
|
* a reset connection is still a wall). */
|
||||||
|
export function classifyNavFailure(error: unknown): NavFailureClass {
|
||||||
|
const msg = String((error as { message?: string })?.message ?? error ?? "");
|
||||||
|
if (WALL_RE.test(msg)) return "wall";
|
||||||
|
// Session death / crash / transient network — a fresh context can recover these.
|
||||||
|
if (
|
||||||
|
/Target (page, context or browser|closed)|context or browser has been closed|page(?:,)? .*has been closed|browser has been closed|page closed|has crashed|Navigation .*interrupted|net::ERR_(?:CONNECTION_RESET|CONNECTION_CLOSED|TIMED_OUT|ABORTED|EMPTY_RESPONSE|NETWORK_CHANGED|SOCKET_NOT_CONNECTED)|Timeout .*exceeded|Navigation timeout/i.test(
|
||||||
|
msg,
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return "retryable";
|
||||||
|
}
|
||||||
|
return "terminal";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether a classified failure earns the single fresh-context retry. */
|
||||||
|
export function isRetryableNavFailure(error: unknown): boolean {
|
||||||
|
return classifyNavFailure(error) === "retryable";
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
/** Deterministic capture-environment shim (Stage 0, runs before ANY page script).
|
||||||
|
*
|
||||||
|
* Two entropy sources make a page render differently on each load — Math.random
|
||||||
|
* (shuffled carousels, random ids, A/B jitter) and the wall clock ("posted N
|
||||||
|
* minutes ago", schedule/countdown logic). Seeding the first and pinning the
|
||||||
|
* second makes a capture reproducible: the same site captured under the same
|
||||||
|
* {seed, epoch} paints the same DOM.
|
||||||
|
*
|
||||||
|
* CAUTIONS (all honored below):
|
||||||
|
* - performance.now() is left REAL. Motion capture (marquee/rotator sampling in
|
||||||
|
* capture/motion.ts) measures real velocities and observation windows off it;
|
||||||
|
* pinning it would corrupt those measurements. Only the Date wall clock moves.
|
||||||
|
* - The Date epoch is a PARAMETER, recorded in capture metadata — a recapture
|
||||||
|
* uses a fresh-but-recorded value. Generation determinism only requires a
|
||||||
|
* frozen capture to produce byte-stable output, which is unaffected by which
|
||||||
|
* epoch was chosen (the epoch is frozen into the capture, not re-derived).
|
||||||
|
* - The clock still ADVANCES from the pinned epoch (via the real-clock delta),
|
||||||
|
* so elapsed-time logic (setTimeout-driven reveals, animation timing that
|
||||||
|
* reads Date, lazy-load debounces) behaves — only the absolute origin is
|
||||||
|
* pinned, not the passage of time.
|
||||||
|
* - Date is patched consistently across constructor / Date.now() / getTime() /
|
||||||
|
* valueOf() / Symbol.toPrimitive by shifting the underlying instant for BOTH
|
||||||
|
* the no-arg constructor and Date.now(); every instance method then reads the
|
||||||
|
* shifted instant through the native prototype, so cookie-banner / schedule
|
||||||
|
* gates that call getTime()/valueOf() see the same pinned clock as now().
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Default fixed epoch: 2026-01-01T00:00:00.000Z. Callers should pass an explicit,
|
||||||
|
* recorded epoch (see captureEpochMs); this is only the fallback. */
|
||||||
|
export const DEFAULT_CAPTURE_EPOCH_MS = 1767225600000;
|
||||||
|
|
||||||
|
/** Default PRNG seed (mulberry32 initial state). A non-zero constant so the first
|
||||||
|
* draw is well-mixed; recorded alongside the epoch for reproducibility. */
|
||||||
|
export const DEFAULT_PRNG_SEED = 0x9e3779b9;
|
||||||
|
|
||||||
|
/** Resolve the epoch a capture run should pin to. Env override lets a recapture
|
||||||
|
* reuse a recorded value; otherwise a fresh timestamp is taken at run start and
|
||||||
|
* recorded in metadata. Determinism of GENERATION does not depend on this value —
|
||||||
|
* only on the capture being frozen once chosen. */
|
||||||
|
export function captureEpochMs(override?: number | string): number {
|
||||||
|
if (typeof override === "number" && Number.isFinite(override)) return override;
|
||||||
|
if (typeof override === "string" && override.trim()) {
|
||||||
|
const parsed = Number(override);
|
||||||
|
if (Number.isFinite(parsed)) return parsed;
|
||||||
|
const asDate = Date.parse(override);
|
||||||
|
if (Number.isFinite(asDate)) return asDate;
|
||||||
|
}
|
||||||
|
return DEFAULT_CAPTURE_EPOCH_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the init-script SOURCE STRING injected via page.addInitScript before any
|
||||||
|
* page script runs. Returned as a raw string (not a function) so tsx/esbuild never
|
||||||
|
* transforms it — the browser receives exactly these bytes.
|
||||||
|
*
|
||||||
|
* Interpolation is numeric-only (Number.isFinite-guarded), so no string escaping /
|
||||||
|
* injection surface exists.
|
||||||
|
*/
|
||||||
|
export function buildDeterministicEnvShim(opts?: { seed?: number; epochMs?: number }): string {
|
||||||
|
const seed = Number.isFinite(opts?.seed) ? Math.trunc(opts!.seed!) : DEFAULT_PRNG_SEED;
|
||||||
|
const epoch = Number.isFinite(opts?.epochMs) ? Math.trunc(opts!.epochMs!) : DEFAULT_CAPTURE_EPOCH_MS;
|
||||||
|
// NOTE: `delta` is computed once at shim-eval time (the earliest possible moment,
|
||||||
|
// before page scripts) so the pinned origin is stable; the real clock then adds
|
||||||
|
// elapsed time on top, keeping relative time truthful.
|
||||||
|
return `(() => {
|
||||||
|
"use strict";
|
||||||
|
// ---- Seeded PRNG (mulberry32): deterministic, well-distributed, cheap ----
|
||||||
|
let __s = ${seed} | 0;
|
||||||
|
const __rand = function random() {
|
||||||
|
__s = (__s + 0x6d2b79f5) | 0;
|
||||||
|
let t = Math.imul(__s ^ (__s >>> 15), 1 | __s);
|
||||||
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
||||||
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||||
|
};
|
||||||
|
try { Object.defineProperty(Math, "random", { value: __rand, writable: true, configurable: true }); }
|
||||||
|
catch (e) { try { Math.random = __rand; } catch (e2) {} }
|
||||||
|
|
||||||
|
// ---- Pinned-but-advancing wall clock. performance.now() stays REAL. ----
|
||||||
|
const RealDate = Date;
|
||||||
|
const realNow = () => RealDate.now();
|
||||||
|
// Shift so the FIRST observed instant is the pinned epoch; real elapsed time is
|
||||||
|
// then added on top (delta is negative if the machine clock is ahead of epoch).
|
||||||
|
const delta = ${epoch} - realNow();
|
||||||
|
const shiftedNow = () => realNow() + delta;
|
||||||
|
function DittoDate(...args) {
|
||||||
|
// \`new Date()\` with no args => pinned clock; every other form is delegated to
|
||||||
|
// the native constructor unchanged (parsing a string, y/m/d, a millis value).
|
||||||
|
const inst = args.length === 0 ? new RealDate(shiftedNow()) : new RealDate(...args);
|
||||||
|
// Re-tag the prototype so instanceof + inherited getTime/valueOf/toISOString work.
|
||||||
|
Object.setPrototypeOf(inst, DittoDate.prototype);
|
||||||
|
return inst;
|
||||||
|
}
|
||||||
|
// Inherit the full native Date prototype (getTime, valueOf, getFullYear,
|
||||||
|
// toISOString, Symbol.toPrimitive, ...) so every read is consistent with now().
|
||||||
|
DittoDate.prototype = Object.create(RealDate.prototype);
|
||||||
|
DittoDate.prototype.constructor = DittoDate;
|
||||||
|
DittoDate.now = shiftedNow;
|
||||||
|
DittoDate.parse = RealDate.parse;
|
||||||
|
DittoDate.UTC = RealDate.UTC;
|
||||||
|
try { Object.defineProperty(globalThis, "Date", { value: DittoDate, writable: true, configurable: true }); }
|
||||||
|
catch (e) { try { globalThis.Date = DittoDate; } catch (e2) {} }
|
||||||
|
})();`;
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import type { AssetGraph } from "../infer/assets.js";
|
|||||||
import type { FontGraph } from "../infer/fonts.js";
|
import type { FontGraph } from "../infer/fonts.js";
|
||||||
import type { Section } from "../infer/sections.js";
|
import type { Section } from "../infer/sections.js";
|
||||||
import type { CaptureResult } from "../capture/capture.js";
|
import type { CaptureResult } from "../capture/capture.js";
|
||||||
|
import { WALL_RE } from "../util/captureFailure.js";
|
||||||
|
|
||||||
export type GateResult = {
|
export type GateResult = {
|
||||||
gate: string;
|
gate: string;
|
||||||
@@ -117,7 +118,8 @@ export function countVisibleInCaptureHiddenInClone(
|
|||||||
// page: an egress/bot wall, a near-empty shell, or a cookie/consent modal that was
|
// page: an egress/bot wall, a near-empty shell, or a cookie/consent modal that was
|
||||||
// never dismissed (the clone then reproduces the modal, so the perceptual gate is
|
// never dismissed (the clone then reproduces the modal, so the perceptual gate is
|
||||||
// fooled too). This gate flags those captures so a "perfect" score can't hide them.
|
// fooled too). This gate flags those captures so a "perfect" score can't hide them.
|
||||||
const WALL_RE = /blocked by egress|access denied|access to this page has been denied|are you a (human|robot)|verify you are human|enable javascript to|please enable javascript|checking your browser|just a moment|attention required|request blocked|why have i been blocked|captcha|cf-browser-verification|ddos protection by/i;
|
// WALL_RE lives in util/captureFailure.ts — shared with the capture-side fast-fail so
|
||||||
|
// the abort and the grade can never drift on what counts as a wall.
|
||||||
|
|
||||||
export function gatePollution(ir: IR, capture: CaptureResult, viewports: number[]): GateResult {
|
export function gatePollution(ir: IR, capture: CaptureResult, viewports: number[]): GateResult {
|
||||||
const issues: string[] = [];
|
const issues: string[] = [];
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
import { describe, it } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import {
|
||||||
|
WALL_RE,
|
||||||
|
isWallText,
|
||||||
|
isBotWall,
|
||||||
|
WALL_MAX_NODES,
|
||||||
|
classifyNavFailure,
|
||||||
|
isRetryableNavFailure,
|
||||||
|
} from "../src/util/captureFailure.js";
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Item 3: capture-side fast-fail predicates (pure — no browser).
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
describe("captureFailure: wall-text detection", () => {
|
||||||
|
it("matches common bot/auth-wall fingerprints", () => {
|
||||||
|
for (const t of [
|
||||||
|
"Just a moment...",
|
||||||
|
"Checking your browser before accessing",
|
||||||
|
"Please enable JavaScript to continue",
|
||||||
|
"Access to this page has been denied",
|
||||||
|
"Are you a human?",
|
||||||
|
"Verify you are human",
|
||||||
|
"Attention Required! | Cloudflare",
|
||||||
|
"Please complete the CAPTCHA",
|
||||||
|
"DDoS protection by Cloudflare",
|
||||||
|
]) {
|
||||||
|
assert.ok(isWallText(t), `expected wall text to match: ${t}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not match ordinary marketing/body copy", () => {
|
||||||
|
for (const t of [
|
||||||
|
"Welcome to our store — shop the new collection",
|
||||||
|
"Our team of humans is here to help you 24/7",
|
||||||
|
"Enable notifications to get the latest deals",
|
||||||
|
"About us · Careers · Contact",
|
||||||
|
]) {
|
||||||
|
assert.equal(isWallText(t), false, `expected NOT to match: ${t}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("WALL_RE is the same instance shared with the pollution gate (single source of truth)", () => {
|
||||||
|
// gates.ts imports WALL_RE from this module; assert the export is a real RegExp
|
||||||
|
// so a refactor that turns it into a function would fail here.
|
||||||
|
assert.ok(WALL_RE instanceof RegExp);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("captureFailure: isBotWall (matches the pollution gate's small+wall rule)", () => {
|
||||||
|
it("flags a small page with wall text", () => {
|
||||||
|
assert.ok(isBotWall({ text: "Just a moment...", nodes: 8 }));
|
||||||
|
assert.ok(isBotWall({ text: "checking your browser", nodes: WALL_MAX_NODES - 1 }));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does NOT flag a large page that merely mentions a wall phrase", () => {
|
||||||
|
// A real page can say "captcha" in its help docs — node count is the discriminator.
|
||||||
|
assert.equal(isBotWall({ text: "how our captcha works", nodes: WALL_MAX_NODES }), false);
|
||||||
|
assert.equal(isBotWall({ text: "how our captcha works", nodes: 5000 }), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does NOT flag a small page with no wall text", () => {
|
||||||
|
assert.equal(isBotWall({ text: "Home · About · Contact", nodes: 10 }), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is null-safe", () => {
|
||||||
|
assert.equal(isBotWall(null), false);
|
||||||
|
assert.equal(isBotWall(undefined), false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("captureFailure: classifyNavFailure", () => {
|
||||||
|
it("classifies session-death / crash / transient network as retryable", () => {
|
||||||
|
for (const msg of [
|
||||||
|
"Target page, context or browser has been closed",
|
||||||
|
"Target closed",
|
||||||
|
"browser has been closed",
|
||||||
|
"page has crashed",
|
||||||
|
"net::ERR_CONNECTION_RESET at https://example.com",
|
||||||
|
"net::ERR_TIMED_OUT",
|
||||||
|
"Timeout 45000ms exceeded",
|
||||||
|
"Navigation timeout of 30000 ms exceeded",
|
||||||
|
]) {
|
||||||
|
assert.equal(classifyNavFailure(new Error(msg)), "retryable", msg);
|
||||||
|
assert.equal(isRetryableNavFailure(new Error(msg)), true, msg);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("classifies wall fingerprints as wall (not retryable — a fresh context can't help)", () => {
|
||||||
|
const e = new Error("navigation failed: just a moment... checking your browser");
|
||||||
|
assert.equal(classifyNavFailure(e), "wall");
|
||||||
|
assert.equal(isRetryableNavFailure(e), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("classifies hard failures as terminal (not retryable)", () => {
|
||||||
|
for (const msg of [
|
||||||
|
"net::ERR_NAME_NOT_RESOLVED",
|
||||||
|
"net::ERR_CERT_AUTHORITY_INVALID",
|
||||||
|
"net::ERR_CONNECTION_REFUSED",
|
||||||
|
"something totally unexpected",
|
||||||
|
]) {
|
||||||
|
assert.equal(classifyNavFailure(new Error(msg)), "terminal", msg);
|
||||||
|
assert.equal(isRetryableNavFailure(new Error(msg)), false, msg);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("wall wins over a transient signature (a wall served over a reset is still a wall)", () => {
|
||||||
|
const e = new Error("net::ERR_CONNECTION_RESET — access denied");
|
||||||
|
assert.equal(classifyNavFailure(e), "wall");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts a bare string or non-Error value", () => {
|
||||||
|
assert.equal(classifyNavFailure("Target closed"), "retryable");
|
||||||
|
assert.equal(classifyNavFailure(undefined), "terminal");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
import { describe, it, before, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { chromium, type Browser, type Page } from "playwright";
|
||||||
|
import {
|
||||||
|
buildDeterministicEnvShim,
|
||||||
|
captureEpochMs,
|
||||||
|
DEFAULT_CAPTURE_EPOCH_MS,
|
||||||
|
DEFAULT_PRNG_SEED,
|
||||||
|
} from "../src/util/envShim.js";
|
||||||
|
|
||||||
|
const ESBUILD_SHIM =
|
||||||
|
"globalThis.__name = globalThis.__name || ((fn) => fn);" +
|
||||||
|
"globalThis.__defProp = globalThis.__defProp || Object.defineProperty;";
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Item 1: deterministic-env shim — pure epoch resolution + in-browser behavior.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
describe("envShim: captureEpochMs resolution", () => {
|
||||||
|
it("defaults to the fixed epoch when no override is given", () => {
|
||||||
|
assert.equal(captureEpochMs(), DEFAULT_CAPTURE_EPOCH_MS);
|
||||||
|
assert.equal(captureEpochMs(undefined), DEFAULT_CAPTURE_EPOCH_MS);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes a finite numeric override through unchanged", () => {
|
||||||
|
assert.equal(captureEpochMs(1234567890), 1234567890);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("parses a numeric string and an ISO date string", () => {
|
||||||
|
assert.equal(captureEpochMs("1234567890"), 1234567890);
|
||||||
|
assert.equal(captureEpochMs("2020-01-01T00:00:00Z"), Date.parse("2020-01-01T00:00:00Z"));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to the default for garbage input", () => {
|
||||||
|
assert.equal(captureEpochMs("not-a-date"), DEFAULT_CAPTURE_EPOCH_MS);
|
||||||
|
assert.equal(captureEpochMs(NaN), DEFAULT_CAPTURE_EPOCH_MS);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("envShim: buildDeterministicEnvShim source safety", () => {
|
||||||
|
it("interpolates only numeric literals (no injection surface)", () => {
|
||||||
|
const src = buildDeterministicEnvShim({ seed: 42, epochMs: 1000 });
|
||||||
|
assert.match(src, /let __s = 42 \| 0;/);
|
||||||
|
assert.match(src, /const delta = 1000 - realNow\(\);/);
|
||||||
|
// performance.now must remain untouched — the shim only patches Math.random + Date,
|
||||||
|
// never assigns to performance.now (motion.ts velocity sampling depends on the real one).
|
||||||
|
assert.equal(/performance\s*\.\s*now\s*=/.test(src), false);
|
||||||
|
assert.equal(/defineProperty\s*\(\s*performance/.test(src), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("truncates non-integer seed/epoch and uses defaults for non-finite", () => {
|
||||||
|
const src = buildDeterministicEnvShim({ seed: 3.9, epochMs: 5.9 });
|
||||||
|
assert.match(src, /let __s = 3 \| 0;/);
|
||||||
|
assert.match(src, /const delta = 5 - realNow\(\);/);
|
||||||
|
const dflt = buildDeterministicEnvShim({ seed: NaN, epochMs: Infinity });
|
||||||
|
assert.match(dflt, new RegExp(`let __s = ${DEFAULT_PRNG_SEED} \\| 0;`));
|
||||||
|
assert.match(dflt, new RegExp(`const delta = ${DEFAULT_CAPTURE_EPOCH_MS} - realNow\\(\\);`));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const VW = 1280, VH = 800;
|
||||||
|
|
||||||
|
describe("envShim: in-browser behavior", () => {
|
||||||
|
let browser: Browser;
|
||||||
|
before(async () => { browser = await chromium.launch(); });
|
||||||
|
after(async () => { await browser.close(); });
|
||||||
|
|
||||||
|
// Fresh page with the shim installed BEFORE any page script (via addInitScript),
|
||||||
|
// exactly as capture.ts wires it. We navigate to a data: URL rather than setContent
|
||||||
|
// so the document is CREATED via navigation — addInitScript fires on document
|
||||||
|
// creation, which setContent (a content swap on the existing about:blank) skips.
|
||||||
|
// The inline <script> then runs after the init scripts and samples the pinned env.
|
||||||
|
const loadWithShim = async (opts: { seed?: number; epochMs?: number }, bodyScript: string): Promise<Page> => {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
await page.setViewportSize({ width: VW, height: VH });
|
||||||
|
await page.addInitScript(ESBUILD_SHIM);
|
||||||
|
await page.addInitScript(buildDeterministicEnvShim(opts));
|
||||||
|
const html = `<!doctype html><html><body><script>${bodyScript}</script></body></html>`;
|
||||||
|
await page.goto("data:text/html," + encodeURIComponent(html));
|
||||||
|
return page;
|
||||||
|
};
|
||||||
|
|
||||||
|
it("seeds Math.random reproducibly across two independent loads (same seed → same sequence)", async () => {
|
||||||
|
const script = "window.__draws = Array.from({length: 10}, () => Math.random());";
|
||||||
|
const p1 = await loadWithShim({ seed: 12345, epochMs: DEFAULT_CAPTURE_EPOCH_MS }, script);
|
||||||
|
const p2 = await loadWithShim({ seed: 12345, epochMs: DEFAULT_CAPTURE_EPOCH_MS }, script);
|
||||||
|
const [a, b] = await Promise.all([
|
||||||
|
p1.evaluate(() => (window as unknown as { __draws: number[] }).__draws),
|
||||||
|
p2.evaluate(() => (window as unknown as { __draws: number[] }).__draws),
|
||||||
|
]);
|
||||||
|
assert.deepEqual(a, b, "two loads with the same seed produce identical Math.random sequences");
|
||||||
|
assert.ok(a.every((n) => n >= 0 && n < 1), "draws are in [0,1)");
|
||||||
|
// Not a constant stream — a broken PRNG returning the same value would also be
|
||||||
|
// "stable"; assert genuine variation so the test can't pass trivially.
|
||||||
|
assert.ok(new Set(a).size > 1, "the sequence varies");
|
||||||
|
await p1.close(); await p2.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a different seed yields a different sequence", async () => {
|
||||||
|
const script = "window.__draws = Array.from({length: 10}, () => Math.random());";
|
||||||
|
const p1 = await loadWithShim({ seed: 1, epochMs: DEFAULT_CAPTURE_EPOCH_MS }, script);
|
||||||
|
const p2 = await loadWithShim({ seed: 2, epochMs: DEFAULT_CAPTURE_EPOCH_MS }, script);
|
||||||
|
const [a, b] = await Promise.all([
|
||||||
|
p1.evaluate(() => (window as unknown as { __draws: number[] }).__draws),
|
||||||
|
p2.evaluate(() => (window as unknown as { __draws: number[] }).__draws),
|
||||||
|
]);
|
||||||
|
assert.notDeepEqual(a, b);
|
||||||
|
await p1.close(); await p2.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("pins the Date epoch consistently across constructor / now() / getTime() / valueOf() / toISOString()", async () => {
|
||||||
|
const epoch = Date.parse("2021-06-15T12:00:00Z");
|
||||||
|
const page = await loadWithShim(
|
||||||
|
{ seed: 1, epochMs: epoch },
|
||||||
|
`window.__t = {
|
||||||
|
now: Date.now(),
|
||||||
|
ctorGetTime: new Date().getTime(),
|
||||||
|
ctorValueOf: new Date().valueOf(),
|
||||||
|
iso: new Date().toISOString(),
|
||||||
|
};`,
|
||||||
|
);
|
||||||
|
const t = await page.evaluate(() => (window as unknown as { __t: { now: number; ctorGetTime: number; ctorValueOf: number; iso: string } }).__t);
|
||||||
|
// All reads land at the pinned epoch (within a few ms of real elapsed time between them).
|
||||||
|
assert.ok(Math.abs(t.now - epoch) < 5000, `Date.now near epoch: ${t.now} vs ${epoch}`);
|
||||||
|
assert.ok(Math.abs(t.ctorGetTime - epoch) < 5000, "new Date().getTime() near epoch");
|
||||||
|
assert.ok(Math.abs(t.ctorValueOf - epoch) < 5000, "new Date().valueOf() near epoch");
|
||||||
|
assert.match(t.iso, /^2021-06-15T12:00:0/, `toISOString pinned: ${t.iso}`);
|
||||||
|
// now() and getTime() must agree (both read the same shifted instant).
|
||||||
|
assert.ok(Math.abs(t.now - t.ctorGetTime) < 5000, "now() and getTime() agree");
|
||||||
|
await page.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the clock ADVANCING from the pinned epoch (timers still behave)", async () => {
|
||||||
|
const epoch = DEFAULT_CAPTURE_EPOCH_MS;
|
||||||
|
const page = await loadWithShim(
|
||||||
|
{ seed: 1, epochMs: epoch },
|
||||||
|
`window.__adv = new Promise((res) => {
|
||||||
|
const t0 = Date.now();
|
||||||
|
setTimeout(() => res({ t0, t1: Date.now() }), 60);
|
||||||
|
});`,
|
||||||
|
);
|
||||||
|
const { t0, t1 } = await page.evaluate(() => (window as unknown as { __adv: Promise<{ t0: number; t1: number }> }).__adv);
|
||||||
|
assert.ok(t0 >= epoch - 5000 && t0 <= epoch + 5000, "start is pinned near epoch");
|
||||||
|
assert.ok(t1 > t0, "time advances across a real setTimeout delay");
|
||||||
|
await page.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves performance.now() REAL (motion sampling depends on it) and advancing", async () => {
|
||||||
|
const page = await loadWithShim(
|
||||||
|
{ seed: 1, epochMs: DEFAULT_CAPTURE_EPOCH_MS },
|
||||||
|
`window.__perf = new Promise((res) => {
|
||||||
|
const p0 = performance.now();
|
||||||
|
setTimeout(() => res({ p0, p1: performance.now() }), 40);
|
||||||
|
});`,
|
||||||
|
);
|
||||||
|
const { p0, p1 } = await page.evaluate(() => (window as unknown as { __perf: Promise<{ p0: number; p1: number }> }).__perf);
|
||||||
|
// performance.now() is a real monotonic clock from navigation start — small, not the epoch.
|
||||||
|
assert.ok(p0 >= 0 && p0 < 60_000, `performance.now() is real (not pinned to epoch): ${p0}`);
|
||||||
|
assert.ok(p1 > p0, "performance.now() advances");
|
||||||
|
await page.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
import { describe, it, before, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { chromium, type Browser, type Page } from "playwright";
|
||||||
|
import { discoverLazyAssetsInPage } from "../src/capture/capture.js";
|
||||||
|
|
||||||
|
const ESBUILD_SHIM =
|
||||||
|
"globalThis.__name = globalThis.__name || ((fn) => fn);" +
|
||||||
|
"globalThis.__defProp = globalThis.__defProp || Object.defineProperty;";
|
||||||
|
|
||||||
|
const VW = 1280, VH = 800;
|
||||||
|
const BASE = "https://example.com/page/";
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Item 2: extended lazy-asset discovery sweep (in-page discovery function).
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
describe("lazySweep: discoverLazyAssetsInPage", () => {
|
||||||
|
let browser: Browser;
|
||||||
|
before(async () => { browser = await chromium.launch(); });
|
||||||
|
after(async () => { await browser.close(); });
|
||||||
|
|
||||||
|
const discover = async (bodyHtml: string): Promise<string[]> => {
|
||||||
|
const page: Page = await browser.newPage();
|
||||||
|
await page.setViewportSize({ width: VW, height: VH });
|
||||||
|
await page.addInitScript(ESBUILD_SHIM);
|
||||||
|
// A <base> gives relative refs a stable absolute form to assert against.
|
||||||
|
await page.setContent(`<!doctype html><html><head><base href="${BASE}"></head><body>${bodyHtml}</body></html>`);
|
||||||
|
await page.evaluate(ESBUILD_SHIM);
|
||||||
|
const refs = await page.evaluate(discoverLazyAssetsInPage);
|
||||||
|
await page.close();
|
||||||
|
return refs;
|
||||||
|
};
|
||||||
|
|
||||||
|
it("discovers img/source src + srcset inside <noscript> fallback markup", async () => {
|
||||||
|
const refs = await discover(`
|
||||||
|
<noscript>
|
||||||
|
<img src="hero.jpg" srcset="hero-2x.jpg 2x, hero-3x.jpg 3x">
|
||||||
|
<picture><source srcset="alt.webp 1x, alt-2x.webp 2x"></picture>
|
||||||
|
</noscript>
|
||||||
|
`);
|
||||||
|
assert.ok(refs.includes(BASE + "hero.jpg"), "noscript img src");
|
||||||
|
assert.ok(refs.includes(BASE + "hero-2x.jpg"), "first srcset variant from noscript img");
|
||||||
|
assert.ok(refs.includes(BASE + "alt.webp"), "first srcset variant from noscript source");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("discovers url(...) values from an element's inline style attribute", async () => {
|
||||||
|
const refs = await discover(`
|
||||||
|
<div style="background-image: url('bg.png'); color: red"></div>
|
||||||
|
<span style='background: url("sprite.svg") no-repeat'></span>
|
||||||
|
`);
|
||||||
|
assert.ok(refs.includes(BASE + "bg.png"), "inline background-image url()");
|
||||||
|
assert.ok(refs.includes(BASE + "sprite.svg"), "inline background shorthand url()");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("discovers data-background / data-background-image (raw URL and url(...) forms)", async () => {
|
||||||
|
const refs = await discover(`
|
||||||
|
<div data-background="lazybg.jpg"></div>
|
||||||
|
<div data-background-image="url('lazybg2.png')"></div>
|
||||||
|
`);
|
||||||
|
assert.ok(refs.includes(BASE + "lazybg.jpg"), "raw data-background URL");
|
||||||
|
assert.ok(refs.includes(BASE + "lazybg2.png"), "url()-wrapped data-background-image");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("discovers the resolved src of an img[loading=lazy]", async () => {
|
||||||
|
const refs = await discover(`<img loading="lazy" src="belowfold.jpg">`);
|
||||||
|
assert.ok(refs.includes(BASE + "belowfold.jpg"));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops data: URIs and dedupes + sorts", async () => {
|
||||||
|
const refs = await discover(`
|
||||||
|
<div style="background: url('dup.png')"></div>
|
||||||
|
<div data-background="dup.png"></div>
|
||||||
|
<img loading="lazy" src="data:image/gif;base64,R0lGOD">
|
||||||
|
`);
|
||||||
|
assert.equal(refs.filter((u) => u.endsWith("dup.png")).length, 1, "deduped across channels");
|
||||||
|
assert.equal(refs.some((u) => u.startsWith("data:")), false, "data: URIs dropped");
|
||||||
|
assert.deepEqual(refs, [...refs].sort(), "sorted");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns nothing for a page with no lazy references", async () => {
|
||||||
|
const refs = await discover(`<div><p>hello</p><img src="eager.jpg"></div>`);
|
||||||
|
// The eager <img src> is the walker's job, not this sweep — the sweep only harvests
|
||||||
|
// the channels the walker misses.
|
||||||
|
assert.deepEqual(refs, []);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user