Initial commit
This commit is contained in:
Samraaj Bath
@@ -0,0 +1,996 @@
|
||||
import { chromium, type Browser, type BrowserContext } from "playwright";
|
||||
import { join } from "node:path";
|
||||
import { collectPage, type PageSnapshot, type FontFace } from "./walker.js";
|
||||
import { tagElements, captureInteractions, type InteractionCapture } from "./interactions.js";
|
||||
import { captureMotion, probeReveals, type MotionCapture } from "./motion.js";
|
||||
import { discoverBreakpoints } from "./breakpoints.js";
|
||||
import { writeJSON, writeJSONCompact, writeBytes, ensureDir } from "../util/fsx.js";
|
||||
import { sha1_12, round } from "../util/canonical.js";
|
||||
|
||||
export const REQUIRED_VIEWPORTS = [375, 768, 1280, 1920] as const;
|
||||
// The dense width set captured for SIZE INFERENCE: a node sampled at 9 widths reveals its sizing
|
||||
// law (constant / proportional / clamped / flex-distributed) far better than 4 — enough to drop a
|
||||
// baked px for a relative construct with confidence (and to surface a shrunk item's natural size,
|
||||
// which needs widths beyond 1920). REQUIRED_VIEWPORTS ⊂ this; only those 4 carry responsive bands.
|
||||
export const SAMPLE_VIEWPORTS = [375, 480, 640, 768, 1024, 1280, 1536, 1920, 2560] as const;
|
||||
|
||||
const VIEWPORT_HEIGHTS: Record<number, number> = {
|
||||
375: 812,
|
||||
480: 854,
|
||||
640: 960,
|
||||
768: 1024,
|
||||
1024: 768,
|
||||
1280: 800,
|
||||
1536: 864,
|
||||
1920: 1080,
|
||||
2560: 1440,
|
||||
};
|
||||
|
||||
const DESKTOP_UA =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
||||
|
||||
// Defines esbuild/tsx runtime helpers in the page so serialized evaluate
|
||||
// callbacks that reference them don't throw ReferenceError.
|
||||
const ESBUILD_SHIM =
|
||||
"globalThis.__name = globalThis.__name || ((fn) => fn);" +
|
||||
"globalThis.__defProp = globalThis.__defProp || Object.defineProperty;";
|
||||
|
||||
export type DiscoveredAsset = {
|
||||
url: string;
|
||||
type: string; // image|svg|video|font|lottie|css|manifest|other
|
||||
contentType: string | null;
|
||||
status: number | null;
|
||||
storedAs: string | null; // sha1-named file in assets-store, or null if not downloaded
|
||||
bytes: number; // size of stored file
|
||||
via: string[]; // discovery sources
|
||||
};
|
||||
|
||||
export type SeoResource = {
|
||||
kind: "robots" | "sitemap" | "llms" | "llms-full";
|
||||
url: string;
|
||||
status: number | null;
|
||||
contentType: string | null;
|
||||
text?: string;
|
||||
};
|
||||
|
||||
export type CaptureResult = {
|
||||
sourceUrl: string;
|
||||
capturedAt: string;
|
||||
viewports: number[];
|
||||
// Browser-as-oracle: the widths where the SOURCE layout actually restructures (display /
|
||||
// flex-direction / wrap / grid-track-count / position / visibility flips), found by sweeping the
|
||||
// viewport and binary-searching each discrete-signature change. Ground truth for the real
|
||||
// responsive band edges — distinct from REQUIRED_VIEWPORTS (our fixed capture widths). Absent if
|
||||
// discovery was disabled or the sweep failed. See breakpoints.ts.
|
||||
breakpoints?: number[];
|
||||
perViewport: Array<{
|
||||
viewport: number;
|
||||
height: number;
|
||||
scrollHeight: number;
|
||||
nodeCount: number;
|
||||
truncated: boolean;
|
||||
overlaysRemaining?: number;
|
||||
blocking?: boolean;
|
||||
quiescent?: boolean;
|
||||
}>;
|
||||
assets: DiscoveredAsset[];
|
||||
seoResources?: SeoResource[];
|
||||
fontFaces: FontFace[];
|
||||
cssTexts: string[]; // sha1 names of stored css files
|
||||
// Stage 2: overlay/popup dismissal audit (union of actions across viewports).
|
||||
dismissal?: { dismissed: string[]; overlaysRemaining: number; removed: number; videoStills: number; blocking: boolean };
|
||||
// Stage 4: optional interaction capture (hover/focus + recognized patterns).
|
||||
interaction?: InteractionCapture;
|
||||
// Stage 5: optional motion capture (WAAPI animations + rotating text). CSS @keyframes
|
||||
// motion is reconstructed from the IR, so it isn't re-captured here.
|
||||
motion?: MotionCapture;
|
||||
};
|
||||
|
||||
function viewportHeight(width: number): number {
|
||||
return VIEWPORT_HEIGHTS[width] ?? Math.round(width * 0.66);
|
||||
}
|
||||
|
||||
function extFromUrl(url: string): string {
|
||||
try {
|
||||
const p = new URL(url).pathname;
|
||||
const dot = p.lastIndexOf(".");
|
||||
if (dot >= 0 && dot > p.lastIndexOf("/")) {
|
||||
const ext = p.slice(dot + 1).toLowerCase().slice(0, 5);
|
||||
if (/^[a-z0-9]+$/.test(ext)) return ext;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
return "";
|
||||
}
|
||||
|
||||
const CONTENT_TYPE_EXT: Record<string, string> = {
|
||||
"image/webp": "webp", "image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg",
|
||||
"image/avif": "avif", "image/gif": "gif", "image/svg+xml": "svg",
|
||||
"image/x-icon": "ico", "image/vnd.microsoft.icon": "ico", "image/bmp": "bmp",
|
||||
"video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", "video/ogg": "ogv",
|
||||
"font/woff2": "woff2", "font/woff": "woff", "font/ttf": "ttf", "font/otf": "otf",
|
||||
"application/font-woff2": "woff2", "application/font-woff": "woff",
|
||||
"application/x-font-ttf": "ttf", "application/vnd.ms-fontobject": "eot",
|
||||
"text/css": "css", "application/json": "json", "application/manifest+json": "webmanifest",
|
||||
};
|
||||
|
||||
function extFromContentType(contentType: string | null): string {
|
||||
if (!contentType) return "";
|
||||
const ct = contentType.split(";")[0]!.trim().toLowerCase();
|
||||
return CONTENT_TYPE_EXT[ct] ?? "";
|
||||
}
|
||||
|
||||
function classifyAsset(url: string, contentType: string | null): string | null {
|
||||
const u = url.toLowerCase().split("?")[0]!;
|
||||
const ct = (contentType || "").toLowerCase();
|
||||
if (u.endsWith(".svg") || ct === "image/svg+xml") return "svg";
|
||||
if (/\.(jpg|jpeg|png|webp|avif|gif|ico|bmp)$/.test(u) || ct.startsWith("image/")) return "image";
|
||||
if (/\.(mp4|mov|webm|m4v|ogv)$/.test(u) || ct.startsWith("video/")) return "video";
|
||||
if (/\.(woff2|woff|ttf|otf|eot)$/.test(u) || ct.startsWith("font/") ||
|
||||
ct.includes("font-woff") || ct.includes("x-font")) return "font";
|
||||
if (u.endsWith(".json") && (u.includes("lottie") || u.includes("animation"))) return "lottie";
|
||||
if (u.endsWith(".webmanifest") || /(?:^|\/)manifest\.json$/.test(u) || ct.includes("manifest+json")) return "manifest";
|
||||
if (u.endsWith(".css") || ct.startsWith("text/css")) return "css";
|
||||
return null;
|
||||
}
|
||||
|
||||
function isCss(url: string, contentType: string | null): boolean {
|
||||
return classifyAsset(url, contentType) === "css";
|
||||
}
|
||||
|
||||
async function autoScroll(page: import("playwright").Page, vpHeight: number): Promise<void> {
|
||||
// Scroll through the page to trigger lazy-loaded images/backgrounds, then return
|
||||
// to the top so document-coordinate bboxes are measured from a settled layout.
|
||||
await page.evaluate(async (step: number) => {
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const maxScroll = () => document.documentElement.scrollHeight - window.innerHeight;
|
||||
let y = 0;
|
||||
let guard = 0;
|
||||
while (y < maxScroll() && guard < 100) {
|
||||
y += step;
|
||||
window.scrollTo(0, y);
|
||||
await sleep(60);
|
||||
guard++;
|
||||
}
|
||||
window.scrollTo(0, 0);
|
||||
await sleep(120);
|
||||
}, Math.max(Math.round(vpHeight * 0.8), 300));
|
||||
}
|
||||
|
||||
async function settle(page: import("playwright").Page, maxMs = 2500): Promise<void> {
|
||||
try {
|
||||
await page.waitForLoadState("networkidle", { timeout: Math.max(maxMs / 2, 800) });
|
||||
} catch { /* ignore */ }
|
||||
try {
|
||||
// document.fonts.ready can never resolve when a font request hangs (heavy SaaS
|
||||
// pages), and page.evaluate has no default timeout — so bound it explicitly.
|
||||
await Promise.race([
|
||||
page.evaluate(() => (document as Document).fonts?.ready as unknown as Promise<void>),
|
||||
new Promise<void>((r) => setTimeout(r, 6000)),
|
||||
]);
|
||||
} catch { /* ignore */ }
|
||||
await page.waitForTimeout(250);
|
||||
}
|
||||
|
||||
export type DismissResult = { dismissed: string[]; overlaysRemaining: number; removed: number; blocking: boolean };
|
||||
|
||||
/**
|
||||
* Stage 2 — overlay/popup dismissal, phase 1: click the accept/close affordance.
|
||||
* Cookie-consent walls, newsletter modals, region/age/app-install interstitials
|
||||
* cover the real page; the capture must see the state a returning user sees.
|
||||
* Deterministic + replayable: click the same known/accept controls in DOM order.
|
||||
* Removal of a stuck overlay happens later (finalizeOverlays) AFTER a settle, so a
|
||||
* just-clicked dialog has time to close and unlock scrolling before we judge it.
|
||||
*/
|
||||
async function clickDismiss(page: import("playwright").Page): Promise<string[]> {
|
||||
try {
|
||||
return await Promise.race([
|
||||
page.evaluate(() => {
|
||||
const dismissed: string[] = [];
|
||||
const vis = (el: Element): boolean => {
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) return false;
|
||||
const r = (el as HTMLElement).getBoundingClientRect();
|
||||
return r.width > 0 && r.height > 0;
|
||||
};
|
||||
const click = (el: Element): void => { try { (el as HTMLElement).click(); } catch { /* ignore */ } };
|
||||
|
||||
// 1) Known consent-framework / generic close affordances, in priority order.
|
||||
const KNOWN = [
|
||||
"#onetrust-accept-btn-handler", "#accept-recommended-btn-handler", ".onetrust-close-btn-handler",
|
||||
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", "#CybotCookiebotDialogBodyButtonAccept",
|
||||
"#truste-consent-button", ".osano-cm-accept-all", ".osano-cm-dialog__close",
|
||||
"[data-testid='uc-accept-all-button']", "[data-testid='uc-deny-all-button']",
|
||||
"#didomi-notice-agree-button", ".didomi-continue-without-agreeing",
|
||||
".qc-cmp2-summary-buttons button[mode='primary']", "button[aria-label='Consent']",
|
||||
".cc-allow", ".cookie-consent-accept", "#hs-eu-confirmation-button", "#gdpr-consent-tool-wrapper button",
|
||||
];
|
||||
for (const sel of KNOWN) {
|
||||
try {
|
||||
for (const el of Array.from(document.querySelectorAll(sel))) {
|
||||
if (vis(el)) { click(el); dismissed.push(sel); break; }
|
||||
}
|
||||
} catch { /* invalid selector in this browser */ }
|
||||
}
|
||||
|
||||
// 2) Scoped text-button pass: only inside overlay-ish containers (dialogs,
|
||||
// or id/class naming cookie/consent/modal/popup/newsletter) so we never
|
||||
// click an ordinary page button.
|
||||
const ACCEPT = new Set([
|
||||
"accept", "accept all", "accept all cookies", "accept cookies", "accept & close",
|
||||
"i accept", "i agree", "agree", "agree and continue", "allow all", "allow cookies",
|
||||
"allow all cookies", "got it", "ok", "okay", "continue", "no thanks", "no, thanks",
|
||||
"dismiss", "close", "got it!", "understood", "yes, i agree",
|
||||
]);
|
||||
const containerSel =
|
||||
"[role='dialog'],[aria-modal='true'],[id*='cookie' i],[class*='cookie' i],[id*='consent' i]," +
|
||||
"[class*='consent' i],[class*='gdpr' i],[id*='gdpr' i],[class*='modal' i],[class*='popup' i]," +
|
||||
"[class*='newsletter' i],[class*='interstitial' i]";
|
||||
let containers: Element[] = [];
|
||||
try { containers = Array.from(document.querySelectorAll(containerSel)); } catch { /* ignore */ }
|
||||
for (const c of containers) {
|
||||
if (!vis(c)) continue;
|
||||
const btns = Array.from(c.querySelectorAll("button,[role='button'],a,input[type='button'],input[type='submit']"));
|
||||
for (const b of btns) {
|
||||
const t = (b.textContent || (b as HTMLInputElement).value || "").replace(/\s+/g, " ").trim().toLowerCase();
|
||||
if (t && ACCEPT.has(t) && vis(b)) { click(b); dismissed.push("text:" + t); break; }
|
||||
}
|
||||
}
|
||||
return dismissed;
|
||||
}),
|
||||
new Promise<string[]>((res) => setTimeout(() => res([]), 6000)),
|
||||
]);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 2 — overlay/popup dismissal, phase 2 (run after a settle). Detect any
|
||||
* full-viewport, high-z, fixed/sticky overlay still present. A fixed nav is wide
|
||||
* but short; a sticky sidebar is tall but narrow — both excluded; a consent wall /
|
||||
* modal backdrop covers most of both axes. If the page is *still scroll-locked*
|
||||
* (the modal is genuinely blocking) remove the overlay — but ONLY when its id/class
|
||||
* looks like a consent/modal layer, never a header/nav/main/footer, so legitimate
|
||||
* sticky chrome is never stripped. Reports `blocking` = a scroll-locking overlay we
|
||||
* could not clear (the pollution gate keys off this, not mere overlay presence).
|
||||
*/
|
||||
async function finalizeOverlays(page: import("playwright").Page): Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }> {
|
||||
try {
|
||||
return await Promise.race([
|
||||
page.evaluate(() => {
|
||||
const vw = window.innerWidth, vh = window.innerHeight;
|
||||
const bigOverlays = (): HTMLElement[] => {
|
||||
const out: HTMLElement[] = [];
|
||||
for (const el of Array.from(document.body.querySelectorAll("*"))) {
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.position !== "fixed" && cs.position !== "sticky") continue;
|
||||
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) continue;
|
||||
const r = (el as HTMLElement).getBoundingClientRect();
|
||||
const z = parseInt(cs.zIndex || "0", 10) || 0;
|
||||
const area = (r.width * r.height) / (vw * vh);
|
||||
if (area >= 0.5 && z >= 100 && r.width >= vw * 0.7 && r.height >= vh * 0.5) out.push(el as HTMLElement);
|
||||
}
|
||||
return out.filter((el) => !out.some((o) => o !== el && o.contains(el)));
|
||||
};
|
||||
const isLocked = (): boolean => {
|
||||
const b = document.body, h = document.documentElement;
|
||||
return getComputedStyle(b).overflow === "hidden" || getComputedStyle(h).overflow === "hidden" ||
|
||||
getComputedStyle(b).position === "fixed";
|
||||
};
|
||||
// A scroll-locked page behind a full-viewport overlay IS a blocking modal by
|
||||
// definition (legit pages don't scroll-lock). So remove ANY such overlay that
|
||||
// isn't page chrome — many modals/drawers carry no consent/modal keyword and
|
||||
// an icon-only close, so a keyword/aria allowlist misses them (ruggable's
|
||||
// z-[1001] drawer). PROTECTED guards real chrome (header/nav/footer) only.
|
||||
const PROTECTED = /header|navbar|nav-|site-nav|topbar|masthead|footer/i;
|
||||
const sig = (el: HTMLElement): string => `${el.id} ${el.className}`.toString();
|
||||
|
||||
const removedLabels: string[] = [];
|
||||
let removed = 0;
|
||||
let remaining = bigOverlays();
|
||||
if (remaining.length && isLocked()) {
|
||||
for (const el of remaining) {
|
||||
const s = sig(el);
|
||||
const z = parseInt(getComputedStyle(el).zIndex || "0", 10) || 0;
|
||||
// Scroll-locked + full-viewport ⇒ blocking modal; remove unless it's page
|
||||
// chrome. Always remove iframes (cross-origin close, unclickable) and the
|
||||
// max-z-index popup trick.
|
||||
const removable = !PROTECTED.test(s) || el.getAttribute("aria-modal") === "true" ||
|
||||
el.tagName === "IFRAME" || z >= 2_000_000_000;
|
||||
if (removable) { el.remove(); removed++; removedLabels.push((el.id || el.className || el.tagName).toString().slice(0, 40)); }
|
||||
}
|
||||
if (removed) { document.body.style.overflow = "visible"; document.documentElement.style.overflow = "visible"; document.body.style.position = "static"; }
|
||||
remaining = bigOverlays();
|
||||
}
|
||||
return { overlaysRemaining: remaining.length, removed, blocking: remaining.length > 0 && isLocked(), removedLabels };
|
||||
}),
|
||||
new Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }>((res) => setTimeout(() => res({ overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }), 6000)),
|
||||
]);
|
||||
} catch {
|
||||
return { overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 2 — animation settling. Wait until layout stops moving (entrance/scroll
|
||||
* reveals finished) before measuring, so geometry isn't sampled mid-transition
|
||||
* ("sizes off due to animations in progress"). Samples large-box geometry across
|
||||
* frames; resolves when stable for several windows or the bound elapses.
|
||||
*/
|
||||
async function waitForQuiescence(page: import("playwright").Page, maxMs = 4000): Promise<boolean> {
|
||||
try {
|
||||
return await Promise.race([
|
||||
page.evaluate(async (budget: number) => {
|
||||
const sample = (): string => {
|
||||
const els = Array.from(document.body.querySelectorAll("*")).filter((e) => {
|
||||
const r = (e as HTMLElement).getBoundingClientRect();
|
||||
return r.width > 80 && r.height > 40;
|
||||
}).slice(0, 240);
|
||||
return els.map((e) => {
|
||||
const r = (e as HTMLElement).getBoundingClientRect();
|
||||
return `${Math.round(r.x)},${Math.round(r.y)},${Math.round(r.width)},${Math.round(r.height)}`;
|
||||
}).join("|");
|
||||
};
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const deadline = Date.now() + budget;
|
||||
let prev = sample();
|
||||
let stable = 0;
|
||||
while (Date.now() < deadline && stable < 3) {
|
||||
await sleep(130);
|
||||
const cur = sample();
|
||||
if (cur === prev) stable++; else { stable = 0; prev = cur; }
|
||||
}
|
||||
return stable >= 3;
|
||||
}, maxMs),
|
||||
new Promise<boolean>((res) => setTimeout(() => res(false), maxMs + 1500)),
|
||||
]);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 2 — dynamic-media first frame. A streamed `<video>` has no deterministic
|
||||
* frame and its request aborts at snapshot time, so the element would render blank.
|
||||
* For videos lacking a `poster`, materialize a representative still and point the
|
||||
* element's poster at it (via a synthetic URL whose bytes the normal asset pipeline
|
||||
* rewrites to a local file). Two acquisition paths:
|
||||
* 1. canvas — draw the decoded frame; exact, but THROWS for cross-origin/tainted
|
||||
* videos (common: CDN-hosted hero videos with no CORS header).
|
||||
* 2. element screenshot (the fallback) — rasterize the element's painted region
|
||||
* via the DevTools protocol; works regardless of CORS and even when the video
|
||||
* decoded late, because it reads composited pixels rather than the media buffer.
|
||||
* This closes the "poster-less video renders blank" gap (e.g. descript's hero).
|
||||
*
|
||||
* Returns canvas stills (bytes ready) + the videos that still need a node-side
|
||||
* element screenshot (the page can't screenshot itself), each tagged with a stable
|
||||
* selector. Videos with no usable surface (no poster obtainable) fall back to the
|
||||
* transparent placeholder, same as before.
|
||||
*/
|
||||
type VideoStillPlan = { stills: Array<{ url: string; dataUrl: string }>; shots: Array<{ url: string; sel: string }> };
|
||||
async function captureVideoStills(page: import("playwright").Page): Promise<VideoStillPlan> {
|
||||
try {
|
||||
const plan = await Promise.race([
|
||||
page.evaluate(async () => {
|
||||
const stills: Array<{ url: string; dataUrl: string }> = [];
|
||||
const shots: Array<{ url: string; sel: string }> = [];
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const hash = (s: string): string => { let h = 5381; for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0; return h.toString(36); };
|
||||
const vids = Array.from(document.querySelectorAll("video"));
|
||||
let i = 0;
|
||||
for (const v of vids) {
|
||||
if (v.getAttribute("poster")) continue; // real poster already discovered
|
||||
const r = v.getBoundingClientRect();
|
||||
if (r.width < 16 || r.height < 16) continue; // not a visible media surface
|
||||
const idx = i++;
|
||||
v.setAttribute("data-clone-vid", String(idx));
|
||||
const key = v.currentSrc || (v.querySelector("source") as HTMLSourceElement | null)?.src || String(idx);
|
||||
const url = `https://clone-still.local/${idx}-${hash(key)}.jpg`;
|
||||
try { v.pause(); } catch { /* ignore */ }
|
||||
try { v.currentTime = 0; } catch { /* ignore */ }
|
||||
await sleep(100);
|
||||
let done = false;
|
||||
const w = v.videoWidth, h = v.videoHeight;
|
||||
if (w && h && v.readyState >= 2) {
|
||||
try {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = w; canvas.height = h;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (ctx) {
|
||||
ctx.drawImage(v, 0, 0, w, h);
|
||||
const dataUrl = canvas.toDataURL("image/jpeg", 0.82); // throws if tainted
|
||||
if (dataUrl.startsWith("data:image/jpeg")) { stills.push({ url, dataUrl }); done = true; }
|
||||
}
|
||||
} catch { /* tainted/cross-origin — fall through to the element screenshot */ }
|
||||
}
|
||||
if (done) {
|
||||
v.setAttribute("poster", url);
|
||||
} else {
|
||||
// Element screenshots capture composited pixels. For background hero
|
||||
// videos with text/CTAs overlaid, that would bake the foreground into
|
||||
// the poster and then render the live DOM on top again.
|
||||
let occluded = false;
|
||||
for (const el of Array.from(document.querySelectorAll("h1,h2,h3,h4,h5,p,span,a,button,li"))) {
|
||||
if (v.contains(el) || (el.textContent || "").trim().length < 3) continue;
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.visibility === "hidden" || cs.display === "none" || parseFloat(cs.opacity || "1") < 0.05) continue;
|
||||
const er = el.getBoundingClientRect();
|
||||
if (er.width < 2 || er.height < 2) continue;
|
||||
const ix = Math.min(er.right, r.right) - Math.max(er.left, r.left);
|
||||
const iy = Math.min(er.bottom, r.bottom) - Math.max(er.top, r.top);
|
||||
if (ix > 0 && iy > 0 && ix * iy > 0.05 * er.width * er.height) { occluded = true; break; }
|
||||
}
|
||||
if (!occluded) {
|
||||
v.setAttribute("poster", url);
|
||||
shots.push({ url, sel: `video[data-clone-vid="${idx}"]` });
|
||||
}
|
||||
}
|
||||
}
|
||||
return { stills, shots };
|
||||
}),
|
||||
new Promise<VideoStillPlan>((res) => setTimeout(() => res({ stills: [], shots: [] }), 12000)),
|
||||
]);
|
||||
return plan;
|
||||
} catch {
|
||||
return { stills: [], shots: [] };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-page screenshot with robustness for heavy/animated pages. The default 30s
|
||||
* timeout is exceeded by tall SaaS pages (Playwright also waits for web fonts);
|
||||
* use a long timeout, freeze animations (also improves determinism), and retry.
|
||||
* As a last resort take a viewport-only shot so the file exists (the capture gate
|
||||
* checks presence; a partial image still beats none).
|
||||
*/
|
||||
async function captureScreenshot(
|
||||
page: import("playwright").Page,
|
||||
path: string,
|
||||
vw: number,
|
||||
log: (e: Record<string, unknown>) => void,
|
||||
): Promise<void> {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
await page.screenshot({ path, fullPage: true, timeout: 90_000, animations: "disabled" });
|
||||
return;
|
||||
} catch (e) {
|
||||
if (attempt === 1) {
|
||||
try {
|
||||
await page.screenshot({ path, fullPage: false, timeout: 30_000, animations: "disabled" });
|
||||
log({ event: "screenshot_fallback_viewport", viewport: vw });
|
||||
return;
|
||||
} catch (e2) {
|
||||
log({ event: "screenshot_error", viewport: vw, error: String(e2) });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureSite(opts: {
|
||||
url: string;
|
||||
outDir: string; // source/ directory
|
||||
viewports?: number[];
|
||||
interactions?: boolean; // Stage 4: opt-in interaction capture (hover/focus + patterns)
|
||||
motion?: boolean; // Stage 5: opt-in motion capture (WAAPI + rotating text)
|
||||
breakpoints?: boolean; // discover the source's real responsive band edges (default on; read-only sweep)
|
||||
screenshots?: boolean; // write per-viewport full-page PNGs (default on). ONLY the validator reads these
|
||||
// (generation never touches pixels), and full-page shots of tall pages are the
|
||||
// dominant capture cost — so a production clone that won't be perceptually graded
|
||||
// can skip them. The cheap poster-less-video element stills are always kept
|
||||
// (generation needs a first frame), this only gates the per-viewport page shots.
|
||||
log?: (event: Record<string, unknown>) => void;
|
||||
}): Promise<CaptureResult> {
|
||||
const viewports = opts.viewports ?? [...REQUIRED_VIEWPORTS];
|
||||
const log = opts.log ?? (() => {});
|
||||
const captureDir = join(opts.outDir, "capture");
|
||||
const screenshotsDir = join(opts.outDir, "screenshots");
|
||||
const cssDir = join(captureDir, "css");
|
||||
const storeDir = join(opts.outDir, "assets-store");
|
||||
ensureDir(captureDir);
|
||||
ensureDir(screenshotsDir);
|
||||
|
||||
// url -> discovered asset (merged across viewports)
|
||||
const assetMap = new Map<string, DiscoveredAsset>();
|
||||
const cssStored = new Set<string>();
|
||||
const fontFaceMap = new Map<string, FontFace>();
|
||||
const seoResourceUrls = new Map<string, SeoResource["kind"]>();
|
||||
|
||||
const sourceOrigin = (() => {
|
||||
try {
|
||||
const u = new URL(opts.url);
|
||||
return u.protocol === "http:" || u.protocol === "https:" ? u.origin : "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
})();
|
||||
const addSeoResource = (url: string, kind: SeoResource["kind"], base = opts.url): void => {
|
||||
try {
|
||||
const abs = new URL(url, base).href;
|
||||
if (!/^https?:\/\//i.test(abs)) return;
|
||||
if (!seoResourceUrls.has(abs)) seoResourceUrls.set(abs, kind);
|
||||
} catch { /* ignore malformed resource hints */ }
|
||||
};
|
||||
if (sourceOrigin) {
|
||||
addSeoResource(sourceOrigin + "/robots.txt", "robots");
|
||||
addSeoResource(sourceOrigin + "/sitemap.xml", "sitemap");
|
||||
addSeoResource(sourceOrigin + "/sitemap_index.xml", "sitemap");
|
||||
addSeoResource(sourceOrigin + "/llms.txt", "llms");
|
||||
addSeoResource(sourceOrigin + "/llms-full.txt", "llms-full");
|
||||
}
|
||||
|
||||
const recordAsset = (url: string, type: string, contentType: string | null, status: number | null, via: string): DiscoveredAsset => {
|
||||
let a = assetMap.get(url);
|
||||
if (!a) {
|
||||
a = { url, type, contentType, status, storedAs: null, bytes: 0, via: [] };
|
||||
assetMap.set(url, a);
|
||||
}
|
||||
if (!a.via.includes(via)) a.via.push(via);
|
||||
if (contentType && !a.contentType) a.contentType = contentType;
|
||||
if (status != null && a.status == null) a.status = status;
|
||||
// SVG/specific types win over generic
|
||||
if (type && (a.type === "other" || (a.type === "image" && type === "svg"))) a.type = type;
|
||||
return a;
|
||||
};
|
||||
|
||||
const storeBytes = (url: string, type: string, bytes: Buffer): void => {
|
||||
if (!bytes || bytes.length === 0) return;
|
||||
const a = assetMap.get(url) ?? recordAsset(url, type, null, null, "network");
|
||||
if (a.storedAs) return;
|
||||
const ext = extFromUrl(url) || extFromContentType(a.contentType) ||
|
||||
(type === "css" ? "css" : type === "font" ? "woff2" : type === "svg" ? "svg" :
|
||||
type === "video" ? "mp4" : type === "lottie" ? "json" : "png");
|
||||
const name = `${sha1_12(url)}.${ext}`;
|
||||
if (type === "css") {
|
||||
writeBytes(join(cssDir, name), bytes);
|
||||
cssStored.add(name);
|
||||
} else {
|
||||
writeBytes(join(storeDir, name), bytes);
|
||||
}
|
||||
a.storedAs = name;
|
||||
a.bytes = bytes.length;
|
||||
};
|
||||
|
||||
const parseManifestForAssets = (text: string, baseUrl: string): void => {
|
||||
let parsed: unknown;
|
||||
try { parsed = JSON.parse(text); } catch { return; }
|
||||
if (!parsed || typeof parsed !== "object") return;
|
||||
const push = (raw: unknown, via: string): void => {
|
||||
if (typeof raw !== "string" || !raw.trim() || raw.trim().startsWith("data:")) return;
|
||||
let abs = raw;
|
||||
try { abs = new URL(raw, baseUrl).href; } catch { /* keep raw */ }
|
||||
const t = classifyAsset(abs, null) ?? "image";
|
||||
recordAsset(abs, t, null, null, via);
|
||||
};
|
||||
const iconList = (parsed as { icons?: unknown }).icons;
|
||||
if (Array.isArray(iconList)) {
|
||||
for (const icon of iconList) if (icon && typeof icon === "object") push((icon as { src?: unknown }).src, "manifest:icons");
|
||||
}
|
||||
const screenshots = (parsed as { screenshots?: unknown }).screenshots;
|
||||
if (Array.isArray(screenshots)) {
|
||||
for (const shot of screenshots) if (shot && typeof shot === "object") push((shot as { src?: unknown }).src, "manifest:screenshots");
|
||||
}
|
||||
const shortcuts = (parsed as { shortcuts?: unknown }).shortcuts;
|
||||
if (Array.isArray(shortcuts)) {
|
||||
for (const shortcut of shortcuts) {
|
||||
const icons = shortcut && typeof shortcut === "object" ? (shortcut as { icons?: unknown }).icons : undefined;
|
||||
if (Array.isArray(icons)) for (const icon of icons) if (icon && typeof icon === "object") push((icon as { src?: unknown }).src, "manifest:shortcuts");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Honor the standard HTTPS_PROXY env so capture works behind an egress proxy
|
||||
// (sandboxed/CI environments). The proxy re-terminates TLS, so the per-context
|
||||
// `ignoreHTTPSErrors` below covers its CA. Capture is not part of the determinism
|
||||
// gate, so this never affects generated output.
|
||||
// Honor HTTPS_PROXY for real remote captures behind an egress proxy (sandboxed/CI),
|
||||
// but ONLY for non-loopback http(s) targets — file:// and localhost fixtures (tests,
|
||||
// the validator's static server) must be fetched directly. Playwright otherwise routes
|
||||
// even loopback through the proxy (its default `<-loopback>`), which would replace a
|
||||
// local fixture with the proxy's error page. Capture is not part of the determinism
|
||||
// gate, so this never affects generated output.
|
||||
let proxyServer = process.env.HTTPS_PROXY || process.env.https_proxy || "";
|
||||
try {
|
||||
const u = new URL(opts.url);
|
||||
// Skip the proxy ONLY for a localhost http(s) target (test fixtures, the validator's
|
||||
// static server) — Playwright otherwise routes loopback through the proxy and replaces
|
||||
// the page with the proxy error page. file:// keeps the proxy on so any REMOTE assets
|
||||
// the local page references fast-fail through the proxy instead of hanging on a blocked
|
||||
// direct connection; external targets obviously keep it.
|
||||
const isHttp = u.protocol === "http:" || u.protocol === "https:";
|
||||
const isLoopback = /^(localhost|127\.0\.0\.1|\[?::1\]?|0\.0\.0\.0)$/.test(u.hostname);
|
||||
if (isHttp && isLoopback) proxyServer = "";
|
||||
} catch { /* keep proxy */ }
|
||||
const browser: Browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ["--disable-blink-features=AutomationControlled", "--disable-dev-shm-usage"],
|
||||
...(proxyServer ? { proxy: { server: proxyServer } } : {}),
|
||||
});
|
||||
|
||||
const perViewport: CaptureResult["perViewport"] = [];
|
||||
let interaction: InteractionCapture | undefined;
|
||||
let motion: MotionCapture | undefined;
|
||||
let discoveredBreakpoints: number[] | undefined;
|
||||
const captureSeoResources: SeoResource[] = [];
|
||||
const cssTextsForParsing: Array<{ baseUrl: string; text: string }> = [];
|
||||
const dismissUnion = { dismissed: [] as string[], overlaysRemaining: 0, removed: 0, videoStills: 0, blocking: false };
|
||||
// Carry cookies/localStorage across the per-viewport contexts so the SAME page
|
||||
// is captured at every width: A/B-test buckets and consent state are usually
|
||||
// session-persisted, so without this each fresh context can load a different
|
||||
// variant (grammarly served a different hero per viewport) and the cross-
|
||||
// viewport IR alignment then can't reconcile them. Sharing the session also
|
||||
// means a banner dismissed at the first width stays dismissed at the rest.
|
||||
const canonical = viewports.includes(1280) ? 1280 : viewports[Math.floor(viewports.length / 2)]!;
|
||||
|
||||
try {
|
||||
// Single context + single navigation; every viewport is captured by RESIZING
|
||||
// the same loaded page (no reload). This guarantees identical content across
|
||||
// widths — eliminating both A/B-per-load variance (grammarly served a different
|
||||
// hero on each fresh load) and session-reuse degeneration (warbyparker returned
|
||||
// a 13-node shell when reloaded with a carried session). The IR's cross-viewport
|
||||
// alignment then operates on one logical DOM that CSS merely reflows.
|
||||
const context: BrowserContext = await browser.newContext({
|
||||
ignoreHTTPSErrors: true,
|
||||
viewport: { width: canonical, height: viewportHeight(canonical) },
|
||||
deviceScaleFactor: 1,
|
||||
userAgent: DESKTOP_UA,
|
||||
javaScriptEnabled: true,
|
||||
});
|
||||
const page = await context.newPage();
|
||||
// tsx/esbuild wraps functions with a __name() helper for stack traces; that
|
||||
// helper does not exist in the browser when we serialize page.evaluate
|
||||
// callbacks. Shim it (as a raw string so it isn't itself transformed).
|
||||
await page.addInitScript(ESBUILD_SHIM);
|
||||
const bodyPromises: Promise<void>[] = [];
|
||||
|
||||
page.on("response", (resp) => {
|
||||
try {
|
||||
const url = resp.url();
|
||||
if (url.startsWith("data:") || url.startsWith("blob:")) return;
|
||||
const ct = resp.headers()["content-type"] || null;
|
||||
const type = classifyAsset(url, ct);
|
||||
if (!type) return;
|
||||
const status = resp.status();
|
||||
recordAsset(url, type, ct, status, "network");
|
||||
const existing = assetMap.get(url);
|
||||
if (existing?.storedAs) return;
|
||||
if (status >= 400) return;
|
||||
bodyPromises.push(
|
||||
(async () => {
|
||||
try {
|
||||
const buf = await resp.body();
|
||||
storeBytes(url, type, buf);
|
||||
if (type === "css") cssTextsForParsing.push({ baseUrl: url, text: buf.toString("utf8") });
|
||||
if (type === "manifest") parseManifestForAssets(buf.toString("utf8"), url);
|
||||
} catch { /* body unavailable */ }
|
||||
})(),
|
||||
);
|
||||
} catch { /* ignore */ }
|
||||
});
|
||||
|
||||
// Single navigation at the canonical width; every viewport below is a resize.
|
||||
log({ event: "goto", url: opts.url });
|
||||
let navigated = false;
|
||||
let navErr: unknown = null;
|
||||
for (let attempt = 0; attempt < 3 && !navigated; attempt++) {
|
||||
try {
|
||||
await page.goto(opts.url, { waitUntil: attempt === 0 ? "load" : "domcontentloaded", timeout: 45000 });
|
||||
navigated = true;
|
||||
} catch (e) {
|
||||
navErr = e;
|
||||
await page.waitForTimeout(1000);
|
||||
}
|
||||
}
|
||||
if (!navigated) throw navErr;
|
||||
await settle(page);
|
||||
|
||||
for (const vw of viewports) {
|
||||
const vh = viewportHeight(vw);
|
||||
await page.setViewportSize({ width: vw, height: vh });
|
||||
await settle(page, 1500); // let the resize reflow + any width-triggered content settle
|
||||
|
||||
// Stage 2: dismiss cookie/consent/newsletter/region overlays. Run TWICE —
|
||||
// once after initial load (cookie/consent walls appear immediately), and
|
||||
// again after the scroll pass (newsletter/email-capture modals are usually
|
||||
// timer- or scroll-triggered and only mount a few seconds in). Each pass
|
||||
// clicks the accept/close control, settles (so a just-closed dialog unlocks
|
||||
// scrolling), THEN removes only a genuinely-stuck consent/modal layer.
|
||||
let overlaysRemaining = 0;
|
||||
let blocking = false;
|
||||
const applyDismiss = async (phase: string): Promise<void> => {
|
||||
const clicked = await clickDismiss(page);
|
||||
if (clicked.length) await settle(page, 1000);
|
||||
const fin = await finalizeOverlays(page);
|
||||
for (const d of clicked) if (!dismissUnion.dismissed.includes(d)) dismissUnion.dismissed.push(d);
|
||||
for (const d of fin.removedLabels) { const k = "removed:" + d; if (!dismissUnion.dismissed.includes(k)) dismissUnion.dismissed.push(k); }
|
||||
dismissUnion.removed += fin.removed;
|
||||
overlaysRemaining = fin.overlaysRemaining;
|
||||
blocking = blocking || fin.blocking;
|
||||
if (clicked.length || fin.removed) { await settle(page, 1000); log({ event: "dismissed", viewport: vw, phase, count: clicked.length, removed: fin.removed, remaining: fin.overlaysRemaining, blocking: fin.blocking }); }
|
||||
};
|
||||
await applyDismiss("load");
|
||||
|
||||
// Stage 5 (scroll reveals): at the FIRST viewport, before any scroll, tag elements
|
||||
// and snapshot the pre-reveal hidden state — scroll reveals fire on the first
|
||||
// autoScroll and stay revealed, so this is the only moment their hidden state is
|
||||
// observable. Idempotent + motion-gated; the settled snapshot is unchanged.
|
||||
if (opts.motion && vw === viewports[0]) { await tagElements(page); await probeReveals(page); }
|
||||
|
||||
await autoScroll(page, vh);
|
||||
await settle(page, 1500);
|
||||
await applyDismiss("post-scroll");
|
||||
dismissUnion.overlaysRemaining = Math.max(dismissUnion.overlaysRemaining, overlaysRemaining);
|
||||
dismissUnion.blocking = dismissUnion.blocking || blocking;
|
||||
// Stage 2: wait for entrance/scroll animations to settle so geometry isn't
|
||||
// sampled mid-transition.
|
||||
const quiescent = await waitForQuiescence(page);
|
||||
|
||||
// Reset window + all inner scrollable containers to scroll 0 so captured
|
||||
// positions match the generated app's default (un-scrolled) render. Inner
|
||||
// scroll state is runtime JS state and otherwise non-deterministic.
|
||||
await page.evaluate(() => {
|
||||
window.scrollTo(0, 0);
|
||||
for (const el of Array.from(document.querySelectorAll("*"))) {
|
||||
if (el.scrollLeft) el.scrollLeft = 0;
|
||||
if (el.scrollTop) el.scrollTop = 0;
|
||||
}
|
||||
});
|
||||
await page.waitForTimeout(150);
|
||||
|
||||
// Stage 2: dynamic-media first frame. Materialize a still for poster-less
|
||||
// videos so they don't render blank — canvas where the frame is readable, else
|
||||
// an element screenshot (the page cannot screenshot itself). Done once at the
|
||||
// canonical viewport: it's the highest-fidelity width, the poster attr persists
|
||||
// across the later resizes, and the IR reads attrs from the canonical snapshot.
|
||||
if (vw === canonical) {
|
||||
const plan = await captureVideoStills(page);
|
||||
for (const s of plan.stills) {
|
||||
const comma = s.dataUrl.indexOf(",");
|
||||
if (comma < 0) continue;
|
||||
try {
|
||||
const buf = Buffer.from(s.dataUrl.slice(comma + 1), "base64");
|
||||
recordAsset(s.url, "image", "image/jpeg", 200, "video-still");
|
||||
storeBytes(s.url, "image", buf);
|
||||
dismissUnion.videoStills++;
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
for (const s of plan.shots) {
|
||||
try {
|
||||
const buf = await page.locator(s.sel).first().screenshot({ type: "jpeg", quality: 82, timeout: 5000, animations: "disabled" });
|
||||
recordAsset(s.url, "image", "image/jpeg", 200, "video-still");
|
||||
storeBytes(s.url, "image", buf);
|
||||
dismissUnion.videoStills++;
|
||||
} catch { /* element not shootable (off-screen/detached) — poster falls back to placeholder */ }
|
||||
}
|
||||
// Element screenshots scroll the target into view; restore scroll 0 so the
|
||||
// canonical DOM walk + screenshot match the generated app's default render.
|
||||
if (plan.shots.length) {
|
||||
await page.evaluate(() => window.scrollTo(0, 0));
|
||||
await page.waitForTimeout(80);
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 4/5: stamp capture-ids before the canonical snapshot so the IR carries
|
||||
// them (whitelisted), enabling interaction deltas + motion specs to map to cids.
|
||||
if ((opts.interactions || opts.motion) && vw === canonical) await tagElements(page);
|
||||
|
||||
// Bound the in-page DOM walk: page.evaluate has no default timeout, so a
|
||||
// pathologically large/animated DOM (e.g. asana.com) could hang forever.
|
||||
const snapshot: PageSnapshot = await Promise.race([
|
||||
page.evaluate(collectPage),
|
||||
new Promise<never>((_, rej) => setTimeout(() => rej(new Error(`collectPage timeout vp${vw}`)), 60_000)),
|
||||
]);
|
||||
|
||||
// Merge discovered references from the snapshot (DOM + accessible CSS).
|
||||
for (const da of snapshot.domAssets) {
|
||||
const t = classifyAsset(da.url, null) ?? (da.kind === "manifest" ? "manifest" : da.kind === "video" ? "video" : da.kind === "svg" ? "svg" : "image");
|
||||
recordAsset(da.url, t, null, null, da.via);
|
||||
}
|
||||
for (const link of snapshot.doc.head?.links ?? []) {
|
||||
const rel = (link.rel || "").toLowerCase();
|
||||
const href = link.href || "";
|
||||
if (!href) continue;
|
||||
if (/\bsitemap\b/.test(rel)) addSeoResource(href, "sitemap", snapshot.doc.url);
|
||||
if (/llms-full\.txt(?:$|[?#])/i.test(href) || /\bllms-full\b/.test(rel)) addSeoResource(href, "llms-full", snapshot.doc.url);
|
||||
else if (/llms\.txt(?:$|[?#])/i.test(href) || /\bllms\b/.test(rel)) addSeoResource(href, "llms", snapshot.doc.url);
|
||||
}
|
||||
for (const u of snapshot.cssUrls) {
|
||||
const t = classifyAsset(u, null) ?? "other";
|
||||
recordAsset(u, t, null, null, "css-url");
|
||||
}
|
||||
for (const ff of snapshot.fontFaces) {
|
||||
const key = `${ff.family}|${ff.weight ?? ""}|${ff.style ?? ""}|${ff.src}`;
|
||||
if (!fontFaceMap.has(key)) fontFaceMap.set(key, ff);
|
||||
}
|
||||
|
||||
// Cap body collection: resp.body() has no timeout, so a streaming/long-poll
|
||||
// response (common on heavy SaaS pages) would hang allSettled forever. By
|
||||
// now bodies have been downloading throughout navigation+scroll; stragglers
|
||||
// are dropped (the post-pass fallback fetch re-fetches anything missing).
|
||||
await Promise.race([
|
||||
Promise.allSettled(bodyPromises),
|
||||
new Promise((r) => setTimeout(r, 20_000)),
|
||||
]);
|
||||
|
||||
// Persist DOM snapshot, and (unless skipped for a production clone) the full-page screenshot.
|
||||
writeJSONCompact(join(captureDir, `dom-${vw}.json`), snapshot);
|
||||
if (opts.screenshots !== false) await captureScreenshot(page, join(screenshotsDir, `${vw}.png`), vw, log);
|
||||
|
||||
// Stage 4: drive recognized affordances at the canonical viewport (opt-in).
|
||||
if (opts.interactions && vw === canonical) {
|
||||
try { interaction = await captureInteractions(page, { log }); }
|
||||
catch (e) { log({ event: "interactions_error", error: String(e).slice(0, 200) }); }
|
||||
}
|
||||
|
||||
// Stage 5: capture motion (WAAPI + rotating text) at the canonical viewport.
|
||||
if (opts.motion && vw === canonical) {
|
||||
try { motion = await captureMotion(page, { log }); }
|
||||
catch (e) { log({ event: "motion_error", error: String(e).slice(0, 200) }); }
|
||||
}
|
||||
|
||||
perViewport.push({
|
||||
viewport: vw,
|
||||
height: vh,
|
||||
scrollHeight: round(snapshot.doc.scrollHeight),
|
||||
nodeCount: snapshot.doc.nodeCount,
|
||||
truncated: snapshot.doc.truncated,
|
||||
overlaysRemaining,
|
||||
blocking,
|
||||
quiescent,
|
||||
});
|
||||
log({ event: "captured", viewport: vw, nodes: snapshot.doc.nodeCount, scrollHeight: snapshot.doc.scrollHeight });
|
||||
}
|
||||
|
||||
// Browser-as-oracle: discover the source's real responsive band edges by sweeping the viewport
|
||||
// and binary-searching each width where the discrete (media-query-toggled) layout signature
|
||||
// changes. Read-only and bounded; runs once here — overlays are dismissed and the DOM settled, so
|
||||
// the signature is the real page, not a consent wall. Never fatal: a failed sweep just omits the
|
||||
// field. The context closes right after, so the swept viewport size needs no restore.
|
||||
if (opts.breakpoints ?? true) {
|
||||
try {
|
||||
const edges = await Promise.race([
|
||||
discoverBreakpoints(page, { min: 320, max: 1920 }),
|
||||
new Promise<number[]>((_, rej) => setTimeout(() => rej(new Error("breakpoint sweep timeout")), 60_000)),
|
||||
]);
|
||||
discoveredBreakpoints = edges;
|
||||
log({ event: "breakpoints_discovered", edges });
|
||||
} catch (e) {
|
||||
log({ event: "breakpoints_error", error: String(e).slice(0, 200) });
|
||||
}
|
||||
}
|
||||
await context.close();
|
||||
|
||||
// Parse @font-face + url() from cross-origin CSS texts we fetched at the
|
||||
// network layer (the in-page walker can only read same-origin sheets).
|
||||
for (const { baseUrl, text } of cssTextsForParsing) {
|
||||
parseCssForFonts(text, baseUrl, fontFaceMap, (u) => {
|
||||
const t = classifyAsset(u, null) ?? "other";
|
||||
recordAsset(u, t, null, null, "css-text");
|
||||
});
|
||||
}
|
||||
|
||||
// Fallback: download any referenced-but-not-yet-stored asset using the
|
||||
// browser context (shares TLS handling). Fonts and CSS-referenced images are
|
||||
// commonly missed by the response listener when served from cache.
|
||||
const fallbackCtx = await browser.newContext({ ignoreHTTPSErrors: true, userAgent: DESKTOP_UA });
|
||||
for (const a of assetMap.values()) {
|
||||
if (a.storedAs) continue;
|
||||
if (a.url.startsWith("data:")) continue;
|
||||
if (!["image", "svg", "video", "font", "lottie", "css", "manifest"].includes(a.type)) continue;
|
||||
try {
|
||||
const resp = await fallbackCtx.request.get(a.url, { timeout: 30000 });
|
||||
a.status = resp.status();
|
||||
if (resp.ok()) {
|
||||
const buf = await resp.body();
|
||||
a.contentType = a.contentType ?? (resp.headers()["content-type"] || null);
|
||||
storeBytes(a.url, a.type, buf);
|
||||
if (a.type === "css") parseCssForFonts(buf.toString("utf8"), a.url, fontFaceMap, (u) => {
|
||||
const t = classifyAsset(u, null) ?? "other";
|
||||
recordAsset(u, t, null, null, "css-text");
|
||||
});
|
||||
if (a.type === "manifest") parseManifestForAssets(buf.toString("utf8"), a.url);
|
||||
}
|
||||
} catch { /* unreachable/signed — left as skipped */ }
|
||||
}
|
||||
const seoResources: SeoResource[] = [];
|
||||
const fetchedSeo = new Set<string>();
|
||||
const fetchSeoResource = async (url: string, kind: SeoResource["kind"]): Promise<void> => {
|
||||
if (fetchedSeo.has(url)) return;
|
||||
fetchedSeo.add(url);
|
||||
try {
|
||||
const resp = await fallbackCtx.request.get(url, { timeout: 15000 });
|
||||
const contentType = resp.headers()["content-type"] || null;
|
||||
const resource: SeoResource = { kind, url, status: resp.status(), contentType };
|
||||
if (resp.ok() && (kind === "llms" || kind === "llms-full" || kind === "robots")) {
|
||||
const text = await resp.text();
|
||||
resource.text = text;
|
||||
if (kind === "robots") {
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
const m = /^sitemap:\s*(\S+)/i.exec(line.trim());
|
||||
if (m) addSeoResource(m[1]!, "sitemap", url);
|
||||
}
|
||||
}
|
||||
}
|
||||
seoResources.push(resource);
|
||||
} catch {
|
||||
seoResources.push({ kind, url, status: null, contentType: null });
|
||||
}
|
||||
};
|
||||
for (let pass = 0; pass < 2; pass++) {
|
||||
const entries = [...seoResourceUrls.entries()].sort((a, b) => a[0].localeCompare(b[0]));
|
||||
for (const [url, kind] of entries) await fetchSeoResource(url, kind);
|
||||
}
|
||||
await fallbackCtx.close();
|
||||
|
||||
if (seoResources.length) {
|
||||
(captureSeoResources as SeoResource[]).push(...seoResources.sort((a, b) => a.url.localeCompare(b.url) || a.kind.localeCompare(b.kind)));
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
const assets = [...assetMap.values()].sort((a, b) => a.url.localeCompare(b.url));
|
||||
const fontFaces = [...fontFaceMap.values()].sort((a, b) =>
|
||||
`${a.family}${a.weight}${a.style}`.localeCompare(`${b.family}${b.weight}${b.style}`),
|
||||
);
|
||||
|
||||
const result: CaptureResult = {
|
||||
sourceUrl: opts.url,
|
||||
capturedAt: new Date().toISOString(),
|
||||
viewports,
|
||||
breakpoints: discoveredBreakpoints,
|
||||
perViewport,
|
||||
assets,
|
||||
...(captureSeoResources.length ? { seoResources: captureSeoResources } : {}),
|
||||
fontFaces,
|
||||
cssTexts: [...cssStored].sort(),
|
||||
dismissal: dismissUnion,
|
||||
interaction,
|
||||
motion,
|
||||
};
|
||||
|
||||
if (interaction) writeJSON(join(opts.outDir, "interaction.json"), interaction);
|
||||
if (motion) writeJSON(join(opts.outDir, "motion.json"), motion);
|
||||
writeJSON(join(opts.outDir, "assets-discovered.json"), assets);
|
||||
writeJSON(join(opts.outDir, "fonts-discovered.json"), fontFaces);
|
||||
writeJSON(join(captureDir, "capture-result.json"), result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseCssForFonts(
|
||||
cssText: string,
|
||||
baseUrl: string,
|
||||
fontFaceMap: Map<string, FontFace>,
|
||||
onUrl: (url: string) => void,
|
||||
): void {
|
||||
const abs = (u: string): string => {
|
||||
try { return new URL(u, baseUrl).href; } catch { return u; }
|
||||
};
|
||||
// Extract @font-face blocks. src urls are rewritten to absolute so they can be
|
||||
// resolved/downloaded regardless of where the CSS file lived.
|
||||
const faceRe = /@font-face\s*\{([^}]*)\}/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = faceRe.exec(cssText)) !== null) {
|
||||
const block = m[1]!;
|
||||
const family = /font-family\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim().replace(/^['"]|['"]$/g, "");
|
||||
let src = /src\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
if (!family || !src) continue;
|
||||
src = src.replace(/url\(\s*(['"]?)([^'")]+)\1\s*\)/gi, (_full, _q, u) =>
|
||||
u.startsWith("data:") ? `url(${u})` : `url(${abs(u)})`);
|
||||
const weight = /font-weight\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
const style = /font-style\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
const display = /font-display\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
const unicodeRange = /unicode-range\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
const key = `${family}|${weight ?? ""}|${style ?? ""}|${src}`;
|
||||
if (!fontFaceMap.has(key)) {
|
||||
fontFaceMap.set(key, { family, src, weight, style, display, unicodeRange });
|
||||
}
|
||||
}
|
||||
// Extract all url() references for asset discovery.
|
||||
const urlRe = /url\(\s*(['"]?)([^'")]+)\1\s*\)/gi;
|
||||
while ((m = urlRe.exec(cssText)) !== null) {
|
||||
const raw = m[2];
|
||||
if (raw && !raw.startsWith("data:")) onUrl(abs(raw));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user