devteamaegisandclaude-flow
9a89fed365
Two follow-ups to the single-page Lottie support: 1. Multi-page (clone-site). generateSiteApp emitted only DittoWire; it now wires DittoLottie per route exactly like wires — builds the route's lottie spec, emits the import + JSX, writes DittoLottie.tsx once, and injects lottie-web into the site package.json when any route replays an animation (the write moved after the route loop so the dep reflects actual usage). cloneSite now captures motion on full route captures (motion: full) so lotties are observed; light sibling probes skip it. Per-route gating verified: a route with no animation gets no DittoLottie. 2. Inline animationData. When a lottie is only in memory (data-driven init, no fetchable URL), capture now writes the JSON to the assets store as a real, content-hashed .json so it materializes to /assets/cloned/lottie like any source — instead of embedding the blob in the page spec. buildLottieSpec then resolves it to a local path; the spec carries path, not animationData. Verified end-to-end: a 2-page fixture emits DittoLottie only on the animated route with lottie-web added; a data-driven inline fixture materializes fd7c…json and the spec references it by path. Co-Authored-By: claude-flow <ruv@ruv.net>
1015 lines
49 KiB
TypeScript
1015 lines
49 KiB
TypeScript
import { chromium, type Browser, type BrowserContext } from "playwright";
|
|
import { join } from "node:path";
|
|
import { collectPage, type PageSnapshot, type FontFace } from "./walker.js";
|
|
import { tagElements, captureInteractions, type InteractionCapture } from "./interactions.js";
|
|
import { captureMotion, probeReveals, type MotionCapture } from "./motion.js";
|
|
import { discoverBreakpoints } from "./breakpoints.js";
|
|
import { writeJSON, writeJSONCompact, writeBytes, ensureDir } from "../util/fsx.js";
|
|
import { sha1_12, round } from "../util/canonical.js";
|
|
|
|
export const REQUIRED_VIEWPORTS = [375, 768, 1280, 1920] as const;
|
|
// The dense width set captured for SIZE INFERENCE: a node sampled at 9 widths reveals its sizing
|
|
// law (constant / proportional / clamped / flex-distributed) far better than 4 — enough to drop a
|
|
// baked px for a relative construct with confidence (and to surface a shrunk item's natural size,
|
|
// which needs widths beyond 1920). REQUIRED_VIEWPORTS ⊂ this; only those 4 carry responsive bands.
|
|
export const SAMPLE_VIEWPORTS = [375, 480, 640, 768, 1024, 1280, 1536, 1920, 2560] as const;
|
|
|
|
const VIEWPORT_HEIGHTS: Record<number, number> = {
|
|
375: 812,
|
|
480: 854,
|
|
640: 960,
|
|
768: 1024,
|
|
1024: 768,
|
|
1280: 800,
|
|
1536: 864,
|
|
1920: 1080,
|
|
2560: 1440,
|
|
};
|
|
|
|
const DESKTOP_UA =
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
|
|
|
// Defines esbuild/tsx runtime helpers in the page so serialized evaluate
|
|
// callbacks that reference them don't throw ReferenceError.
|
|
const ESBUILD_SHIM =
|
|
"globalThis.__name = globalThis.__name || ((fn) => fn);" +
|
|
"globalThis.__defProp = globalThis.__defProp || Object.defineProperty;";
|
|
|
|
export type DiscoveredAsset = {
|
|
url: string;
|
|
type: string; // image|svg|video|font|lottie|css|manifest|other
|
|
contentType: string | null;
|
|
status: number | null;
|
|
storedAs: string | null; // sha1-named file in assets-store, or null if not downloaded
|
|
bytes: number; // size of stored file
|
|
via: string[]; // discovery sources
|
|
};
|
|
|
|
export type SeoResource = {
|
|
kind: "robots" | "sitemap" | "llms" | "llms-full";
|
|
url: string;
|
|
status: number | null;
|
|
contentType: string | null;
|
|
text?: string;
|
|
};
|
|
|
|
export type CaptureResult = {
|
|
sourceUrl: string;
|
|
capturedAt: string;
|
|
viewports: number[];
|
|
// Browser-as-oracle: the widths where the SOURCE layout actually restructures (display /
|
|
// flex-direction / wrap / grid-track-count / position / visibility flips), found by sweeping the
|
|
// viewport and binary-searching each discrete-signature change. Ground truth for the real
|
|
// responsive band edges — distinct from REQUIRED_VIEWPORTS (our fixed capture widths). Absent if
|
|
// discovery was disabled or the sweep failed. See breakpoints.ts.
|
|
breakpoints?: number[];
|
|
perViewport: Array<{
|
|
viewport: number;
|
|
height: number;
|
|
scrollHeight: number;
|
|
nodeCount: number;
|
|
truncated: boolean;
|
|
overlaysRemaining?: number;
|
|
blocking?: boolean;
|
|
quiescent?: boolean;
|
|
}>;
|
|
assets: DiscoveredAsset[];
|
|
seoResources?: SeoResource[];
|
|
fontFaces: FontFace[];
|
|
cssTexts: string[]; // sha1 names of stored css files
|
|
// Stage 2: overlay/popup dismissal audit (union of actions across viewports).
|
|
dismissal?: { dismissed: string[]; overlaysRemaining: number; removed: number; videoStills: number; blocking: boolean };
|
|
// Stage 4: optional interaction capture (hover/focus + recognized patterns).
|
|
interaction?: InteractionCapture;
|
|
// Stage 5: optional motion capture (WAAPI animations + rotating text). CSS @keyframes
|
|
// motion is reconstructed from the IR, so it isn't re-captured here.
|
|
motion?: MotionCapture;
|
|
};
|
|
|
|
function viewportHeight(width: number): number {
|
|
return VIEWPORT_HEIGHTS[width] ?? Math.round(width * 0.66);
|
|
}
|
|
|
|
function extFromUrl(url: string): string {
|
|
try {
|
|
const p = new URL(url).pathname;
|
|
const dot = p.lastIndexOf(".");
|
|
if (dot >= 0 && dot > p.lastIndexOf("/")) {
|
|
const ext = p.slice(dot + 1).toLowerCase().slice(0, 5);
|
|
if (/^[a-z0-9]+$/.test(ext)) return ext;
|
|
}
|
|
} catch { /* ignore */ }
|
|
return "";
|
|
}
|
|
|
|
const CONTENT_TYPE_EXT: Record<string, string> = {
|
|
"image/webp": "webp", "image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg",
|
|
"image/avif": "avif", "image/gif": "gif", "image/svg+xml": "svg",
|
|
"image/x-icon": "ico", "image/vnd.microsoft.icon": "ico", "image/bmp": "bmp",
|
|
"video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", "video/ogg": "ogv",
|
|
"font/woff2": "woff2", "font/woff": "woff", "font/ttf": "ttf", "font/otf": "otf",
|
|
"application/font-woff2": "woff2", "application/font-woff": "woff",
|
|
"application/x-font-ttf": "ttf", "application/vnd.ms-fontobject": "eot",
|
|
"text/css": "css", "application/json": "json", "application/manifest+json": "webmanifest",
|
|
};
|
|
|
|
function extFromContentType(contentType: string | null): string {
|
|
if (!contentType) return "";
|
|
const ct = contentType.split(";")[0]!.trim().toLowerCase();
|
|
return CONTENT_TYPE_EXT[ct] ?? "";
|
|
}
|
|
|
|
function classifyAsset(url: string, contentType: string | null): string | null {
|
|
const u = url.toLowerCase().split("?")[0]!;
|
|
const ct = (contentType || "").toLowerCase();
|
|
if (u.endsWith(".svg") || ct === "image/svg+xml") return "svg";
|
|
if (/\.(jpg|jpeg|png|webp|avif|gif|ico|bmp)$/.test(u) || ct.startsWith("image/")) return "image";
|
|
if (/\.(mp4|mov|webm|m4v|ogv)$/.test(u) || ct.startsWith("video/")) return "video";
|
|
if (/\.(woff2|woff|ttf|otf|eot)$/.test(u) || ct.startsWith("font/") ||
|
|
ct.includes("font-woff") || ct.includes("x-font")) return "font";
|
|
if (u.endsWith(".json") && (u.includes("lottie") || u.includes("animation"))) return "lottie";
|
|
if (u.endsWith(".webmanifest") || /(?:^|\/)manifest\.json$/.test(u) || ct.includes("manifest+json")) return "manifest";
|
|
if (u.endsWith(".css") || ct.startsWith("text/css")) return "css";
|
|
return null;
|
|
}
|
|
|
|
function isCss(url: string, contentType: string | null): boolean {
|
|
return classifyAsset(url, contentType) === "css";
|
|
}
|
|
|
|
async function autoScroll(page: import("playwright").Page, vpHeight: number): Promise<void> {
|
|
// Scroll through the page to trigger lazy-loaded images/backgrounds, then return
|
|
// to the top so document-coordinate bboxes are measured from a settled layout.
|
|
await page.evaluate(async (step: number) => {
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
const maxScroll = () => document.documentElement.scrollHeight - window.innerHeight;
|
|
let y = 0;
|
|
let guard = 0;
|
|
while (y < maxScroll() && guard < 100) {
|
|
y += step;
|
|
window.scrollTo(0, y);
|
|
await sleep(60);
|
|
guard++;
|
|
}
|
|
window.scrollTo(0, 0);
|
|
await sleep(120);
|
|
}, Math.max(Math.round(vpHeight * 0.8), 300));
|
|
}
|
|
|
|
async function settle(page: import("playwright").Page, maxMs = 2500): Promise<void> {
|
|
try {
|
|
await page.waitForLoadState("networkidle", { timeout: Math.max(maxMs / 2, 800) });
|
|
} catch { /* ignore */ }
|
|
try {
|
|
// document.fonts.ready can never resolve when a font request hangs (heavy SaaS
|
|
// pages), and page.evaluate has no default timeout — so bound it explicitly.
|
|
await Promise.race([
|
|
page.evaluate(() => (document as Document).fonts?.ready as unknown as Promise<void>),
|
|
new Promise<void>((r) => setTimeout(r, 6000)),
|
|
]);
|
|
} catch { /* ignore */ }
|
|
await page.waitForTimeout(250);
|
|
}
|
|
|
|
export type DismissResult = { dismissed: string[]; overlaysRemaining: number; removed: number; blocking: boolean };
|
|
|
|
/**
|
|
* Stage 2 — overlay/popup dismissal, phase 1: click the accept/close affordance.
|
|
* Cookie-consent walls, newsletter modals, region/age/app-install interstitials
|
|
* cover the real page; the capture must see the state a returning user sees.
|
|
* Deterministic + replayable: click the same known/accept controls in DOM order.
|
|
* Removal of a stuck overlay happens later (finalizeOverlays) AFTER a settle, so a
|
|
* just-clicked dialog has time to close and unlock scrolling before we judge it.
|
|
*/
|
|
async function clickDismiss(page: import("playwright").Page): Promise<string[]> {
|
|
try {
|
|
return await Promise.race([
|
|
page.evaluate(() => {
|
|
const dismissed: string[] = [];
|
|
const vis = (el: Element): boolean => {
|
|
const cs = getComputedStyle(el);
|
|
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) return false;
|
|
const r = (el as HTMLElement).getBoundingClientRect();
|
|
return r.width > 0 && r.height > 0;
|
|
};
|
|
const click = (el: Element): void => { try { (el as HTMLElement).click(); } catch { /* ignore */ } };
|
|
|
|
// 1) Known consent-framework / generic close affordances, in priority order.
|
|
const KNOWN = [
|
|
"#onetrust-accept-btn-handler", "#accept-recommended-btn-handler", ".onetrust-close-btn-handler",
|
|
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", "#CybotCookiebotDialogBodyButtonAccept",
|
|
"#truste-consent-button", ".osano-cm-accept-all", ".osano-cm-dialog__close",
|
|
"[data-testid='uc-accept-all-button']", "[data-testid='uc-deny-all-button']",
|
|
"#didomi-notice-agree-button", ".didomi-continue-without-agreeing",
|
|
".qc-cmp2-summary-buttons button[mode='primary']", "button[aria-label='Consent']",
|
|
".cc-allow", ".cookie-consent-accept", "#hs-eu-confirmation-button", "#gdpr-consent-tool-wrapper button",
|
|
];
|
|
for (const sel of KNOWN) {
|
|
try {
|
|
for (const el of Array.from(document.querySelectorAll(sel))) {
|
|
if (vis(el)) { click(el); dismissed.push(sel); break; }
|
|
}
|
|
} catch { /* invalid selector in this browser */ }
|
|
}
|
|
|
|
// 2) Scoped text-button pass: only inside overlay-ish containers (dialogs,
|
|
// or id/class naming cookie/consent/modal/popup/newsletter) so we never
|
|
// click an ordinary page button.
|
|
const ACCEPT = new Set([
|
|
"accept", "accept all", "accept all cookies", "accept cookies", "accept & close",
|
|
"i accept", "i agree", "agree", "agree and continue", "allow all", "allow cookies",
|
|
"allow all cookies", "got it", "ok", "okay", "continue", "no thanks", "no, thanks",
|
|
"dismiss", "close", "got it!", "understood", "yes, i agree",
|
|
]);
|
|
const containerSel =
|
|
"[role='dialog'],[aria-modal='true'],[id*='cookie' i],[class*='cookie' i],[id*='consent' i]," +
|
|
"[class*='consent' i],[class*='gdpr' i],[id*='gdpr' i],[class*='modal' i],[class*='popup' i]," +
|
|
"[class*='newsletter' i],[class*='interstitial' i]";
|
|
let containers: Element[] = [];
|
|
try { containers = Array.from(document.querySelectorAll(containerSel)); } catch { /* ignore */ }
|
|
for (const c of containers) {
|
|
if (!vis(c)) continue;
|
|
const btns = Array.from(c.querySelectorAll("button,[role='button'],a,input[type='button'],input[type='submit']"));
|
|
for (const b of btns) {
|
|
const t = (b.textContent || (b as HTMLInputElement).value || "").replace(/\s+/g, " ").trim().toLowerCase();
|
|
if (t && ACCEPT.has(t) && vis(b)) { click(b); dismissed.push("text:" + t); break; }
|
|
}
|
|
}
|
|
return dismissed;
|
|
}),
|
|
new Promise<string[]>((res) => setTimeout(() => res([]), 6000)),
|
|
]);
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stage 2 — overlay/popup dismissal, phase 2 (run after a settle). Detect any
|
|
* full-viewport, high-z, fixed/sticky overlay still present. A fixed nav is wide
|
|
* but short; a sticky sidebar is tall but narrow — both excluded; a consent wall /
|
|
* modal backdrop covers most of both axes. If the page is *still scroll-locked*
|
|
* (the modal is genuinely blocking) remove the overlay — but ONLY when its id/class
|
|
* looks like a consent/modal layer, never a header/nav/main/footer, so legitimate
|
|
* sticky chrome is never stripped. Reports `blocking` = a scroll-locking overlay we
|
|
* could not clear (the pollution gate keys off this, not mere overlay presence).
|
|
*/
|
|
async function finalizeOverlays(page: import("playwright").Page): Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }> {
|
|
try {
|
|
return await Promise.race([
|
|
page.evaluate(() => {
|
|
const vw = window.innerWidth, vh = window.innerHeight;
|
|
const bigOverlays = (): HTMLElement[] => {
|
|
const out: HTMLElement[] = [];
|
|
for (const el of Array.from(document.body.querySelectorAll("*"))) {
|
|
const cs = getComputedStyle(el);
|
|
if (cs.position !== "fixed" && cs.position !== "sticky") continue;
|
|
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) continue;
|
|
const r = (el as HTMLElement).getBoundingClientRect();
|
|
const z = parseInt(cs.zIndex || "0", 10) || 0;
|
|
const area = (r.width * r.height) / (vw * vh);
|
|
if (area >= 0.5 && z >= 100 && r.width >= vw * 0.7 && r.height >= vh * 0.5) out.push(el as HTMLElement);
|
|
}
|
|
return out.filter((el) => !out.some((o) => o !== el && o.contains(el)));
|
|
};
|
|
const isLocked = (): boolean => {
|
|
const b = document.body, h = document.documentElement;
|
|
return getComputedStyle(b).overflow === "hidden" || getComputedStyle(h).overflow === "hidden" ||
|
|
getComputedStyle(b).position === "fixed";
|
|
};
|
|
// A scroll-locked page behind a full-viewport overlay IS a blocking modal by
|
|
// definition (legit pages don't scroll-lock). So remove ANY such overlay that
|
|
// isn't page chrome — many modals/drawers carry no consent/modal keyword and
|
|
// an icon-only close, so a keyword/aria allowlist misses them (ruggable's
|
|
// z-[1001] drawer). PROTECTED guards real chrome (header/nav/footer) only.
|
|
const PROTECTED = /header|navbar|nav-|site-nav|topbar|masthead|footer/i;
|
|
const sig = (el: HTMLElement): string => `${el.id} ${el.className}`.toString();
|
|
|
|
const removedLabels: string[] = [];
|
|
let removed = 0;
|
|
let remaining = bigOverlays();
|
|
if (remaining.length && isLocked()) {
|
|
for (const el of remaining) {
|
|
const s = sig(el);
|
|
const z = parseInt(getComputedStyle(el).zIndex || "0", 10) || 0;
|
|
// Scroll-locked + full-viewport ⇒ blocking modal; remove unless it's page
|
|
// chrome. Always remove iframes (cross-origin close, unclickable) and the
|
|
// max-z-index popup trick.
|
|
const removable = !PROTECTED.test(s) || el.getAttribute("aria-modal") === "true" ||
|
|
el.tagName === "IFRAME" || z >= 2_000_000_000;
|
|
if (removable) { el.remove(); removed++; removedLabels.push((el.id || el.className || el.tagName).toString().slice(0, 40)); }
|
|
}
|
|
if (removed) { document.body.style.overflow = "visible"; document.documentElement.style.overflow = "visible"; document.body.style.position = "static"; }
|
|
remaining = bigOverlays();
|
|
}
|
|
return { overlaysRemaining: remaining.length, removed, blocking: remaining.length > 0 && isLocked(), removedLabels };
|
|
}),
|
|
new Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }>((res) => setTimeout(() => res({ overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }), 6000)),
|
|
]);
|
|
} catch {
|
|
return { overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stage 2 — animation settling. Wait until layout stops moving (entrance/scroll
|
|
* reveals finished) before measuring, so geometry isn't sampled mid-transition
|
|
* ("sizes off due to animations in progress"). Samples large-box geometry across
|
|
* frames; resolves when stable for several windows or the bound elapses.
|
|
*/
|
|
async function waitForQuiescence(page: import("playwright").Page, maxMs = 4000): Promise<boolean> {
|
|
try {
|
|
return await Promise.race([
|
|
page.evaluate(async (budget: number) => {
|
|
const sample = (): string => {
|
|
const els = Array.from(document.body.querySelectorAll("*")).filter((e) => {
|
|
const r = (e as HTMLElement).getBoundingClientRect();
|
|
return r.width > 80 && r.height > 40;
|
|
}).slice(0, 240);
|
|
return els.map((e) => {
|
|
const r = (e as HTMLElement).getBoundingClientRect();
|
|
return `${Math.round(r.x)},${Math.round(r.y)},${Math.round(r.width)},${Math.round(r.height)}`;
|
|
}).join("|");
|
|
};
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
const deadline = Date.now() + budget;
|
|
let prev = sample();
|
|
let stable = 0;
|
|
while (Date.now() < deadline && stable < 3) {
|
|
await sleep(130);
|
|
const cur = sample();
|
|
if (cur === prev) stable++; else { stable = 0; prev = cur; }
|
|
}
|
|
return stable >= 3;
|
|
}, maxMs),
|
|
new Promise<boolean>((res) => setTimeout(() => res(false), maxMs + 1500)),
|
|
]);
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stage 2 — dynamic-media first frame. A streamed `<video>` has no deterministic
|
|
* frame and its request aborts at snapshot time, so the element would render blank.
|
|
* For videos lacking a `poster`, materialize a representative still and point the
|
|
* element's poster at it (via a synthetic URL whose bytes the normal asset pipeline
|
|
* rewrites to a local file). Two acquisition paths:
|
|
* 1. canvas — draw the decoded frame; exact, but THROWS for cross-origin/tainted
|
|
* videos (common: CDN-hosted hero videos with no CORS header).
|
|
* 2. element screenshot (the fallback) — rasterize the element's painted region
|
|
* via the DevTools protocol; works regardless of CORS and even when the video
|
|
* decoded late, because it reads composited pixels rather than the media buffer.
|
|
* This closes the "poster-less video renders blank" gap (e.g. descript's hero).
|
|
*
|
|
* Returns canvas stills (bytes ready) + the videos that still need a node-side
|
|
* element screenshot (the page can't screenshot itself), each tagged with a stable
|
|
* selector. Videos with no usable surface (no poster obtainable) fall back to the
|
|
* transparent placeholder, same as before.
|
|
*/
|
|
type VideoStillPlan = { stills: Array<{ url: string; dataUrl: string }>; shots: Array<{ url: string; sel: string }> };
|
|
async function captureVideoStills(page: import("playwright").Page): Promise<VideoStillPlan> {
|
|
try {
|
|
const plan = await Promise.race([
|
|
page.evaluate(async () => {
|
|
const stills: Array<{ url: string; dataUrl: string }> = [];
|
|
const shots: Array<{ url: string; sel: string }> = [];
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
const hash = (s: string): string => { let h = 5381; for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0; return h.toString(36); };
|
|
const vids = Array.from(document.querySelectorAll("video"));
|
|
let i = 0;
|
|
for (const v of vids) {
|
|
if (v.getAttribute("poster")) continue; // real poster already discovered
|
|
const r = v.getBoundingClientRect();
|
|
if (r.width < 16 || r.height < 16) continue; // not a visible media surface
|
|
const idx = i++;
|
|
v.setAttribute("data-clone-vid", String(idx));
|
|
const key = v.currentSrc || (v.querySelector("source") as HTMLSourceElement | null)?.src || String(idx);
|
|
const url = `https://clone-still.local/${idx}-${hash(key)}.jpg`;
|
|
try { v.pause(); } catch { /* ignore */ }
|
|
try { v.currentTime = 0; } catch { /* ignore */ }
|
|
await sleep(100);
|
|
let done = false;
|
|
const w = v.videoWidth, h = v.videoHeight;
|
|
if (w && h && v.readyState >= 2) {
|
|
try {
|
|
const canvas = document.createElement("canvas");
|
|
canvas.width = w; canvas.height = h;
|
|
const ctx = canvas.getContext("2d");
|
|
if (ctx) {
|
|
ctx.drawImage(v, 0, 0, w, h);
|
|
const dataUrl = canvas.toDataURL("image/jpeg", 0.82); // throws if tainted
|
|
if (dataUrl.startsWith("data:image/jpeg")) { stills.push({ url, dataUrl }); done = true; }
|
|
}
|
|
} catch { /* tainted/cross-origin — fall through to the element screenshot */ }
|
|
}
|
|
if (done) {
|
|
v.setAttribute("poster", url);
|
|
} else {
|
|
// Element screenshots capture composited pixels. For background hero
|
|
// videos with text/CTAs overlaid, that would bake the foreground into
|
|
// the poster and then render the live DOM on top again.
|
|
let occluded = false;
|
|
for (const el of Array.from(document.querySelectorAll("h1,h2,h3,h4,h5,p,span,a,button,li"))) {
|
|
if (v.contains(el) || (el.textContent || "").trim().length < 3) continue;
|
|
const cs = getComputedStyle(el);
|
|
if (cs.visibility === "hidden" || cs.display === "none" || parseFloat(cs.opacity || "1") < 0.05) continue;
|
|
const er = el.getBoundingClientRect();
|
|
if (er.width < 2 || er.height < 2) continue;
|
|
const ix = Math.min(er.right, r.right) - Math.max(er.left, r.left);
|
|
const iy = Math.min(er.bottom, r.bottom) - Math.max(er.top, r.top);
|
|
if (ix > 0 && iy > 0 && ix * iy > 0.05 * er.width * er.height) { occluded = true; break; }
|
|
}
|
|
if (!occluded) {
|
|
v.setAttribute("poster", url);
|
|
shots.push({ url, sel: `video[data-clone-vid="${idx}"]` });
|
|
}
|
|
}
|
|
}
|
|
return { stills, shots };
|
|
}),
|
|
new Promise<VideoStillPlan>((res) => setTimeout(() => res({ stills: [], shots: [] }), 12000)),
|
|
]);
|
|
return plan;
|
|
} catch {
|
|
return { stills: [], shots: [] };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Full-page screenshot with robustness for heavy/animated pages. The default 30s
|
|
* timeout is exceeded by tall SaaS pages (Playwright also waits for web fonts);
|
|
* use a long timeout, freeze animations (also improves determinism), and retry.
|
|
* As a last resort take a viewport-only shot so the file exists (the capture gate
|
|
* checks presence; a partial image still beats none).
|
|
*/
|
|
async function captureScreenshot(
|
|
page: import("playwright").Page,
|
|
path: string,
|
|
vw: number,
|
|
log: (e: Record<string, unknown>) => void,
|
|
): Promise<void> {
|
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
try {
|
|
await page.screenshot({ path, fullPage: true, timeout: 90_000, animations: "disabled" });
|
|
return;
|
|
} catch (e) {
|
|
if (attempt === 1) {
|
|
try {
|
|
await page.screenshot({ path, fullPage: false, timeout: 30_000, animations: "disabled" });
|
|
log({ event: "screenshot_fallback_viewport", viewport: vw });
|
|
return;
|
|
} catch (e2) {
|
|
log({ event: "screenshot_error", viewport: vw, error: String(e2) });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function captureSite(opts: {
|
|
url: string;
|
|
outDir: string; // source/ directory
|
|
viewports?: number[];
|
|
interactions?: boolean; // Stage 4: opt-in interaction capture (hover/focus + patterns)
|
|
motion?: boolean; // Stage 5: opt-in motion capture (WAAPI + rotating text)
|
|
breakpoints?: boolean; // discover the source's real responsive band edges (default on; read-only sweep)
|
|
screenshots?: boolean; // write per-viewport full-page PNGs (default on). ONLY the validator reads these
|
|
// (generation never touches pixels), and full-page shots of tall pages are the
|
|
// dominant capture cost — so a production clone that won't be perceptually graded
|
|
// can skip them. The cheap poster-less-video element stills are always kept
|
|
// (generation needs a first frame), this only gates the per-viewport page shots.
|
|
log?: (event: Record<string, unknown>) => void;
|
|
}): Promise<CaptureResult> {
|
|
const viewports = opts.viewports ?? [...REQUIRED_VIEWPORTS];
|
|
const log = opts.log ?? (() => {});
|
|
const captureDir = join(opts.outDir, "capture");
|
|
const screenshotsDir = join(opts.outDir, "screenshots");
|
|
const cssDir = join(captureDir, "css");
|
|
const storeDir = join(opts.outDir, "assets-store");
|
|
ensureDir(captureDir);
|
|
ensureDir(screenshotsDir);
|
|
|
|
// url -> discovered asset (merged across viewports)
|
|
const assetMap = new Map<string, DiscoveredAsset>();
|
|
const cssStored = new Set<string>();
|
|
const fontFaceMap = new Map<string, FontFace>();
|
|
const seoResourceUrls = new Map<string, SeoResource["kind"]>();
|
|
|
|
const sourceOrigin = (() => {
|
|
try {
|
|
const u = new URL(opts.url);
|
|
return u.protocol === "http:" || u.protocol === "https:" ? u.origin : "";
|
|
} catch {
|
|
return "";
|
|
}
|
|
})();
|
|
const addSeoResource = (url: string, kind: SeoResource["kind"], base = opts.url): void => {
|
|
try {
|
|
const abs = new URL(url, base).href;
|
|
if (!/^https?:\/\//i.test(abs)) return;
|
|
if (!seoResourceUrls.has(abs)) seoResourceUrls.set(abs, kind);
|
|
} catch { /* ignore malformed resource hints */ }
|
|
};
|
|
if (sourceOrigin) {
|
|
addSeoResource(sourceOrigin + "/robots.txt", "robots");
|
|
addSeoResource(sourceOrigin + "/sitemap.xml", "sitemap");
|
|
addSeoResource(sourceOrigin + "/sitemap_index.xml", "sitemap");
|
|
addSeoResource(sourceOrigin + "/llms.txt", "llms");
|
|
addSeoResource(sourceOrigin + "/llms-full.txt", "llms-full");
|
|
}
|
|
|
|
const recordAsset = (url: string, type: string, contentType: string | null, status: number | null, via: string): DiscoveredAsset => {
|
|
let a = assetMap.get(url);
|
|
if (!a) {
|
|
a = { url, type, contentType, status, storedAs: null, bytes: 0, via: [] };
|
|
assetMap.set(url, a);
|
|
}
|
|
if (!a.via.includes(via)) a.via.push(via);
|
|
if (contentType && !a.contentType) a.contentType = contentType;
|
|
if (status != null && a.status == null) a.status = status;
|
|
// SVG/specific types win over generic
|
|
if (type && (a.type === "other" || (a.type === "image" && type === "svg"))) a.type = type;
|
|
return a;
|
|
};
|
|
|
|
const storeBytes = (url: string, type: string, bytes: Buffer): void => {
|
|
if (!bytes || bytes.length === 0) return;
|
|
const a = assetMap.get(url) ?? recordAsset(url, type, null, null, "network");
|
|
if (a.storedAs) return;
|
|
const ext = extFromUrl(url) || extFromContentType(a.contentType) ||
|
|
(type === "css" ? "css" : type === "font" ? "woff2" : type === "svg" ? "svg" :
|
|
type === "video" ? "mp4" : type === "lottie" ? "json" : "png");
|
|
const name = `${sha1_12(url)}.${ext}`;
|
|
if (type === "css") {
|
|
writeBytes(join(cssDir, name), bytes);
|
|
cssStored.add(name);
|
|
} else {
|
|
writeBytes(join(storeDir, name), bytes);
|
|
}
|
|
a.storedAs = name;
|
|
a.bytes = bytes.length;
|
|
};
|
|
|
|
const parseManifestForAssets = (text: string, baseUrl: string): void => {
|
|
let parsed: unknown;
|
|
try { parsed = JSON.parse(text); } catch { return; }
|
|
if (!parsed || typeof parsed !== "object") return;
|
|
const push = (raw: unknown, via: string): void => {
|
|
if (typeof raw !== "string" || !raw.trim() || raw.trim().startsWith("data:")) return;
|
|
let abs = raw;
|
|
try { abs = new URL(raw, baseUrl).href; } catch { /* keep raw */ }
|
|
const t = classifyAsset(abs, null) ?? "image";
|
|
recordAsset(abs, t, null, null, via);
|
|
};
|
|
const iconList = (parsed as { icons?: unknown }).icons;
|
|
if (Array.isArray(iconList)) {
|
|
for (const icon of iconList) if (icon && typeof icon === "object") push((icon as { src?: unknown }).src, "manifest:icons");
|
|
}
|
|
const screenshots = (parsed as { screenshots?: unknown }).screenshots;
|
|
if (Array.isArray(screenshots)) {
|
|
for (const shot of screenshots) if (shot && typeof shot === "object") push((shot as { src?: unknown }).src, "manifest:screenshots");
|
|
}
|
|
const shortcuts = (parsed as { shortcuts?: unknown }).shortcuts;
|
|
if (Array.isArray(shortcuts)) {
|
|
for (const shortcut of shortcuts) {
|
|
const icons = shortcut && typeof shortcut === "object" ? (shortcut as { icons?: unknown }).icons : undefined;
|
|
if (Array.isArray(icons)) for (const icon of icons) if (icon && typeof icon === "object") push((icon as { src?: unknown }).src, "manifest:shortcuts");
|
|
}
|
|
}
|
|
};
|
|
|
|
// Honor the standard HTTPS_PROXY env so capture works behind an egress proxy
|
|
// (sandboxed/CI environments). The proxy re-terminates TLS, so the per-context
|
|
// `ignoreHTTPSErrors` below covers its CA. Capture is not part of the determinism
|
|
// gate, so this never affects generated output.
|
|
// Honor HTTPS_PROXY for real remote captures behind an egress proxy (sandboxed/CI),
|
|
// but ONLY for non-loopback http(s) targets — file:// and localhost fixtures (tests,
|
|
// the validator's static server) must be fetched directly. Playwright otherwise routes
|
|
// even loopback through the proxy (its default `<-loopback>`), which would replace a
|
|
// local fixture with the proxy's error page. Capture is not part of the determinism
|
|
// gate, so this never affects generated output.
|
|
let proxyServer = process.env.HTTPS_PROXY || process.env.https_proxy || "";
|
|
try {
|
|
const u = new URL(opts.url);
|
|
// Skip the proxy ONLY for a localhost http(s) target (test fixtures, the validator's
|
|
// static server) — Playwright otherwise routes loopback through the proxy and replaces
|
|
// the page with the proxy error page. file:// keeps the proxy on so any REMOTE assets
|
|
// the local page references fast-fail through the proxy instead of hanging on a blocked
|
|
// direct connection; external targets obviously keep it.
|
|
const isHttp = u.protocol === "http:" || u.protocol === "https:";
|
|
const isLoopback = /^(localhost|127\.0\.0\.1|\[?::1\]?|0\.0\.0\.0)$/.test(u.hostname);
|
|
if (isHttp && isLoopback) proxyServer = "";
|
|
} catch { /* keep proxy */ }
|
|
const browser: Browser = await chromium.launch({
|
|
headless: true,
|
|
args: ["--disable-blink-features=AutomationControlled", "--disable-dev-shm-usage"],
|
|
...(proxyServer ? { proxy: { server: proxyServer } } : {}),
|
|
});
|
|
|
|
const perViewport: CaptureResult["perViewport"] = [];
|
|
let interaction: InteractionCapture | undefined;
|
|
let motion: MotionCapture | undefined;
|
|
let discoveredBreakpoints: number[] | undefined;
|
|
const captureSeoResources: SeoResource[] = [];
|
|
const cssTextsForParsing: Array<{ baseUrl: string; text: string }> = [];
|
|
const dismissUnion = { dismissed: [] as string[], overlaysRemaining: 0, removed: 0, videoStills: 0, blocking: false };
|
|
// Carry cookies/localStorage across the per-viewport contexts so the SAME page
|
|
// is captured at every width: A/B-test buckets and consent state are usually
|
|
// session-persisted, so without this each fresh context can load a different
|
|
// variant (grammarly served a different hero per viewport) and the cross-
|
|
// viewport IR alignment then can't reconcile them. Sharing the session also
|
|
// means a banner dismissed at the first width stays dismissed at the rest.
|
|
const canonical = viewports.includes(1280) ? 1280 : viewports[Math.floor(viewports.length / 2)]!;
|
|
|
|
try {
|
|
// Single context + single navigation; every viewport is captured by RESIZING
|
|
// the same loaded page (no reload). This guarantees identical content across
|
|
// widths — eliminating both A/B-per-load variance (grammarly served a different
|
|
// hero on each fresh load) and session-reuse degeneration (warbyparker returned
|
|
// a 13-node shell when reloaded with a carried session). The IR's cross-viewport
|
|
// alignment then operates on one logical DOM that CSS merely reflows.
|
|
const context: BrowserContext = await browser.newContext({
|
|
ignoreHTTPSErrors: true,
|
|
viewport: { width: canonical, height: viewportHeight(canonical) },
|
|
deviceScaleFactor: 1,
|
|
userAgent: DESKTOP_UA,
|
|
javaScriptEnabled: true,
|
|
});
|
|
const page = await context.newPage();
|
|
// tsx/esbuild wraps functions with a __name() helper for stack traces; that
|
|
// helper does not exist in the browser when we serialize page.evaluate
|
|
// callbacks. Shim it (as a raw string so it isn't itself transformed).
|
|
await page.addInitScript(ESBUILD_SHIM);
|
|
const bodyPromises: Promise<void>[] = [];
|
|
|
|
page.on("response", (resp) => {
|
|
try {
|
|
const url = resp.url();
|
|
if (url.startsWith("data:") || url.startsWith("blob:")) return;
|
|
const ct = resp.headers()["content-type"] || null;
|
|
const type = classifyAsset(url, ct);
|
|
if (!type) return;
|
|
const status = resp.status();
|
|
recordAsset(url, type, ct, status, "network");
|
|
const existing = assetMap.get(url);
|
|
if (existing?.storedAs) return;
|
|
if (status >= 400) return;
|
|
bodyPromises.push(
|
|
(async () => {
|
|
try {
|
|
const buf = await resp.body();
|
|
storeBytes(url, type, buf);
|
|
if (type === "css") cssTextsForParsing.push({ baseUrl: url, text: buf.toString("utf8") });
|
|
if (type === "manifest") parseManifestForAssets(buf.toString("utf8"), url);
|
|
} catch { /* body unavailable */ }
|
|
})(),
|
|
);
|
|
} catch { /* ignore */ }
|
|
});
|
|
|
|
// Single navigation at the canonical width; every viewport below is a resize.
|
|
log({ event: "goto", url: opts.url });
|
|
let navigated = false;
|
|
let navErr: unknown = null;
|
|
for (let attempt = 0; attempt < 3 && !navigated; attempt++) {
|
|
try {
|
|
await page.goto(opts.url, { waitUntil: attempt === 0 ? "load" : "domcontentloaded", timeout: 45000 });
|
|
navigated = true;
|
|
} catch (e) {
|
|
navErr = e;
|
|
await page.waitForTimeout(1000);
|
|
}
|
|
}
|
|
if (!navigated) throw navErr;
|
|
await settle(page);
|
|
|
|
for (const vw of viewports) {
|
|
const vh = viewportHeight(vw);
|
|
await page.setViewportSize({ width: vw, height: vh });
|
|
await settle(page, 1500); // let the resize reflow + any width-triggered content settle
|
|
|
|
// Stage 2: dismiss cookie/consent/newsletter/region overlays. Run TWICE —
|
|
// once after initial load (cookie/consent walls appear immediately), and
|
|
// again after the scroll pass (newsletter/email-capture modals are usually
|
|
// timer- or scroll-triggered and only mount a few seconds in). Each pass
|
|
// clicks the accept/close control, settles (so a just-closed dialog unlocks
|
|
// scrolling), THEN removes only a genuinely-stuck consent/modal layer.
|
|
let overlaysRemaining = 0;
|
|
let blocking = false;
|
|
const applyDismiss = async (phase: string): Promise<void> => {
|
|
const clicked = await clickDismiss(page);
|
|
if (clicked.length) await settle(page, 1000);
|
|
const fin = await finalizeOverlays(page);
|
|
for (const d of clicked) if (!dismissUnion.dismissed.includes(d)) dismissUnion.dismissed.push(d);
|
|
for (const d of fin.removedLabels) { const k = "removed:" + d; if (!dismissUnion.dismissed.includes(k)) dismissUnion.dismissed.push(k); }
|
|
dismissUnion.removed += fin.removed;
|
|
overlaysRemaining = fin.overlaysRemaining;
|
|
blocking = blocking || fin.blocking;
|
|
if (clicked.length || fin.removed) { await settle(page, 1000); log({ event: "dismissed", viewport: vw, phase, count: clicked.length, removed: fin.removed, remaining: fin.overlaysRemaining, blocking: fin.blocking }); }
|
|
};
|
|
await applyDismiss("load");
|
|
|
|
// Stage 5 (scroll reveals): at the FIRST viewport, before any scroll, tag elements
|
|
// and snapshot the pre-reveal hidden state — scroll reveals fire on the first
|
|
// autoScroll and stay revealed, so this is the only moment their hidden state is
|
|
// observable. Idempotent + motion-gated; the settled snapshot is unchanged.
|
|
if (opts.motion && vw === viewports[0]) { await tagElements(page); await probeReveals(page); }
|
|
|
|
await autoScroll(page, vh);
|
|
await settle(page, 1500);
|
|
await applyDismiss("post-scroll");
|
|
dismissUnion.overlaysRemaining = Math.max(dismissUnion.overlaysRemaining, overlaysRemaining);
|
|
dismissUnion.blocking = dismissUnion.blocking || blocking;
|
|
// Stage 2: wait for entrance/scroll animations to settle so geometry isn't
|
|
// sampled mid-transition.
|
|
const quiescent = await waitForQuiescence(page);
|
|
|
|
// Reset window + all inner scrollable containers to scroll 0 so captured
|
|
// positions match the generated app's default (un-scrolled) render. Inner
|
|
// scroll state is runtime JS state and otherwise non-deterministic.
|
|
await page.evaluate(() => {
|
|
window.scrollTo(0, 0);
|
|
for (const el of Array.from(document.querySelectorAll("*"))) {
|
|
if (el.scrollLeft) el.scrollLeft = 0;
|
|
if (el.scrollTop) el.scrollTop = 0;
|
|
}
|
|
});
|
|
await page.waitForTimeout(150);
|
|
|
|
// Stage 2: dynamic-media first frame. Materialize a still for poster-less
|
|
// videos so they don't render blank — canvas where the frame is readable, else
|
|
// an element screenshot (the page cannot screenshot itself). Done once at the
|
|
// canonical viewport: it's the highest-fidelity width, the poster attr persists
|
|
// across the later resizes, and the IR reads attrs from the canonical snapshot.
|
|
if (vw === canonical) {
|
|
const plan = await captureVideoStills(page);
|
|
for (const s of plan.stills) {
|
|
const comma = s.dataUrl.indexOf(",");
|
|
if (comma < 0) continue;
|
|
try {
|
|
const buf = Buffer.from(s.dataUrl.slice(comma + 1), "base64");
|
|
recordAsset(s.url, "image", "image/jpeg", 200, "video-still");
|
|
storeBytes(s.url, "image", buf);
|
|
dismissUnion.videoStills++;
|
|
} catch { /* ignore */ }
|
|
}
|
|
for (const s of plan.shots) {
|
|
try {
|
|
const buf = await page.locator(s.sel).first().screenshot({ type: "jpeg", quality: 82, timeout: 5000, animations: "disabled" });
|
|
recordAsset(s.url, "image", "image/jpeg", 200, "video-still");
|
|
storeBytes(s.url, "image", buf);
|
|
dismissUnion.videoStills++;
|
|
} catch { /* element not shootable (off-screen/detached) — poster falls back to placeholder */ }
|
|
}
|
|
// Element screenshots scroll the target into view; restore scroll 0 so the
|
|
// canonical DOM walk + screenshot match the generated app's default render.
|
|
if (plan.shots.length) {
|
|
await page.evaluate(() => window.scrollTo(0, 0));
|
|
await page.waitForTimeout(80);
|
|
}
|
|
}
|
|
|
|
// Stage 4/5: stamp capture-ids before the canonical snapshot so the IR carries
|
|
// them (whitelisted), enabling interaction deltas + motion specs to map to cids.
|
|
if ((opts.interactions || opts.motion) && vw === canonical) await tagElements(page);
|
|
|
|
// Bound the in-page DOM walk: page.evaluate has no default timeout, so a
|
|
// pathologically large/animated DOM (e.g. asana.com) could hang forever.
|
|
const snapshot: PageSnapshot = await Promise.race([
|
|
page.evaluate(collectPage),
|
|
new Promise<never>((_, rej) => setTimeout(() => rej(new Error(`collectPage timeout vp${vw}`)), 60_000)),
|
|
]);
|
|
|
|
// Merge discovered references from the snapshot (DOM + accessible CSS).
|
|
for (const da of snapshot.domAssets) {
|
|
const t = classifyAsset(da.url, null) ?? (da.kind === "manifest" ? "manifest" : da.kind === "video" ? "video" : da.kind === "svg" ? "svg" : "image");
|
|
recordAsset(da.url, t, null, null, da.via);
|
|
}
|
|
for (const link of snapshot.doc.head?.links ?? []) {
|
|
const rel = (link.rel || "").toLowerCase();
|
|
const href = link.href || "";
|
|
if (!href) continue;
|
|
if (/\bsitemap\b/.test(rel)) addSeoResource(href, "sitemap", snapshot.doc.url);
|
|
if (/llms-full\.txt(?:$|[?#])/i.test(href) || /\bllms-full\b/.test(rel)) addSeoResource(href, "llms-full", snapshot.doc.url);
|
|
else if (/llms\.txt(?:$|[?#])/i.test(href) || /\bllms\b/.test(rel)) addSeoResource(href, "llms", snapshot.doc.url);
|
|
}
|
|
for (const u of snapshot.cssUrls) {
|
|
const t = classifyAsset(u, null) ?? "other";
|
|
recordAsset(u, t, null, null, "css-url");
|
|
}
|
|
for (const ff of snapshot.fontFaces) {
|
|
const key = `${ff.family}|${ff.weight ?? ""}|${ff.style ?? ""}|${ff.src}`;
|
|
if (!fontFaceMap.has(key)) fontFaceMap.set(key, ff);
|
|
}
|
|
|
|
// Cap body collection: resp.body() has no timeout, so a streaming/long-poll
|
|
// response (common on heavy SaaS pages) would hang allSettled forever. By
|
|
// now bodies have been downloading throughout navigation+scroll; stragglers
|
|
// are dropped (the post-pass fallback fetch re-fetches anything missing).
|
|
await Promise.race([
|
|
Promise.allSettled(bodyPromises),
|
|
new Promise((r) => setTimeout(r, 20_000)),
|
|
]);
|
|
|
|
// Persist DOM snapshot, and (unless skipped for a production clone) the full-page screenshot.
|
|
writeJSONCompact(join(captureDir, `dom-${vw}.json`), snapshot);
|
|
if (opts.screenshots !== false) await captureScreenshot(page, join(screenshotsDir, `${vw}.png`), vw, log);
|
|
|
|
// Stage 4: drive recognized affordances at the canonical viewport (opt-in).
|
|
if (opts.interactions && vw === canonical) {
|
|
try { interaction = await captureInteractions(page, { log }); }
|
|
catch (e) { log({ event: "interactions_error", error: String(e).slice(0, 200) }); }
|
|
}
|
|
|
|
// Stage 5: capture motion (WAAPI + rotating text) at the canonical viewport.
|
|
if (opts.motion && vw === canonical) {
|
|
try { motion = await captureMotion(page, { log }); }
|
|
catch (e) { log({ event: "motion_error", error: String(e).slice(0, 200) }); }
|
|
// Register lottie source JSONs as assets so the asset stage downloads + materializes
|
|
// them (the in-page detector only records the URL; the fallback fetch grabs the file).
|
|
for (const l of motion?.lotties ?? []) {
|
|
if (l.src) { recordAsset(l.src, "lottie", null, null, "lottie"); continue; }
|
|
// Inline animationData (no fetchable URL): store it as a real .json asset so it
|
|
// materializes to /assets/cloned/lottie like any other source, instead of being
|
|
// embedded in the page spec. Content-hashed so output stays deterministic.
|
|
if (l.inlineKey && motion?.lottieInline) {
|
|
const data = motion.lottieInline[l.inlineKey];
|
|
if (data === undefined) continue;
|
|
const buf = Buffer.from(JSON.stringify(data), "utf8");
|
|
const synthUrl = `ditto-inline:/lottie/${sha1_12(buf.toString("utf8"))}.json`;
|
|
recordAsset(synthUrl, "lottie", "application/json", 200, "lottie-inline");
|
|
storeBytes(synthUrl, "lottie", buf);
|
|
l.src = synthUrl; // now resolves to a local path through the asset graph
|
|
l.inlineKey = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
perViewport.push({
|
|
viewport: vw,
|
|
height: vh,
|
|
scrollHeight: round(snapshot.doc.scrollHeight),
|
|
nodeCount: snapshot.doc.nodeCount,
|
|
truncated: snapshot.doc.truncated,
|
|
overlaysRemaining,
|
|
blocking,
|
|
quiescent,
|
|
});
|
|
log({ event: "captured", viewport: vw, nodes: snapshot.doc.nodeCount, scrollHeight: snapshot.doc.scrollHeight });
|
|
}
|
|
|
|
// Browser-as-oracle: discover the source's real responsive band edges by sweeping the viewport
|
|
// and binary-searching each width where the discrete (media-query-toggled) layout signature
|
|
// changes. Read-only and bounded; runs once here — overlays are dismissed and the DOM settled, so
|
|
// the signature is the real page, not a consent wall. Never fatal: a failed sweep just omits the
|
|
// field. The context closes right after, so the swept viewport size needs no restore.
|
|
if (opts.breakpoints ?? true) {
|
|
try {
|
|
const edges = await Promise.race([
|
|
discoverBreakpoints(page, { min: 320, max: 1920 }),
|
|
new Promise<number[]>((_, rej) => setTimeout(() => rej(new Error("breakpoint sweep timeout")), 60_000)),
|
|
]);
|
|
discoveredBreakpoints = edges;
|
|
log({ event: "breakpoints_discovered", edges });
|
|
} catch (e) {
|
|
log({ event: "breakpoints_error", error: String(e).slice(0, 200) });
|
|
}
|
|
}
|
|
await context.close();
|
|
|
|
// Parse @font-face + url() from cross-origin CSS texts we fetched at the
|
|
// network layer (the in-page walker can only read same-origin sheets).
|
|
for (const { baseUrl, text } of cssTextsForParsing) {
|
|
parseCssForFonts(text, baseUrl, fontFaceMap, (u) => {
|
|
const t = classifyAsset(u, null) ?? "other";
|
|
recordAsset(u, t, null, null, "css-text");
|
|
});
|
|
}
|
|
|
|
// Fallback: download any referenced-but-not-yet-stored asset using the
|
|
// browser context (shares TLS handling). Fonts and CSS-referenced images are
|
|
// commonly missed by the response listener when served from cache.
|
|
const fallbackCtx = await browser.newContext({ ignoreHTTPSErrors: true, userAgent: DESKTOP_UA });
|
|
for (const a of assetMap.values()) {
|
|
if (a.storedAs) continue;
|
|
if (a.url.startsWith("data:")) continue;
|
|
if (!["image", "svg", "video", "font", "lottie", "css", "manifest"].includes(a.type)) continue;
|
|
try {
|
|
const resp = await fallbackCtx.request.get(a.url, { timeout: 30000 });
|
|
a.status = resp.status();
|
|
if (resp.ok()) {
|
|
const buf = await resp.body();
|
|
a.contentType = a.contentType ?? (resp.headers()["content-type"] || null);
|
|
storeBytes(a.url, a.type, buf);
|
|
if (a.type === "css") parseCssForFonts(buf.toString("utf8"), a.url, fontFaceMap, (u) => {
|
|
const t = classifyAsset(u, null) ?? "other";
|
|
recordAsset(u, t, null, null, "css-text");
|
|
});
|
|
if (a.type === "manifest") parseManifestForAssets(buf.toString("utf8"), a.url);
|
|
}
|
|
} catch { /* unreachable/signed — left as skipped */ }
|
|
}
|
|
const seoResources: SeoResource[] = [];
|
|
const fetchedSeo = new Set<string>();
|
|
const fetchSeoResource = async (url: string, kind: SeoResource["kind"]): Promise<void> => {
|
|
if (fetchedSeo.has(url)) return;
|
|
fetchedSeo.add(url);
|
|
try {
|
|
const resp = await fallbackCtx.request.get(url, { timeout: 15000 });
|
|
const contentType = resp.headers()["content-type"] || null;
|
|
const resource: SeoResource = { kind, url, status: resp.status(), contentType };
|
|
if (resp.ok() && (kind === "llms" || kind === "llms-full" || kind === "robots")) {
|
|
const text = await resp.text();
|
|
resource.text = text;
|
|
if (kind === "robots") {
|
|
for (const line of text.split(/\r?\n/)) {
|
|
const m = /^sitemap:\s*(\S+)/i.exec(line.trim());
|
|
if (m) addSeoResource(m[1]!, "sitemap", url);
|
|
}
|
|
}
|
|
}
|
|
seoResources.push(resource);
|
|
} catch {
|
|
seoResources.push({ kind, url, status: null, contentType: null });
|
|
}
|
|
};
|
|
for (let pass = 0; pass < 2; pass++) {
|
|
const entries = [...seoResourceUrls.entries()].sort((a, b) => a[0].localeCompare(b[0]));
|
|
for (const [url, kind] of entries) await fetchSeoResource(url, kind);
|
|
}
|
|
await fallbackCtx.close();
|
|
|
|
if (seoResources.length) {
|
|
(captureSeoResources as SeoResource[]).push(...seoResources.sort((a, b) => a.url.localeCompare(b.url) || a.kind.localeCompare(b.kind)));
|
|
}
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
|
|
const assets = [...assetMap.values()].sort((a, b) => a.url.localeCompare(b.url));
|
|
const fontFaces = [...fontFaceMap.values()].sort((a, b) =>
|
|
`${a.family}${a.weight}${a.style}`.localeCompare(`${b.family}${b.weight}${b.style}`),
|
|
);
|
|
|
|
const result: CaptureResult = {
|
|
sourceUrl: opts.url,
|
|
capturedAt: new Date().toISOString(),
|
|
viewports,
|
|
breakpoints: discoveredBreakpoints,
|
|
perViewport,
|
|
assets,
|
|
...(captureSeoResources.length ? { seoResources: captureSeoResources } : {}),
|
|
fontFaces,
|
|
cssTexts: [...cssStored].sort(),
|
|
dismissal: dismissUnion,
|
|
interaction,
|
|
motion,
|
|
};
|
|
|
|
if (interaction) writeJSON(join(opts.outDir, "interaction.json"), interaction);
|
|
if (motion) writeJSON(join(opts.outDir, "motion.json"), motion);
|
|
writeJSON(join(opts.outDir, "assets-discovered.json"), assets);
|
|
writeJSON(join(opts.outDir, "fonts-discovered.json"), fontFaces);
|
|
writeJSON(join(captureDir, "capture-result.json"), result);
|
|
|
|
return result;
|
|
}
|
|
|
|
function parseCssForFonts(
|
|
cssText: string,
|
|
baseUrl: string,
|
|
fontFaceMap: Map<string, FontFace>,
|
|
onUrl: (url: string) => void,
|
|
): void {
|
|
const abs = (u: string): string => {
|
|
try { return new URL(u, baseUrl).href; } catch { return u; }
|
|
};
|
|
// Extract @font-face blocks. src urls are rewritten to absolute so they can be
|
|
// resolved/downloaded regardless of where the CSS file lived.
|
|
const faceRe = /@font-face\s*\{([^}]*)\}/gi;
|
|
let m: RegExpExecArray | null;
|
|
while ((m = faceRe.exec(cssText)) !== null) {
|
|
const block = m[1]!;
|
|
const family = /font-family\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim().replace(/^['"]|['"]$/g, "");
|
|
let src = /src\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
|
if (!family || !src) continue;
|
|
src = src.replace(/url\(\s*(['"]?)([^'")]+)\1\s*\)/gi, (_full, _q, u) =>
|
|
u.startsWith("data:") ? `url(${u})` : `url(${abs(u)})`);
|
|
const weight = /font-weight\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
|
const style = /font-style\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
|
const display = /font-display\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
|
const unicodeRange = /unicode-range\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
|
const key = `${family}|${weight ?? ""}|${style ?? ""}|${src}`;
|
|
if (!fontFaceMap.has(key)) {
|
|
fontFaceMap.set(key, { family, src, weight, style, display, unicodeRange });
|
|
}
|
|
}
|
|
// Extract all url() references for asset discovery.
|
|
const urlRe = /url\(\s*(['"]?)([^'")]+)\1\s*\)/gi;
|
|
while ((m = urlRe.exec(cssText)) !== null) {
|
|
const raw = m[2];
|
|
if (raw && !raw.startsWith("data:")) onUrl(abs(raw));
|
|
}
|
|
}
|