Initial commit
This commit is contained in:
Samraaj Bath
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Breakpoint discovery using the browser as the layout oracle.
|
||||
*
|
||||
* We capture at a few fixed widths and force the responsive bands onto Tailwind's default
|
||||
* breakpoints — which is wrong for any site whose real cut points differ, and can spawn extra bands
|
||||
* when our sample widths happen to straddle a breakpoint that isn't really there. Instead, ASK the
|
||||
* page: sweep the viewport width and find the exact widths where the layout actually changes.
|
||||
*
|
||||
* Media queries cause DISCRETE jumps in structural properties (display, flex-direction, flex-wrap,
|
||||
* grid track count, position, float, text-align, visibility) — those flip only at a breakpoint,
|
||||
* whereas widths/heights scale smoothly. So we hash that discrete signature across a coarse width
|
||||
* scan, then binary-search each interval where the hash changed down to 1px. The result is the
|
||||
* site's REAL breakpoint set, to drive both the capture sample widths and the emitted band edges.
|
||||
*/
|
||||
import type { Page } from "playwright";
|
||||
|
||||
/** In-page: a hash of every element's discrete (media-query-toggled) layout properties. Pure
|
||||
* structural — excludes widths/heights/font-size, which vary continuously and would mask the steps. */
|
||||
const DISCRETE_SIGNATURE = (): string => {
|
||||
let s = "";
|
||||
const walk = (el: Element): void => {
|
||||
const cs = getComputedStyle(el);
|
||||
const cols = (cs.gridTemplateColumns || "none") === "none" ? 0 : cs.gridTemplateColumns.split(" ").filter(Boolean).length;
|
||||
const painted = (el as HTMLElement).offsetParent !== null || cs.position === "fixed" ? 1 : 0;
|
||||
s += `${cs.display}|${cs.flexDirection}|${cs.flexWrap}|${cols}|${cs.position}|${cs.float}|${cs.textAlign}|${cs.visibility}|${painted};`;
|
||||
for (let i = 0; i < el.children.length; i++) walk(el.children[i]!);
|
||||
};
|
||||
walk(document.body);
|
||||
let h = 0;
|
||||
for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0;
|
||||
return `${s.length}:${h}`;
|
||||
};
|
||||
|
||||
export async function discoverBreakpoints(
|
||||
page: Page,
|
||||
opts?: { min?: number; max?: number; coarseStep?: number; height?: number },
|
||||
): Promise<number[]> {
|
||||
const min = opts?.min ?? 320;
|
||||
const max = opts?.max ?? 1920;
|
||||
const step = opts?.coarseStep ?? 16;
|
||||
const height = opts?.height ?? 1200;
|
||||
const sig = async (w: number): Promise<string> => {
|
||||
await page.setViewportSize({ width: w, height });
|
||||
return page.evaluate(DISCRETE_SIGNATURE);
|
||||
};
|
||||
|
||||
// Coarse scan: record where the signature changes between adjacent samples.
|
||||
const coarse: { w: number; sig: string }[] = [];
|
||||
for (let w = min; w <= max; w += step) coarse.push({ w, sig: await sig(w) });
|
||||
|
||||
// Binary-search each changed interval down to 1px — the edge where the NEW layout begins.
|
||||
const edges: number[] = [];
|
||||
for (let i = 1; i < coarse.length; i++) {
|
||||
if (coarse[i]!.sig === coarse[i - 1]!.sig) continue;
|
||||
let lo = coarse[i - 1]!.w, hi = coarse[i]!.w; const loSig = coarse[i - 1]!.sig;
|
||||
while (hi - lo > 1) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
if ((await sig(mid)) === loSig) lo = mid; else hi = mid;
|
||||
}
|
||||
edges.push(hi);
|
||||
}
|
||||
return edges;
|
||||
}
|
||||
@@ -0,0 +1,996 @@
|
||||
import { chromium, type Browser, type BrowserContext } from "playwright";
|
||||
import { join } from "node:path";
|
||||
import { collectPage, type PageSnapshot, type FontFace } from "./walker.js";
|
||||
import { tagElements, captureInteractions, type InteractionCapture } from "./interactions.js";
|
||||
import { captureMotion, probeReveals, type MotionCapture } from "./motion.js";
|
||||
import { discoverBreakpoints } from "./breakpoints.js";
|
||||
import { writeJSON, writeJSONCompact, writeBytes, ensureDir } from "../util/fsx.js";
|
||||
import { sha1_12, round } from "../util/canonical.js";
|
||||
|
||||
export const REQUIRED_VIEWPORTS = [375, 768, 1280, 1920] as const;
|
||||
// The dense width set captured for SIZE INFERENCE: a node sampled at 9 widths reveals its sizing
|
||||
// law (constant / proportional / clamped / flex-distributed) far better than 4 — enough to drop a
|
||||
// baked px for a relative construct with confidence (and to surface a shrunk item's natural size,
|
||||
// which needs widths beyond 1920). REQUIRED_VIEWPORTS ⊂ this; only those 4 carry responsive bands.
|
||||
export const SAMPLE_VIEWPORTS = [375, 480, 640, 768, 1024, 1280, 1536, 1920, 2560] as const;
|
||||
|
||||
const VIEWPORT_HEIGHTS: Record<number, number> = {
|
||||
375: 812,
|
||||
480: 854,
|
||||
640: 960,
|
||||
768: 1024,
|
||||
1024: 768,
|
||||
1280: 800,
|
||||
1536: 864,
|
||||
1920: 1080,
|
||||
2560: 1440,
|
||||
};
|
||||
|
||||
const DESKTOP_UA =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
||||
|
||||
// Defines esbuild/tsx runtime helpers in the page so serialized evaluate
|
||||
// callbacks that reference them don't throw ReferenceError.
|
||||
const ESBUILD_SHIM =
|
||||
"globalThis.__name = globalThis.__name || ((fn) => fn);" +
|
||||
"globalThis.__defProp = globalThis.__defProp || Object.defineProperty;";
|
||||
|
||||
export type DiscoveredAsset = {
|
||||
url: string;
|
||||
type: string; // image|svg|video|font|lottie|css|manifest|other
|
||||
contentType: string | null;
|
||||
status: number | null;
|
||||
storedAs: string | null; // sha1-named file in assets-store, or null if not downloaded
|
||||
bytes: number; // size of stored file
|
||||
via: string[]; // discovery sources
|
||||
};
|
||||
|
||||
export type SeoResource = {
|
||||
kind: "robots" | "sitemap" | "llms" | "llms-full";
|
||||
url: string;
|
||||
status: number | null;
|
||||
contentType: string | null;
|
||||
text?: string;
|
||||
};
|
||||
|
||||
export type CaptureResult = {
|
||||
sourceUrl: string;
|
||||
capturedAt: string;
|
||||
viewports: number[];
|
||||
// Browser-as-oracle: the widths where the SOURCE layout actually restructures (display /
|
||||
// flex-direction / wrap / grid-track-count / position / visibility flips), found by sweeping the
|
||||
// viewport and binary-searching each discrete-signature change. Ground truth for the real
|
||||
// responsive band edges — distinct from REQUIRED_VIEWPORTS (our fixed capture widths). Absent if
|
||||
// discovery was disabled or the sweep failed. See breakpoints.ts.
|
||||
breakpoints?: number[];
|
||||
perViewport: Array<{
|
||||
viewport: number;
|
||||
height: number;
|
||||
scrollHeight: number;
|
||||
nodeCount: number;
|
||||
truncated: boolean;
|
||||
overlaysRemaining?: number;
|
||||
blocking?: boolean;
|
||||
quiescent?: boolean;
|
||||
}>;
|
||||
assets: DiscoveredAsset[];
|
||||
seoResources?: SeoResource[];
|
||||
fontFaces: FontFace[];
|
||||
cssTexts: string[]; // sha1 names of stored css files
|
||||
// Stage 2: overlay/popup dismissal audit (union of actions across viewports).
|
||||
dismissal?: { dismissed: string[]; overlaysRemaining: number; removed: number; videoStills: number; blocking: boolean };
|
||||
// Stage 4: optional interaction capture (hover/focus + recognized patterns).
|
||||
interaction?: InteractionCapture;
|
||||
// Stage 5: optional motion capture (WAAPI animations + rotating text). CSS @keyframes
|
||||
// motion is reconstructed from the IR, so it isn't re-captured here.
|
||||
motion?: MotionCapture;
|
||||
};
|
||||
|
||||
function viewportHeight(width: number): number {
|
||||
return VIEWPORT_HEIGHTS[width] ?? Math.round(width * 0.66);
|
||||
}
|
||||
|
||||
function extFromUrl(url: string): string {
|
||||
try {
|
||||
const p = new URL(url).pathname;
|
||||
const dot = p.lastIndexOf(".");
|
||||
if (dot >= 0 && dot > p.lastIndexOf("/")) {
|
||||
const ext = p.slice(dot + 1).toLowerCase().slice(0, 5);
|
||||
if (/^[a-z0-9]+$/.test(ext)) return ext;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
return "";
|
||||
}
|
||||
|
||||
const CONTENT_TYPE_EXT: Record<string, string> = {
|
||||
"image/webp": "webp", "image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg",
|
||||
"image/avif": "avif", "image/gif": "gif", "image/svg+xml": "svg",
|
||||
"image/x-icon": "ico", "image/vnd.microsoft.icon": "ico", "image/bmp": "bmp",
|
||||
"video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", "video/ogg": "ogv",
|
||||
"font/woff2": "woff2", "font/woff": "woff", "font/ttf": "ttf", "font/otf": "otf",
|
||||
"application/font-woff2": "woff2", "application/font-woff": "woff",
|
||||
"application/x-font-ttf": "ttf", "application/vnd.ms-fontobject": "eot",
|
||||
"text/css": "css", "application/json": "json", "application/manifest+json": "webmanifest",
|
||||
};
|
||||
|
||||
function extFromContentType(contentType: string | null): string {
|
||||
if (!contentType) return "";
|
||||
const ct = contentType.split(";")[0]!.trim().toLowerCase();
|
||||
return CONTENT_TYPE_EXT[ct] ?? "";
|
||||
}
|
||||
|
||||
function classifyAsset(url: string, contentType: string | null): string | null {
|
||||
const u = url.toLowerCase().split("?")[0]!;
|
||||
const ct = (contentType || "").toLowerCase();
|
||||
if (u.endsWith(".svg") || ct === "image/svg+xml") return "svg";
|
||||
if (/\.(jpg|jpeg|png|webp|avif|gif|ico|bmp)$/.test(u) || ct.startsWith("image/")) return "image";
|
||||
if (/\.(mp4|mov|webm|m4v|ogv)$/.test(u) || ct.startsWith("video/")) return "video";
|
||||
if (/\.(woff2|woff|ttf|otf|eot)$/.test(u) || ct.startsWith("font/") ||
|
||||
ct.includes("font-woff") || ct.includes("x-font")) return "font";
|
||||
if (u.endsWith(".json") && (u.includes("lottie") || u.includes("animation"))) return "lottie";
|
||||
if (u.endsWith(".webmanifest") || /(?:^|\/)manifest\.json$/.test(u) || ct.includes("manifest+json")) return "manifest";
|
||||
if (u.endsWith(".css") || ct.startsWith("text/css")) return "css";
|
||||
return null;
|
||||
}
|
||||
|
||||
function isCss(url: string, contentType: string | null): boolean {
|
||||
return classifyAsset(url, contentType) === "css";
|
||||
}
|
||||
|
||||
async function autoScroll(page: import("playwright").Page, vpHeight: number): Promise<void> {
|
||||
// Scroll through the page to trigger lazy-loaded images/backgrounds, then return
|
||||
// to the top so document-coordinate bboxes are measured from a settled layout.
|
||||
await page.evaluate(async (step: number) => {
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const maxScroll = () => document.documentElement.scrollHeight - window.innerHeight;
|
||||
let y = 0;
|
||||
let guard = 0;
|
||||
while (y < maxScroll() && guard < 100) {
|
||||
y += step;
|
||||
window.scrollTo(0, y);
|
||||
await sleep(60);
|
||||
guard++;
|
||||
}
|
||||
window.scrollTo(0, 0);
|
||||
await sleep(120);
|
||||
}, Math.max(Math.round(vpHeight * 0.8), 300));
|
||||
}
|
||||
|
||||
async function settle(page: import("playwright").Page, maxMs = 2500): Promise<void> {
|
||||
try {
|
||||
await page.waitForLoadState("networkidle", { timeout: Math.max(maxMs / 2, 800) });
|
||||
} catch { /* ignore */ }
|
||||
try {
|
||||
// document.fonts.ready can never resolve when a font request hangs (heavy SaaS
|
||||
// pages), and page.evaluate has no default timeout — so bound it explicitly.
|
||||
await Promise.race([
|
||||
page.evaluate(() => (document as Document).fonts?.ready as unknown as Promise<void>),
|
||||
new Promise<void>((r) => setTimeout(r, 6000)),
|
||||
]);
|
||||
} catch { /* ignore */ }
|
||||
await page.waitForTimeout(250);
|
||||
}
|
||||
|
||||
export type DismissResult = { dismissed: string[]; overlaysRemaining: number; removed: number; blocking: boolean };
|
||||
|
||||
/**
|
||||
* Stage 2 — overlay/popup dismissal, phase 1: click the accept/close affordance.
|
||||
* Cookie-consent walls, newsletter modals, region/age/app-install interstitials
|
||||
* cover the real page; the capture must see the state a returning user sees.
|
||||
* Deterministic + replayable: click the same known/accept controls in DOM order.
|
||||
* Removal of a stuck overlay happens later (finalizeOverlays) AFTER a settle, so a
|
||||
* just-clicked dialog has time to close and unlock scrolling before we judge it.
|
||||
*/
|
||||
async function clickDismiss(page: import("playwright").Page): Promise<string[]> {
|
||||
try {
|
||||
return await Promise.race([
|
||||
page.evaluate(() => {
|
||||
const dismissed: string[] = [];
|
||||
const vis = (el: Element): boolean => {
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) return false;
|
||||
const r = (el as HTMLElement).getBoundingClientRect();
|
||||
return r.width > 0 && r.height > 0;
|
||||
};
|
||||
const click = (el: Element): void => { try { (el as HTMLElement).click(); } catch { /* ignore */ } };
|
||||
|
||||
// 1) Known consent-framework / generic close affordances, in priority order.
|
||||
const KNOWN = [
|
||||
"#onetrust-accept-btn-handler", "#accept-recommended-btn-handler", ".onetrust-close-btn-handler",
|
||||
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", "#CybotCookiebotDialogBodyButtonAccept",
|
||||
"#truste-consent-button", ".osano-cm-accept-all", ".osano-cm-dialog__close",
|
||||
"[data-testid='uc-accept-all-button']", "[data-testid='uc-deny-all-button']",
|
||||
"#didomi-notice-agree-button", ".didomi-continue-without-agreeing",
|
||||
".qc-cmp2-summary-buttons button[mode='primary']", "button[aria-label='Consent']",
|
||||
".cc-allow", ".cookie-consent-accept", "#hs-eu-confirmation-button", "#gdpr-consent-tool-wrapper button",
|
||||
];
|
||||
for (const sel of KNOWN) {
|
||||
try {
|
||||
for (const el of Array.from(document.querySelectorAll(sel))) {
|
||||
if (vis(el)) { click(el); dismissed.push(sel); break; }
|
||||
}
|
||||
} catch { /* invalid selector in this browser */ }
|
||||
}
|
||||
|
||||
// 2) Scoped text-button pass: only inside overlay-ish containers (dialogs,
|
||||
// or id/class naming cookie/consent/modal/popup/newsletter) so we never
|
||||
// click an ordinary page button.
|
||||
const ACCEPT = new Set([
|
||||
"accept", "accept all", "accept all cookies", "accept cookies", "accept & close",
|
||||
"i accept", "i agree", "agree", "agree and continue", "allow all", "allow cookies",
|
||||
"allow all cookies", "got it", "ok", "okay", "continue", "no thanks", "no, thanks",
|
||||
"dismiss", "close", "got it!", "understood", "yes, i agree",
|
||||
]);
|
||||
const containerSel =
|
||||
"[role='dialog'],[aria-modal='true'],[id*='cookie' i],[class*='cookie' i],[id*='consent' i]," +
|
||||
"[class*='consent' i],[class*='gdpr' i],[id*='gdpr' i],[class*='modal' i],[class*='popup' i]," +
|
||||
"[class*='newsletter' i],[class*='interstitial' i]";
|
||||
let containers: Element[] = [];
|
||||
try { containers = Array.from(document.querySelectorAll(containerSel)); } catch { /* ignore */ }
|
||||
for (const c of containers) {
|
||||
if (!vis(c)) continue;
|
||||
const btns = Array.from(c.querySelectorAll("button,[role='button'],a,input[type='button'],input[type='submit']"));
|
||||
for (const b of btns) {
|
||||
const t = (b.textContent || (b as HTMLInputElement).value || "").replace(/\s+/g, " ").trim().toLowerCase();
|
||||
if (t && ACCEPT.has(t) && vis(b)) { click(b); dismissed.push("text:" + t); break; }
|
||||
}
|
||||
}
|
||||
return dismissed;
|
||||
}),
|
||||
new Promise<string[]>((res) => setTimeout(() => res([]), 6000)),
|
||||
]);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 2 — overlay/popup dismissal, phase 2 (run after a settle). Detect any
|
||||
* full-viewport, high-z, fixed/sticky overlay still present. A fixed nav is wide
|
||||
* but short; a sticky sidebar is tall but narrow — both excluded; a consent wall /
|
||||
* modal backdrop covers most of both axes. If the page is *still scroll-locked*
|
||||
* (the modal is genuinely blocking) remove the overlay — but ONLY when its id/class
|
||||
* looks like a consent/modal layer, never a header/nav/main/footer, so legitimate
|
||||
* sticky chrome is never stripped. Reports `blocking` = a scroll-locking overlay we
|
||||
* could not clear (the pollution gate keys off this, not mere overlay presence).
|
||||
*/
|
||||
async function finalizeOverlays(page: import("playwright").Page): Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }> {
|
||||
try {
|
||||
return await Promise.race([
|
||||
page.evaluate(() => {
|
||||
const vw = window.innerWidth, vh = window.innerHeight;
|
||||
const bigOverlays = (): HTMLElement[] => {
|
||||
const out: HTMLElement[] = [];
|
||||
for (const el of Array.from(document.body.querySelectorAll("*"))) {
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.position !== "fixed" && cs.position !== "sticky") continue;
|
||||
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) continue;
|
||||
const r = (el as HTMLElement).getBoundingClientRect();
|
||||
const z = parseInt(cs.zIndex || "0", 10) || 0;
|
||||
const area = (r.width * r.height) / (vw * vh);
|
||||
if (area >= 0.5 && z >= 100 && r.width >= vw * 0.7 && r.height >= vh * 0.5) out.push(el as HTMLElement);
|
||||
}
|
||||
return out.filter((el) => !out.some((o) => o !== el && o.contains(el)));
|
||||
};
|
||||
const isLocked = (): boolean => {
|
||||
const b = document.body, h = document.documentElement;
|
||||
return getComputedStyle(b).overflow === "hidden" || getComputedStyle(h).overflow === "hidden" ||
|
||||
getComputedStyle(b).position === "fixed";
|
||||
};
|
||||
// A scroll-locked page behind a full-viewport overlay IS a blocking modal by
|
||||
// definition (legit pages don't scroll-lock). So remove ANY such overlay that
|
||||
// isn't page chrome — many modals/drawers carry no consent/modal keyword and
|
||||
// an icon-only close, so a keyword/aria allowlist misses them (ruggable's
|
||||
// z-[1001] drawer). PROTECTED guards real chrome (header/nav/footer) only.
|
||||
const PROTECTED = /header|navbar|nav-|site-nav|topbar|masthead|footer/i;
|
||||
const sig = (el: HTMLElement): string => `${el.id} ${el.className}`.toString();
|
||||
|
||||
const removedLabels: string[] = [];
|
||||
let removed = 0;
|
||||
let remaining = bigOverlays();
|
||||
if (remaining.length && isLocked()) {
|
||||
for (const el of remaining) {
|
||||
const s = sig(el);
|
||||
const z = parseInt(getComputedStyle(el).zIndex || "0", 10) || 0;
|
||||
// Scroll-locked + full-viewport ⇒ blocking modal; remove unless it's page
|
||||
// chrome. Always remove iframes (cross-origin close, unclickable) and the
|
||||
// max-z-index popup trick.
|
||||
const removable = !PROTECTED.test(s) || el.getAttribute("aria-modal") === "true" ||
|
||||
el.tagName === "IFRAME" || z >= 2_000_000_000;
|
||||
if (removable) { el.remove(); removed++; removedLabels.push((el.id || el.className || el.tagName).toString().slice(0, 40)); }
|
||||
}
|
||||
if (removed) { document.body.style.overflow = "visible"; document.documentElement.style.overflow = "visible"; document.body.style.position = "static"; }
|
||||
remaining = bigOverlays();
|
||||
}
|
||||
return { overlaysRemaining: remaining.length, removed, blocking: remaining.length > 0 && isLocked(), removedLabels };
|
||||
}),
|
||||
new Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }>((res) => setTimeout(() => res({ overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }), 6000)),
|
||||
]);
|
||||
} catch {
|
||||
return { overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 2 — animation settling. Wait until layout stops moving (entrance/scroll
|
||||
* reveals finished) before measuring, so geometry isn't sampled mid-transition
|
||||
* ("sizes off due to animations in progress"). Samples large-box geometry across
|
||||
* frames; resolves when stable for several windows or the bound elapses.
|
||||
*/
|
||||
async function waitForQuiescence(page: import("playwright").Page, maxMs = 4000): Promise<boolean> {
|
||||
try {
|
||||
return await Promise.race([
|
||||
page.evaluate(async (budget: number) => {
|
||||
const sample = (): string => {
|
||||
const els = Array.from(document.body.querySelectorAll("*")).filter((e) => {
|
||||
const r = (e as HTMLElement).getBoundingClientRect();
|
||||
return r.width > 80 && r.height > 40;
|
||||
}).slice(0, 240);
|
||||
return els.map((e) => {
|
||||
const r = (e as HTMLElement).getBoundingClientRect();
|
||||
return `${Math.round(r.x)},${Math.round(r.y)},${Math.round(r.width)},${Math.round(r.height)}`;
|
||||
}).join("|");
|
||||
};
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const deadline = Date.now() + budget;
|
||||
let prev = sample();
|
||||
let stable = 0;
|
||||
while (Date.now() < deadline && stable < 3) {
|
||||
await sleep(130);
|
||||
const cur = sample();
|
||||
if (cur === prev) stable++; else { stable = 0; prev = cur; }
|
||||
}
|
||||
return stable >= 3;
|
||||
}, maxMs),
|
||||
new Promise<boolean>((res) => setTimeout(() => res(false), maxMs + 1500)),
|
||||
]);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 2 — dynamic-media first frame. A streamed `<video>` has no deterministic
|
||||
* frame and its request aborts at snapshot time, so the element would render blank.
|
||||
* For videos lacking a `poster`, materialize a representative still and point the
|
||||
* element's poster at it (via a synthetic URL whose bytes the normal asset pipeline
|
||||
* rewrites to a local file). Two acquisition paths:
|
||||
* 1. canvas — draw the decoded frame; exact, but THROWS for cross-origin/tainted
|
||||
* videos (common: CDN-hosted hero videos with no CORS header).
|
||||
* 2. element screenshot (the fallback) — rasterize the element's painted region
|
||||
* via the DevTools protocol; works regardless of CORS and even when the video
|
||||
* decoded late, because it reads composited pixels rather than the media buffer.
|
||||
* This closes the "poster-less video renders blank" gap (e.g. descript's hero).
|
||||
*
|
||||
* Returns canvas stills (bytes ready) + the videos that still need a node-side
|
||||
* element screenshot (the page can't screenshot itself), each tagged with a stable
|
||||
* selector. Videos with no usable surface (no poster obtainable) fall back to the
|
||||
* transparent placeholder, same as before.
|
||||
*/
|
||||
type VideoStillPlan = { stills: Array<{ url: string; dataUrl: string }>; shots: Array<{ url: string; sel: string }> };
|
||||
async function captureVideoStills(page: import("playwright").Page): Promise<VideoStillPlan> {
|
||||
try {
|
||||
const plan = await Promise.race([
|
||||
page.evaluate(async () => {
|
||||
const stills: Array<{ url: string; dataUrl: string }> = [];
|
||||
const shots: Array<{ url: string; sel: string }> = [];
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const hash = (s: string): string => { let h = 5381; for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0; return h.toString(36); };
|
||||
const vids = Array.from(document.querySelectorAll("video"));
|
||||
let i = 0;
|
||||
for (const v of vids) {
|
||||
if (v.getAttribute("poster")) continue; // real poster already discovered
|
||||
const r = v.getBoundingClientRect();
|
||||
if (r.width < 16 || r.height < 16) continue; // not a visible media surface
|
||||
const idx = i++;
|
||||
v.setAttribute("data-clone-vid", String(idx));
|
||||
const key = v.currentSrc || (v.querySelector("source") as HTMLSourceElement | null)?.src || String(idx);
|
||||
const url = `https://clone-still.local/${idx}-${hash(key)}.jpg`;
|
||||
try { v.pause(); } catch { /* ignore */ }
|
||||
try { v.currentTime = 0; } catch { /* ignore */ }
|
||||
await sleep(100);
|
||||
let done = false;
|
||||
const w = v.videoWidth, h = v.videoHeight;
|
||||
if (w && h && v.readyState >= 2) {
|
||||
try {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = w; canvas.height = h;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (ctx) {
|
||||
ctx.drawImage(v, 0, 0, w, h);
|
||||
const dataUrl = canvas.toDataURL("image/jpeg", 0.82); // throws if tainted
|
||||
if (dataUrl.startsWith("data:image/jpeg")) { stills.push({ url, dataUrl }); done = true; }
|
||||
}
|
||||
} catch { /* tainted/cross-origin — fall through to the element screenshot */ }
|
||||
}
|
||||
if (done) {
|
||||
v.setAttribute("poster", url);
|
||||
} else {
|
||||
// Element screenshots capture composited pixels. For background hero
|
||||
// videos with text/CTAs overlaid, that would bake the foreground into
|
||||
// the poster and then render the live DOM on top again.
|
||||
let occluded = false;
|
||||
for (const el of Array.from(document.querySelectorAll("h1,h2,h3,h4,h5,p,span,a,button,li"))) {
|
||||
if (v.contains(el) || (el.textContent || "").trim().length < 3) continue;
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.visibility === "hidden" || cs.display === "none" || parseFloat(cs.opacity || "1") < 0.05) continue;
|
||||
const er = el.getBoundingClientRect();
|
||||
if (er.width < 2 || er.height < 2) continue;
|
||||
const ix = Math.min(er.right, r.right) - Math.max(er.left, r.left);
|
||||
const iy = Math.min(er.bottom, r.bottom) - Math.max(er.top, r.top);
|
||||
if (ix > 0 && iy > 0 && ix * iy > 0.05 * er.width * er.height) { occluded = true; break; }
|
||||
}
|
||||
if (!occluded) {
|
||||
v.setAttribute("poster", url);
|
||||
shots.push({ url, sel: `video[data-clone-vid="${idx}"]` });
|
||||
}
|
||||
}
|
||||
}
|
||||
return { stills, shots };
|
||||
}),
|
||||
new Promise<VideoStillPlan>((res) => setTimeout(() => res({ stills: [], shots: [] }), 12000)),
|
||||
]);
|
||||
return plan;
|
||||
} catch {
|
||||
return { stills: [], shots: [] };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-page screenshot with robustness for heavy/animated pages. The default 30s
|
||||
* timeout is exceeded by tall SaaS pages (Playwright also waits for web fonts);
|
||||
* use a long timeout, freeze animations (also improves determinism), and retry.
|
||||
* As a last resort take a viewport-only shot so the file exists (the capture gate
|
||||
* checks presence; a partial image still beats none).
|
||||
*/
|
||||
async function captureScreenshot(
|
||||
page: import("playwright").Page,
|
||||
path: string,
|
||||
vw: number,
|
||||
log: (e: Record<string, unknown>) => void,
|
||||
): Promise<void> {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
await page.screenshot({ path, fullPage: true, timeout: 90_000, animations: "disabled" });
|
||||
return;
|
||||
} catch (e) {
|
||||
if (attempt === 1) {
|
||||
try {
|
||||
await page.screenshot({ path, fullPage: false, timeout: 30_000, animations: "disabled" });
|
||||
log({ event: "screenshot_fallback_viewport", viewport: vw });
|
||||
return;
|
||||
} catch (e2) {
|
||||
log({ event: "screenshot_error", viewport: vw, error: String(e2) });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureSite(opts: {
|
||||
url: string;
|
||||
outDir: string; // source/ directory
|
||||
viewports?: number[];
|
||||
interactions?: boolean; // Stage 4: opt-in interaction capture (hover/focus + patterns)
|
||||
motion?: boolean; // Stage 5: opt-in motion capture (WAAPI + rotating text)
|
||||
breakpoints?: boolean; // discover the source's real responsive band edges (default on; read-only sweep)
|
||||
screenshots?: boolean; // write per-viewport full-page PNGs (default on). ONLY the validator reads these
|
||||
// (generation never touches pixels), and full-page shots of tall pages are the
|
||||
// dominant capture cost — so a production clone that won't be perceptually graded
|
||||
// can skip them. The cheap poster-less-video element stills are always kept
|
||||
// (generation needs a first frame), this only gates the per-viewport page shots.
|
||||
log?: (event: Record<string, unknown>) => void;
|
||||
}): Promise<CaptureResult> {
|
||||
const viewports = opts.viewports ?? [...REQUIRED_VIEWPORTS];
|
||||
const log = opts.log ?? (() => {});
|
||||
const captureDir = join(opts.outDir, "capture");
|
||||
const screenshotsDir = join(opts.outDir, "screenshots");
|
||||
const cssDir = join(captureDir, "css");
|
||||
const storeDir = join(opts.outDir, "assets-store");
|
||||
ensureDir(captureDir);
|
||||
ensureDir(screenshotsDir);
|
||||
|
||||
// url -> discovered asset (merged across viewports)
|
||||
const assetMap = new Map<string, DiscoveredAsset>();
|
||||
const cssStored = new Set<string>();
|
||||
const fontFaceMap = new Map<string, FontFace>();
|
||||
const seoResourceUrls = new Map<string, SeoResource["kind"]>();
|
||||
|
||||
const sourceOrigin = (() => {
|
||||
try {
|
||||
const u = new URL(opts.url);
|
||||
return u.protocol === "http:" || u.protocol === "https:" ? u.origin : "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
})();
|
||||
const addSeoResource = (url: string, kind: SeoResource["kind"], base = opts.url): void => {
|
||||
try {
|
||||
const abs = new URL(url, base).href;
|
||||
if (!/^https?:\/\//i.test(abs)) return;
|
||||
if (!seoResourceUrls.has(abs)) seoResourceUrls.set(abs, kind);
|
||||
} catch { /* ignore malformed resource hints */ }
|
||||
};
|
||||
if (sourceOrigin) {
|
||||
addSeoResource(sourceOrigin + "/robots.txt", "robots");
|
||||
addSeoResource(sourceOrigin + "/sitemap.xml", "sitemap");
|
||||
addSeoResource(sourceOrigin + "/sitemap_index.xml", "sitemap");
|
||||
addSeoResource(sourceOrigin + "/llms.txt", "llms");
|
||||
addSeoResource(sourceOrigin + "/llms-full.txt", "llms-full");
|
||||
}
|
||||
|
||||
const recordAsset = (url: string, type: string, contentType: string | null, status: number | null, via: string): DiscoveredAsset => {
|
||||
let a = assetMap.get(url);
|
||||
if (!a) {
|
||||
a = { url, type, contentType, status, storedAs: null, bytes: 0, via: [] };
|
||||
assetMap.set(url, a);
|
||||
}
|
||||
if (!a.via.includes(via)) a.via.push(via);
|
||||
if (contentType && !a.contentType) a.contentType = contentType;
|
||||
if (status != null && a.status == null) a.status = status;
|
||||
// SVG/specific types win over generic
|
||||
if (type && (a.type === "other" || (a.type === "image" && type === "svg"))) a.type = type;
|
||||
return a;
|
||||
};
|
||||
|
||||
const storeBytes = (url: string, type: string, bytes: Buffer): void => {
|
||||
if (!bytes || bytes.length === 0) return;
|
||||
const a = assetMap.get(url) ?? recordAsset(url, type, null, null, "network");
|
||||
if (a.storedAs) return;
|
||||
const ext = extFromUrl(url) || extFromContentType(a.contentType) ||
|
||||
(type === "css" ? "css" : type === "font" ? "woff2" : type === "svg" ? "svg" :
|
||||
type === "video" ? "mp4" : type === "lottie" ? "json" : "png");
|
||||
const name = `${sha1_12(url)}.${ext}`;
|
||||
if (type === "css") {
|
||||
writeBytes(join(cssDir, name), bytes);
|
||||
cssStored.add(name);
|
||||
} else {
|
||||
writeBytes(join(storeDir, name), bytes);
|
||||
}
|
||||
a.storedAs = name;
|
||||
a.bytes = bytes.length;
|
||||
};
|
||||
|
||||
const parseManifestForAssets = (text: string, baseUrl: string): void => {
|
||||
let parsed: unknown;
|
||||
try { parsed = JSON.parse(text); } catch { return; }
|
||||
if (!parsed || typeof parsed !== "object") return;
|
||||
const push = (raw: unknown, via: string): void => {
|
||||
if (typeof raw !== "string" || !raw.trim() || raw.trim().startsWith("data:")) return;
|
||||
let abs = raw;
|
||||
try { abs = new URL(raw, baseUrl).href; } catch { /* keep raw */ }
|
||||
const t = classifyAsset(abs, null) ?? "image";
|
||||
recordAsset(abs, t, null, null, via);
|
||||
};
|
||||
const iconList = (parsed as { icons?: unknown }).icons;
|
||||
if (Array.isArray(iconList)) {
|
||||
for (const icon of iconList) if (icon && typeof icon === "object") push((icon as { src?: unknown }).src, "manifest:icons");
|
||||
}
|
||||
const screenshots = (parsed as { screenshots?: unknown }).screenshots;
|
||||
if (Array.isArray(screenshots)) {
|
||||
for (const shot of screenshots) if (shot && typeof shot === "object") push((shot as { src?: unknown }).src, "manifest:screenshots");
|
||||
}
|
||||
const shortcuts = (parsed as { shortcuts?: unknown }).shortcuts;
|
||||
if (Array.isArray(shortcuts)) {
|
||||
for (const shortcut of shortcuts) {
|
||||
const icons = shortcut && typeof shortcut === "object" ? (shortcut as { icons?: unknown }).icons : undefined;
|
||||
if (Array.isArray(icons)) for (const icon of icons) if (icon && typeof icon === "object") push((icon as { src?: unknown }).src, "manifest:shortcuts");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Honor the standard HTTPS_PROXY env so capture works behind an egress proxy
|
||||
// (sandboxed/CI environments). The proxy re-terminates TLS, so the per-context
|
||||
// `ignoreHTTPSErrors` below covers its CA. Capture is not part of the determinism
|
||||
// gate, so this never affects generated output.
|
||||
// Honor HTTPS_PROXY for real remote captures behind an egress proxy (sandboxed/CI),
|
||||
// but ONLY for non-loopback http(s) targets — file:// and localhost fixtures (tests,
|
||||
// the validator's static server) must be fetched directly. Playwright otherwise routes
|
||||
// even loopback through the proxy (its default `<-loopback>`), which would replace a
|
||||
// local fixture with the proxy's error page. Capture is not part of the determinism
|
||||
// gate, so this never affects generated output.
|
||||
let proxyServer = process.env.HTTPS_PROXY || process.env.https_proxy || "";
|
||||
try {
|
||||
const u = new URL(opts.url);
|
||||
// Skip the proxy ONLY for a localhost http(s) target (test fixtures, the validator's
|
||||
// static server) — Playwright otherwise routes loopback through the proxy and replaces
|
||||
// the page with the proxy error page. file:// keeps the proxy on so any REMOTE assets
|
||||
// the local page references fast-fail through the proxy instead of hanging on a blocked
|
||||
// direct connection; external targets obviously keep it.
|
||||
const isHttp = u.protocol === "http:" || u.protocol === "https:";
|
||||
const isLoopback = /^(localhost|127\.0\.0\.1|\[?::1\]?|0\.0\.0\.0)$/.test(u.hostname);
|
||||
if (isHttp && isLoopback) proxyServer = "";
|
||||
} catch { /* keep proxy */ }
|
||||
const browser: Browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ["--disable-blink-features=AutomationControlled", "--disable-dev-shm-usage"],
|
||||
...(proxyServer ? { proxy: { server: proxyServer } } : {}),
|
||||
});
|
||||
|
||||
const perViewport: CaptureResult["perViewport"] = [];
|
||||
let interaction: InteractionCapture | undefined;
|
||||
let motion: MotionCapture | undefined;
|
||||
let discoveredBreakpoints: number[] | undefined;
|
||||
const captureSeoResources: SeoResource[] = [];
|
||||
const cssTextsForParsing: Array<{ baseUrl: string; text: string }> = [];
|
||||
const dismissUnion = { dismissed: [] as string[], overlaysRemaining: 0, removed: 0, videoStills: 0, blocking: false };
|
||||
// Carry cookies/localStorage across the per-viewport contexts so the SAME page
|
||||
// is captured at every width: A/B-test buckets and consent state are usually
|
||||
// session-persisted, so without this each fresh context can load a different
|
||||
// variant (grammarly served a different hero per viewport) and the cross-
|
||||
// viewport IR alignment then can't reconcile them. Sharing the session also
|
||||
// means a banner dismissed at the first width stays dismissed at the rest.
|
||||
const canonical = viewports.includes(1280) ? 1280 : viewports[Math.floor(viewports.length / 2)]!;
|
||||
|
||||
try {
|
||||
// Single context + single navigation; every viewport is captured by RESIZING
|
||||
// the same loaded page (no reload). This guarantees identical content across
|
||||
// widths — eliminating both A/B-per-load variance (grammarly served a different
|
||||
// hero on each fresh load) and session-reuse degeneration (warbyparker returned
|
||||
// a 13-node shell when reloaded with a carried session). The IR's cross-viewport
|
||||
// alignment then operates on one logical DOM that CSS merely reflows.
|
||||
const context: BrowserContext = await browser.newContext({
|
||||
ignoreHTTPSErrors: true,
|
||||
viewport: { width: canonical, height: viewportHeight(canonical) },
|
||||
deviceScaleFactor: 1,
|
||||
userAgent: DESKTOP_UA,
|
||||
javaScriptEnabled: true,
|
||||
});
|
||||
const page = await context.newPage();
|
||||
// tsx/esbuild wraps functions with a __name() helper for stack traces; that
|
||||
// helper does not exist in the browser when we serialize page.evaluate
|
||||
// callbacks. Shim it (as a raw string so it isn't itself transformed).
|
||||
await page.addInitScript(ESBUILD_SHIM);
|
||||
const bodyPromises: Promise<void>[] = [];
|
||||
|
||||
page.on("response", (resp) => {
|
||||
try {
|
||||
const url = resp.url();
|
||||
if (url.startsWith("data:") || url.startsWith("blob:")) return;
|
||||
const ct = resp.headers()["content-type"] || null;
|
||||
const type = classifyAsset(url, ct);
|
||||
if (!type) return;
|
||||
const status = resp.status();
|
||||
recordAsset(url, type, ct, status, "network");
|
||||
const existing = assetMap.get(url);
|
||||
if (existing?.storedAs) return;
|
||||
if (status >= 400) return;
|
||||
bodyPromises.push(
|
||||
(async () => {
|
||||
try {
|
||||
const buf = await resp.body();
|
||||
storeBytes(url, type, buf);
|
||||
if (type === "css") cssTextsForParsing.push({ baseUrl: url, text: buf.toString("utf8") });
|
||||
if (type === "manifest") parseManifestForAssets(buf.toString("utf8"), url);
|
||||
} catch { /* body unavailable */ }
|
||||
})(),
|
||||
);
|
||||
} catch { /* ignore */ }
|
||||
});
|
||||
|
||||
// Single navigation at the canonical width; every viewport below is a resize.
|
||||
log({ event: "goto", url: opts.url });
|
||||
let navigated = false;
|
||||
let navErr: unknown = null;
|
||||
for (let attempt = 0; attempt < 3 && !navigated; attempt++) {
|
||||
try {
|
||||
await page.goto(opts.url, { waitUntil: attempt === 0 ? "load" : "domcontentloaded", timeout: 45000 });
|
||||
navigated = true;
|
||||
} catch (e) {
|
||||
navErr = e;
|
||||
await page.waitForTimeout(1000);
|
||||
}
|
||||
}
|
||||
if (!navigated) throw navErr;
|
||||
await settle(page);
|
||||
|
||||
for (const vw of viewports) {
|
||||
const vh = viewportHeight(vw);
|
||||
await page.setViewportSize({ width: vw, height: vh });
|
||||
await settle(page, 1500); // let the resize reflow + any width-triggered content settle
|
||||
|
||||
// Stage 2: dismiss cookie/consent/newsletter/region overlays. Run TWICE —
|
||||
// once after initial load (cookie/consent walls appear immediately), and
|
||||
// again after the scroll pass (newsletter/email-capture modals are usually
|
||||
// timer- or scroll-triggered and only mount a few seconds in). Each pass
|
||||
// clicks the accept/close control, settles (so a just-closed dialog unlocks
|
||||
// scrolling), THEN removes only a genuinely-stuck consent/modal layer.
|
||||
let overlaysRemaining = 0;
|
||||
let blocking = false;
|
||||
const applyDismiss = async (phase: string): Promise<void> => {
|
||||
const clicked = await clickDismiss(page);
|
||||
if (clicked.length) await settle(page, 1000);
|
||||
const fin = await finalizeOverlays(page);
|
||||
for (const d of clicked) if (!dismissUnion.dismissed.includes(d)) dismissUnion.dismissed.push(d);
|
||||
for (const d of fin.removedLabels) { const k = "removed:" + d; if (!dismissUnion.dismissed.includes(k)) dismissUnion.dismissed.push(k); }
|
||||
dismissUnion.removed += fin.removed;
|
||||
overlaysRemaining = fin.overlaysRemaining;
|
||||
blocking = blocking || fin.blocking;
|
||||
if (clicked.length || fin.removed) { await settle(page, 1000); log({ event: "dismissed", viewport: vw, phase, count: clicked.length, removed: fin.removed, remaining: fin.overlaysRemaining, blocking: fin.blocking }); }
|
||||
};
|
||||
await applyDismiss("load");
|
||||
|
||||
// Stage 5 (scroll reveals): at the FIRST viewport, before any scroll, tag elements
|
||||
// and snapshot the pre-reveal hidden state — scroll reveals fire on the first
|
||||
// autoScroll and stay revealed, so this is the only moment their hidden state is
|
||||
// observable. Idempotent + motion-gated; the settled snapshot is unchanged.
|
||||
if (opts.motion && vw === viewports[0]) { await tagElements(page); await probeReveals(page); }
|
||||
|
||||
await autoScroll(page, vh);
|
||||
await settle(page, 1500);
|
||||
await applyDismiss("post-scroll");
|
||||
dismissUnion.overlaysRemaining = Math.max(dismissUnion.overlaysRemaining, overlaysRemaining);
|
||||
dismissUnion.blocking = dismissUnion.blocking || blocking;
|
||||
// Stage 2: wait for entrance/scroll animations to settle so geometry isn't
|
||||
// sampled mid-transition.
|
||||
const quiescent = await waitForQuiescence(page);
|
||||
|
||||
// Reset window + all inner scrollable containers to scroll 0 so captured
|
||||
// positions match the generated app's default (un-scrolled) render. Inner
|
||||
// scroll state is runtime JS state and otherwise non-deterministic.
|
||||
await page.evaluate(() => {
|
||||
window.scrollTo(0, 0);
|
||||
for (const el of Array.from(document.querySelectorAll("*"))) {
|
||||
if (el.scrollLeft) el.scrollLeft = 0;
|
||||
if (el.scrollTop) el.scrollTop = 0;
|
||||
}
|
||||
});
|
||||
await page.waitForTimeout(150);
|
||||
|
||||
// Stage 2: dynamic-media first frame. Materialize a still for poster-less
|
||||
// videos so they don't render blank — canvas where the frame is readable, else
|
||||
// an element screenshot (the page cannot screenshot itself). Done once at the
|
||||
// canonical viewport: it's the highest-fidelity width, the poster attr persists
|
||||
// across the later resizes, and the IR reads attrs from the canonical snapshot.
|
||||
if (vw === canonical) {
|
||||
const plan = await captureVideoStills(page);
|
||||
for (const s of plan.stills) {
|
||||
const comma = s.dataUrl.indexOf(",");
|
||||
if (comma < 0) continue;
|
||||
try {
|
||||
const buf = Buffer.from(s.dataUrl.slice(comma + 1), "base64");
|
||||
recordAsset(s.url, "image", "image/jpeg", 200, "video-still");
|
||||
storeBytes(s.url, "image", buf);
|
||||
dismissUnion.videoStills++;
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
for (const s of plan.shots) {
|
||||
try {
|
||||
const buf = await page.locator(s.sel).first().screenshot({ type: "jpeg", quality: 82, timeout: 5000, animations: "disabled" });
|
||||
recordAsset(s.url, "image", "image/jpeg", 200, "video-still");
|
||||
storeBytes(s.url, "image", buf);
|
||||
dismissUnion.videoStills++;
|
||||
} catch { /* element not shootable (off-screen/detached) — poster falls back to placeholder */ }
|
||||
}
|
||||
// Element screenshots scroll the target into view; restore scroll 0 so the
|
||||
// canonical DOM walk + screenshot match the generated app's default render.
|
||||
if (plan.shots.length) {
|
||||
await page.evaluate(() => window.scrollTo(0, 0));
|
||||
await page.waitForTimeout(80);
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 4/5: stamp capture-ids before the canonical snapshot so the IR carries
|
||||
// them (whitelisted), enabling interaction deltas + motion specs to map to cids.
|
||||
if ((opts.interactions || opts.motion) && vw === canonical) await tagElements(page);
|
||||
|
||||
// Bound the in-page DOM walk: page.evaluate has no default timeout, so a
|
||||
// pathologically large/animated DOM (e.g. asana.com) could hang forever.
|
||||
const snapshot: PageSnapshot = await Promise.race([
|
||||
page.evaluate(collectPage),
|
||||
new Promise<never>((_, rej) => setTimeout(() => rej(new Error(`collectPage timeout vp${vw}`)), 60_000)),
|
||||
]);
|
||||
|
||||
// Merge discovered references from the snapshot (DOM + accessible CSS).
|
||||
for (const da of snapshot.domAssets) {
|
||||
const t = classifyAsset(da.url, null) ?? (da.kind === "manifest" ? "manifest" : da.kind === "video" ? "video" : da.kind === "svg" ? "svg" : "image");
|
||||
recordAsset(da.url, t, null, null, da.via);
|
||||
}
|
||||
for (const link of snapshot.doc.head?.links ?? []) {
|
||||
const rel = (link.rel || "").toLowerCase();
|
||||
const href = link.href || "";
|
||||
if (!href) continue;
|
||||
if (/\bsitemap\b/.test(rel)) addSeoResource(href, "sitemap", snapshot.doc.url);
|
||||
if (/llms-full\.txt(?:$|[?#])/i.test(href) || /\bllms-full\b/.test(rel)) addSeoResource(href, "llms-full", snapshot.doc.url);
|
||||
else if (/llms\.txt(?:$|[?#])/i.test(href) || /\bllms\b/.test(rel)) addSeoResource(href, "llms", snapshot.doc.url);
|
||||
}
|
||||
for (const u of snapshot.cssUrls) {
|
||||
const t = classifyAsset(u, null) ?? "other";
|
||||
recordAsset(u, t, null, null, "css-url");
|
||||
}
|
||||
for (const ff of snapshot.fontFaces) {
|
||||
const key = `${ff.family}|${ff.weight ?? ""}|${ff.style ?? ""}|${ff.src}`;
|
||||
if (!fontFaceMap.has(key)) fontFaceMap.set(key, ff);
|
||||
}
|
||||
|
||||
// Cap body collection: resp.body() has no timeout, so a streaming/long-poll
|
||||
// response (common on heavy SaaS pages) would hang allSettled forever. By
|
||||
// now bodies have been downloading throughout navigation+scroll; stragglers
|
||||
// are dropped (the post-pass fallback fetch re-fetches anything missing).
|
||||
await Promise.race([
|
||||
Promise.allSettled(bodyPromises),
|
||||
new Promise((r) => setTimeout(r, 20_000)),
|
||||
]);
|
||||
|
||||
// Persist DOM snapshot, and (unless skipped for a production clone) the full-page screenshot.
|
||||
writeJSONCompact(join(captureDir, `dom-${vw}.json`), snapshot);
|
||||
if (opts.screenshots !== false) await captureScreenshot(page, join(screenshotsDir, `${vw}.png`), vw, log);
|
||||
|
||||
// Stage 4: drive recognized affordances at the canonical viewport (opt-in).
|
||||
if (opts.interactions && vw === canonical) {
|
||||
try { interaction = await captureInteractions(page, { log }); }
|
||||
catch (e) { log({ event: "interactions_error", error: String(e).slice(0, 200) }); }
|
||||
}
|
||||
|
||||
// Stage 5: capture motion (WAAPI + rotating text) at the canonical viewport.
|
||||
if (opts.motion && vw === canonical) {
|
||||
try { motion = await captureMotion(page, { log }); }
|
||||
catch (e) { log({ event: "motion_error", error: String(e).slice(0, 200) }); }
|
||||
}
|
||||
|
||||
perViewport.push({
|
||||
viewport: vw,
|
||||
height: vh,
|
||||
scrollHeight: round(snapshot.doc.scrollHeight),
|
||||
nodeCount: snapshot.doc.nodeCount,
|
||||
truncated: snapshot.doc.truncated,
|
||||
overlaysRemaining,
|
||||
blocking,
|
||||
quiescent,
|
||||
});
|
||||
log({ event: "captured", viewport: vw, nodes: snapshot.doc.nodeCount, scrollHeight: snapshot.doc.scrollHeight });
|
||||
}
|
||||
|
||||
// Browser-as-oracle: discover the source's real responsive band edges by sweeping the viewport
|
||||
// and binary-searching each width where the discrete (media-query-toggled) layout signature
|
||||
// changes. Read-only and bounded; runs once here — overlays are dismissed and the DOM settled, so
|
||||
// the signature is the real page, not a consent wall. Never fatal: a failed sweep just omits the
|
||||
// field. The context closes right after, so the swept viewport size needs no restore.
|
||||
if (opts.breakpoints ?? true) {
|
||||
try {
|
||||
const edges = await Promise.race([
|
||||
discoverBreakpoints(page, { min: 320, max: 1920 }),
|
||||
new Promise<number[]>((_, rej) => setTimeout(() => rej(new Error("breakpoint sweep timeout")), 60_000)),
|
||||
]);
|
||||
discoveredBreakpoints = edges;
|
||||
log({ event: "breakpoints_discovered", edges });
|
||||
} catch (e) {
|
||||
log({ event: "breakpoints_error", error: String(e).slice(0, 200) });
|
||||
}
|
||||
}
|
||||
await context.close();
|
||||
|
||||
// Parse @font-face + url() from cross-origin CSS texts we fetched at the
|
||||
// network layer (the in-page walker can only read same-origin sheets).
|
||||
for (const { baseUrl, text } of cssTextsForParsing) {
|
||||
parseCssForFonts(text, baseUrl, fontFaceMap, (u) => {
|
||||
const t = classifyAsset(u, null) ?? "other";
|
||||
recordAsset(u, t, null, null, "css-text");
|
||||
});
|
||||
}
|
||||
|
||||
// Fallback: download any referenced-but-not-yet-stored asset using the
|
||||
// browser context (shares TLS handling). Fonts and CSS-referenced images are
|
||||
// commonly missed by the response listener when served from cache.
|
||||
const fallbackCtx = await browser.newContext({ ignoreHTTPSErrors: true, userAgent: DESKTOP_UA });
|
||||
for (const a of assetMap.values()) {
|
||||
if (a.storedAs) continue;
|
||||
if (a.url.startsWith("data:")) continue;
|
||||
if (!["image", "svg", "video", "font", "lottie", "css", "manifest"].includes(a.type)) continue;
|
||||
try {
|
||||
const resp = await fallbackCtx.request.get(a.url, { timeout: 30000 });
|
||||
a.status = resp.status();
|
||||
if (resp.ok()) {
|
||||
const buf = await resp.body();
|
||||
a.contentType = a.contentType ?? (resp.headers()["content-type"] || null);
|
||||
storeBytes(a.url, a.type, buf);
|
||||
if (a.type === "css") parseCssForFonts(buf.toString("utf8"), a.url, fontFaceMap, (u) => {
|
||||
const t = classifyAsset(u, null) ?? "other";
|
||||
recordAsset(u, t, null, null, "css-text");
|
||||
});
|
||||
if (a.type === "manifest") parseManifestForAssets(buf.toString("utf8"), a.url);
|
||||
}
|
||||
} catch { /* unreachable/signed — left as skipped */ }
|
||||
}
|
||||
const seoResources: SeoResource[] = [];
|
||||
const fetchedSeo = new Set<string>();
|
||||
const fetchSeoResource = async (url: string, kind: SeoResource["kind"]): Promise<void> => {
|
||||
if (fetchedSeo.has(url)) return;
|
||||
fetchedSeo.add(url);
|
||||
try {
|
||||
const resp = await fallbackCtx.request.get(url, { timeout: 15000 });
|
||||
const contentType = resp.headers()["content-type"] || null;
|
||||
const resource: SeoResource = { kind, url, status: resp.status(), contentType };
|
||||
if (resp.ok() && (kind === "llms" || kind === "llms-full" || kind === "robots")) {
|
||||
const text = await resp.text();
|
||||
resource.text = text;
|
||||
if (kind === "robots") {
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
const m = /^sitemap:\s*(\S+)/i.exec(line.trim());
|
||||
if (m) addSeoResource(m[1]!, "sitemap", url);
|
||||
}
|
||||
}
|
||||
}
|
||||
seoResources.push(resource);
|
||||
} catch {
|
||||
seoResources.push({ kind, url, status: null, contentType: null });
|
||||
}
|
||||
};
|
||||
for (let pass = 0; pass < 2; pass++) {
|
||||
const entries = [...seoResourceUrls.entries()].sort((a, b) => a[0].localeCompare(b[0]));
|
||||
for (const [url, kind] of entries) await fetchSeoResource(url, kind);
|
||||
}
|
||||
await fallbackCtx.close();
|
||||
|
||||
if (seoResources.length) {
|
||||
(captureSeoResources as SeoResource[]).push(...seoResources.sort((a, b) => a.url.localeCompare(b.url) || a.kind.localeCompare(b.kind)));
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
const assets = [...assetMap.values()].sort((a, b) => a.url.localeCompare(b.url));
|
||||
const fontFaces = [...fontFaceMap.values()].sort((a, b) =>
|
||||
`${a.family}${a.weight}${a.style}`.localeCompare(`${b.family}${b.weight}${b.style}`),
|
||||
);
|
||||
|
||||
const result: CaptureResult = {
|
||||
sourceUrl: opts.url,
|
||||
capturedAt: new Date().toISOString(),
|
||||
viewports,
|
||||
breakpoints: discoveredBreakpoints,
|
||||
perViewport,
|
||||
assets,
|
||||
...(captureSeoResources.length ? { seoResources: captureSeoResources } : {}),
|
||||
fontFaces,
|
||||
cssTexts: [...cssStored].sort(),
|
||||
dismissal: dismissUnion,
|
||||
interaction,
|
||||
motion,
|
||||
};
|
||||
|
||||
if (interaction) writeJSON(join(opts.outDir, "interaction.json"), interaction);
|
||||
if (motion) writeJSON(join(opts.outDir, "motion.json"), motion);
|
||||
writeJSON(join(opts.outDir, "assets-discovered.json"), assets);
|
||||
writeJSON(join(opts.outDir, "fonts-discovered.json"), fontFaces);
|
||||
writeJSON(join(captureDir, "capture-result.json"), result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseCssForFonts(
|
||||
cssText: string,
|
||||
baseUrl: string,
|
||||
fontFaceMap: Map<string, FontFace>,
|
||||
onUrl: (url: string) => void,
|
||||
): void {
|
||||
const abs = (u: string): string => {
|
||||
try { return new URL(u, baseUrl).href; } catch { return u; }
|
||||
};
|
||||
// Extract @font-face blocks. src urls are rewritten to absolute so they can be
|
||||
// resolved/downloaded regardless of where the CSS file lived.
|
||||
const faceRe = /@font-face\s*\{([^}]*)\}/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = faceRe.exec(cssText)) !== null) {
|
||||
const block = m[1]!;
|
||||
const family = /font-family\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim().replace(/^['"]|['"]$/g, "");
|
||||
let src = /src\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
if (!family || !src) continue;
|
||||
src = src.replace(/url\(\s*(['"]?)([^'")]+)\1\s*\)/gi, (_full, _q, u) =>
|
||||
u.startsWith("data:") ? `url(${u})` : `url(${abs(u)})`);
|
||||
const weight = /font-weight\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
const style = /font-style\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
const display = /font-display\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
const unicodeRange = /unicode-range\s*:\s*([^;]+)/i.exec(block)?.[1]?.trim();
|
||||
const key = `${family}|${weight ?? ""}|${style ?? ""}|${src}`;
|
||||
if (!fontFaceMap.has(key)) {
|
||||
fontFaceMap.set(key, { family, src, weight, style, display, unicodeRange });
|
||||
}
|
||||
}
|
||||
// Extract all url() references for asset discovery.
|
||||
const urlRe = /url\(\s*(['"]?)([^'")]+)\1\s*\)/gi;
|
||||
while ((m = urlRe.exec(cssText)) !== null) {
|
||||
const raw = m[2];
|
||||
if (raw && !raw.startsWith("data:")) onUrl(abs(raw));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,350 @@
|
||||
import type { Page } from "playwright";
|
||||
|
||||
/**
|
||||
* Stage 5 motion capture. Runs at the canonical viewport AFTER `tagElements` (so every
|
||||
* element carries a `data-cid-cap` the IR threads through to a cid). Two families that
|
||||
* the declarative CSS path (per-node `animation-name` + captured `@keyframes`, handled
|
||||
* entirely in the IR/generator) does NOT cover:
|
||||
*
|
||||
* - **WAAPI** (`element.animate(...)`, e.g. Framer Motion): only ever visible via
|
||||
* `document.getAnimations()`. Captured here as keyframes + timing so a fixed client
|
||||
* controller can replay it. (Entrance WAAPI that already finished by capture time is
|
||||
* not observable — persistent/looping WAAPI is; that's the deterministic subset.)
|
||||
* - **Rotating text**: a JS interval swapping an element's text (shopify "category
|
||||
* creator → global empire", notion "Ship → Create"). Observed with a MutationObserver
|
||||
* over a short window; captured as the text cycle + cadence for a fixed controller.
|
||||
*
|
||||
* CSS @keyframes/transition motion is intentionally NOT re-captured here — it is fully
|
||||
* reconstructable from the static evidence already in the IR, which is the most
|
||||
* deterministic path.
|
||||
*/
|
||||
|
||||
export type WaapiAnim = {
|
||||
cap: string; // data-cid-cap of the animated element (→ cid at generation)
|
||||
keyframes: Array<Record<string, string | number>>; // effect.getKeyframes()
|
||||
duration: number; // ms
|
||||
delay: number; // ms
|
||||
easing: string;
|
||||
iterations: number; // -1 encodes Infinity (JSON-safe)
|
||||
direction: string;
|
||||
fill: string;
|
||||
};
|
||||
|
||||
export type RotatorSpec = {
|
||||
cap: string; // data-cid-cap of the element whose text cycles
|
||||
texts: string[]; // the observed cycle of trimmed text values (in order)
|
||||
intervalMs: number; // approximate cadence between swaps
|
||||
};
|
||||
|
||||
export type RevealSpec = {
|
||||
cap: string; // data-cid-cap of a scroll-revealed element
|
||||
opacity: string; // its hidden (pre-reveal) opacity, e.g. "0"
|
||||
transform: string; // its hidden transform (slide/scale offset), or "none"
|
||||
transition: string; // the transition to animate the reveal with
|
||||
};
|
||||
|
||||
export type MarqueeSpec = {
|
||||
cap: string; // data-cid-cap of the continuously-translating track
|
||||
axis: "x"; // horizontal marquee (the common case; vertical reserved)
|
||||
pxPerSec: number; // signed scroll velocity (negative = leftward)
|
||||
periodPx: number; // distance per seamless loop (one duplicated copy ≈ scrollWidth/2)
|
||||
};
|
||||
|
||||
export type MotionCapture = {
|
||||
waapi: WaapiAnim[];
|
||||
rotators: RotatorSpec[];
|
||||
reveals: RevealSpec[]; // scroll-triggered entrance reveals (start hidden, reveal on view)
|
||||
marquees: MarqueeSpec[]; // rAF-driven continuous tickers (Framer Motion etc.) — not in getAnimations()
|
||||
cssAnimated: number; // elements with a computed animation-name (informational)
|
||||
};
|
||||
|
||||
/**
|
||||
* Stage 5 (scroll reveals) — pre-scroll probe. Scroll-triggered reveals start hidden
|
||||
* (opacity:0 / transform offset) and animate in when scrolled into view; by the time the
|
||||
* settled snapshot is taken (after `autoScroll` has walked the page) they are already
|
||||
* revealed, so their hidden state must be sampled BEFORE the first scroll. Records, on
|
||||
* `window.__cloneReveal`, the pre-scroll opacity/transform/transition of every tagged
|
||||
* element that is hidden-but-boxed with a real transition — the candidate reveal set
|
||||
* `captureMotion` later confirms (kept only if the element ends up visible). Idempotent;
|
||||
* call once at the first viewport, after settle, before `autoScroll`.
|
||||
*/
|
||||
export async function probeReveals(page: Page): Promise<void> {
|
||||
try {
|
||||
await page.evaluate(() => {
|
||||
const out: Record<string, { opacity: string; transform: string; transition: string }> = {};
|
||||
for (const el of Array.from(document.querySelectorAll("[data-cid-cap]"))) {
|
||||
const cap = el.getAttribute("data-cid-cap"); if (!cap) continue;
|
||||
let cs: CSSStyleDeclaration; try { cs = getComputedStyle(el); } catch { continue; }
|
||||
const op = parseFloat(cs.opacity || "1");
|
||||
if (op > 0.05) continue; // only currently-hidden elements are reveal candidates
|
||||
const r = (el as HTMLElement).getBoundingClientRect();
|
||||
if (r.width < 8 || r.height < 8) continue; // must occupy real space (not a 0-box hidden node)
|
||||
// must have a transition on opacity/transform/all (so the reveal animates, not snaps)
|
||||
const tp = (cs.transitionProperty || "").toLowerCase();
|
||||
const td = cs.transitionDuration || "0s";
|
||||
const animates = /opacity|transform|all/.test(tp) && !/^0s(,\s*0s)*$/.test(td);
|
||||
if (!animates) continue;
|
||||
out[cap] = {
|
||||
opacity: cs.opacity || "0",
|
||||
transform: cs.transform && cs.transform !== "none" ? cs.transform : "none",
|
||||
transition: cs.transition && cs.transition !== "all 0s ease 0s" ? cs.transition : `opacity ${td}, transform ${td}`,
|
||||
};
|
||||
}
|
||||
(window as unknown as { __cloneReveal?: unknown }).__cloneReveal = out;
|
||||
});
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 5 (marquees) — rAF-driven continuous tickers. Framer Motion (and similar)
|
||||
* drive infinite horizontal scrollers by mutating an element's `transform` every frame
|
||||
* via requestAnimationFrame, so they are invisible to `getAnimations()` (the WAAPI path)
|
||||
* and have no CSS `@keyframes` (the declarative path). They are also paused while
|
||||
* off-screen, so the velocity is only observable after scrolling the track into view.
|
||||
* Detect a track by a translateX that changes steadily in one direction once visible,
|
||||
* and record its signed velocity + seamless-loop period (one duplicated copy ≈ half the
|
||||
* scroll width) so the clone can replay the loop. Read-only beyond scrolling (restores
|
||||
* scroll to top); does not touch the captured snapshot/IR.
|
||||
*/
|
||||
async function detectMarquees(page: Page): Promise<MarqueeSpec[]> {
|
||||
try {
|
||||
return await page.evaluate(async () => {
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const txOf = (el: Element): number => { try { return new DOMMatrixReadOnly(getComputedStyle(el).transform).m41; } catch { return 0; } };
|
||||
const inClip = (el: Element): boolean => {
|
||||
let p = el.parentElement, depth = 0;
|
||||
while (p && depth < 6) { const ox = getComputedStyle(p).overflowX; if (ox === "hidden" || ox === "clip") return true; p = p.parentElement; depth++; }
|
||||
return false;
|
||||
};
|
||||
// Candidates: tagged, already translated (paused tickers keep their offset), with
|
||||
// duplicated content (≥2 children) inside an overflow-clip viewport — the marquee shape.
|
||||
const cand: Element[] = [];
|
||||
for (const el of Array.from(document.querySelectorAll("[data-cid-cap]"))) {
|
||||
if (Math.abs(txOf(el)) <= 0.5) continue;
|
||||
if (el.children.length < 2) continue;
|
||||
if (!inClip(el)) continue;
|
||||
cand.push(el);
|
||||
if (cand.length >= 16) break;
|
||||
}
|
||||
const out: Array<{ cap: string; axis: "x"; pxPerSec: number; periodPx: number }> = [];
|
||||
const seen = new Set<string>();
|
||||
for (const el of cand) {
|
||||
const cap = el.getAttribute("data-cid-cap"); if (!cap || seen.has(cap)) continue;
|
||||
try { el.scrollIntoView({ block: "center" }); } catch { /* ignore */ }
|
||||
await sleep(450); // wake the off-screen-paused ticker + let the scroll settle
|
||||
const xs: number[] = [];
|
||||
for (let i = 0; i < 6; i++) { xs.push(txOf(el)); await sleep(120); }
|
||||
const dxs: number[] = []; for (let i = 1; i < xs.length; i++) dxs.push(xs[i]! - xs[i - 1]!);
|
||||
if (dxs.length < 3) continue;
|
||||
// velocity = median per-120ms delta (median ignores the single wrap-reset outlier).
|
||||
const med = [...dxs].sort((a, b) => a - b)[Math.floor(dxs.length / 2)]!;
|
||||
const pxPerSec = Math.round((med / 120) * 1000);
|
||||
if (Math.abs(pxPerSec) < 4) continue; // not actually moving (e.g. a frozen scroll-scrub offset)
|
||||
// direction must be consistent in all but (at most) the one wrap step.
|
||||
if (dxs.filter((d) => Math.sign(d) === Math.sign(med)).length < dxs.length - 1) continue;
|
||||
const periodPx = Math.round(el.scrollWidth / 2);
|
||||
if (periodPx < 40) continue;
|
||||
seen.add(cap);
|
||||
out.push({ cap, axis: "x", pxPerSec, periodPx });
|
||||
}
|
||||
try { window.scrollTo(0, 0); } catch { /* ignore */ }
|
||||
return out;
|
||||
});
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 5 (scroll-scrub reveals) — scroll-LINKED entrance animations. Some sections
|
||||
* (framer's "Agents" sticky-pinned panels) drive opacity/transform as a continuous
|
||||
* function of scroll position rather than a one-way IntersectionObserver reveal, so they
|
||||
* start only PARTIALLY hidden (opacity ~0.2, a translate offset) and `probeReveals`
|
||||
* (which requires opacity ≤ 0.05 pre-scroll) never sees them — and the settled snapshot
|
||||
* catches them frozen MID-scrub (opacity 0.63), which the clone then bakes as a dimmed,
|
||||
* offset panel. Detect them by sampling each panel's opacity/transform across a full
|
||||
* scroll pass: a panel that is dimmed/slid at some scroll position but reaches full
|
||||
* opacity at another is a scroll reveal. Reconstruct as a normal reveal (hide at the
|
||||
* most-dimmed state, transition to full on view) — the deterministic, gate-reconciled
|
||||
* path (the validator force-reveals before grading, so gates measure the revealed frame).
|
||||
*/
|
||||
async function detectScrubReveals(page: Page): Promise<RevealSpec[]> {
|
||||
try {
|
||||
return await page.evaluate(async () => {
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const txOf = (cs: CSSStyleDeclaration): number => { try { return new DOMMatrixReadOnly(cs.transform).m41; } catch { return 0; } };
|
||||
// Panel-like candidates: tagged, sized, content-bearing, in normal flow.
|
||||
const cands: HTMLElement[] = [];
|
||||
for (const el of Array.from(document.querySelectorAll("[data-cid-cap]")) as HTMLElement[]) {
|
||||
const r = el.getBoundingClientRect();
|
||||
if (r.width < 120 || r.height < 60) continue;
|
||||
const pos = getComputedStyle(el).position;
|
||||
if (pos === "fixed" || pos === "sticky") continue;
|
||||
if (!el.querySelector("img, p, h1, h2, h3, span")) continue; // has real content
|
||||
cands.push(el);
|
||||
if (cands.length >= 1200) break;
|
||||
}
|
||||
const samples = new Map<HTMLElement, Array<{ op: number; tx: number }>>();
|
||||
const maxY = Math.max(0, document.documentElement.scrollHeight - window.innerHeight);
|
||||
const steps = 14;
|
||||
for (let i = 0; i <= steps; i++) {
|
||||
window.scrollTo(0, Math.round((maxY * i) / steps));
|
||||
await sleep(120);
|
||||
for (const el of cands) {
|
||||
const cs = getComputedStyle(el);
|
||||
let a = samples.get(el); if (!a) { a = []; samples.set(el, a); }
|
||||
a.push({ op: parseFloat(cs.opacity || "1"), tx: txOf(cs) });
|
||||
}
|
||||
}
|
||||
window.scrollTo(0, 0);
|
||||
const found: Array<{ el: HTMLElement; cap: string; opacity: string; transform: string; transition: string }> = [];
|
||||
for (const [el, a] of samples) {
|
||||
const ops = a.map((s) => s.op), absTx = a.map((s) => Math.abs(s.tx));
|
||||
const maxOp = Math.max(...ops), minOp = Math.min(...ops);
|
||||
const maxTx = Math.max(...absTx), minTx = Math.min(...absTx);
|
||||
const opReveal = maxOp > 0.9 && minOp < 0.85; // fades in across scroll
|
||||
const txReveal = maxOp > 0.9 && maxTx > 6 && minTx < 2; // slides in across scroll
|
||||
if (!opReveal && !txReveal) continue;
|
||||
// hidden = the most-dimmed sample (lowest opacity, tie-break largest offset).
|
||||
let hidden = a[0]!;
|
||||
for (const s of a) if (s.op < hidden.op - 0.001 || (Math.abs(s.op - hidden.op) <= 0.001 && Math.abs(s.tx) > Math.abs(hidden.tx))) hidden = s;
|
||||
const cap = el.getAttribute("data-cid-cap"); if (!cap) continue;
|
||||
found.push({
|
||||
el, cap,
|
||||
opacity: String(Math.max(0, Math.min(hidden.op, 0.5))),
|
||||
transform: Math.abs(hidden.tx) > 2 ? `translateX(${Math.round(hidden.tx)}px)` : "none",
|
||||
transition: "opacity 0.6s ease, transform 0.6s ease",
|
||||
});
|
||||
}
|
||||
// Keep only the OUTERMOST scrub per nesting (a panel and its scrubbed children both match).
|
||||
const outer = found.filter((f) => !found.some((o) => o !== f && o.el.contains(f.el)));
|
||||
return outer.slice(0, 40).map(({ cap, opacity, transform, transition }) => ({ cap, opacity, transform, transition }));
|
||||
});
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
export async function captureMotion(page: Page, opts?: { observeMs?: number; log?: (e: Record<string, unknown>) => void }): Promise<MotionCapture> {
|
||||
const log = opts?.log ?? (() => {});
|
||||
const observeMs = opts?.observeMs ?? 2600;
|
||||
try {
|
||||
const result = await Promise.race([
|
||||
page.evaluate(async (budget: number) => {
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
// ---- WAAPI: pure script-driven animations (exclude CSS animations/transitions,
|
||||
// which the declarative path reproduces). Capture persistent/looping ones — the
|
||||
// deterministic subset still alive at capture time. ----
|
||||
const waapi: Array<{
|
||||
cap: string; keyframes: Array<Record<string, string | number>>;
|
||||
duration: number; delay: number; easing: string; iterations: number; direction: string; fill: string;
|
||||
}> = [];
|
||||
try {
|
||||
const anims = (document as Document).getAnimations ? document.getAnimations() : [];
|
||||
const seen = new Set<string>();
|
||||
for (const a of anims) {
|
||||
const ctor = a.constructor.name;
|
||||
if (ctor === "CSSAnimation" || ctor === "CSSTransition") continue; // declarative path owns these
|
||||
const eff = a.effect as KeyframeEffect | null;
|
||||
const target = eff?.target as Element | undefined;
|
||||
const cap = target?.getAttribute?.("data-cid-cap");
|
||||
if (!cap || seen.has(cap)) continue;
|
||||
let kfs: Array<Record<string, string | number>> = [];
|
||||
try { kfs = (eff!.getKeyframes() as unknown as Array<Record<string, string | number>>); } catch { kfs = []; }
|
||||
if (!kfs.length) continue;
|
||||
const t = eff!.getTiming();
|
||||
const iters = t.iterations === Infinity ? -1 : (typeof t.iterations === "number" ? t.iterations : 1);
|
||||
seen.add(cap);
|
||||
waapi.push({
|
||||
cap,
|
||||
keyframes: kfs,
|
||||
duration: typeof t.duration === "number" ? t.duration : 0,
|
||||
delay: t.delay ?? 0,
|
||||
easing: String(t.easing ?? "linear"),
|
||||
iterations: iters,
|
||||
direction: String(t.direction ?? "normal"),
|
||||
fill: String(t.fill ?? "none"),
|
||||
});
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// ---- Rotating text: watch for elements whose text content cycles. Record the
|
||||
// smallest text-bearing element that changes (so we cycle the word, not a whole
|
||||
// section), the ordered set of values it shows, and the cadence. ----
|
||||
const changes = new Map<string, { texts: string[]; times: number[] }>();
|
||||
const norm = (s: string) => s.replace(/\s+/g, " ").trim();
|
||||
const obs = new MutationObserver((records) => {
|
||||
for (const rec of records) {
|
||||
// climb to the nearest element with a data-cid-cap
|
||||
let el: Node | null = rec.target;
|
||||
while (el && !(el as Element).getAttribute?.("data-cid-cap")) el = el.parentNode;
|
||||
const cap = el && (el as Element).getAttribute?.("data-cid-cap");
|
||||
if (!cap) continue;
|
||||
const elem = el as Element;
|
||||
// only track small text-bearing elements (a rotating word/phrase, not a big block)
|
||||
if (elem.querySelector("[data-cid-cap]")) continue; // has element children → not a leaf word
|
||||
const txt = norm(elem.textContent || "");
|
||||
if (!txt || txt.length > 80) continue;
|
||||
const e = changes.get(cap) ?? { texts: [], times: [] };
|
||||
if (e.texts[e.texts.length - 1] !== txt) { e.texts.push(txt); e.times.push(performance.now()); }
|
||||
changes.set(cap, e);
|
||||
}
|
||||
});
|
||||
obs.observe(document.body, { subtree: true, childList: true, characterData: true });
|
||||
await sleep(budget);
|
||||
obs.disconnect();
|
||||
|
||||
const rotators: Array<{ cap: string; texts: string[]; intervalMs: number }> = [];
|
||||
for (const [cap, e] of changes) {
|
||||
// a genuine rotator shows ≥2 distinct values
|
||||
const distinct = Array.from(new Set(e.texts));
|
||||
if (distinct.length < 2) continue;
|
||||
let interval = 0;
|
||||
if (e.times.length >= 2) {
|
||||
const gaps: number[] = [];
|
||||
for (let i = 1; i < e.times.length; i++) gaps.push(e.times[i]! - e.times[i - 1]!);
|
||||
gaps.sort((a, b) => a - b);
|
||||
interval = Math.round(gaps[Math.floor(gaps.length / 2)]!);
|
||||
}
|
||||
rotators.push({ cap, texts: distinct.slice(0, 12), intervalMs: interval || 2000 });
|
||||
}
|
||||
|
||||
// ---- Scroll reveals: confirm the pre-scroll candidates (probeReveals) that are
|
||||
// NOW visible — those genuinely revealed on scroll (vs. elements that stayed hidden).
|
||||
const reveals: Array<{ cap: string; opacity: string; transform: string; transition: string }> = [];
|
||||
try {
|
||||
const probed = (window as unknown as { __cloneReveal?: Record<string, { opacity: string; transform: string; transition: string }> }).__cloneReveal || {};
|
||||
for (const cap of Object.keys(probed)) {
|
||||
const el = document.querySelector(`[data-cid-cap="${cap}"]`);
|
||||
if (!el) continue;
|
||||
const cs = getComputedStyle(el);
|
||||
if (parseFloat(cs.opacity || "1") <= 0.05) continue; // still hidden → not a reveal, just hidden
|
||||
const r = (el as HTMLElement).getBoundingClientRect();
|
||||
if (r.width < 8 || r.height < 8) continue;
|
||||
reveals.push({ cap, ...probed[cap]! });
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
|
||||
let cssAnimated = 0;
|
||||
for (const el of Array.from(document.querySelectorAll("*"))) {
|
||||
try { const an = getComputedStyle(el).animationName; if (an && an !== "none") cssAnimated++; } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
return { waapi, rotators, reveals, cssAnimated };
|
||||
}, observeMs),
|
||||
new Promise<Omit<MotionCapture, "marquees">>((res) => setTimeout(() => res({ waapi: [], rotators: [], reveals: [], cssAnimated: 0 }), observeMs + 4000)),
|
||||
]);
|
||||
// Scroll-scrub reveals: scroll-linked panels probeReveals can't see (they start only
|
||||
// partially hidden). Merge into reveals, preferring the probe-confirmed entry when a cap
|
||||
// appears in both.
|
||||
const scrubReveals = await detectScrubReveals(page);
|
||||
const reveals = [...result.reveals];
|
||||
const haveCap = new Set(reveals.map((r) => r.cap));
|
||||
for (const s of scrubReveals) if (!haveCap.has(s.cap)) { reveals.push(s); haveCap.add(s.cap); }
|
||||
// Marquees run as a separate pass (scrolls each track into view to wake the paused
|
||||
// rAF ticker; kept after the rotator MutationObserver window so it can't add false text rotators).
|
||||
const marquees = await detectMarquees(page);
|
||||
log({ event: "motion_captured", waapi: result.waapi.length, rotators: result.rotators.length, reveals: reveals.length, marquees: marquees.length, cssAnimated: result.cssAnimated });
|
||||
return { ...result, reveals, marquees };
|
||||
} catch (e) {
|
||||
log({ event: "motion_error", error: String(e).slice(0, 200) });
|
||||
return { waapi: [], rotators: [], reveals: [], marquees: [], cssAnimated: 0 };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,608 @@
|
||||
/**
|
||||
* In-page capture walker. This function is serialized and executed inside the
|
||||
* browser via page.evaluate, so it must be fully self-contained (no imports, no
|
||||
* outer references). It returns a serializable snapshot of the rendered page:
|
||||
* the visible DOM tree with computed styles, bounding boxes (in document
|
||||
* coordinates), pseudo-element styles, plus discovered CSS/font/asset references.
|
||||
*
|
||||
* Design notes:
|
||||
* - We serialize ALL elements (including display:none) so the tree structure is
|
||||
* identical across viewports and nodes can be aligned by pre-order index.
|
||||
* - Inline <svg> is captured as raw outerHTML and not recursed — reconstructing
|
||||
* SVG node-by-node is lossy; replaying the markup is exact.
|
||||
* - script/style/link/meta/noscript/template are skipped (we generate our own
|
||||
* CSS and never reproduce JS).
|
||||
*/
|
||||
|
||||
export type RawBBox = { x: number; y: number; width: number; height: number };
|
||||
export type RawStyle = Record<string, string>;
|
||||
/** Sizing-intent probe results: does the browser re-derive this
|
||||
* element's width/height when we drop the authored value? Ground truth for "is this dimension
|
||||
* load-bearing". For out-of-flow (absolute/fixed) elements, `insetDrop` instead records, per side,
|
||||
* whether setting that inset to `auto` leaves the box exactly in place — i.e. it's a filled-in used
|
||||
* value (the browser resolved `bottom`/`right` from the containing-block size) that we should NOT
|
||||
* bake, vs an authored anchor that pins the box. */
|
||||
export type RawSizing = {
|
||||
wAuto: boolean; wFill: boolean; hAuto: boolean;
|
||||
// Does `height:100%` re-derive this box (within 0.5px) at this viewport? The HEIGHT analogue of
|
||||
// wFill: ground truth that the element FILLS a definite-height containing block, so the faithful
|
||||
// emission is `h-full` rather than a baked per-vp px band. Distinguished from hAuto (content-sized):
|
||||
// a node with hFill && !hAuto genuinely fills a definite parent (the source's `h-full`), whereas
|
||||
// hAuto means it's content-sized (drop to auto). Present only for in-flow probed elements.
|
||||
hFill?: boolean;
|
||||
// Intrinsic-size anchors: the element's min-content and max-content widths,
|
||||
// measured live. They are the anchors a fluid width LAW is fit from — a load-bearing width that
|
||||
// VARIES across viewports but rides between these bounds is a fluid rule (`clamp()`/`%`/`flex`),
|
||||
// not four baked px. Present only for in-flow probed elements (px, rounded).
|
||||
wMin?: number; wMax?: number;
|
||||
insetDrop?: { top: boolean; right: boolean; bottom: boolean; left: boolean };
|
||||
};
|
||||
|
||||
export type RawNode = {
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
computed: RawStyle;
|
||||
bbox: RawBBox;
|
||||
visible: boolean;
|
||||
sizing?: RawSizing;
|
||||
before?: RawStyle;
|
||||
after?: RawStyle;
|
||||
rawHTML?: string; // set for inline <svg>
|
||||
children: RawChild[];
|
||||
};
|
||||
|
||||
export type RawChild = RawNode | { text: string };
|
||||
|
||||
export type FontFace = {
|
||||
family: string;
|
||||
src: string; // raw src descriptor (may contain multiple url())
|
||||
weight?: string;
|
||||
style?: string;
|
||||
display?: string;
|
||||
unicodeRange?: string;
|
||||
stretch?: string;
|
||||
};
|
||||
|
||||
export type PageSnapshot = {
|
||||
doc: {
|
||||
url: string;
|
||||
title: string;
|
||||
head: {
|
||||
description: string; canonical: string; ogTitle: string; ogDescription: string;
|
||||
ogImage: string; ogType: string; ogSiteName: string; twitterCard: string; themeColor: string;
|
||||
keywords?: string; robots?: string; referrer?: string; colorScheme?: string;
|
||||
meta?: Array<{ name?: string; property?: string; httpEquiv?: string; content: string }>;
|
||||
links?: Array<{
|
||||
rel: string; href: string; as?: string; type?: string; sizes?: string; media?: string;
|
||||
color?: string; hrefLang?: string; title?: string; crossOrigin?: string; referrerPolicy?: string;
|
||||
}>;
|
||||
jsonLd?: Array<{ id?: string; text: string }>;
|
||||
};
|
||||
lang: string;
|
||||
charset: string;
|
||||
viewportWidth: number;
|
||||
viewportHeight: number;
|
||||
scrollWidth: number;
|
||||
scrollHeight: number;
|
||||
htmlBg: string;
|
||||
bodyBg: string;
|
||||
bodyColor: string;
|
||||
bodyFont: string;
|
||||
metaViewport: string;
|
||||
nodeCount: number;
|
||||
truncated: boolean;
|
||||
};
|
||||
root: RawNode;
|
||||
cssVars: Record<string, string>;
|
||||
fontFaces: FontFace[];
|
||||
cssUrls: string[];
|
||||
domAssets: Array<{ kind: string; url: string; via: string }>;
|
||||
keyframes: string[]; // raw @keyframes blocks from accessible sheets
|
||||
};
|
||||
|
||||
export function collectPage(): PageSnapshot {
|
||||
// NOTE: This function is serialized and run in the browser via page.evaluate,
|
||||
// so every constant/helper it uses must be declared INSIDE it (no module-scope
|
||||
// closure is available in the page).
|
||||
|
||||
// Curated computed-style property set. Comprehensive enough to replay layout,
|
||||
// box model, typography, visuals, layering, and safe transitions/animations.
|
||||
const PROPS: string[] = [
|
||||
"display", "position", "top", "right", "bottom", "left", "float", "clear",
|
||||
"zIndex", "boxSizing", "visibility", "opacity", "isolation",
|
||||
"width", "height", "minWidth", "minHeight", "maxWidth", "maxHeight",
|
||||
"marginTop", "marginRight", "marginBottom", "marginLeft",
|
||||
"paddingTop", "paddingRight", "paddingBottom", "paddingLeft",
|
||||
"borderTopWidth", "borderRightWidth", "borderBottomWidth", "borderLeftWidth",
|
||||
"borderTopStyle", "borderRightStyle", "borderBottomStyle", "borderLeftStyle",
|
||||
"borderTopColor", "borderRightColor", "borderBottomColor", "borderLeftColor",
|
||||
"borderTopLeftRadius", "borderTopRightRadius", "borderBottomRightRadius", "borderBottomLeftRadius",
|
||||
"flexDirection", "flexWrap", "justifyContent", "alignItems", "alignContent",
|
||||
"alignSelf", "flexGrow", "flexShrink", "flexBasis", "order",
|
||||
"gap", "rowGap", "columnGap",
|
||||
"gridTemplateColumns", "gridTemplateRows", "gridTemplateAreas", "gridAutoFlow", "gridAutoRows",
|
||||
"gridAutoColumns", "justifyItems", "placeItems", "placeContent",
|
||||
"gridColumnStart", "gridColumnEnd", "gridRowStart", "gridRowEnd",
|
||||
"overflow", "overflowX", "overflowY",
|
||||
"objectFit", "objectPosition", "aspectRatio", "verticalAlign",
|
||||
"color", "fontFamily", "fontSize", "fontWeight", "fontStyle", "lineHeight",
|
||||
"letterSpacing", "wordSpacing", "textAlign", "textTransform",
|
||||
"textDecorationLine", "textDecorationColor", "textDecorationStyle",
|
||||
"whiteSpace", "wordBreak", "overflowWrap", "textOverflow", "textIndent",
|
||||
"textShadow", "fontVariantCaps", "fontFeatureSettings",
|
||||
// Line clamping (`display:-webkit-box; -webkit-box-orient:vertical; -webkit-line-clamp:N`):
|
||||
// the mechanism that keeps cards equal height regardless of text length. Without it the engine
|
||||
// sees only the RESULTING px height and bakes/drops it per card — uneven.
|
||||
"webkitLineClamp", "webkitBoxOrient",
|
||||
"listStyleType", "listStylePosition", "writingMode", "direction",
|
||||
"backgroundColor", "backgroundImage", "backgroundSize", "backgroundPosition",
|
||||
"backgroundRepeat", "backgroundClip", "backgroundOrigin", "backgroundAttachment",
|
||||
"backgroundBlendMode",
|
||||
"boxShadow", "filter", "backdropFilter", "mixBlendMode",
|
||||
"transform", "transformOrigin", "transformStyle", "perspective",
|
||||
// Individual transform properties (CSS Transforms Level 2) — modern sites centre/offset with
|
||||
// `translate: -50% -50%` etc. INSTEAD of `transform`. getComputedStyle reports them separately
|
||||
// (transform stays "none"), so without capturing them a translate-centred box loses its shift.
|
||||
// Emitted as raw `[translate:...]` properties.
|
||||
"translate", "rotate", "scale",
|
||||
"clipPath", "maskImage",
|
||||
"webkitTextStroke", "webkitTextFillColor", "webkitBackgroundClip",
|
||||
"transition", "animationName", "animationDuration", "animationTimingFunction",
|
||||
"animationDelay", "animationIterationCount", "animationDirection", "animationFillMode",
|
||||
"cursor", "pointerEvents", "userSelect", "tableLayout", "borderCollapse", "borderSpacing",
|
||||
];
|
||||
|
||||
const PSEUDO_PROPS: string[] = [
|
||||
"content", "display", "position", "top", "right", "bottom", "left", "zIndex",
|
||||
"width", "height", "marginTop", "marginRight", "marginBottom", "marginLeft",
|
||||
"paddingTop", "paddingRight", "paddingBottom", "paddingLeft",
|
||||
"borderTopWidth", "borderRightWidth", "borderBottomWidth", "borderLeftWidth",
|
||||
"borderTopStyle", "borderRightStyle", "borderBottomStyle", "borderLeftStyle",
|
||||
"borderTopColor", "borderRightColor", "borderBottomColor", "borderLeftColor",
|
||||
"borderTopLeftRadius", "borderTopRightRadius", "borderBottomRightRadius", "borderBottomLeftRadius",
|
||||
"color", "fontFamily", "fontSize", "fontWeight", "lineHeight", "letterSpacing",
|
||||
"textAlign", "textTransform",
|
||||
"backgroundColor", "backgroundImage", "backgroundSize", "backgroundPosition", "backgroundRepeat",
|
||||
"boxShadow", "opacity", "transform", "transformOrigin", "translate", "rotate", "scale", "filter",
|
||||
"overflow", "objectFit",
|
||||
];
|
||||
|
||||
const SKIP_TAGS = new Set([
|
||||
"script", "style", "link", "meta", "noscript", "template", "base", "title", "head",
|
||||
]);
|
||||
|
||||
const MAX_NODES = 12000;
|
||||
|
||||
const round2 = (n: number): number => Math.round((n || 0) * 100) / 100;
|
||||
const scrollX = window.scrollX;
|
||||
const scrollY = window.scrollY;
|
||||
|
||||
// Resolve `line-height: normal` to a concrete px value by probing the actual
|
||||
// line-box height for each (font-family, font-size, font-weight, font-style).
|
||||
// getComputedStyle reports the keyword "normal", which renders font-metric-
|
||||
// dependently and drifts sub-pixel over many lines (e.g. dense text tables);
|
||||
// emitting the resolved px makes the clone deterministic and exact.
|
||||
const lhCache = new Map<string, string>();
|
||||
const lhProbe = document.createElement("div");
|
||||
lhProbe.style.cssText = "position:absolute;visibility:hidden;left:-99999px;top:-99999px;padding:0;border:0;margin:0;white-space:nowrap;line-height:normal;";
|
||||
lhProbe.textContent = "Mgy";
|
||||
document.body.appendChild(lhProbe);
|
||||
const resolveNormalLineHeight = (ff: string, fs: string, fw: string, fst: string): string => {
|
||||
const key = `${ff}|${fs}|${fw}|${fst}`;
|
||||
const cached = lhCache.get(key);
|
||||
if (cached !== undefined) return cached;
|
||||
lhProbe.style.fontFamily = ff;
|
||||
lhProbe.style.fontSize = fs;
|
||||
lhProbe.style.fontWeight = fw;
|
||||
lhProbe.style.fontStyle = fst;
|
||||
const h = lhProbe.getBoundingClientRect().height;
|
||||
const v = h > 0 ? `${Math.round(h * 100) / 100}px` : "normal";
|
||||
lhCache.set(key, v);
|
||||
return v;
|
||||
};
|
||||
let nodeCount = 0;
|
||||
let truncated = false;
|
||||
|
||||
const grabStyle = (cs: CSSStyleDeclaration, props: string[]): RawStyle => {
|
||||
const out: RawStyle = {};
|
||||
for (const p of props) {
|
||||
// CSSStyleDeclaration is indexable by camelCase property names.
|
||||
const v = (cs as unknown as Record<string, string>)[p];
|
||||
if (v !== undefined && v !== null && v !== "") out[p] = v;
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
const isVisible = (el: Element, cs: CSSStyleDeclaration, bbox: RawBBox): boolean => {
|
||||
if (cs.display === "none") return false;
|
||||
if (cs.visibility === "hidden" || cs.visibility === "collapse") return false;
|
||||
if (parseFloat(cs.opacity || "1") === 0) return false;
|
||||
if (bbox.width === 0 && bbox.height === 0) {
|
||||
// zero-size but might still matter (e.g. absolutely positioned icon); treat
|
||||
// as not visible for matching purposes.
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
const serializeElement = (el: Element): RawNode | null => {
|
||||
if (nodeCount >= MAX_NODES) { truncated = true; return null; }
|
||||
const tag = el.tagName.toLowerCase();
|
||||
nodeCount++;
|
||||
|
||||
const cs = window.getComputedStyle(el);
|
||||
const r = el.getBoundingClientRect();
|
||||
const bbox: RawBBox = {
|
||||
x: round2(r.x + scrollX),
|
||||
y: round2(r.y + scrollY),
|
||||
width: round2(r.width),
|
||||
height: round2(r.height),
|
||||
};
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
for (const a of Array.from(el.attributes)) {
|
||||
// Drop inline event handlers and inline style (we replay computed style).
|
||||
const name = a.name;
|
||||
if (name.startsWith("on")) continue;
|
||||
attrs[name] = a.value;
|
||||
}
|
||||
|
||||
const resolveLH = (style: RawStyle): void => {
|
||||
if (style.lineHeight === "normal" && style.fontSize) {
|
||||
style.lineHeight = resolveNormalLineHeight(
|
||||
style.fontFamily || cs.fontFamily,
|
||||
style.fontSize,
|
||||
style.fontWeight || "400",
|
||||
style.fontStyle || "normal",
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const computed = grabStyle(cs, PROPS);
|
||||
resolveLH(computed);
|
||||
|
||||
// Sizing-intent probe: does the browser re-derive this box when we drop its
|
||||
// width/height? Ground truth for whether the generator can omit the baked dimension. Run AFTER
|
||||
// the canonical cs/bbox are recorded and fully restored after, so the capture is unchanged — it
|
||||
// only ADDS three booleans. Skipped for out-of-flow, replaced, hidden, and animated elements.
|
||||
let sizing: RawSizing | undefined;
|
||||
const probePos = cs.position || "static";
|
||||
if (
|
||||
(probePos === "static" || probePos === "relative") && r.width > 1 && cs.display !== "none" &&
|
||||
(cs.animationName || "none") === "none" && (el as HTMLElement).style &&
|
||||
!/^(img|svg|video|canvas|picture|iframe|input|textarea|select|hr|object|embed|br)$/.test(tag)
|
||||
) {
|
||||
const h = el as HTMLElement;
|
||||
const sw = h.style.getPropertyValue("width"), swp = h.style.getPropertyPriority("width");
|
||||
const sh = h.style.getPropertyValue("height"), shp = h.style.getPropertyPriority("height");
|
||||
const restore = () => {
|
||||
h.style.removeProperty("width"); if (sw) h.style.setProperty("width", sw, swp);
|
||||
h.style.removeProperty("height"); if (sh) h.style.setProperty("height", sh, shp);
|
||||
};
|
||||
try {
|
||||
h.style.setProperty("width", "auto", "important");
|
||||
const wa = el.getBoundingClientRect().width;
|
||||
h.style.setProperty("width", "100%", "important");
|
||||
const wf = el.getBoundingClientRect().width;
|
||||
// Intrinsic-size anchors (probe 3): min-content (longest unbreakable run) and max-content
|
||||
// (no-wrap natural width). A load-bearing width that varies across viewports but stays
|
||||
// between these is a fluid law; these are the anchors css.ts fits it from.
|
||||
h.style.setProperty("width", "min-content", "important");
|
||||
const wmin = el.getBoundingClientRect().width;
|
||||
h.style.setProperty("width", "max-content", "important");
|
||||
const wmax = el.getBoundingClientRect().width;
|
||||
h.style.removeProperty("width"); if (sw) h.style.setProperty("width", sw, swp);
|
||||
h.style.setProperty("height", "auto", "important");
|
||||
const ha = el.getBoundingClientRect().height;
|
||||
h.style.setProperty("height", "100%", "important");
|
||||
const hf = el.getBoundingClientRect().height;
|
||||
// Tight 0.5px: only call a dimension "reproduced" when auto/100% lands essentially exactly,
|
||||
// so a drop can't accumulate a visible shift across many elements (favours fidelity).
|
||||
sizing = {
|
||||
wAuto: Math.abs(wa - r.width) <= 0.5, wFill: Math.abs(wf - r.width) <= 0.5, hAuto: Math.abs(ha - r.height) <= 0.5,
|
||||
hFill: Math.abs(hf - r.height) <= 0.5,
|
||||
wMin: Math.round(wmin * 100) / 100, wMax: Math.round(wmax * 100) / 100,
|
||||
};
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
} else if (
|
||||
(probePos === "absolute" || probePos === "fixed") && cs.display !== "none" &&
|
||||
(cs.animationName || "none") === "none" && (el as HTMLElement).style
|
||||
) {
|
||||
// Inset-anchor probe: for an out-of-flow box, per side, does setting that
|
||||
// inset to `auto` leave the box exactly in place? If so the inset was a filled-in USED value
|
||||
// (the browser resolved it from the containing-block size — `bottom` = page height, `right` =
|
||||
// viewport width) that bakes a huge per-viewport number and a band; if the box MOVES, the inset
|
||||
// is the authored anchor and must stay. Out-of-flow, so dropping never cascades the in-flow page.
|
||||
const h = el as HTMLElement;
|
||||
const r0 = el.getBoundingClientRect();
|
||||
const drop = { top: false, right: false, bottom: false, left: false };
|
||||
for (const side of ["top", "right", "bottom", "left"] as const) {
|
||||
const cur = cs[side as "top"];
|
||||
if (cur === "auto" || cur == null || cur === "") { drop[side] = true; continue; } // nothing to emit
|
||||
const sv = h.style.getPropertyValue(side), svp = h.style.getPropertyPriority(side);
|
||||
try {
|
||||
h.style.setProperty(side, "auto", "important");
|
||||
const r1 = el.getBoundingClientRect();
|
||||
drop[side] = Math.abs(r1.left - r0.left) <= 0.5 && Math.abs(r1.top - r0.top) <= 0.5 &&
|
||||
Math.abs(r1.width - r0.width) <= 0.5 && Math.abs(r1.height - r0.height) <= 0.5;
|
||||
} finally {
|
||||
h.style.removeProperty(side); if (sv) h.style.setProperty(side, sv, svp);
|
||||
}
|
||||
}
|
||||
sizing = { wAuto: false, wFill: false, hAuto: false, insetDrop: drop };
|
||||
}
|
||||
|
||||
const node: RawNode = {
|
||||
tag,
|
||||
attrs,
|
||||
computed,
|
||||
bbox,
|
||||
visible: isVisible(el, cs, bbox),
|
||||
...(sizing ? { sizing } : {}),
|
||||
children: [],
|
||||
};
|
||||
|
||||
// Pseudo-elements
|
||||
try {
|
||||
const before = window.getComputedStyle(el, "::before");
|
||||
if (before.content && before.content !== "none" && before.content !== "normal") {
|
||||
node.before = grabStyle(before, PSEUDO_PROPS);
|
||||
resolveLH(node.before);
|
||||
}
|
||||
const after = window.getComputedStyle(el, "::after");
|
||||
if (after.content && after.content !== "none" && after.content !== "normal") {
|
||||
node.after = grabStyle(after, PSEUDO_PROPS);
|
||||
resolveLH(node.after);
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// Inline SVG → raw markup, no recursion.
|
||||
if (tag === "svg") {
|
||||
node.rawHTML = el.outerHTML;
|
||||
return node;
|
||||
}
|
||||
|
||||
// Whitespace inside pre/pre-wrap/break-spaces is significant (e.g. multi-line
|
||||
// code blocks with highlighted spans separated by newlines).
|
||||
const preserveWs = /^(pre|pre-wrap|break-spaces)/.test(cs.whiteSpace || "");
|
||||
|
||||
// Recurse children (elements + text nodes), preserving order.
|
||||
for (const child of Array.from(el.childNodes)) {
|
||||
if (child.nodeType === Node.TEXT_NODE) {
|
||||
const t = child.textContent || "";
|
||||
if (preserveWs && t.length > 0) {
|
||||
node.children.push({ text: t });
|
||||
} else if (t.trim().length > 0) {
|
||||
node.children.push({ text: t });
|
||||
} else if (t.length > 0 && node.children.length > 0) {
|
||||
// Preserve a single significant space between inline elements.
|
||||
node.children.push({ text: " " });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (child.nodeType !== Node.ELEMENT_NODE) continue;
|
||||
const childEl = child as Element;
|
||||
if (SKIP_TAGS.has(childEl.tagName.toLowerCase())) continue;
|
||||
const sn = serializeElement(childEl);
|
||||
if (sn) node.children.push(sn);
|
||||
}
|
||||
|
||||
return node;
|
||||
};
|
||||
|
||||
// ---- Stylesheet introspection (font-faces, css url(), keyframes, vars) ----
|
||||
const fontFaces: FontFace[] = [];
|
||||
const cssUrlSet = new Set<string>();
|
||||
const keyframes: string[] = [];
|
||||
const cssVars: Record<string, string> = {};
|
||||
|
||||
const absUrl = (u: string): string => {
|
||||
try { return new URL(u, document.baseURI).href; } catch { return u; }
|
||||
};
|
||||
|
||||
const harvestUrlsFromText = (text: string): void => {
|
||||
const re = /url\(\s*(['"]?)([^'")]+)\1\s*\)/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
const raw = m[2];
|
||||
if (!raw || raw.startsWith("data:")) continue;
|
||||
cssUrlSet.add(absUrl(raw));
|
||||
}
|
||||
};
|
||||
|
||||
const readRules = (rules: CSSRuleList): void => {
|
||||
for (const rule of Array.from(rules)) {
|
||||
const type = rule.constructor.name;
|
||||
if (type === "CSSFontFaceRule") {
|
||||
const r = rule as CSSFontFaceRule;
|
||||
const s = r.style;
|
||||
const family = (s.getPropertyValue("font-family") || "").trim().replace(/^['"]|['"]$/g, "");
|
||||
const src = (s.getPropertyValue("src") || "").trim();
|
||||
if (family && src) {
|
||||
fontFaces.push({
|
||||
family,
|
||||
src,
|
||||
weight: s.getPropertyValue("font-weight") || undefined,
|
||||
style: s.getPropertyValue("font-style") || undefined,
|
||||
display: s.getPropertyValue("font-display") || undefined,
|
||||
unicodeRange: s.getPropertyValue("unicode-range") || undefined,
|
||||
stretch: s.getPropertyValue("font-stretch") || undefined,
|
||||
});
|
||||
harvestUrlsFromText(src);
|
||||
}
|
||||
} else if (type === "CSSKeyframesRule") {
|
||||
keyframes.push((rule as CSSKeyframesRule).cssText);
|
||||
} else if (type === "CSSStyleRule") {
|
||||
const r = rule as CSSStyleRule;
|
||||
if (r.style && r.style.cssText.includes("url(")) harvestUrlsFromText(r.style.cssText);
|
||||
} else if (type === "CSSMediaRule" || type === "CSSSupportsRule") {
|
||||
try { readRules((rule as CSSGroupingRule).cssRules); } catch { /* ignore */ }
|
||||
} else if (type === "CSSImportRule") {
|
||||
const imp = rule as CSSImportRule;
|
||||
try { if (imp.styleSheet) readRules(imp.styleSheet.cssRules); } catch { /* cross-origin */ }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (const sheet of Array.from(document.styleSheets)) {
|
||||
try {
|
||||
readRules(sheet.cssRules);
|
||||
} catch {
|
||||
// Cross-origin sheet — record its href so the capture layer can fetch the
|
||||
// raw text out-of-band and parse font-faces/urls from it.
|
||||
if (sheet.href) cssUrlSet.add(absUrl(sheet.href));
|
||||
}
|
||||
}
|
||||
|
||||
// CSS custom properties declared on :root.
|
||||
try {
|
||||
const rootCs = window.getComputedStyle(document.documentElement);
|
||||
for (let i = 0; i < rootCs.length; i++) {
|
||||
const prop = rootCs.item(i);
|
||||
if (prop.startsWith("--")) {
|
||||
const val = rootCs.getPropertyValue(prop).trim();
|
||||
if (val) cssVars[prop] = val;
|
||||
}
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// ---- DOM asset references ----
|
||||
const domAssets: Array<{ kind: string; url: string; via: string }> = [];
|
||||
const pushAsset = (kind: string, url: string | null, via: string): void => {
|
||||
if (!url) return;
|
||||
const trimmed = url.trim();
|
||||
if (!trimmed || trimmed.startsWith("data:")) return;
|
||||
domAssets.push({ kind, url: absUrl(trimmed), via });
|
||||
};
|
||||
// Lazy loaders often keep a transparent data: placeholder in src/srcset and park
|
||||
// the real URL in data-* until the element scrolls near the viewport. Harvest all
|
||||
// known carriers; pushAsset skips placeholders, so this is safe for normal images.
|
||||
const harvestSrcset = (kind: string, srcset: string | null, via: string): void => {
|
||||
if (!srcset) return;
|
||||
for (const part of srcset.split(",")) {
|
||||
const u = part.trim().split(/\s+/)[0];
|
||||
if (u) pushAsset(kind, u, via);
|
||||
}
|
||||
};
|
||||
for (const img of Array.from(document.querySelectorAll("img"))) {
|
||||
pushAsset("image", img.getAttribute("src"), "img[src]");
|
||||
pushAsset("image", img.getAttribute("data-lazy-src"), "img[data-lazy-src]");
|
||||
pushAsset("image", img.getAttribute("data-src"), "img[data-src]");
|
||||
pushAsset("image", img.getAttribute("data-original"), "img[data-original]");
|
||||
pushAsset("image", img.getAttribute("data-ll-src"), "img[data-ll-src]");
|
||||
harvestSrcset("image", img.getAttribute("srcset"), "img[srcset]");
|
||||
harvestSrcset("image", img.getAttribute("data-lazy-srcset"), "img[data-lazy-srcset]");
|
||||
harvestSrcset("image", img.getAttribute("data-srcset"), "img[data-srcset]");
|
||||
}
|
||||
for (const source of Array.from(document.querySelectorAll("source"))) {
|
||||
pushAsset("media", source.getAttribute("src"), "source[src]");
|
||||
pushAsset("media", source.getAttribute("data-src"), "source[data-src]");
|
||||
harvestSrcset("image", source.getAttribute("srcset"), "source[srcset]");
|
||||
harvestSrcset("image", source.getAttribute("data-lazy-srcset"), "source[data-lazy-srcset]");
|
||||
harvestSrcset("image", source.getAttribute("data-srcset"), "source[data-srcset]");
|
||||
}
|
||||
for (const video of Array.from(document.querySelectorAll("video"))) {
|
||||
pushAsset("video", video.getAttribute("src"), "video[src]");
|
||||
pushAsset("image", video.getAttribute("poster"), "video[poster]");
|
||||
}
|
||||
for (const use of Array.from(document.querySelectorAll("use"))) {
|
||||
const href = use.getAttribute("href") || use.getAttribute("xlink:href");
|
||||
if (href && !href.startsWith("#")) pushAsset("svg", href, "use[href]");
|
||||
}
|
||||
|
||||
const body = document.body;
|
||||
const root = serializeElement(body)!;
|
||||
lhProbe.remove();
|
||||
|
||||
const htmlCs = window.getComputedStyle(document.documentElement);
|
||||
const bodyCs = window.getComputedStyle(body);
|
||||
|
||||
const metaContent = (sel: string): string => (document.querySelector(sel) as HTMLMetaElement | null)?.content || "";
|
||||
const linkHref = (sel: string): string => (document.querySelector(sel) as HTMLLinkElement | null)?.href || "";
|
||||
const attr = (el: Element, name: string): string | undefined => {
|
||||
const v = el.getAttribute(name);
|
||||
return v && v.trim() ? v.trim() : undefined;
|
||||
};
|
||||
const headMeta = Array.from(document.head.querySelectorAll("meta")).map((m) => ({
|
||||
...(attr(m, "name") ? { name: attr(m, "name") } : {}),
|
||||
...(attr(m, "property") ? { property: attr(m, "property") } : {}),
|
||||
...(attr(m, "http-equiv") ? { httpEquiv: attr(m, "http-equiv") } : {}),
|
||||
content: (m.getAttribute("content") || "").trim(),
|
||||
})).filter((m) => m.content || m.name || m.property || m.httpEquiv);
|
||||
const headLinks = Array.from(document.head.querySelectorAll("link[href]")).map((l) => {
|
||||
const link = l as HTMLLinkElement;
|
||||
return {
|
||||
rel: (link.getAttribute("rel") || "").trim(),
|
||||
href: link.href || absUrl(link.getAttribute("href") || ""),
|
||||
...(attr(link, "as") ? { as: attr(link, "as") } : {}),
|
||||
...(attr(link, "type") ? { type: attr(link, "type") } : {}),
|
||||
...(attr(link, "sizes") ? { sizes: attr(link, "sizes") } : {}),
|
||||
...(attr(link, "media") ? { media: attr(link, "media") } : {}),
|
||||
...(attr(link, "color") ? { color: attr(link, "color") } : {}),
|
||||
...(attr(link, "hreflang") ? { hrefLang: attr(link, "hreflang") } : {}),
|
||||
...(attr(link, "title") ? { title: attr(link, "title") } : {}),
|
||||
...(attr(link, "crossorigin") ? { crossOrigin: attr(link, "crossorigin") } : {}),
|
||||
...(attr(link, "referrerpolicy") ? { referrerPolicy: attr(link, "referrerpolicy") } : {}),
|
||||
};
|
||||
}).filter((l) => l.rel || l.href);
|
||||
const jsonLd = Array.from(document.querySelectorAll("script[type]")).filter((s) => {
|
||||
const type = (s.getAttribute("type") || "").toLowerCase().split(";")[0]!.trim();
|
||||
return type === "application/ld+json";
|
||||
}).map((s) => ({
|
||||
...(attr(s, "id") ? { id: attr(s, "id") } : {}),
|
||||
text: (s.textContent || "").trim(),
|
||||
})).filter((s) => s.text);
|
||||
|
||||
for (const link of headLinks) {
|
||||
const rel = link.rel.toLowerCase();
|
||||
if (/\b(?:icon|shortcut icon|apple-touch-icon|mask-icon)\b/.test(rel)) pushAsset("image", link.href, `head link[rel="${link.rel}"]`);
|
||||
if (/\bmanifest\b/.test(rel)) pushAsset("manifest", link.href, `head link[rel="${link.rel}"]`);
|
||||
}
|
||||
|
||||
return {
|
||||
doc: {
|
||||
url: location.href,
|
||||
title: document.title || "",
|
||||
// SEO head metadata (for the generated app's <metadata>, robots, llms.txt).
|
||||
head: {
|
||||
description: metaContent('meta[name="description"]'),
|
||||
canonical: linkHref('link[rel="canonical"]'),
|
||||
ogTitle: metaContent('meta[property="og:title"]'),
|
||||
ogDescription: metaContent('meta[property="og:description"]'),
|
||||
ogImage: metaContent('meta[property="og:image"]'),
|
||||
ogType: metaContent('meta[property="og:type"]'),
|
||||
ogSiteName: metaContent('meta[property="og:site_name"]'),
|
||||
twitterCard: metaContent('meta[name="twitter:card"]'),
|
||||
themeColor: metaContent('meta[name="theme-color"]'),
|
||||
keywords: metaContent('meta[name="keywords"]'),
|
||||
robots: metaContent('meta[name="robots"]'),
|
||||
referrer: metaContent('meta[name="referrer"]'),
|
||||
colorScheme: metaContent('meta[name="color-scheme"]'),
|
||||
meta: headMeta,
|
||||
links: headLinks,
|
||||
jsonLd,
|
||||
},
|
||||
lang: document.documentElement.getAttribute("lang") || "",
|
||||
charset: document.characterSet || "UTF-8",
|
||||
viewportWidth: window.innerWidth,
|
||||
viewportHeight: window.innerHeight,
|
||||
scrollWidth: round2(document.documentElement.scrollWidth),
|
||||
scrollHeight: round2(document.documentElement.scrollHeight),
|
||||
htmlBg: htmlCs.backgroundColor,
|
||||
bodyBg: bodyCs.backgroundColor,
|
||||
bodyColor: bodyCs.color,
|
||||
bodyFont: bodyCs.fontFamily,
|
||||
metaViewport: (document.querySelector('meta[name="viewport"]') as HTMLMetaElement | null)?.content || "",
|
||||
nodeCount,
|
||||
truncated,
|
||||
},
|
||||
root,
|
||||
cssVars,
|
||||
fontFaces,
|
||||
cssUrls: Array.from(cssUrlSet).sort(),
|
||||
domAssets,
|
||||
keyframes,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user