Popup dismissal hardening: shadow roots, overlay units, scroll-lock recovery
- Overlay detection resolves effective z-index through stacking ancestors, treats a zero-size max-z wrapper plus its full-viewport descendant as one removable unit, pierces open shadow roots, and relaxes the z floor when the page is scroll-locked; a lock persisting after dismissal reports blocking even with no detected overlay - clickDismiss runs in every frame (close buttons live in cross-origin popup iframes), matches decline/no-thanks/aria-label close affordances, and re-runs before every snapshot AND screenshot to catch late-mounting dialogs - rootUnclamp triggers from IR content extent (a scroll-lock collapses captured scrollHeight) and strips vendor-injected body position/overflow locks when firing - Pollution gate fails on the scroll-lock contradiction: scrollHeight pinned to one viewport while IR content spans multiple - Frame graft skips popup-creative vendor hosts; inline embedded signup forms remain graftable (regression-tested) 322 tests pass (24 new), typecheck clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Samraaj Bath
co-authored by
Claude Fable 5
Claude Fable 5
parent
23505ee229
commit
0dd71a68d9
+269
-104
@@ -261,64 +261,102 @@ export type DismissResult = { dismissed: string[]; overlaysRemaining: number; re
|
||||
* Removal of a stuck overlay happens later (finalizeOverlays) AFTER a settle, so a
|
||||
* just-clicked dialog has time to close and unlock scrolling before we judge it.
|
||||
*/
|
||||
/**
|
||||
* In-page dismissal click pass (exported for fixture tests). Runs in ONE document — the main
|
||||
* page OR a cross-origin frame (Attentive/Recart/Klaviyo creatives host their Decline/× inside
|
||||
* an iframe, so the caller runs this in every `page.frames()`). Traverses open shadow roots
|
||||
* (Recart mounts its popup in a shadow tree) when scanning both containers and buttons.
|
||||
*/
|
||||
export function clickDismissInPage(): string[] {
|
||||
const dismissed: string[] = [];
|
||||
// Deep collector across open shadow roots.
|
||||
const deepEls = (root: ParentNode, sel: string): Element[] => {
|
||||
const out: Element[] = [];
|
||||
const walk = (r: ParentNode): void => {
|
||||
let hits: Element[] = [];
|
||||
try { hits = Array.from(r.querySelectorAll(sel)); } catch { /* invalid selector */ }
|
||||
out.push(...hits);
|
||||
for (const el of Array.from(r.querySelectorAll("*"))) {
|
||||
const sr = (el as HTMLElement).shadowRoot;
|
||||
if (sr) walk(sr);
|
||||
}
|
||||
};
|
||||
walk(root);
|
||||
return out;
|
||||
};
|
||||
const vis = (el: Element): boolean => {
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) return false;
|
||||
const r = (el as HTMLElement).getBoundingClientRect();
|
||||
return r.width > 0 && r.height > 0;
|
||||
};
|
||||
const click = (el: Element): void => { try { (el as HTMLElement).click(); } catch { /* ignore */ } };
|
||||
|
||||
// 1) Known consent-framework / generic close affordances, in priority order.
|
||||
const KNOWN = [
|
||||
"#onetrust-accept-btn-handler", "#accept-recommended-btn-handler", ".onetrust-close-btn-handler",
|
||||
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", "#CybotCookiebotDialogBodyButtonAccept",
|
||||
"#truste-consent-button", ".osano-cm-accept-all", ".osano-cm-dialog__close",
|
||||
"[data-testid='uc-accept-all-button']", "[data-testid='uc-deny-all-button']",
|
||||
"#didomi-notice-agree-button", ".didomi-continue-without-agreeing",
|
||||
".qc-cmp2-summary-buttons button[mode='primary']", "button[aria-label='Consent']",
|
||||
".cc-allow", ".cookie-consent-accept", "#hs-eu-confirmation-button", "#gdpr-consent-tool-wrapper button",
|
||||
];
|
||||
for (const sel of KNOWN) {
|
||||
for (const el of deepEls(document, sel)) {
|
||||
if (vis(el)) { click(el); dismissed.push(sel); break; }
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Scoped text-button pass: only inside overlay-ish containers (dialogs, or id/class naming
|
||||
// cookie/consent/modal/popup/newsletter/overlay/ccpa/privacy) so we never click an ordinary
|
||||
// page button. ACCEPT now also covers email-capture DECLINE affordances ("decline",
|
||||
// "no thanks", "not now", …) — a popup's own opt-out is the surest deterministic close.
|
||||
const ACCEPT = new Set([
|
||||
"accept", "accept all", "accept all cookies", "accept cookies", "accept & close",
|
||||
"i accept", "i agree", "agree", "agree and continue", "allow all", "allow cookies",
|
||||
"allow all cookies", "got it", "ok", "okay", "continue", "no thanks", "no, thanks",
|
||||
"dismiss", "close", "got it!", "understood", "yes, i agree",
|
||||
"decline", "no, thank you", "not now", "maybe later", "no thank you", "skip", "reject all",
|
||||
]);
|
||||
const containerSel =
|
||||
"[role='dialog'],[aria-modal='true'],[id*='cookie' i],[class*='cookie' i],[id*='consent' i]," +
|
||||
"[class*='consent' i],[class*='gdpr' i],[id*='gdpr' i],[class*='modal' i],[class*='popup' i]," +
|
||||
"[class*='newsletter' i],[class*='interstitial' i],[class*='overlay' i],[id*='overlay' i]," +
|
||||
"[class*='ccpa' i],[id*='ccpa' i],[class*='privacy' i],[id*='pop' i]";
|
||||
const containers = deepEls(document, containerSel);
|
||||
for (const c of containers) {
|
||||
if (!vis(c)) continue;
|
||||
// Text/value-matched accept/decline buttons.
|
||||
const btns = deepEls(c, "button,[role='button'],a,input[type='button'],input[type='submit']");
|
||||
let clicked = false;
|
||||
for (const b of btns) {
|
||||
const t = (b.textContent || (b as HTMLInputElement).value || "").replace(/\s+/g, " ").trim().toLowerCase();
|
||||
if (t && ACCEPT.has(t) && vis(b)) { click(b); dismissed.push("text:" + t); clicked = true; break; }
|
||||
}
|
||||
if (clicked) continue;
|
||||
// aria-label / title close affordance (icon-only ×, no text content to match).
|
||||
const closers = deepEls(c, "[aria-label*='close' i],[aria-label*='dismiss' i],[title*='close' i],button.close,.close-button,[class*='close' i][role='button']");
|
||||
for (const x of closers) {
|
||||
if (vis(x)) { click(x); dismissed.push("close:" + ((x.getAttribute("aria-label") || x.getAttribute("title") || "x")).slice(0, 24)); break; }
|
||||
}
|
||||
}
|
||||
return dismissed;
|
||||
}
|
||||
|
||||
async function clickDismiss(page: import("playwright").Page): Promise<string[]> {
|
||||
try {
|
||||
return await Promise.race([
|
||||
page.evaluate(() => {
|
||||
const dismissed: string[] = [];
|
||||
const vis = (el: Element): boolean => {
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) return false;
|
||||
const r = (el as HTMLElement).getBoundingClientRect();
|
||||
return r.width > 0 && r.height > 0;
|
||||
};
|
||||
const click = (el: Element): void => { try { (el as HTMLElement).click(); } catch { /* ignore */ } };
|
||||
|
||||
// 1) Known consent-framework / generic close affordances, in priority order.
|
||||
const KNOWN = [
|
||||
"#onetrust-accept-btn-handler", "#accept-recommended-btn-handler", ".onetrust-close-btn-handler",
|
||||
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", "#CybotCookiebotDialogBodyButtonAccept",
|
||||
"#truste-consent-button", ".osano-cm-accept-all", ".osano-cm-dialog__close",
|
||||
"[data-testid='uc-accept-all-button']", "[data-testid='uc-deny-all-button']",
|
||||
"#didomi-notice-agree-button", ".didomi-continue-without-agreeing",
|
||||
".qc-cmp2-summary-buttons button[mode='primary']", "button[aria-label='Consent']",
|
||||
".cc-allow", ".cookie-consent-accept", "#hs-eu-confirmation-button", "#gdpr-consent-tool-wrapper button",
|
||||
];
|
||||
for (const sel of KNOWN) {
|
||||
try {
|
||||
for (const el of Array.from(document.querySelectorAll(sel))) {
|
||||
if (vis(el)) { click(el); dismissed.push(sel); break; }
|
||||
}
|
||||
} catch { /* invalid selector in this browser */ }
|
||||
}
|
||||
|
||||
// 2) Scoped text-button pass: only inside overlay-ish containers (dialogs,
|
||||
// or id/class naming cookie/consent/modal/popup/newsletter) so we never
|
||||
// click an ordinary page button.
|
||||
const ACCEPT = new Set([
|
||||
"accept", "accept all", "accept all cookies", "accept cookies", "accept & close",
|
||||
"i accept", "i agree", "agree", "agree and continue", "allow all", "allow cookies",
|
||||
"allow all cookies", "got it", "ok", "okay", "continue", "no thanks", "no, thanks",
|
||||
"dismiss", "close", "got it!", "understood", "yes, i agree",
|
||||
]);
|
||||
const containerSel =
|
||||
"[role='dialog'],[aria-modal='true'],[id*='cookie' i],[class*='cookie' i],[id*='consent' i]," +
|
||||
"[class*='consent' i],[class*='gdpr' i],[id*='gdpr' i],[class*='modal' i],[class*='popup' i]," +
|
||||
"[class*='newsletter' i],[class*='interstitial' i]";
|
||||
let containers: Element[] = [];
|
||||
try { containers = Array.from(document.querySelectorAll(containerSel)); } catch { /* ignore */ }
|
||||
for (const c of containers) {
|
||||
if (!vis(c)) continue;
|
||||
const btns = Array.from(c.querySelectorAll("button,[role='button'],a,input[type='button'],input[type='submit']"));
|
||||
for (const b of btns) {
|
||||
const t = (b.textContent || (b as HTMLInputElement).value || "").replace(/\s+/g, " ").trim().toLowerCase();
|
||||
if (t && ACCEPT.has(t) && vis(b)) { click(b); dismissed.push("text:" + t); break; }
|
||||
}
|
||||
}
|
||||
return dismissed;
|
||||
}),
|
||||
// Run the click pass in the main document AND every frame — cross-origin popup creatives
|
||||
// (Attentive/Recart/…) host their Decline/close controls inside an iframe that the top
|
||||
// document cannot reach, but Playwright can evaluate inside each frame from Node.
|
||||
const runOne = (frame: import("playwright").Frame): Promise<string[]> =>
|
||||
Promise.race([
|
||||
frame.evaluate(clickDismissInPage).catch(() => [] as string[]),
|
||||
new Promise<string[]>((res) => setTimeout(() => res([]), 6000)),
|
||||
]);
|
||||
try {
|
||||
const frames = page.frames();
|
||||
const results = await Promise.all(frames.map((f) => runOne(f).catch(() => [] as string[])));
|
||||
return results.flat();
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
@@ -334,57 +372,168 @@ async function clickDismiss(page: import("playwright").Page): Promise<string[]>
|
||||
* sticky chrome is never stripped. Reports `blocking` = a scroll-locking overlay we
|
||||
* could not clear (the pollution gate keys off this, not mere overlay presence).
|
||||
*/
|
||||
async function finalizeOverlays(page: import("playwright").Page): Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }> {
|
||||
export type FinalizeOverlaysResult = { overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] };
|
||||
|
||||
/**
|
||||
* In-page overlay finalizer (exported so fixture tests can page.evaluate it directly).
|
||||
* Detects any full-viewport, fixed/sticky BLOCKING layer still present and — when the page
|
||||
* is scroll-locked (a state legit pages never enter) — removes it. Key robustness rules:
|
||||
*
|
||||
* (a) EFFECTIVE z-index: an element with `z-index:auto` inherits its stacking position from
|
||||
* the nearest positioned/opacity/transform ancestor that establishes a stacking context.
|
||||
* A vendor popup iframe is z:auto but sits inside a z=INT_MAX wrapper, so per-element
|
||||
* `parseInt("auto")` under-reads it — resolve z through the ancestor chain instead.
|
||||
* (b) LOCK relaxes the z gate: on a scroll-locked page, ANY fixed/sticky full-viewport layer
|
||||
* is blocking regardless of z (catches a z:auto creative and a z=50 CCPA dialog). Off a
|
||||
* locked page we keep the z>=100 floor to avoid stripping legit fixed content.
|
||||
* (c) OVERLAY UNIT: a 0-size (height:0) max-z wrapper whose descendant is a full-viewport
|
||||
* fixed iframe is ONE overlay — the wrapper carries the z, the iframe carries the pixels.
|
||||
* Detecting either means removing BOTH (climb from a removable iframe to its high-z
|
||||
* wrapper ancestor; and treat a max-z 0-size wrapper as an overlay via its full-viewport
|
||||
* descendant). Neither alone passes the area+z gate, so they must be grouped.
|
||||
* (d) LOUD FAILURE: if the lock persists after removal (or was locked with nothing detected),
|
||||
* report blocking=true so the pollution gate fails rather than shipping a polluted capture.
|
||||
*/
|
||||
export function finalizeOverlaysInPage(): FinalizeOverlaysResult {
|
||||
const vw = window.innerWidth, vh = window.innerHeight;
|
||||
// Deep element walk that descends into OPEN shadow roots. A shadow host reports 0 light-DOM
|
||||
// children, so a popup mounted inside a shadow tree (Recart's `#recart-popup-root`) is invisible
|
||||
// to a plain `querySelectorAll("*")` — its full-viewport fixed layer would never be detected and
|
||||
// would paint into every screenshot. Traverse `element.shadowRoot` (open only; closed roots are
|
||||
// unreachable — the host itself is still caught by the geometry gate below and removed whole).
|
||||
const deepEls = (root: ParentNode): Element[] => {
|
||||
const out: Element[] = [];
|
||||
const push = (r: ParentNode): void => {
|
||||
for (const el of Array.from(r.querySelectorAll("*"))) {
|
||||
out.push(el);
|
||||
const sr = (el as HTMLElement).shadowRoot;
|
||||
if (sr) push(sr);
|
||||
}
|
||||
};
|
||||
push(root);
|
||||
return out;
|
||||
};
|
||||
const isLocked = (): boolean => {
|
||||
const b = document.body, h = document.documentElement;
|
||||
const bs = getComputedStyle(b), hs = getComputedStyle(h);
|
||||
return bs.overflow === "hidden" || hs.overflow === "hidden" ||
|
||||
bs.overflowY === "hidden" || hs.overflowY === "hidden" ||
|
||||
bs.position === "fixed" || bs.position === "absolute";
|
||||
};
|
||||
// Effective z-index: climb to the nearest ancestor that both is positioned/creates a stacking
|
||||
// context AND reports a numeric z-index. `z-index:auto` on a positioned child paints in its
|
||||
// parent stacking context, so the parent's z is the layer's true stacking rank.
|
||||
const effectiveZ = (el: Element): number => {
|
||||
let cur: Element | null = el;
|
||||
let hops = 0;
|
||||
while (cur && hops < 20) {
|
||||
const cs = getComputedStyle(cur);
|
||||
const zi = parseInt(cs.zIndex || "", 10);
|
||||
const positioned = cs.position !== "static";
|
||||
if (Number.isFinite(zi) && positioned) return zi;
|
||||
cur = cur.parentElement;
|
||||
hops++;
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
const rectOf = (el: Element) => (el as HTMLElement).getBoundingClientRect();
|
||||
const fullViewport = (r: DOMRect): boolean => r.width >= vw * 0.7 && r.height >= vh * 0.5 && (r.width * r.height) / (vw * vh) >= 0.5;
|
||||
const painting = (cs: CSSStyleDeclaration): boolean =>
|
||||
cs.display !== "none" && cs.visibility !== "hidden" && parseFloat(cs.opacity || "1") !== 0;
|
||||
|
||||
const locked = isLocked();
|
||||
|
||||
// Candidate blocking layers. A layer qualifies when it is fixed/sticky, painting, and either
|
||||
// (1) itself full-viewport, or (2) a 0-size/high-z wrapper whose descendant is full-viewport
|
||||
// (the overlay-unit case — the wrapper holds the z, an inner iframe holds the pixels).
|
||||
// Full-viewport descendant test that pierces shadow roots (Recart mounts the covering layer
|
||||
// inside the host's shadow tree, so a light-DOM `querySelectorAll` would find nothing).
|
||||
const hasFullDescendant = (el: Element): boolean => {
|
||||
const scope: ParentNode = (el as HTMLElement).shadowRoot ?? el;
|
||||
return deepEls(scope).some((d) => { const dcs = getComputedStyle(d); return painting(dcs) && (dcs.position === "fixed" || dcs.position === "sticky" || dcs.position === "absolute") && fullViewport(rectOf(d)); });
|
||||
};
|
||||
const bigOverlays = (): HTMLElement[] => {
|
||||
const out: HTMLElement[] = [];
|
||||
for (const el of deepEls(document.body)) {
|
||||
const cs = getComputedStyle(el);
|
||||
const isShadowHost = !!(el as HTMLElement).shadowRoot;
|
||||
// A shadow host is usually position:static with a fixed layer INSIDE its root; still test it
|
||||
// (via its shadow descendants) so the whole host can be removed as one overlay unit.
|
||||
if (cs.position !== "fixed" && cs.position !== "sticky" && !isShadowHost) continue;
|
||||
if (!painting(cs)) continue;
|
||||
const r = rectOf(el);
|
||||
const z = effectiveZ(el);
|
||||
// Off a locked page keep the z>=100 floor; on a locked page any full-viewport fixed
|
||||
// layer is blocking (rule b). A max-z wrapper is a blocking layer even at 0 size when it
|
||||
// wraps a full-viewport descendant (rule c) — legit chrome never reaches the INT_MAX band.
|
||||
const positioned = cs.position === "fixed" || cs.position === "sticky";
|
||||
const selfFull = positioned && fullViewport(r);
|
||||
const wrapsFull = (isShadowHost || r.width < 4 || r.height < 4) && (isShadowHost || z >= 100) && hasFullDescendant(el);
|
||||
if (!selfFull && !wrapsFull) continue;
|
||||
const zPass = locked || z >= 100 || (wrapsFull && isShadowHost);
|
||||
if (zPass) out.push(el as HTMLElement);
|
||||
}
|
||||
// Keep outermost only (a wrapper subsumes its inner full-viewport iframe — the overlay unit).
|
||||
// `Node.contains` does NOT cross shadow boundaries, so also treat a candidate that lives inside
|
||||
// another candidate's shadow tree as contained — otherwise a shadow host AND its inner layer
|
||||
// both survive and we double-remove (or leave the inner layer behind).
|
||||
const shadowContains = (host: Element, node: Element): boolean => {
|
||||
let cur: Node | null = node;
|
||||
while (cur) {
|
||||
const rootNode = cur.getRootNode();
|
||||
if (rootNode instanceof ShadowRoot) {
|
||||
if (rootNode.host === host || host.contains(rootNode.host)) return true;
|
||||
cur = rootNode.host;
|
||||
} else break;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
return out.filter((el) => !out.some((o) => o !== el && (o.contains(el) || shadowContains(o, el))));
|
||||
};
|
||||
|
||||
// A scroll-locked page behind a full-viewport overlay IS a blocking modal by definition
|
||||
// (legit pages don't scroll-lock). Remove ANY such overlay that isn't page chrome — many
|
||||
// modals/drawers carry no consent/modal keyword and an icon-only close, so a keyword/aria
|
||||
// allowlist misses them. PROTECTED guards real chrome (header/nav/footer) only.
|
||||
const PROTECTED = /header|navbar|nav-|site-nav|topbar|masthead|footer/i;
|
||||
const sig = (el: HTMLElement): string => `${el.id} ${el.className}`.toString();
|
||||
|
||||
const removedLabels: string[] = [];
|
||||
let removed = 0;
|
||||
let remaining = bigOverlays();
|
||||
if (remaining.length && locked) {
|
||||
for (const el of remaining) {
|
||||
const s = sig(el);
|
||||
const z = effectiveZ(el);
|
||||
// Scroll-locked + full-viewport ⇒ blocking modal; remove unless it's page chrome. Always
|
||||
// remove iframes (cross-origin close, unclickable), max-z wrappers, and the overlay unit:
|
||||
// when the layer IS or CONTAINS a full-viewport fixed iframe, remove the whole subtree
|
||||
// (rule c — the 0-size wrapper + its iframe come out together as one node).
|
||||
const isShadowHost = !!(el as HTMLElement).shadowRoot;
|
||||
const containsIframe = el.tagName === "IFRAME" || !!el.querySelector("iframe") ||
|
||||
(isShadowHost && !!(el as HTMLElement).shadowRoot!.querySelector("iframe"));
|
||||
const removable = !PROTECTED.test(s) || el.getAttribute("aria-modal") === "true" ||
|
||||
containsIframe || isShadowHost || z >= 100;
|
||||
if (removable) { el.remove(); removed++; removedLabels.push((el.id || el.className || el.tagName).toString().slice(0, 40)); }
|
||||
}
|
||||
if (removed) {
|
||||
document.body.style.overflow = "visible"; document.documentElement.style.overflow = "visible";
|
||||
document.body.style.overflowY = "visible"; document.documentElement.style.overflowY = "visible";
|
||||
document.body.style.position = "static";
|
||||
}
|
||||
remaining = bigOverlays();
|
||||
}
|
||||
// Loud failure (rule d): a persisting lock is blocking whether or not a specific overlay was
|
||||
// still detectable — a scroll-locked capture is polluted, so surface it to the pollution gate.
|
||||
const stillLocked = isLocked();
|
||||
return { overlaysRemaining: remaining.length, removed, blocking: stillLocked && (remaining.length > 0 || locked), removedLabels };
|
||||
}
|
||||
|
||||
async function finalizeOverlays(page: import("playwright").Page): Promise<FinalizeOverlaysResult> {
|
||||
try {
|
||||
return await Promise.race([
|
||||
page.evaluate(() => {
|
||||
const vw = window.innerWidth, vh = window.innerHeight;
|
||||
const bigOverlays = (): HTMLElement[] => {
|
||||
const out: HTMLElement[] = [];
|
||||
for (const el of Array.from(document.body.querySelectorAll("*"))) {
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.position !== "fixed" && cs.position !== "sticky") continue;
|
||||
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) continue;
|
||||
const r = (el as HTMLElement).getBoundingClientRect();
|
||||
const z = parseInt(cs.zIndex || "0", 10) || 0;
|
||||
const area = (r.width * r.height) / (vw * vh);
|
||||
if (area >= 0.5 && z >= 100 && r.width >= vw * 0.7 && r.height >= vh * 0.5) out.push(el as HTMLElement);
|
||||
}
|
||||
return out.filter((el) => !out.some((o) => o !== el && o.contains(el)));
|
||||
};
|
||||
const isLocked = (): boolean => {
|
||||
const b = document.body, h = document.documentElement;
|
||||
return getComputedStyle(b).overflow === "hidden" || getComputedStyle(h).overflow === "hidden" ||
|
||||
getComputedStyle(b).position === "fixed";
|
||||
};
|
||||
// A scroll-locked page behind a full-viewport overlay IS a blocking modal by
|
||||
// definition (legit pages don't scroll-lock). So remove ANY such overlay that
|
||||
// isn't page chrome — many modals/drawers carry no consent/modal keyword and
|
||||
// an icon-only close, so a keyword/aria allowlist misses them (ruggable's
|
||||
// z-[1001] drawer). PROTECTED guards real chrome (header/nav/footer) only.
|
||||
const PROTECTED = /header|navbar|nav-|site-nav|topbar|masthead|footer/i;
|
||||
const sig = (el: HTMLElement): string => `${el.id} ${el.className}`.toString();
|
||||
|
||||
const removedLabels: string[] = [];
|
||||
let removed = 0;
|
||||
let remaining = bigOverlays();
|
||||
if (remaining.length && isLocked()) {
|
||||
for (const el of remaining) {
|
||||
const s = sig(el);
|
||||
const z = parseInt(getComputedStyle(el).zIndex || "0", 10) || 0;
|
||||
// Scroll-locked + full-viewport ⇒ blocking modal; remove unless it's page
|
||||
// chrome. Always remove iframes (cross-origin close, unclickable) and the
|
||||
// max-z-index popup trick.
|
||||
const removable = !PROTECTED.test(s) || el.getAttribute("aria-modal") === "true" ||
|
||||
el.tagName === "IFRAME" || z >= 2_000_000_000;
|
||||
if (removable) { el.remove(); removed++; removedLabels.push((el.id || el.className || el.tagName).toString().slice(0, 40)); }
|
||||
}
|
||||
if (removed) { document.body.style.overflow = "visible"; document.documentElement.style.overflow = "visible"; document.body.style.position = "static"; }
|
||||
remaining = bigOverlays();
|
||||
}
|
||||
return { overlaysRemaining: remaining.length, removed, blocking: remaining.length > 0 && isLocked(), removedLabels };
|
||||
}),
|
||||
new Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }>((res) => setTimeout(() => res({ overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }), 6000)),
|
||||
page.evaluate(finalizeOverlaysInPage),
|
||||
new Promise<FinalizeOverlaysResult>((res) => setTimeout(() => res({ overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }), 6000)),
|
||||
]);
|
||||
} catch {
|
||||
return { overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] };
|
||||
@@ -1206,6 +1355,15 @@ export async function captureSite(opts: {
|
||||
// let scroll-linked effects settle so the snapshot records the genuine at-rest values.
|
||||
await settleScrollTopBeforeSnapshot(page);
|
||||
|
||||
// Late-mounting dialogs (a CCPA opt-out that opens between the canonical and last
|
||||
// viewport; a Recart/Attentive promo that fires on a timer) mount AFTER the load /
|
||||
// post-scroll passes and would otherwise pollute this viewport's DOM snapshot. Re-run
|
||||
// the dismissal pass immediately before the walk. Cheap when nothing matches — the
|
||||
// settle only runs when a control was actually clicked or an overlay removed.
|
||||
await applyDismiss("pre-snapshot");
|
||||
dismissUnion.overlaysRemaining = Math.max(dismissUnion.overlaysRemaining, overlaysRemaining);
|
||||
dismissUnion.blocking = dismissUnion.blocking || blocking;
|
||||
|
||||
// Bound the in-page DOM walk: page.evaluate has no default timeout, so a
|
||||
// pathologically large/animated DOM (e.g. asana.com) could hang forever.
|
||||
const snapshot: PageSnapshot = await Promise.race([
|
||||
@@ -1254,6 +1412,13 @@ export async function captureSite(opts: {
|
||||
// Persist DOM snapshot, and (unless skipped for a production clone) the full-page screenshot.
|
||||
writeJSONCompact(join(captureDir, `dom-${vw}.json`), snapshot);
|
||||
if (opts.screenshots !== false) {
|
||||
// A promo popup can mount in the window between the DOM walk and the screenshot (Recart
|
||||
// fires into a shadow root on a short timer). The screenshot is the ground-truth channel
|
||||
// the perceptual gate compares against, so re-run dismissal right before it — otherwise a
|
||||
// late popup paints over every shot even though the DOM snapshot came out clean. Cheap
|
||||
// when nothing matches.
|
||||
await applyDismiss("pre-screenshot");
|
||||
dismissUnion.blocking = dismissUnion.blocking || blocking;
|
||||
// Normalize every video to frame 0 (paused) at THIS viewport before the shot — the clone is
|
||||
// always at frame 0, so pinning the source there too makes the two channels comparable
|
||||
// regardless of the playback time the viewport happened to catch.
|
||||
|
||||
@@ -29,8 +29,14 @@ export const MIN_FRAME_DIM = 48;
|
||||
export type FramePlan = "skip" | "still" | "graft";
|
||||
|
||||
// Ad/analytics/consent plumbing frames render nothing a visitor values — skip entirely.
|
||||
// The trailing group is email-capture / promo POPUP CREATIVE hosts: a vendor whose creative
|
||||
// iframe IS a full-viewport interstitial ("Enjoy 15% off" over a dimmed backdrop). Grafting it
|
||||
// pours the popup's copy into the DOM/text channel and paints the modal over the real page.
|
||||
// CAUTION: these hosts serve OVERLAY creatives, distinct from the vendor's INLINE-form embed
|
||||
// hosts (e.g. static-forms.klaviyo.com), which stay graftable — inline signup forms are a
|
||||
// deliberately-grafted feature (see iframeGraft tests). Match creative/overlay hosts only.
|
||||
const FRAME_SKIP_RE =
|
||||
/(?:doubleclick\.net|googlesyndication\.com|googletagmanager\.com|google-analytics\.com|googleadservices\.com|adservice\.google|facebook\.com\/tr|connect\.facebook\.net|\brecaptcha\b|hcaptcha\.com|challenges\.cloudflare\.com|adsrvr\.org|amazon-adsystem\.com)/i;
|
||||
/(?:doubleclick\.net|googlesyndication\.com|googletagmanager\.com|google-analytics\.com|googleadservices\.com|adservice\.google|facebook\.com\/tr|connect\.facebook\.net|\brecaptcha\b|hcaptcha\.com|challenges\.cloudflare\.com|adsrvr\.org|amazon-adsystem\.com|creatives?\.attn\.tv|\.attentivemobile\.com|bounceexchange\.com|bouncex\.net|\.wunderkind\.co|wknd\.ai|cdn\.justuno\.com\/mkjs|privy\.com\/.*popup|widget\.privy\.com|\.recart\.com|recart\.io)/i;
|
||||
// Media players are JS-built canvases/videos whose DOM graft is meaningless; an element
|
||||
// screenshot (the poster frame + play chrome) is the faithful static paint.
|
||||
const FRAME_STILL_RE =
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { IR, IRNode, StyleMap } from "../normalize/ir.js";
|
||||
import { isTextChild } from "../normalize/ir.js";
|
||||
import { isTextChild, irContentExtent } from "../normalize/ir.js";
|
||||
import type { TokenResolver } from "../infer/tokens.js";
|
||||
|
||||
/** Finalize a decls map: reference design tokens (var(--…)) where the value is tokenized
|
||||
@@ -2447,14 +2447,20 @@ function declsForViewport(
|
||||
// intro — body/html { height:100vh; overflow-y:scroll|hidden } — can be captured in
|
||||
// that transient state, pinning the root to one viewport so it CLIPS its overflowing
|
||||
// content; the clone's document.scrollHeight then collapses to one viewport and the
|
||||
// page renders truncated (paco.me). The lock is provably transient when the source's
|
||||
// OWN captured scrollHeight exceeds the pinned height (the real document scrolled
|
||||
// taller) — unlike a genuine internal-scroll shell, whose scrollHeight matches. Drop
|
||||
// the clamp (height + clipping overflow-y) so the root grows to its content.
|
||||
// page renders truncated (paco.me). Two independent signals prove the lock is transient:
|
||||
// • captured scrollHeight exceeds the pinned height (the real document scrolled taller); OR
|
||||
// • the IR's own IN-FLOW content extent (max descendant border-box bottom) exceeds the clamp.
|
||||
// The SECOND signal is essential for an email-capture popup that scroll-LOCKS the page: the
|
||||
// lock collapses `document.scrollHeight` down to the viewport (== clamp), so the scrollHeight
|
||||
// test can never fire in exactly the worst case — but the in-flow sections still carry their
|
||||
// real coordinates, so the content extent (5000px+ on a real homepage) still betrays the clamp.
|
||||
// A genuine internal-scroll shell has matching content extent AND scrollHeight, so neither fires.
|
||||
const clampH = parseFloat(cs.height || "");
|
||||
const contentExtent = (tag === "body" || tag === "html") ? irContentExtent(node, vp) : 0;
|
||||
const rootUnclamp =
|
||||
(tag === "body" || tag === "html") &&
|
||||
!!rootScrollHeight && clampH > 0 && rootScrollHeight > clampH + 4 &&
|
||||
clampH > 0 &&
|
||||
((!!rootScrollHeight && rootScrollHeight > clampH + 4) || contentExtent > clampH + 4) &&
|
||||
/^(scroll|hidden|auto|clip)$/.test(cs.overflowY || cs.overflow || "visible");
|
||||
// A pure-text leaf's height is fully content-derived (padding/border + lines × line-height).
|
||||
// We reproduce the font, line-height, padding and width, so its height auto-resolves to the
|
||||
@@ -2622,8 +2628,20 @@ function declsForViewport(
|
||||
}
|
||||
|
||||
// Root un-clamp: relax the clipping vertical overflow so the document scrolls at the
|
||||
// window level (no spurious scrollbar gutter) and grows to its content height.
|
||||
if (rootUnclamp) out.set("overflow-y", "visible");
|
||||
// window level (no spurious scrollbar gutter) and grows to its content height. Popup
|
||||
// vendors also inject `position:absolute|fixed` + `overflow:hidden` on <body> to freeze
|
||||
// the page under their modal; that lock state gets captured as the body's computed style
|
||||
// and, unstripped, hard-clips the clone to one screen (it can't scroll). Strip both — the
|
||||
// real body is `position:static; overflow:visible` — so the un-clamped root scrolls normally.
|
||||
if (rootUnclamp) {
|
||||
out.set("overflow-y", "visible");
|
||||
out.set("overflow-x", "visible");
|
||||
out.delete("overflow");
|
||||
out.delete("height");
|
||||
const posV = out.get("position");
|
||||
if (posV === "absolute" || posV === "fixed") out.delete("position");
|
||||
for (const side of ["top", "right", "bottom", "left"]) out.delete(side);
|
||||
}
|
||||
|
||||
// Fluid grid columns (fr) replace the baked per-viewport px tracks so the grid scales.
|
||||
if (gridCols !== undefined) out.set("grid-template-columns", gridCols);
|
||||
|
||||
@@ -83,6 +83,33 @@ export function isTextChild(c: IRChild): c is IRTextNode {
|
||||
return (c as IRTextNode).text !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* In-flow content extent at a viewport: the largest border-box bottom (`bbox.y + bbox.height`)
|
||||
* over every VISIBLE, IN-FLOW descendant. This is the true height the real page laid out to,
|
||||
* independent of a scroll-lock that clips the root to one viewport (a popup vendor that sets
|
||||
* `body{overflow:hidden;height:100vh}` collapses `document.scrollHeight` to the viewport, but
|
||||
* the in-flow sections still carry their real coordinates). Out-of-flow (absolute/fixed) and
|
||||
* floated boxes are excluded — a fixed overlay/footer badge or a floated aside is not part of
|
||||
* the document flow that determines page height. Pure + deterministic (reads only the IR).
|
||||
*/
|
||||
export function irContentExtent(root: IRNode, vp: number): number {
|
||||
let maxBottom = 0;
|
||||
const visit = (n: IRNode): void => {
|
||||
for (const c of n.children) {
|
||||
if (isTextChild(c)) continue;
|
||||
const cs = c.computedByVp[vp];
|
||||
const bb = c.bboxByVp[vp];
|
||||
if (!cs || !bb || !c.visibleByVp[vp]) { visit(c); continue; }
|
||||
const pos = cs.position || "static";
|
||||
const inFlow = pos !== "absolute" && pos !== "fixed" && (cs.float || "none") === "none";
|
||||
if (inFlow) maxBottom = Math.max(maxBottom, bb.y + bb.height);
|
||||
visit(c);
|
||||
}
|
||||
};
|
||||
visit(root);
|
||||
return maxBottom;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover lazy-loaded CSS backgrounds dropped by uneven per-viewport capture.
|
||||
* If an element has exactly one URL background at some sampled widths and
|
||||
@@ -175,9 +202,24 @@ const NOISE_TAGS = new Set(["next-route-announcer"]);
|
||||
// width where capture happened to read it open → an empty 400×700 white rectangle at 2xl). Drop
|
||||
// the whole subtree so it never reaches generation.
|
||||
const THIRD_PARTY_OVERLAY = /(?:^|[\s_-])(?:intercom|drift-|hubspot-messages|zendesk|zd-|onetrust|ot-sdk-|usercentrics|grecaptcha|crisp-client|tawk-|livechat|helpscout|beacon-container|cookiebot|cky-)/;
|
||||
|
||||
/**
|
||||
* Email-capture / promo POPUP overlay containers (Attentive, Wunderkind/Bounce Exchange,
|
||||
* Justuno, Privy, Omnisend, Sailthru, Wisepops, etc.). These vendors inject a full-viewport
|
||||
* fixed overlay + backdrop that scroll-locks the page — third-party chrome, not site content.
|
||||
*
|
||||
* CAUTION: these vendors ALSO ship inline, embedded signup forms that ARE real page content
|
||||
* and are deliberately grafted (see iframeGraft tests). So this matches ONLY the OVERLAY
|
||||
* CONTAINER markers each vendor uses for its popup (`attentive_overlay`, `bx-wrapper` /
|
||||
* `wk-`, `justuno`-`container`, `privy-`popup, …) — never a bare vendor name that an inline
|
||||
* form embed would also carry. Overlay-container tokens only.
|
||||
*/
|
||||
export const POPUP_OVERLAY_CONTAINER =
|
||||
/(?:^|[\s_-])(?:attentive_overlay|attentive_creative|attn-|bx-wrapper|bx-window|bounce-?exchange|wknd-|wunderkind|justuno_container|jw-overlay|privy-popup|privy-container|klaviyo-form-.*overlay|sailthru-overlay|wisepops-root|recart-popup|recart-modal)/;
|
||||
function isThirdPartyOverlay(raw: RawNode): boolean {
|
||||
const idClass = `${raw.attrs?.id ?? ""} ${raw.attrs?.class ?? ""}`.toLowerCase();
|
||||
if (THIRD_PARTY_OVERLAY.test(idClass)) return true;
|
||||
if (POPUP_OVERLAY_CONTAINER.test(idClass)) return true;
|
||||
// INT_MAX band: overlay libraries stack above everything. Real content should not reach here,
|
||||
// so this cannot swallow site chrome under normal captures.
|
||||
const zi = parseInt(raw.computed?.zIndex ?? "", 10);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { IR, IRNode } from "../normalize/ir.js";
|
||||
import { isTextChild } from "../normalize/ir.js";
|
||||
import { isTextChild, irContentExtent } from "../normalize/ir.js";
|
||||
import type { PageSnapshot } from "../capture/walker.js";
|
||||
import type { GenNode } from "./render.js";
|
||||
import { indexByCid } from "./render.js";
|
||||
@@ -119,11 +119,30 @@ export function gatePollution(ir: IR, capture: CaptureResult, viewports: number[
|
||||
let blocking = capture.dismissal?.blocking ?? false;
|
||||
for (const pv of capture.perViewport) { overlaysRemaining = Math.max(overlaysRemaining, pv.overlaysRemaining ?? 0); blocking = blocking || !!pv.blocking; }
|
||||
let minHeightRatio = Infinity;
|
||||
let maxHeightRatio = 0;
|
||||
for (const pv of capture.perViewport) {
|
||||
if (pv.height > 0) minHeightRatio = Math.min(minHeightRatio, pv.scrollHeight / pv.height);
|
||||
if (pv.height > 0) {
|
||||
const ratio = pv.scrollHeight / pv.height;
|
||||
minHeightRatio = Math.min(minHeightRatio, ratio);
|
||||
maxHeightRatio = Math.max(maxHeightRatio, ratio);
|
||||
}
|
||||
}
|
||||
if (!Number.isFinite(minHeightRatio)) minHeightRatio = 1;
|
||||
|
||||
// Scroll-locked-capture contradiction: an email-capture/promo popup that sets
|
||||
// body{overflow:hidden;height:100vh} collapses `document.scrollHeight` to EXACTLY the viewport
|
||||
// height at EVERY width (ratio ~1.0 across the board) — yet the real page's IN-FLOW content
|
||||
// (the IR's sections) still lays out several viewports tall. A genuine one-screen landing page
|
||||
// has content extent ~= its scrollHeight, so this only fires when the two disagree: captured
|
||||
// scrollHeight pinned to one viewport WHILE the IR content spans multiple. That is a
|
||||
// scroll-locked, polluted capture — the overlay detector should have caught it, so fail loudly.
|
||||
let maxContentRatio = 0;
|
||||
for (const pv of capture.perViewport) {
|
||||
if (pv.height > 0) maxContentRatio = Math.max(maxContentRatio, irContentExtent(ir.root, pv.viewport) / pv.height);
|
||||
}
|
||||
// scrollHeight never exceeds ~1 viewport at any width, but the IR content is 2+ viewports tall.
|
||||
const scrollLockedContradiction = maxHeightRatio > 0 && maxHeightRatio < 1.15 && maxContentRatio >= 2;
|
||||
|
||||
// Degenerate signals. Calibrated against real captures: an egress/bot wall is
|
||||
// ~3 nodes / ~24 chars; the most minimal legitimate page in the suite
|
||||
// (michaelcole.me) is 26 nodes / 640 chars. Thresholds sit safely between.
|
||||
@@ -131,6 +150,7 @@ export function gatePollution(ir: IR, capture: CaptureResult, viewports: number[
|
||||
if (wall && nodeCount < 220) issues.push("bot/egress wall text on a small page");
|
||||
if (textChars < 60 && nodeCount < 50) issues.push(`near-empty page: ${textChars} visible text chars, ${nodeCount} nodes`);
|
||||
if (blocking) issues.push("a full-viewport modal still scroll-locks the page after dismissal");
|
||||
if (scrollLockedContradiction) issues.push(`scroll-locked capture: scrollHeight pinned to ~1 viewport at every width while IR content spans ${round2(maxContentRatio)} viewports`);
|
||||
|
||||
return {
|
||||
gate: "pollution",
|
||||
@@ -138,6 +158,7 @@ export function gatePollution(ir: IR, capture: CaptureResult, viewports: number[
|
||||
metrics: {
|
||||
nodeCount, visibleTextChars: textChars, wallTextDetected: wall,
|
||||
overlaysRemaining, blocking, minScrollHeightRatio: round2(minHeightRatio),
|
||||
maxScrollHeightRatio: round2(maxHeightRatio), maxContentExtentRatio: round2(maxContentRatio),
|
||||
dismissedCount: capture.dismissal?.dismissed.length ?? 0,
|
||||
overlaysRemoved: capture.dismissal?.removed ?? 0,
|
||||
videoStills: capture.dismissal?.videoStills ?? 0,
|
||||
|
||||
@@ -49,6 +49,42 @@ function baseRule(css: string, id: string): string {
|
||||
return m?.[1] ?? "";
|
||||
}
|
||||
|
||||
// Root scroll-lock un-clamp: a popup vendor locks the page with
|
||||
// body{position:absolute;overflow:hidden;height:100vh}, which the capture bakes as the body's
|
||||
// computed style AND collapses document.scrollHeight down to the clamp (so the captured-scrollHeight
|
||||
// trigger can never fire). The IR's in-flow CONTENT EXTENT (a footer laid out at y=5000) still
|
||||
// exceeds the clamp, so the un-clamp must fire off that signal and strip the lock.
|
||||
describe("generateCss root scroll-lock un-clamp (content-extent trigger)", () => {
|
||||
it("drops the body height/position/overflow lock when IR content extent exceeds the clamp", () => {
|
||||
const footer = node("n1", "footer", computed());
|
||||
// Footer sits far below the one-viewport clamp at every viewport.
|
||||
for (const vp of VPS) footer.bboxByVp[vp] = { x: 0, y: 4000, width: vp, height: 1000 }; // bottom = 5000
|
||||
const bodyCs = computed({ position: "absolute", overflow: "hidden", overflowY: "hidden", height: "812px", top: "0px", left: "0px" });
|
||||
const root = node("n0", "body", bodyCs, [footer]);
|
||||
for (const vp of VPS) root.bboxByVp[vp] = { x: 0, y: 0, width: vp, height: 812 };
|
||||
// perViewport.scrollHeight stays collapsed to 800 (the lock's doing) — so ONLY the content
|
||||
// extent can trigger the un-clamp.
|
||||
const css = generateCss(irWith(root), new Map());
|
||||
const body = baseRule(css, "n0");
|
||||
assert.ok(!/height:812px/.test(body), `body height clamp dropped (got: ${body})`);
|
||||
assert.ok(!/position:absolute/.test(body), `body position:absolute stripped (got: ${body})`);
|
||||
assert.ok(/overflow-y:visible/.test(body), `overflow-y forced visible (got: ${body})`);
|
||||
assert.ok(!/overflow:hidden/.test(body), `overflow:hidden not emitted (got: ${body})`);
|
||||
});
|
||||
|
||||
it("does NOT un-clamp a genuine one-screen page (content extent within the clamp)", () => {
|
||||
const inner = node("n1", "div", computed());
|
||||
for (const vp of VPS) inner.bboxByVp[vp] = { x: 0, y: 0, width: vp, height: 700 };
|
||||
const bodyCs = computed({ overflow: "hidden", overflowY: "hidden", height: "800px" });
|
||||
const root = node("n0", "body", bodyCs, [inner]);
|
||||
for (const vp of VPS) root.bboxByVp[vp] = { x: 0, y: 0, width: vp, height: 800 };
|
||||
const css = generateCss(irWith(root), new Map());
|
||||
const body = baseRule(css, "n0");
|
||||
// The clamp is legitimate (content fits) — height kept, overflow not forced to visible.
|
||||
assert.ok(/height:800px/.test(body), `legit body height preserved (got: ${body})`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateCss list markers", () => {
|
||||
it("re-establishes list-style on a <ul> whose disc equals the parent's initial value", () => {
|
||||
// list-style-type's initial value is `disc` on EVERY element, so a real <ul disc>
|
||||
|
||||
@@ -72,6 +72,23 @@ describe("planForFrameUrl", () => {
|
||||
assert.equal(planForFrameUrl("https://static-forms.klaviyo.com/forms/abc"), "graft");
|
||||
assert.equal(planForFrameUrl("http://127.0.0.1:4001/iframe-embed.html"), "graft");
|
||||
});
|
||||
|
||||
it("SKIPS full-viewport promo POPUP CREATIVE hosts (Attentive/Recart/Wunderkind overlays)", () => {
|
||||
// These vendor hosts serve a full-viewport interstitial creative — grafting pours the
|
||||
// popup's copy into the DOM/text channel and paints the modal over the real page.
|
||||
assert.equal(planForFrameUrl("https://creatives.attn.tv/creative/12345"), "skip");
|
||||
assert.equal(planForFrameUrl("https://creative.attn.tv/loader/x"), "skip");
|
||||
assert.equal(planForFrameUrl("https://app.recart.com/popup/abc"), "skip");
|
||||
assert.equal(planForFrameUrl("https://tag.wunderkind.co/creative"), "skip");
|
||||
assert.equal(planForFrameUrl("https://api.bounceexchange.com/creative"), "skip");
|
||||
});
|
||||
|
||||
it("STILL grafts INLINE embed hosts even for the same vendors (inline forms are a feature)", () => {
|
||||
// Caution guard: a deliberately-grafted inline signup form must never be caught by the
|
||||
// popup-creative skip list. Inline-form hosts differ from overlay-creative hosts.
|
||||
assert.equal(planForFrameUrl("https://static-forms.klaviyo.com/forms/abc"), "graft");
|
||||
assert.equal(planForFrameUrl("https://manage.kmail-lists.com/subscriptions/subscribe"), "graft");
|
||||
});
|
||||
});
|
||||
|
||||
describe("graftFrameIntoSnapshot", () => {
|
||||
|
||||
@@ -117,6 +117,34 @@ describe("IR drops font-metric probe nodes (fix 4)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("IR drops popup-vendor OVERLAY containers, keeps inline embedded forms", () => {
|
||||
it("drops an email-capture popup overlay container (vendor overlay id) but keeps a real section", () => {
|
||||
const overlay = raw("div", { id: "attentive_overlay" }, [
|
||||
raw("iframe", { id: "attentive_creative", src: "https://creatives.attn.tv/x" }),
|
||||
]);
|
||||
const real = raw("section", { class: "hero" }, [raw("h1", {}, [{ text: "Real content" }])]);
|
||||
const body = raw("body", {}, [real, overlay]);
|
||||
|
||||
const root = buildFixtureIR(body);
|
||||
const kept = root.children.filter((c) => !isTextChild(c)).map((c) => (c as IRNode).tag);
|
||||
assert.deepEqual(kept, ["section"], "the attentive overlay subtree is dropped");
|
||||
assert.equal(findByTag(root, "iframe"), null, "the popup creative iframe never reaches the IR");
|
||||
});
|
||||
|
||||
it("does NOT drop an INLINE embedded signup form that merely carries a vendor name (feature, not popup)", () => {
|
||||
// A deliberately-grafted inline Klaviyo form: a real, sized form embedded in page content. Its
|
||||
// class names the vendor but is NOT an overlay-container marker, so it must survive.
|
||||
const inlineForm = raw("div", { class: "klaviyo-form klaviyo-form-inline" }, [
|
||||
raw("form", { id: "email-signup" }, [raw("input", { type: "email" })]),
|
||||
]);
|
||||
const body = raw("body", {}, [inlineForm]);
|
||||
|
||||
const root = buildFixtureIR(body);
|
||||
assert.ok(findByTag(root, "form"), "the inline signup form survives the prune");
|
||||
assert.ok(findByTag(root, "input"), "its input survives too");
|
||||
});
|
||||
});
|
||||
|
||||
// Defect C (normalize side) — an infinite CSS animation gated to a breakpoint (a Webflow `max-lg`
|
||||
// marquee) is `animation:none` at the widths it does not run, but the browser still reports the last
|
||||
// FROZEN translateX there. `neutralizeAnimatedTransforms` zeroes the transform at EVERY viewport
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
import { describe, it, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { chromium, type Browser, type Page } from "playwright";
|
||||
import { finalizeOverlaysInPage, clickDismissInPage } from "../src/capture/capture.js";
|
||||
import { irContentExtent, type IRNode, type BBox, type StyleMap } from "../src/normalize/ir.js";
|
||||
import { gatePollution } from "../src/validate/gates.js";
|
||||
import type { CaptureResult } from "../src/capture/capture.js";
|
||||
import type { IR } from "../src/normalize/ir.js";
|
||||
|
||||
// tsx/esbuild wraps serialized page functions with __name()/__defProp; shim so page.evaluate works.
|
||||
const ESBUILD_SHIM =
|
||||
"globalThis.__name = globalThis.__name || ((fn) => fn);" +
|
||||
"globalThis.__defProp = globalThis.__defProp || Object.defineProperty;";
|
||||
|
||||
const VW = 1280, VH = 800;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// finalizeOverlaysInPage — effective-z resolution, overlay-unit grouping,
|
||||
// lock-relaxed z gate, shadow-root traversal, loud-failure blocking.
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("capture: finalizeOverlaysInPage overlay detection", () => {
|
||||
let browser: Browser;
|
||||
before(async () => { browser = await chromium.launch(); });
|
||||
after(async () => { await browser.close(); });
|
||||
|
||||
const newPage = async (html: string): Promise<Page> => {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewportSize({ width: VW, height: VH });
|
||||
await page.addInitScript(ESBUILD_SHIM);
|
||||
await page.setContent(html);
|
||||
await page.evaluate(ESBUILD_SHIM);
|
||||
return page;
|
||||
};
|
||||
|
||||
it("groups a 0-height max-z wrapper + its full-viewport z:auto iframe as ONE overlay and removes both (scroll-locked)", async () => {
|
||||
// Mirrors the Attentive structure: body scroll-locked; a fixed z=INT_MAX wrapper of height 0
|
||||
// whose child is a fixed full-viewport iframe with z-index:auto. Neither passes the naive
|
||||
// per-element area+z gate alone (wrapper: area 0; iframe: z parses to 0).
|
||||
const page = await newPage(`
|
||||
<style>
|
||||
html,body{margin:0}
|
||||
body{position:absolute;top:0;left:0;right:0;height:${VH}px;overflow:hidden}
|
||||
#wrap{position:fixed;top:${VH}px;left:0;width:${VW}px;height:0;z-index:2147483647}
|
||||
#creative{position:fixed;top:0;left:0;width:${VW}px;height:${VH}px;z-index:auto;background:#000}
|
||||
</style>
|
||||
<main>real page content</main>
|
||||
<div id="wrap"><iframe id="creative"></iframe></div>
|
||||
`);
|
||||
const res = await page.evaluate(finalizeOverlaysInPage);
|
||||
assert.equal(res.removed, 1, "the wrapper+iframe unit is removed as one node");
|
||||
assert.ok(res.removedLabels.some((l) => l.includes("wrap")), `removed the max-z wrapper (got ${JSON.stringify(res.removedLabels)})`);
|
||||
const gone = await page.evaluate(() => !document.getElementById("wrap") && !document.getElementById("creative"));
|
||||
assert.ok(gone, "both the wrapper and its inner iframe are gone");
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it("detects a z=50 full-viewport fixed dialog ONLY because the page is scroll-locked (lock relaxes the z>=100 gate)", async () => {
|
||||
const page = await newPage(`
|
||||
<style>
|
||||
html,body{margin:0}
|
||||
body{overflow:hidden}
|
||||
#ccpa{position:fixed;inset:0;width:${VW}px;height:${VH}px;z-index:50;background:rgba(0,0,0,.7)}
|
||||
</style>
|
||||
<main>content</main>
|
||||
<div id="ccpa">California Residents Privacy Rights</div>
|
||||
`);
|
||||
const res = await page.evaluate(finalizeOverlaysInPage);
|
||||
assert.equal(res.removed, 1, "the z=50 dialog is removed on a locked page");
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it("does NOT remove a z=50 full-viewport fixed layer when the page is NOT scroll-locked (z floor preserved)", async () => {
|
||||
const page = await newPage(`
|
||||
<style>
|
||||
html,body{margin:0}
|
||||
#hero{position:fixed;inset:0;width:${VW}px;height:${VH}px;z-index:50;background:#eee}
|
||||
</style>
|
||||
<div id="hero">a legit fixed hero, page scrolls normally</div>
|
||||
`);
|
||||
const res = await page.evaluate(finalizeOverlaysInPage);
|
||||
assert.equal(res.removed, 0, "no removal off a locked page — the z>=100 floor guards legit fixed content");
|
||||
assert.equal(res.blocking, false);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it("removes a promo popup mounted inside an OPEN shadow root and reports the host label", async () => {
|
||||
const page = await newPage(`
|
||||
<style>html,body{margin:0}body{overflow:hidden}</style>
|
||||
<main>content</main>
|
||||
<div id="recart-popup-root"></div>
|
||||
<script>
|
||||
const host = document.getElementById("recart-popup-root");
|
||||
const root = host.attachShadow({ mode: "open" });
|
||||
const layer = document.createElement("div");
|
||||
layer.style.cssText = "position:fixed;inset:0;width:${VW}px;height:${VH}px;z-index:999;background:rgba(0,0,0,.6)";
|
||||
layer.textContent = "Enjoy 15% off";
|
||||
root.appendChild(layer);
|
||||
</script>
|
||||
`);
|
||||
const res = await page.evaluate(finalizeOverlaysInPage);
|
||||
assert.equal(res.removed, 1, "the shadow-hosting popup element is removed as one unit");
|
||||
const gone = await page.evaluate(() => !document.getElementById("recart-popup-root"));
|
||||
assert.ok(gone, "the shadow host is gone from the light DOM");
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it("reports blocking=true when the scroll-lock persists even with NO overlay detected (loud failure)", async () => {
|
||||
// Body is scroll-locked but there is no fixed full-viewport layer to find (e.g. a closed
|
||||
// shadow root we cannot pierce). A silently-locked capture is polluted → surface it.
|
||||
const page = await newPage(`
|
||||
<style>html,body{margin:0}body{overflow:hidden;position:absolute;height:${VH}px}</style>
|
||||
<main>content</main>
|
||||
`);
|
||||
const res = await page.evaluate(finalizeOverlaysInPage);
|
||||
assert.equal(res.removed, 0);
|
||||
assert.equal(res.blocking, true, "a persistent lock with nothing removable still reports blocking");
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it("reports blocking=false and removes nothing on an ordinary unlocked page", async () => {
|
||||
const page = await newPage(`
|
||||
<style>html,body{margin:0}</style>
|
||||
<header>nav</header><main>lots of content</main><footer>footer</footer>
|
||||
`);
|
||||
const res = await page.evaluate(finalizeOverlaysInPage);
|
||||
assert.equal(res.removed, 0);
|
||||
assert.equal(res.blocking, false);
|
||||
assert.equal(res.overlaysRemaining, 0);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it("protects real sticky page chrome (header/nav) from removal even when detected", async () => {
|
||||
// A scroll-locked page with a sticky header that happens to be tall — the header is PROTECTED,
|
||||
// and it isn't full-viewport, so it never even qualifies. Assert nothing is stripped.
|
||||
const page = await newPage(`
|
||||
<style>
|
||||
html,body{margin:0}body{overflow:hidden}
|
||||
#masthead{position:sticky;top:0;width:${VW}px;height:80px;z-index:1000;background:#fff}
|
||||
</style>
|
||||
<header id="masthead">site chrome</header>
|
||||
<main>content</main>
|
||||
`);
|
||||
const res = await page.evaluate(finalizeOverlaysInPage);
|
||||
const present = await page.evaluate(() => !!document.getElementById("masthead"));
|
||||
assert.ok(present, "the header survives");
|
||||
await page.close();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// clickDismissInPage — decline/close matchers, shadow-root traversal.
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("capture: clickDismissInPage matchers", () => {
|
||||
let browser: Browser;
|
||||
before(async () => { browser = await chromium.launch(); });
|
||||
after(async () => { await browser.close(); });
|
||||
|
||||
const run = async (html: string): Promise<string[]> => {
|
||||
const page = await browser.newPage();
|
||||
await page.addInitScript(ESBUILD_SHIM);
|
||||
await page.setContent(html);
|
||||
await page.evaluate(ESBUILD_SHIM);
|
||||
const res = await page.evaluate(clickDismissInPage);
|
||||
await page.close();
|
||||
return res;
|
||||
};
|
||||
|
||||
it("clicks a Decline button inside an overlay container (email-capture opt-out)", async () => {
|
||||
const dismissed = await run(`
|
||||
<div class="signup-overlay" style="position:fixed;inset:0;width:400px;height:400px">
|
||||
<button onclick="window.__declined=true">Decline</button>
|
||||
</div>
|
||||
`);
|
||||
assert.ok(dismissed.includes("text:decline"), `clicked Decline (got ${JSON.stringify(dismissed)})`);
|
||||
});
|
||||
|
||||
it("clicks an aria-label close (×) inside an overlay container when no accept/decline text matches", async () => {
|
||||
const dismissed = await run(`
|
||||
<div id="promo-popup" style="position:fixed;inset:0;width:400px;height:400px">
|
||||
<button aria-label="Close dialog">×</button>
|
||||
</div>
|
||||
`);
|
||||
assert.ok(dismissed.some((d) => d.startsWith("close:")), `clicked the aria close (got ${JSON.stringify(dismissed)})`);
|
||||
});
|
||||
|
||||
it("matches an overlay/ccpa/privacy container id/class (extended container selector)", async () => {
|
||||
const dismissed = await run(`
|
||||
<div id="ccpaPop" style="position:fixed;inset:0;width:400px;height:400px">
|
||||
<button>Accept</button>
|
||||
</div>
|
||||
`);
|
||||
assert.ok(dismissed.includes("text:accept"), `matched the ccpa container (got ${JSON.stringify(dismissed)})`);
|
||||
});
|
||||
|
||||
it("traverses an open shadow root to find the close control", async () => {
|
||||
const page = await browser.newPage();
|
||||
await page.addInitScript(ESBUILD_SHIM);
|
||||
await page.setContent(`<div id="host"></div>`);
|
||||
await page.evaluate(ESBUILD_SHIM);
|
||||
await page.evaluate(() => {
|
||||
const host = document.getElementById("host")!;
|
||||
const root = host.attachShadow({ mode: "open" });
|
||||
root.innerHTML = `<div class="popup-modal" style="position:fixed;inset:0;width:400px;height:400px"><button aria-label="close">×</button></div>`;
|
||||
});
|
||||
const dismissed = await page.evaluate(clickDismissInPage);
|
||||
await page.close();
|
||||
assert.ok(dismissed.some((d) => d.startsWith("close:")), `found the shadow-root close (got ${JSON.stringify(dismissed)})`);
|
||||
});
|
||||
|
||||
it("never clicks an ordinary page button outside any overlay container", async () => {
|
||||
const dismissed = await run(`<main><button>Accept</button><a>Continue</a></main>`);
|
||||
assert.deepEqual(dismissed, [], "no overlay container ⇒ no clicks");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// irContentExtent — pure geometry (unclamp / pollution trigger).
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("normalize: irContentExtent", () => {
|
||||
const bbox = (x: number, y: number, w: number, h: number): BBox => ({ x, y, width: w, height: h });
|
||||
const style = (over: Partial<StyleMap> = {}): StyleMap => ({ position: "static", ...over } as StyleMap);
|
||||
const node = (over: Partial<IRNode>): IRNode => ({
|
||||
id: "n", tag: "div", attrs: {},
|
||||
visibleByVp: { 1280: true }, bboxByVp: { 1280: bbox(0, 0, 100, 100) },
|
||||
computedByVp: { 1280: style() }, children: [], ...over,
|
||||
});
|
||||
|
||||
it("returns the max in-flow descendant border-box bottom", () => {
|
||||
const root = node({
|
||||
tag: "body",
|
||||
children: [
|
||||
node({ bboxByVp: { 1280: bbox(0, 0, 1280, 800) } }),
|
||||
node({ bboxByVp: { 1280: bbox(0, 800, 1280, 4398) } }), // footer bottom = 5198
|
||||
],
|
||||
});
|
||||
assert.equal(irContentExtent(root, 1280), 5198);
|
||||
});
|
||||
|
||||
it("excludes out-of-flow (fixed/absolute) and floated boxes", () => {
|
||||
const root = node({
|
||||
tag: "body",
|
||||
children: [
|
||||
node({ bboxByVp: { 1280: bbox(0, 0, 1280, 900) } }),
|
||||
node({ computedByVp: { 1280: style({ position: "fixed" }) }, bboxByVp: { 1280: bbox(0, 0, 1280, 9999) } }),
|
||||
node({ computedByVp: { 1280: style({ float: "left" } as Partial<StyleMap>) }, bboxByVp: { 1280: bbox(0, 0, 1280, 8888) } }),
|
||||
],
|
||||
});
|
||||
assert.equal(irContentExtent(root, 1280), 900, "fixed + float extents ignored");
|
||||
});
|
||||
|
||||
it("descends into nested in-flow subtrees", () => {
|
||||
const root = node({
|
||||
tag: "body",
|
||||
children: [node({ children: [node({ bboxByVp: { 1280: bbox(0, 0, 500, 3000) } })] })],
|
||||
});
|
||||
assert.equal(irContentExtent(root, 1280), 3000);
|
||||
});
|
||||
|
||||
it("ignores nodes invisible at the queried viewport", () => {
|
||||
const root = node({
|
||||
tag: "body",
|
||||
children: [node({ visibleByVp: { 1280: false }, bboxByVp: { 1280: bbox(0, 0, 100, 5000) } })],
|
||||
});
|
||||
assert.equal(irContentExtent(root, 1280), 0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// gatePollution — scroll-locked-capture contradiction.
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("validate: pollution gate scroll-locked contradiction", () => {
|
||||
const bbox = (x: number, y: number, w: number, h: number): BBox => ({ x, y, width: w, height: h });
|
||||
const st = (over: Partial<StyleMap> = {}): StyleMap => ({ position: "static", ...over } as StyleMap);
|
||||
// A body with plenty of text and a footer at y=5198 (content extent ~6.4 viewports @800).
|
||||
const mkIr = (): IR => ({
|
||||
doc: {
|
||||
sourceUrl: "https://x.test", title: "t", lang: "en", charset: "UTF-8", metaViewport: "",
|
||||
viewports: [1280], sampleViewports: [1280], canonicalViewport: 1280,
|
||||
perViewport: { 1280: { scrollHeight: 800, scrollWidth: 1280, htmlBg: "", bodyBg: "", bodyColor: "", bodyFont: "" } },
|
||||
nodeCount: 200, keyframes: [],
|
||||
},
|
||||
root: {
|
||||
id: "n0", tag: "body", attrs: {}, visibleByVp: { 1280: true },
|
||||
bboxByVp: { 1280: bbox(0, 0, 1280, 800) }, computedByVp: { 1280: st() },
|
||||
children: [
|
||||
{ id: "n1", tag: "section", attrs: {}, visibleByVp: { 1280: true }, bboxByVp: { 1280: bbox(0, 0, 1280, 800) }, computedByVp: { 1280: st() },
|
||||
children: [{ text: "hero copy ".repeat(20) }] },
|
||||
{ id: "n2", tag: "footer", attrs: {}, visibleByVp: { 1280: true }, bboxByVp: { 1280: bbox(0, 4074, 1280, 1124) }, computedByVp: { 1280: st() },
|
||||
children: [{ text: "footer copy ".repeat(20) }] },
|
||||
],
|
||||
},
|
||||
});
|
||||
const mkCapture = (scrollHeight: number): CaptureResult => ({
|
||||
perViewport: [
|
||||
{ viewport: 375, height: 812, scrollHeight: 812, nodeCount: 200, truncated: false },
|
||||
{ viewport: 1280, height: 800, scrollHeight, nodeCount: 200, truncated: false },
|
||||
],
|
||||
dismissal: { dismissed: [], overlaysRemaining: 0, removed: 0, videoStills: 0, blocking: false },
|
||||
} as unknown as CaptureResult);
|
||||
|
||||
it("FAILS when scrollHeight is pinned to ~1 viewport at every width but IR content spans multiple viewports", () => {
|
||||
const res = gatePollution(mkIr(), mkCapture(800), [1280]);
|
||||
assert.equal(res.pass, false, "the scroll-locked contradiction is caught");
|
||||
assert.ok(res.issues.some((i) => i.includes("scroll-locked capture")), `issue reported (got ${JSON.stringify(res.issues)})`);
|
||||
});
|
||||
|
||||
it("PASSES when scrollHeight matches the tall content (a genuinely scrollable page)", () => {
|
||||
const res = gatePollution(mkIr(), mkCapture(5198), [1280]);
|
||||
assert.ok(!res.issues.some((i) => i.includes("scroll-locked capture")), "no false positive when the page really scrolled");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user