Popup dismissal hardening: shadow roots, overlay units, scroll-lock recovery

- Overlay detection resolves effective z-index through stacking ancestors,
  treats a zero-size max-z wrapper plus its full-viewport descendant as one
  removable unit, pierces open shadow roots, and relaxes the z floor when
  the page is scroll-locked; a lock persisting after dismissal reports
  blocking even with no detected overlay
- clickDismiss runs in every frame (close buttons live in cross-origin
  popup iframes), matches decline/no-thanks/aria-label close affordances,
  and re-runs before every snapshot AND screenshot to catch late-mounting
  dialogs
- rootUnclamp triggers from IR content extent (a scroll-lock collapses
  captured scrollHeight) and strips vendor-injected body position/overflow
  locks when firing
- Pollution gate fails on the scroll-lock contradiction: scrollHeight
  pinned to one viewport while IR content spans multiple
- Frame graft skips popup-creative vendor hosts; inline embedded signup
  forms remain graftable (regression-tested)

322 tests pass (24 new), typecheck clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Samraaj Bath
2026-07-04 01:44:12 -07:00
co-authored by Claude Fable 5
parent 23505ee229
commit 0dd71a68d9
9 changed files with 759 additions and 115 deletions
+213 -48
View File
@@ -261,11 +261,29 @@ export type DismissResult = { dismissed: string[]; overlaysRemaining: number; re
* Removal of a stuck overlay happens later (finalizeOverlays) AFTER a settle, so a * Removal of a stuck overlay happens later (finalizeOverlays) AFTER a settle, so a
* just-clicked dialog has time to close and unlock scrolling before we judge it. * just-clicked dialog has time to close and unlock scrolling before we judge it.
*/ */
async function clickDismiss(page: import("playwright").Page): Promise<string[]> { /**
try { * In-page dismissal click pass (exported for fixture tests). Runs in ONE document — the main
return await Promise.race([ * page OR a cross-origin frame (Attentive/Recart/Klaviyo creatives host their Decline/× inside
page.evaluate(() => { * an iframe, so the caller runs this in every `page.frames()`). Traverses open shadow roots
* (Recart mounts its popup in a shadow tree) when scanning both containers and buttons.
*/
export function clickDismissInPage(): string[] {
const dismissed: string[] = []; const dismissed: string[] = [];
// Deep collector across open shadow roots.
const deepEls = (root: ParentNode, sel: string): Element[] => {
const out: Element[] = [];
const walk = (r: ParentNode): void => {
let hits: Element[] = [];
try { hits = Array.from(r.querySelectorAll(sel)); } catch { /* invalid selector */ }
out.push(...hits);
for (const el of Array.from(r.querySelectorAll("*"))) {
const sr = (el as HTMLElement).shadowRoot;
if (sr) walk(sr);
}
};
walk(root);
return out;
};
const vis = (el: Element): boolean => { const vis = (el: Element): boolean => {
const cs = getComputedStyle(el); const cs = getComputedStyle(el);
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) return false; if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) return false;
@@ -285,40 +303,60 @@ async function clickDismiss(page: import("playwright").Page): Promise<string[]>
".cc-allow", ".cookie-consent-accept", "#hs-eu-confirmation-button", "#gdpr-consent-tool-wrapper button", ".cc-allow", ".cookie-consent-accept", "#hs-eu-confirmation-button", "#gdpr-consent-tool-wrapper button",
]; ];
for (const sel of KNOWN) { for (const sel of KNOWN) {
try { for (const el of deepEls(document, sel)) {
for (const el of Array.from(document.querySelectorAll(sel))) {
if (vis(el)) { click(el); dismissed.push(sel); break; } if (vis(el)) { click(el); dismissed.push(sel); break; }
} }
} catch { /* invalid selector in this browser */ }
} }
// 2) Scoped text-button pass: only inside overlay-ish containers (dialogs, // 2) Scoped text-button pass: only inside overlay-ish containers (dialogs, or id/class naming
// or id/class naming cookie/consent/modal/popup/newsletter) so we never // cookie/consent/modal/popup/newsletter/overlay/ccpa/privacy) so we never click an ordinary
// click an ordinary page button. // page button. ACCEPT now also covers email-capture DECLINE affordances ("decline",
// "no thanks", "not now", …) — a popup's own opt-out is the surest deterministic close.
const ACCEPT = new Set([ const ACCEPT = new Set([
"accept", "accept all", "accept all cookies", "accept cookies", "accept & close", "accept", "accept all", "accept all cookies", "accept cookies", "accept & close",
"i accept", "i agree", "agree", "agree and continue", "allow all", "allow cookies", "i accept", "i agree", "agree", "agree and continue", "allow all", "allow cookies",
"allow all cookies", "got it", "ok", "okay", "continue", "no thanks", "no, thanks", "allow all cookies", "got it", "ok", "okay", "continue", "no thanks", "no, thanks",
"dismiss", "close", "got it!", "understood", "yes, i agree", "dismiss", "close", "got it!", "understood", "yes, i agree",
"decline", "no, thank you", "not now", "maybe later", "no thank you", "skip", "reject all",
]); ]);
const containerSel = const containerSel =
"[role='dialog'],[aria-modal='true'],[id*='cookie' i],[class*='cookie' i],[id*='consent' i]," + "[role='dialog'],[aria-modal='true'],[id*='cookie' i],[class*='cookie' i],[id*='consent' i]," +
"[class*='consent' i],[class*='gdpr' i],[id*='gdpr' i],[class*='modal' i],[class*='popup' i]," + "[class*='consent' i],[class*='gdpr' i],[id*='gdpr' i],[class*='modal' i],[class*='popup' i]," +
"[class*='newsletter' i],[class*='interstitial' i]"; "[class*='newsletter' i],[class*='interstitial' i],[class*='overlay' i],[id*='overlay' i]," +
let containers: Element[] = []; "[class*='ccpa' i],[id*='ccpa' i],[class*='privacy' i],[id*='pop' i]";
try { containers = Array.from(document.querySelectorAll(containerSel)); } catch { /* ignore */ } const containers = deepEls(document, containerSel);
for (const c of containers) { for (const c of containers) {
if (!vis(c)) continue; if (!vis(c)) continue;
const btns = Array.from(c.querySelectorAll("button,[role='button'],a,input[type='button'],input[type='submit']")); // Text/value-matched accept/decline buttons.
const btns = deepEls(c, "button,[role='button'],a,input[type='button'],input[type='submit']");
let clicked = false;
for (const b of btns) { for (const b of btns) {
const t = (b.textContent || (b as HTMLInputElement).value || "").replace(/\s+/g, " ").trim().toLowerCase(); const t = (b.textContent || (b as HTMLInputElement).value || "").replace(/\s+/g, " ").trim().toLowerCase();
if (t && ACCEPT.has(t) && vis(b)) { click(b); dismissed.push("text:" + t); break; } if (t && ACCEPT.has(t) && vis(b)) { click(b); dismissed.push("text:" + t); clicked = true; break; }
}
if (clicked) continue;
// aria-label / title close affordance (icon-only ×, no text content to match).
const closers = deepEls(c, "[aria-label*='close' i],[aria-label*='dismiss' i],[title*='close' i],button.close,.close-button,[class*='close' i][role='button']");
for (const x of closers) {
if (vis(x)) { click(x); dismissed.push("close:" + ((x.getAttribute("aria-label") || x.getAttribute("title") || "x")).slice(0, 24)); break; }
} }
} }
return dismissed; return dismissed;
}), }
async function clickDismiss(page: import("playwright").Page): Promise<string[]> {
// Run the click pass in the main document AND every frame — cross-origin popup creatives
// (Attentive/Recart/…) host their Decline/close controls inside an iframe that the top
// document cannot reach, but Playwright can evaluate inside each frame from Node.
const runOne = (frame: import("playwright").Frame): Promise<string[]> =>
Promise.race([
frame.evaluate(clickDismissInPage).catch(() => [] as string[]),
new Promise<string[]>((res) => setTimeout(() => res([]), 6000)), new Promise<string[]>((res) => setTimeout(() => res([]), 6000)),
]); ]);
try {
const frames = page.frames();
const results = await Promise.all(frames.map((f) => runOne(f).catch(() => [] as string[])));
return results.flat();
} catch { } catch {
return []; return [];
} }
@@ -334,57 +372,168 @@ async function clickDismiss(page: import("playwright").Page): Promise<string[]>
* sticky chrome is never stripped. Reports `blocking` = a scroll-locking overlay we * sticky chrome is never stripped. Reports `blocking` = a scroll-locking overlay we
* could not clear (the pollution gate keys off this, not mere overlay presence). * could not clear (the pollution gate keys off this, not mere overlay presence).
*/ */
async function finalizeOverlays(page: import("playwright").Page): Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }> { export type FinalizeOverlaysResult = { overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] };
try {
return await Promise.race([ /**
page.evaluate(() => { * In-page overlay finalizer (exported so fixture tests can page.evaluate it directly).
* Detects any full-viewport, fixed/sticky BLOCKING layer still present and — when the page
* is scroll-locked (a state legit pages never enter) — removes it. Key robustness rules:
*
* (a) EFFECTIVE z-index: an element with `z-index:auto` inherits its stacking position from
* the nearest positioned/opacity/transform ancestor that establishes a stacking context.
* A vendor popup iframe is z:auto but sits inside a z=INT_MAX wrapper, so per-element
* `parseInt("auto")` under-reads it — resolve z through the ancestor chain instead.
* (b) LOCK relaxes the z gate: on a scroll-locked page, ANY fixed/sticky full-viewport layer
* is blocking regardless of z (catches a z:auto creative and a z=50 CCPA dialog). Off a
* locked page we keep the z>=100 floor to avoid stripping legit fixed content.
* (c) OVERLAY UNIT: a 0-size (height:0) max-z wrapper whose descendant is a full-viewport
* fixed iframe is ONE overlay — the wrapper carries the z, the iframe carries the pixels.
* Detecting either means removing BOTH (climb from a removable iframe to its high-z
* wrapper ancestor; and treat a max-z 0-size wrapper as an overlay via its full-viewport
* descendant). Neither alone passes the area+z gate, so they must be grouped.
* (d) LOUD FAILURE: if the lock persists after removal (or was locked with nothing detected),
* report blocking=true so the pollution gate fails rather than shipping a polluted capture.
*/
export function finalizeOverlaysInPage(): FinalizeOverlaysResult {
const vw = window.innerWidth, vh = window.innerHeight; const vw = window.innerWidth, vh = window.innerHeight;
const bigOverlays = (): HTMLElement[] => { // Deep element walk that descends into OPEN shadow roots. A shadow host reports 0 light-DOM
const out: HTMLElement[] = []; // children, so a popup mounted inside a shadow tree (Recart's `#recart-popup-root`) is invisible
for (const el of Array.from(document.body.querySelectorAll("*"))) { // to a plain `querySelectorAll("*")` — its full-viewport fixed layer would never be detected and
const cs = getComputedStyle(el); // would paint into every screenshot. Traverse `element.shadowRoot` (open only; closed roots are
if (cs.position !== "fixed" && cs.position !== "sticky") continue; // unreachable — the host itself is still caught by the geometry gate below and removed whole).
if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) continue; const deepEls = (root: ParentNode): Element[] => {
const r = (el as HTMLElement).getBoundingClientRect(); const out: Element[] = [];
const z = parseInt(cs.zIndex || "0", 10) || 0; const push = (r: ParentNode): void => {
const area = (r.width * r.height) / (vw * vh); for (const el of Array.from(r.querySelectorAll("*"))) {
if (area >= 0.5 && z >= 100 && r.width >= vw * 0.7 && r.height >= vh * 0.5) out.push(el as HTMLElement); out.push(el);
const sr = (el as HTMLElement).shadowRoot;
if (sr) push(sr);
} }
return out.filter((el) => !out.some((o) => o !== el && o.contains(el))); };
push(root);
return out;
}; };
const isLocked = (): boolean => { const isLocked = (): boolean => {
const b = document.body, h = document.documentElement; const b = document.body, h = document.documentElement;
return getComputedStyle(b).overflow === "hidden" || getComputedStyle(h).overflow === "hidden" || const bs = getComputedStyle(b), hs = getComputedStyle(h);
getComputedStyle(b).position === "fixed"; return bs.overflow === "hidden" || hs.overflow === "hidden" ||
bs.overflowY === "hidden" || hs.overflowY === "hidden" ||
bs.position === "fixed" || bs.position === "absolute";
}; };
// A scroll-locked page behind a full-viewport overlay IS a blocking modal by // Effective z-index: climb to the nearest ancestor that both is positioned/creates a stacking
// definition (legit pages don't scroll-lock). So remove ANY such overlay that // context AND reports a numeric z-index. `z-index:auto` on a positioned child paints in its
// isn't page chrome — many modals/drawers carry no consent/modal keyword and // parent stacking context, so the parent's z is the layer's true stacking rank.
// an icon-only close, so a keyword/aria allowlist misses them (ruggable's const effectiveZ = (el: Element): number => {
// z-[1001] drawer). PROTECTED guards real chrome (header/nav/footer) only. let cur: Element | null = el;
let hops = 0;
while (cur && hops < 20) {
const cs = getComputedStyle(cur);
const zi = parseInt(cs.zIndex || "", 10);
const positioned = cs.position !== "static";
if (Number.isFinite(zi) && positioned) return zi;
cur = cur.parentElement;
hops++;
}
return 0;
};
const rectOf = (el: Element) => (el as HTMLElement).getBoundingClientRect();
const fullViewport = (r: DOMRect): boolean => r.width >= vw * 0.7 && r.height >= vh * 0.5 && (r.width * r.height) / (vw * vh) >= 0.5;
const painting = (cs: CSSStyleDeclaration): boolean =>
cs.display !== "none" && cs.visibility !== "hidden" && parseFloat(cs.opacity || "1") !== 0;
const locked = isLocked();
// Candidate blocking layers. A layer qualifies when it is fixed/sticky, painting, and either
// (1) itself full-viewport, or (2) a 0-size/high-z wrapper whose descendant is full-viewport
// (the overlay-unit case — the wrapper holds the z, an inner iframe holds the pixels).
// Full-viewport descendant test that pierces shadow roots (Recart mounts the covering layer
// inside the host's shadow tree, so a light-DOM `querySelectorAll` would find nothing).
const hasFullDescendant = (el: Element): boolean => {
const scope: ParentNode = (el as HTMLElement).shadowRoot ?? el;
return deepEls(scope).some((d) => { const dcs = getComputedStyle(d); return painting(dcs) && (dcs.position === "fixed" || dcs.position === "sticky" || dcs.position === "absolute") && fullViewport(rectOf(d)); });
};
const bigOverlays = (): HTMLElement[] => {
const out: HTMLElement[] = [];
for (const el of deepEls(document.body)) {
const cs = getComputedStyle(el);
const isShadowHost = !!(el as HTMLElement).shadowRoot;
// A shadow host is usually position:static with a fixed layer INSIDE its root; still test it
// (via its shadow descendants) so the whole host can be removed as one overlay unit.
if (cs.position !== "fixed" && cs.position !== "sticky" && !isShadowHost) continue;
if (!painting(cs)) continue;
const r = rectOf(el);
const z = effectiveZ(el);
// Off a locked page keep the z>=100 floor; on a locked page any full-viewport fixed
// layer is blocking (rule b). A max-z wrapper is a blocking layer even at 0 size when it
// wraps a full-viewport descendant (rule c) — legit chrome never reaches the INT_MAX band.
const positioned = cs.position === "fixed" || cs.position === "sticky";
const selfFull = positioned && fullViewport(r);
const wrapsFull = (isShadowHost || r.width < 4 || r.height < 4) && (isShadowHost || z >= 100) && hasFullDescendant(el);
if (!selfFull && !wrapsFull) continue;
const zPass = locked || z >= 100 || (wrapsFull && isShadowHost);
if (zPass) out.push(el as HTMLElement);
}
// Keep outermost only (a wrapper subsumes its inner full-viewport iframe — the overlay unit).
// `Node.contains` does NOT cross shadow boundaries, so also treat a candidate that lives inside
// another candidate's shadow tree as contained — otherwise a shadow host AND its inner layer
// both survive and we double-remove (or leave the inner layer behind).
const shadowContains = (host: Element, node: Element): boolean => {
let cur: Node | null = node;
while (cur) {
const rootNode = cur.getRootNode();
if (rootNode instanceof ShadowRoot) {
if (rootNode.host === host || host.contains(rootNode.host)) return true;
cur = rootNode.host;
} else break;
}
return false;
};
return out.filter((el) => !out.some((o) => o !== el && (o.contains(el) || shadowContains(o, el))));
};
// A scroll-locked page behind a full-viewport overlay IS a blocking modal by definition
// (legit pages don't scroll-lock). Remove ANY such overlay that isn't page chrome — many
// modals/drawers carry no consent/modal keyword and an icon-only close, so a keyword/aria
// allowlist misses them. PROTECTED guards real chrome (header/nav/footer) only.
const PROTECTED = /header|navbar|nav-|site-nav|topbar|masthead|footer/i; const PROTECTED = /header|navbar|nav-|site-nav|topbar|masthead|footer/i;
const sig = (el: HTMLElement): string => `${el.id} ${el.className}`.toString(); const sig = (el: HTMLElement): string => `${el.id} ${el.className}`.toString();
const removedLabels: string[] = []; const removedLabels: string[] = [];
let removed = 0; let removed = 0;
let remaining = bigOverlays(); let remaining = bigOverlays();
if (remaining.length && isLocked()) { if (remaining.length && locked) {
for (const el of remaining) { for (const el of remaining) {
const s = sig(el); const s = sig(el);
const z = parseInt(getComputedStyle(el).zIndex || "0", 10) || 0; const z = effectiveZ(el);
// Scroll-locked + full-viewport ⇒ blocking modal; remove unless it's page // Scroll-locked + full-viewport ⇒ blocking modal; remove unless it's page chrome. Always
// chrome. Always remove iframes (cross-origin close, unclickable) and the // remove iframes (cross-origin close, unclickable), max-z wrappers, and the overlay unit:
// max-z-index popup trick. // when the layer IS or CONTAINS a full-viewport fixed iframe, remove the whole subtree
// (rule c — the 0-size wrapper + its iframe come out together as one node).
const isShadowHost = !!(el as HTMLElement).shadowRoot;
const containsIframe = el.tagName === "IFRAME" || !!el.querySelector("iframe") ||
(isShadowHost && !!(el as HTMLElement).shadowRoot!.querySelector("iframe"));
const removable = !PROTECTED.test(s) || el.getAttribute("aria-modal") === "true" || const removable = !PROTECTED.test(s) || el.getAttribute("aria-modal") === "true" ||
el.tagName === "IFRAME" || z >= 2_000_000_000; containsIframe || isShadowHost || z >= 100;
if (removable) { el.remove(); removed++; removedLabels.push((el.id || el.className || el.tagName).toString().slice(0, 40)); } if (removable) { el.remove(); removed++; removedLabels.push((el.id || el.className || el.tagName).toString().slice(0, 40)); }
} }
if (removed) { document.body.style.overflow = "visible"; document.documentElement.style.overflow = "visible"; document.body.style.position = "static"; } if (removed) {
document.body.style.overflow = "visible"; document.documentElement.style.overflow = "visible";
document.body.style.overflowY = "visible"; document.documentElement.style.overflowY = "visible";
document.body.style.position = "static";
}
remaining = bigOverlays(); remaining = bigOverlays();
} }
return { overlaysRemaining: remaining.length, removed, blocking: remaining.length > 0 && isLocked(), removedLabels }; // Loud failure (rule d): a persisting lock is blocking whether or not a specific overlay was
}), // still detectable — a scroll-locked capture is polluted, so surface it to the pollution gate.
new Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }>((res) => setTimeout(() => res({ overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }), 6000)), const stillLocked = isLocked();
return { overlaysRemaining: remaining.length, removed, blocking: stillLocked && (remaining.length > 0 || locked), removedLabels };
}
async function finalizeOverlays(page: import("playwright").Page): Promise<FinalizeOverlaysResult> {
try {
return await Promise.race([
page.evaluate(finalizeOverlaysInPage),
new Promise<FinalizeOverlaysResult>((res) => setTimeout(() => res({ overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }), 6000)),
]); ]);
} catch { } catch {
return { overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }; return { overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] };
@@ -1206,6 +1355,15 @@ export async function captureSite(opts: {
// let scroll-linked effects settle so the snapshot records the genuine at-rest values. // let scroll-linked effects settle so the snapshot records the genuine at-rest values.
await settleScrollTopBeforeSnapshot(page); await settleScrollTopBeforeSnapshot(page);
// Late-mounting dialogs (a CCPA opt-out that opens between the canonical and last
// viewport; a Recart/Attentive promo that fires on a timer) mount AFTER the load /
// post-scroll passes and would otherwise pollute this viewport's DOM snapshot. Re-run
// the dismissal pass immediately before the walk. Cheap when nothing matches — the
// settle only runs when a control was actually clicked or an overlay removed.
await applyDismiss("pre-snapshot");
dismissUnion.overlaysRemaining = Math.max(dismissUnion.overlaysRemaining, overlaysRemaining);
dismissUnion.blocking = dismissUnion.blocking || blocking;
// Bound the in-page DOM walk: page.evaluate has no default timeout, so a // Bound the in-page DOM walk: page.evaluate has no default timeout, so a
// pathologically large/animated DOM (e.g. asana.com) could hang forever. // pathologically large/animated DOM (e.g. asana.com) could hang forever.
const snapshot: PageSnapshot = await Promise.race([ const snapshot: PageSnapshot = await Promise.race([
@@ -1254,6 +1412,13 @@ export async function captureSite(opts: {
// Persist DOM snapshot, and (unless skipped for a production clone) the full-page screenshot. // Persist DOM snapshot, and (unless skipped for a production clone) the full-page screenshot.
writeJSONCompact(join(captureDir, `dom-${vw}.json`), snapshot); writeJSONCompact(join(captureDir, `dom-${vw}.json`), snapshot);
if (opts.screenshots !== false) { if (opts.screenshots !== false) {
// A promo popup can mount in the window between the DOM walk and the screenshot (Recart
// fires into a shadow root on a short timer). The screenshot is the ground-truth channel
// the perceptual gate compares against, so re-run dismissal right before it — otherwise a
// late popup paints over every shot even though the DOM snapshot came out clean. Cheap
// when nothing matches.
await applyDismiss("pre-screenshot");
dismissUnion.blocking = dismissUnion.blocking || blocking;
// Normalize every video to frame 0 (paused) at THIS viewport before the shot — the clone is // Normalize every video to frame 0 (paused) at THIS viewport before the shot — the clone is
// always at frame 0, so pinning the source there too makes the two channels comparable // always at frame 0, so pinning the source there too makes the two channels comparable
// regardless of the playback time the viewport happened to catch. // regardless of the playback time the viewport happened to catch.
+7 -1
View File
@@ -29,8 +29,14 @@ export const MIN_FRAME_DIM = 48;
export type FramePlan = "skip" | "still" | "graft"; export type FramePlan = "skip" | "still" | "graft";
// Ad/analytics/consent plumbing frames render nothing a visitor values — skip entirely. // Ad/analytics/consent plumbing frames render nothing a visitor values — skip entirely.
// The trailing group is email-capture / promo POPUP CREATIVE hosts: a vendor whose creative
// iframe IS a full-viewport interstitial ("Enjoy 15% off" over a dimmed backdrop). Grafting it
// pours the popup's copy into the DOM/text channel and paints the modal over the real page.
// CAUTION: these hosts serve OVERLAY creatives, distinct from the vendor's INLINE-form embed
// hosts (e.g. static-forms.klaviyo.com), which stay graftable — inline signup forms are a
// deliberately-grafted feature (see iframeGraft tests). Match creative/overlay hosts only.
const FRAME_SKIP_RE = const FRAME_SKIP_RE =
/(?:doubleclick\.net|googlesyndication\.com|googletagmanager\.com|google-analytics\.com|googleadservices\.com|adservice\.google|facebook\.com\/tr|connect\.facebook\.net|\brecaptcha\b|hcaptcha\.com|challenges\.cloudflare\.com|adsrvr\.org|amazon-adsystem\.com)/i; /(?:doubleclick\.net|googlesyndication\.com|googletagmanager\.com|google-analytics\.com|googleadservices\.com|adservice\.google|facebook\.com\/tr|connect\.facebook\.net|\brecaptcha\b|hcaptcha\.com|challenges\.cloudflare\.com|adsrvr\.org|amazon-adsystem\.com|creatives?\.attn\.tv|\.attentivemobile\.com|bounceexchange\.com|bouncex\.net|\.wunderkind\.co|wknd\.ai|cdn\.justuno\.com\/mkjs|privy\.com\/.*popup|widget\.privy\.com|\.recart\.com|recart\.io)/i;
// Media players are JS-built canvases/videos whose DOM graft is meaningless; an element // Media players are JS-built canvases/videos whose DOM graft is meaningless; an element
// screenshot (the poster frame + play chrome) is the faithful static paint. // screenshot (the poster frame + play chrome) is the faithful static paint.
const FRAME_STILL_RE = const FRAME_STILL_RE =
+26 -8
View File
@@ -1,5 +1,5 @@
import type { IR, IRNode, StyleMap } from "../normalize/ir.js"; import type { IR, IRNode, StyleMap } from "../normalize/ir.js";
import { isTextChild } from "../normalize/ir.js"; import { isTextChild, irContentExtent } from "../normalize/ir.js";
import type { TokenResolver } from "../infer/tokens.js"; import type { TokenResolver } from "../infer/tokens.js";
/** Finalize a decls map: reference design tokens (var(--…)) where the value is tokenized /** Finalize a decls map: reference design tokens (var(--…)) where the value is tokenized
@@ -2447,14 +2447,20 @@ function declsForViewport(
// intro — body/html { height:100vh; overflow-y:scroll|hidden } — can be captured in // intro — body/html { height:100vh; overflow-y:scroll|hidden } — can be captured in
// that transient state, pinning the root to one viewport so it CLIPS its overflowing // that transient state, pinning the root to one viewport so it CLIPS its overflowing
// content; the clone's document.scrollHeight then collapses to one viewport and the // content; the clone's document.scrollHeight then collapses to one viewport and the
// page renders truncated (paco.me). The lock is provably transient when the source's // page renders truncated (paco.me). Two independent signals prove the lock is transient:
// OWN captured scrollHeight exceeds the pinned height (the real document scrolled // captured scrollHeight exceeds the pinned height (the real document scrolled taller); OR
// taller) — unlike a genuine internal-scroll shell, whose scrollHeight matches. Drop // • the IR's own IN-FLOW content extent (max descendant border-box bottom) exceeds the clamp.
// the clamp (height + clipping overflow-y) so the root grows to its content. // The SECOND signal is essential for an email-capture popup that scroll-LOCKS the page: the
// lock collapses `document.scrollHeight` down to the viewport (== clamp), so the scrollHeight
// test can never fire in exactly the worst case — but the in-flow sections still carry their
// real coordinates, so the content extent (5000px+ on a real homepage) still betrays the clamp.
// A genuine internal-scroll shell has matching content extent AND scrollHeight, so neither fires.
const clampH = parseFloat(cs.height || ""); const clampH = parseFloat(cs.height || "");
const contentExtent = (tag === "body" || tag === "html") ? irContentExtent(node, vp) : 0;
const rootUnclamp = const rootUnclamp =
(tag === "body" || tag === "html") && (tag === "body" || tag === "html") &&
!!rootScrollHeight && clampH > 0 && rootScrollHeight > clampH + 4 && clampH > 0 &&
((!!rootScrollHeight && rootScrollHeight > clampH + 4) || contentExtent > clampH + 4) &&
/^(scroll|hidden|auto|clip)$/.test(cs.overflowY || cs.overflow || "visible"); /^(scroll|hidden|auto|clip)$/.test(cs.overflowY || cs.overflow || "visible");
// A pure-text leaf's height is fully content-derived (padding/border + lines × line-height). // A pure-text leaf's height is fully content-derived (padding/border + lines × line-height).
// We reproduce the font, line-height, padding and width, so its height auto-resolves to the // We reproduce the font, line-height, padding and width, so its height auto-resolves to the
@@ -2622,8 +2628,20 @@ function declsForViewport(
} }
// Root un-clamp: relax the clipping vertical overflow so the document scrolls at the // Root un-clamp: relax the clipping vertical overflow so the document scrolls at the
// window level (no spurious scrollbar gutter) and grows to its content height. // window level (no spurious scrollbar gutter) and grows to its content height. Popup
if (rootUnclamp) out.set("overflow-y", "visible"); // vendors also inject `position:absolute|fixed` + `overflow:hidden` on <body> to freeze
// the page under their modal; that lock state gets captured as the body's computed style
// and, unstripped, hard-clips the clone to one screen (it can't scroll). Strip both — the
// real body is `position:static; overflow:visible` — so the un-clamped root scrolls normally.
if (rootUnclamp) {
out.set("overflow-y", "visible");
out.set("overflow-x", "visible");
out.delete("overflow");
out.delete("height");
const posV = out.get("position");
if (posV === "absolute" || posV === "fixed") out.delete("position");
for (const side of ["top", "right", "bottom", "left"]) out.delete(side);
}
// Fluid grid columns (fr) replace the baked per-viewport px tracks so the grid scales. // Fluid grid columns (fr) replace the baked per-viewport px tracks so the grid scales.
if (gridCols !== undefined) out.set("grid-template-columns", gridCols); if (gridCols !== undefined) out.set("grid-template-columns", gridCols);
+42
View File
@@ -83,6 +83,33 @@ export function isTextChild(c: IRChild): c is IRTextNode {
return (c as IRTextNode).text !== undefined; return (c as IRTextNode).text !== undefined;
} }
/**
* In-flow content extent at a viewport: the largest border-box bottom (`bbox.y + bbox.height`)
* over every VISIBLE, IN-FLOW descendant. This is the true height the real page laid out to,
* independent of a scroll-lock that clips the root to one viewport (a popup vendor that sets
* `body{overflow:hidden;height:100vh}` collapses `document.scrollHeight` to the viewport, but
* the in-flow sections still carry their real coordinates). Out-of-flow (absolute/fixed) and
* floated boxes are excluded a fixed overlay/footer badge or a floated aside is not part of
* the document flow that determines page height. Pure + deterministic (reads only the IR).
*/
export function irContentExtent(root: IRNode, vp: number): number {
let maxBottom = 0;
const visit = (n: IRNode): void => {
for (const c of n.children) {
if (isTextChild(c)) continue;
const cs = c.computedByVp[vp];
const bb = c.bboxByVp[vp];
if (!cs || !bb || !c.visibleByVp[vp]) { visit(c); continue; }
const pos = cs.position || "static";
const inFlow = pos !== "absolute" && pos !== "fixed" && (cs.float || "none") === "none";
if (inFlow) maxBottom = Math.max(maxBottom, bb.y + bb.height);
visit(c);
}
};
visit(root);
return maxBottom;
}
/** /**
* Recover lazy-loaded CSS backgrounds dropped by uneven per-viewport capture. * Recover lazy-loaded CSS backgrounds dropped by uneven per-viewport capture.
* If an element has exactly one URL background at some sampled widths and * If an element has exactly one URL background at some sampled widths and
@@ -175,9 +202,24 @@ const NOISE_TAGS = new Set(["next-route-announcer"]);
// width where capture happened to read it open → an empty 400×700 white rectangle at 2xl). Drop // width where capture happened to read it open → an empty 400×700 white rectangle at 2xl). Drop
// the whole subtree so it never reaches generation. // the whole subtree so it never reaches generation.
const THIRD_PARTY_OVERLAY = /(?:^|[\s_-])(?:intercom|drift-|hubspot-messages|zendesk|zd-|onetrust|ot-sdk-|usercentrics|grecaptcha|crisp-client|tawk-|livechat|helpscout|beacon-container|cookiebot|cky-)/; const THIRD_PARTY_OVERLAY = /(?:^|[\s_-])(?:intercom|drift-|hubspot-messages|zendesk|zd-|onetrust|ot-sdk-|usercentrics|grecaptcha|crisp-client|tawk-|livechat|helpscout|beacon-container|cookiebot|cky-)/;
/**
* Email-capture / promo POPUP overlay containers (Attentive, Wunderkind/Bounce Exchange,
* Justuno, Privy, Omnisend, Sailthru, Wisepops, etc.). These vendors inject a full-viewport
* fixed overlay + backdrop that scroll-locks the page third-party chrome, not site content.
*
* CAUTION: these vendors ALSO ship inline, embedded signup forms that ARE real page content
* and are deliberately grafted (see iframeGraft tests). So this matches ONLY the OVERLAY
* CONTAINER markers each vendor uses for its popup (`attentive_overlay`, `bx-wrapper` /
* `wk-`, `justuno`-`container`, `privy-`popup, ) never a bare vendor name that an inline
* form embed would also carry. Overlay-container tokens only.
*/
export const POPUP_OVERLAY_CONTAINER =
/(?:^|[\s_-])(?:attentive_overlay|attentive_creative|attn-|bx-wrapper|bx-window|bounce-?exchange|wknd-|wunderkind|justuno_container|jw-overlay|privy-popup|privy-container|klaviyo-form-.*overlay|sailthru-overlay|wisepops-root|recart-popup|recart-modal)/;
function isThirdPartyOverlay(raw: RawNode): boolean { function isThirdPartyOverlay(raw: RawNode): boolean {
const idClass = `${raw.attrs?.id ?? ""} ${raw.attrs?.class ?? ""}`.toLowerCase(); const idClass = `${raw.attrs?.id ?? ""} ${raw.attrs?.class ?? ""}`.toLowerCase();
if (THIRD_PARTY_OVERLAY.test(idClass)) return true; if (THIRD_PARTY_OVERLAY.test(idClass)) return true;
if (POPUP_OVERLAY_CONTAINER.test(idClass)) return true;
// INT_MAX band: overlay libraries stack above everything. Real content should not reach here, // INT_MAX band: overlay libraries stack above everything. Real content should not reach here,
// so this cannot swallow site chrome under normal captures. // so this cannot swallow site chrome under normal captures.
const zi = parseInt(raw.computed?.zIndex ?? "", 10); const zi = parseInt(raw.computed?.zIndex ?? "", 10);
+23 -2
View File
@@ -1,5 +1,5 @@
import type { IR, IRNode } from "../normalize/ir.js"; import type { IR, IRNode } from "../normalize/ir.js";
import { isTextChild } from "../normalize/ir.js"; import { isTextChild, irContentExtent } from "../normalize/ir.js";
import type { PageSnapshot } from "../capture/walker.js"; import type { PageSnapshot } from "../capture/walker.js";
import type { GenNode } from "./render.js"; import type { GenNode } from "./render.js";
import { indexByCid } from "./render.js"; import { indexByCid } from "./render.js";
@@ -119,11 +119,30 @@ export function gatePollution(ir: IR, capture: CaptureResult, viewports: number[
let blocking = capture.dismissal?.blocking ?? false; let blocking = capture.dismissal?.blocking ?? false;
for (const pv of capture.perViewport) { overlaysRemaining = Math.max(overlaysRemaining, pv.overlaysRemaining ?? 0); blocking = blocking || !!pv.blocking; } for (const pv of capture.perViewport) { overlaysRemaining = Math.max(overlaysRemaining, pv.overlaysRemaining ?? 0); blocking = blocking || !!pv.blocking; }
let minHeightRatio = Infinity; let minHeightRatio = Infinity;
let maxHeightRatio = 0;
for (const pv of capture.perViewport) { for (const pv of capture.perViewport) {
if (pv.height > 0) minHeightRatio = Math.min(minHeightRatio, pv.scrollHeight / pv.height); if (pv.height > 0) {
const ratio = pv.scrollHeight / pv.height;
minHeightRatio = Math.min(minHeightRatio, ratio);
maxHeightRatio = Math.max(maxHeightRatio, ratio);
}
} }
if (!Number.isFinite(minHeightRatio)) minHeightRatio = 1; if (!Number.isFinite(minHeightRatio)) minHeightRatio = 1;
// Scroll-locked-capture contradiction: an email-capture/promo popup that sets
// body{overflow:hidden;height:100vh} collapses `document.scrollHeight` to EXACTLY the viewport
// height at EVERY width (ratio ~1.0 across the board) — yet the real page's IN-FLOW content
// (the IR's sections) still lays out several viewports tall. A genuine one-screen landing page
// has content extent ~= its scrollHeight, so this only fires when the two disagree: captured
// scrollHeight pinned to one viewport WHILE the IR content spans multiple. That is a
// scroll-locked, polluted capture — the overlay detector should have caught it, so fail loudly.
let maxContentRatio = 0;
for (const pv of capture.perViewport) {
if (pv.height > 0) maxContentRatio = Math.max(maxContentRatio, irContentExtent(ir.root, pv.viewport) / pv.height);
}
// scrollHeight never exceeds ~1 viewport at any width, but the IR content is 2+ viewports tall.
const scrollLockedContradiction = maxHeightRatio > 0 && maxHeightRatio < 1.15 && maxContentRatio >= 2;
// Degenerate signals. Calibrated against real captures: an egress/bot wall is // Degenerate signals. Calibrated against real captures: an egress/bot wall is
// ~3 nodes / ~24 chars; the most minimal legitimate page in the suite // ~3 nodes / ~24 chars; the most minimal legitimate page in the suite
// (michaelcole.me) is 26 nodes / 640 chars. Thresholds sit safely between. // (michaelcole.me) is 26 nodes / 640 chars. Thresholds sit safely between.
@@ -131,6 +150,7 @@ export function gatePollution(ir: IR, capture: CaptureResult, viewports: number[
if (wall && nodeCount < 220) issues.push("bot/egress wall text on a small page"); if (wall && nodeCount < 220) issues.push("bot/egress wall text on a small page");
if (textChars < 60 && nodeCount < 50) issues.push(`near-empty page: ${textChars} visible text chars, ${nodeCount} nodes`); if (textChars < 60 && nodeCount < 50) issues.push(`near-empty page: ${textChars} visible text chars, ${nodeCount} nodes`);
if (blocking) issues.push("a full-viewport modal still scroll-locks the page after dismissal"); if (blocking) issues.push("a full-viewport modal still scroll-locks the page after dismissal");
if (scrollLockedContradiction) issues.push(`scroll-locked capture: scrollHeight pinned to ~1 viewport at every width while IR content spans ${round2(maxContentRatio)} viewports`);
return { return {
gate: "pollution", gate: "pollution",
@@ -138,6 +158,7 @@ export function gatePollution(ir: IR, capture: CaptureResult, viewports: number[
metrics: { metrics: {
nodeCount, visibleTextChars: textChars, wallTextDetected: wall, nodeCount, visibleTextChars: textChars, wallTextDetected: wall,
overlaysRemaining, blocking, minScrollHeightRatio: round2(minHeightRatio), overlaysRemaining, blocking, minScrollHeightRatio: round2(minHeightRatio),
maxScrollHeightRatio: round2(maxHeightRatio), maxContentExtentRatio: round2(maxContentRatio),
dismissedCount: capture.dismissal?.dismissed.length ?? 0, dismissedCount: capture.dismissal?.dismissed.length ?? 0,
overlaysRemoved: capture.dismissal?.removed ?? 0, overlaysRemoved: capture.dismissal?.removed ?? 0,
videoStills: capture.dismissal?.videoStills ?? 0, videoStills: capture.dismissal?.videoStills ?? 0,
+36
View File
@@ -49,6 +49,42 @@ function baseRule(css: string, id: string): string {
return m?.[1] ?? ""; return m?.[1] ?? "";
} }
// Root scroll-lock un-clamp: a popup vendor locks the page with
// body{position:absolute;overflow:hidden;height:100vh}, which the capture bakes as the body's
// computed style AND collapses document.scrollHeight down to the clamp (so the captured-scrollHeight
// trigger can never fire). The IR's in-flow CONTENT EXTENT (a footer laid out at y=5000) still
// exceeds the clamp, so the un-clamp must fire off that signal and strip the lock.
describe("generateCss root scroll-lock un-clamp (content-extent trigger)", () => {
it("drops the body height/position/overflow lock when IR content extent exceeds the clamp", () => {
const footer = node("n1", "footer", computed());
// Footer sits far below the one-viewport clamp at every viewport.
for (const vp of VPS) footer.bboxByVp[vp] = { x: 0, y: 4000, width: vp, height: 1000 }; // bottom = 5000
const bodyCs = computed({ position: "absolute", overflow: "hidden", overflowY: "hidden", height: "812px", top: "0px", left: "0px" });
const root = node("n0", "body", bodyCs, [footer]);
for (const vp of VPS) root.bboxByVp[vp] = { x: 0, y: 0, width: vp, height: 812 };
// perViewport.scrollHeight stays collapsed to 800 (the lock's doing) — so ONLY the content
// extent can trigger the un-clamp.
const css = generateCss(irWith(root), new Map());
const body = baseRule(css, "n0");
assert.ok(!/height:812px/.test(body), `body height clamp dropped (got: ${body})`);
assert.ok(!/position:absolute/.test(body), `body position:absolute stripped (got: ${body})`);
assert.ok(/overflow-y:visible/.test(body), `overflow-y forced visible (got: ${body})`);
assert.ok(!/overflow:hidden/.test(body), `overflow:hidden not emitted (got: ${body})`);
});
it("does NOT un-clamp a genuine one-screen page (content extent within the clamp)", () => {
const inner = node("n1", "div", computed());
for (const vp of VPS) inner.bboxByVp[vp] = { x: 0, y: 0, width: vp, height: 700 };
const bodyCs = computed({ overflow: "hidden", overflowY: "hidden", height: "800px" });
const root = node("n0", "body", bodyCs, [inner]);
for (const vp of VPS) root.bboxByVp[vp] = { x: 0, y: 0, width: vp, height: 800 };
const css = generateCss(irWith(root), new Map());
const body = baseRule(css, "n0");
// The clamp is legitimate (content fits) — height kept, overflow not forced to visible.
assert.ok(/height:800px/.test(body), `legit body height preserved (got: ${body})`);
});
});
describe("generateCss list markers", () => { describe("generateCss list markers", () => {
it("re-establishes list-style on a <ul> whose disc equals the parent's initial value", () => { it("re-establishes list-style on a <ul> whose disc equals the parent's initial value", () => {
// list-style-type's initial value is `disc` on EVERY element, so a real <ul disc> // list-style-type's initial value is `disc` on EVERY element, so a real <ul disc>
+17
View File
@@ -72,6 +72,23 @@ describe("planForFrameUrl", () => {
assert.equal(planForFrameUrl("https://static-forms.klaviyo.com/forms/abc"), "graft"); assert.equal(planForFrameUrl("https://static-forms.klaviyo.com/forms/abc"), "graft");
assert.equal(planForFrameUrl("http://127.0.0.1:4001/iframe-embed.html"), "graft"); assert.equal(planForFrameUrl("http://127.0.0.1:4001/iframe-embed.html"), "graft");
}); });
it("SKIPS full-viewport promo POPUP CREATIVE hosts (Attentive/Recart/Wunderkind overlays)", () => {
// These vendor hosts serve a full-viewport interstitial creative — grafting pours the
// popup's copy into the DOM/text channel and paints the modal over the real page.
assert.equal(planForFrameUrl("https://creatives.attn.tv/creative/12345"), "skip");
assert.equal(planForFrameUrl("https://creative.attn.tv/loader/x"), "skip");
assert.equal(planForFrameUrl("https://app.recart.com/popup/abc"), "skip");
assert.equal(planForFrameUrl("https://tag.wunderkind.co/creative"), "skip");
assert.equal(planForFrameUrl("https://api.bounceexchange.com/creative"), "skip");
});
it("STILL grafts INLINE embed hosts even for the same vendors (inline forms are a feature)", () => {
// Caution guard: a deliberately-grafted inline signup form must never be caught by the
// popup-creative skip list. Inline-form hosts differ from overlay-creative hosts.
assert.equal(planForFrameUrl("https://static-forms.klaviyo.com/forms/abc"), "graft");
assert.equal(planForFrameUrl("https://manage.kmail-lists.com/subscriptions/subscribe"), "graft");
});
}); });
describe("graftFrameIntoSnapshot", () => { describe("graftFrameIntoSnapshot", () => {
+28
View File
@@ -117,6 +117,34 @@ describe("IR drops font-metric probe nodes (fix 4)", () => {
}); });
}); });
describe("IR drops popup-vendor OVERLAY containers, keeps inline embedded forms", () => {
it("drops an email-capture popup overlay container (vendor overlay id) but keeps a real section", () => {
const overlay = raw("div", { id: "attentive_overlay" }, [
raw("iframe", { id: "attentive_creative", src: "https://creatives.attn.tv/x" }),
]);
const real = raw("section", { class: "hero" }, [raw("h1", {}, [{ text: "Real content" }])]);
const body = raw("body", {}, [real, overlay]);
const root = buildFixtureIR(body);
const kept = root.children.filter((c) => !isTextChild(c)).map((c) => (c as IRNode).tag);
assert.deepEqual(kept, ["section"], "the attentive overlay subtree is dropped");
assert.equal(findByTag(root, "iframe"), null, "the popup creative iframe never reaches the IR");
});
it("does NOT drop an INLINE embedded signup form that merely carries a vendor name (feature, not popup)", () => {
// A deliberately-grafted inline Klaviyo form: a real, sized form embedded in page content. Its
// class names the vendor but is NOT an overlay-container marker, so it must survive.
const inlineForm = raw("div", { class: "klaviyo-form klaviyo-form-inline" }, [
raw("form", { id: "email-signup" }, [raw("input", { type: "email" })]),
]);
const body = raw("body", {}, [inlineForm]);
const root = buildFixtureIR(body);
assert.ok(findByTag(root, "form"), "the inline signup form survives the prune");
assert.ok(findByTag(root, "input"), "its input survives too");
});
});
// Defect C (normalize side) — an infinite CSS animation gated to a breakpoint (a Webflow `max-lg` // Defect C (normalize side) — an infinite CSS animation gated to a breakpoint (a Webflow `max-lg`
// marquee) is `animation:none` at the widths it does not run, but the browser still reports the last // marquee) is `animation:none` at the widths it does not run, but the browser still reports the last
// FROZEN translateX there. `neutralizeAnimatedTransforms` zeroes the transform at EVERY viewport // FROZEN translateX there. `neutralizeAnimatedTransforms` zeroes the transform at EVERY viewport
+311
View File
@@ -0,0 +1,311 @@
import { describe, it, before, after } from "node:test";
import assert from "node:assert/strict";
import { chromium, type Browser, type Page } from "playwright";
import { finalizeOverlaysInPage, clickDismissInPage } from "../src/capture/capture.js";
import { irContentExtent, type IRNode, type BBox, type StyleMap } from "../src/normalize/ir.js";
import { gatePollution } from "../src/validate/gates.js";
import type { CaptureResult } from "../src/capture/capture.js";
import type { IR } from "../src/normalize/ir.js";
// tsx/esbuild wraps serialized page functions with __name()/__defProp; shim so page.evaluate works.
const ESBUILD_SHIM =
"globalThis.__name = globalThis.__name || ((fn) => fn);" +
"globalThis.__defProp = globalThis.__defProp || Object.defineProperty;";
const VW = 1280, VH = 800;
// ---------------------------------------------------------------------------
// finalizeOverlaysInPage — effective-z resolution, overlay-unit grouping,
// lock-relaxed z gate, shadow-root traversal, loud-failure blocking.
// ---------------------------------------------------------------------------
describe("capture: finalizeOverlaysInPage overlay detection", () => {
let browser: Browser;
before(async () => { browser = await chromium.launch(); });
after(async () => { await browser.close(); });
const newPage = async (html: string): Promise<Page> => {
const page = await browser.newPage();
await page.setViewportSize({ width: VW, height: VH });
await page.addInitScript(ESBUILD_SHIM);
await page.setContent(html);
await page.evaluate(ESBUILD_SHIM);
return page;
};
it("groups a 0-height max-z wrapper + its full-viewport z:auto iframe as ONE overlay and removes both (scroll-locked)", async () => {
// Mirrors the Attentive structure: body scroll-locked; a fixed z=INT_MAX wrapper of height 0
// whose child is a fixed full-viewport iframe with z-index:auto. Neither passes the naive
// per-element area+z gate alone (wrapper: area 0; iframe: z parses to 0).
const page = await newPage(`
<style>
html,body{margin:0}
body{position:absolute;top:0;left:0;right:0;height:${VH}px;overflow:hidden}
#wrap{position:fixed;top:${VH}px;left:0;width:${VW}px;height:0;z-index:2147483647}
#creative{position:fixed;top:0;left:0;width:${VW}px;height:${VH}px;z-index:auto;background:#000}
</style>
<main>real page content</main>
<div id="wrap"><iframe id="creative"></iframe></div>
`);
const res = await page.evaluate(finalizeOverlaysInPage);
assert.equal(res.removed, 1, "the wrapper+iframe unit is removed as one node");
assert.ok(res.removedLabels.some((l) => l.includes("wrap")), `removed the max-z wrapper (got ${JSON.stringify(res.removedLabels)})`);
const gone = await page.evaluate(() => !document.getElementById("wrap") && !document.getElementById("creative"));
assert.ok(gone, "both the wrapper and its inner iframe are gone");
await page.close();
});
it("detects a z=50 full-viewport fixed dialog ONLY because the page is scroll-locked (lock relaxes the z>=100 gate)", async () => {
const page = await newPage(`
<style>
html,body{margin:0}
body{overflow:hidden}
#ccpa{position:fixed;inset:0;width:${VW}px;height:${VH}px;z-index:50;background:rgba(0,0,0,.7)}
</style>
<main>content</main>
<div id="ccpa">California Residents Privacy Rights</div>
`);
const res = await page.evaluate(finalizeOverlaysInPage);
assert.equal(res.removed, 1, "the z=50 dialog is removed on a locked page");
await page.close();
});
it("does NOT remove a z=50 full-viewport fixed layer when the page is NOT scroll-locked (z floor preserved)", async () => {
const page = await newPage(`
<style>
html,body{margin:0}
#hero{position:fixed;inset:0;width:${VW}px;height:${VH}px;z-index:50;background:#eee}
</style>
<div id="hero">a legit fixed hero, page scrolls normally</div>
`);
const res = await page.evaluate(finalizeOverlaysInPage);
assert.equal(res.removed, 0, "no removal off a locked page — the z>=100 floor guards legit fixed content");
assert.equal(res.blocking, false);
await page.close();
});
it("removes a promo popup mounted inside an OPEN shadow root and reports the host label", async () => {
const page = await newPage(`
<style>html,body{margin:0}body{overflow:hidden}</style>
<main>content</main>
<div id="recart-popup-root"></div>
<script>
const host = document.getElementById("recart-popup-root");
const root = host.attachShadow({ mode: "open" });
const layer = document.createElement("div");
layer.style.cssText = "position:fixed;inset:0;width:${VW}px;height:${VH}px;z-index:999;background:rgba(0,0,0,.6)";
layer.textContent = "Enjoy 15% off";
root.appendChild(layer);
</script>
`);
const res = await page.evaluate(finalizeOverlaysInPage);
assert.equal(res.removed, 1, "the shadow-hosting popup element is removed as one unit");
const gone = await page.evaluate(() => !document.getElementById("recart-popup-root"));
assert.ok(gone, "the shadow host is gone from the light DOM");
await page.close();
});
it("reports blocking=true when the scroll-lock persists even with NO overlay detected (loud failure)", async () => {
// Body is scroll-locked but there is no fixed full-viewport layer to find (e.g. a closed
// shadow root we cannot pierce). A silently-locked capture is polluted → surface it.
const page = await newPage(`
<style>html,body{margin:0}body{overflow:hidden;position:absolute;height:${VH}px}</style>
<main>content</main>
`);
const res = await page.evaluate(finalizeOverlaysInPage);
assert.equal(res.removed, 0);
assert.equal(res.blocking, true, "a persistent lock with nothing removable still reports blocking");
await page.close();
});
it("reports blocking=false and removes nothing on an ordinary unlocked page", async () => {
const page = await newPage(`
<style>html,body{margin:0}</style>
<header>nav</header><main>lots of content</main><footer>footer</footer>
`);
const res = await page.evaluate(finalizeOverlaysInPage);
assert.equal(res.removed, 0);
assert.equal(res.blocking, false);
assert.equal(res.overlaysRemaining, 0);
await page.close();
});
it("protects real sticky page chrome (header/nav) from removal even when detected", async () => {
// A scroll-locked page with a sticky header that happens to be tall — the header is PROTECTED,
// and it isn't full-viewport, so it never even qualifies. Assert nothing is stripped.
const page = await newPage(`
<style>
html,body{margin:0}body{overflow:hidden}
#masthead{position:sticky;top:0;width:${VW}px;height:80px;z-index:1000;background:#fff}
</style>
<header id="masthead">site chrome</header>
<main>content</main>
`);
const res = await page.evaluate(finalizeOverlaysInPage);
const present = await page.evaluate(() => !!document.getElementById("masthead"));
assert.ok(present, "the header survives");
await page.close();
});
});
// ---------------------------------------------------------------------------
// clickDismissInPage — decline/close matchers, shadow-root traversal.
// ---------------------------------------------------------------------------
describe("capture: clickDismissInPage matchers", () => {
let browser: Browser;
before(async () => { browser = await chromium.launch(); });
after(async () => { await browser.close(); });
const run = async (html: string): Promise<string[]> => {
const page = await browser.newPage();
await page.addInitScript(ESBUILD_SHIM);
await page.setContent(html);
await page.evaluate(ESBUILD_SHIM);
const res = await page.evaluate(clickDismissInPage);
await page.close();
return res;
};
it("clicks a Decline button inside an overlay container (email-capture opt-out)", async () => {
const dismissed = await run(`
<div class="signup-overlay" style="position:fixed;inset:0;width:400px;height:400px">
<button onclick="window.__declined=true">Decline</button>
</div>
`);
assert.ok(dismissed.includes("text:decline"), `clicked Decline (got ${JSON.stringify(dismissed)})`);
});
it("clicks an aria-label close (×) inside an overlay container when no accept/decline text matches", async () => {
const dismissed = await run(`
<div id="promo-popup" style="position:fixed;inset:0;width:400px;height:400px">
<button aria-label="Close dialog">×</button>
</div>
`);
assert.ok(dismissed.some((d) => d.startsWith("close:")), `clicked the aria close (got ${JSON.stringify(dismissed)})`);
});
it("matches an overlay/ccpa/privacy container id/class (extended container selector)", async () => {
const dismissed = await run(`
<div id="ccpaPop" style="position:fixed;inset:0;width:400px;height:400px">
<button>Accept</button>
</div>
`);
assert.ok(dismissed.includes("text:accept"), `matched the ccpa container (got ${JSON.stringify(dismissed)})`);
});
it("traverses an open shadow root to find the close control", async () => {
const page = await browser.newPage();
await page.addInitScript(ESBUILD_SHIM);
await page.setContent(`<div id="host"></div>`);
await page.evaluate(ESBUILD_SHIM);
await page.evaluate(() => {
const host = document.getElementById("host")!;
const root = host.attachShadow({ mode: "open" });
root.innerHTML = `<div class="popup-modal" style="position:fixed;inset:0;width:400px;height:400px"><button aria-label="close">×</button></div>`;
});
const dismissed = await page.evaluate(clickDismissInPage);
await page.close();
assert.ok(dismissed.some((d) => d.startsWith("close:")), `found the shadow-root close (got ${JSON.stringify(dismissed)})`);
});
it("never clicks an ordinary page button outside any overlay container", async () => {
const dismissed = await run(`<main><button>Accept</button><a>Continue</a></main>`);
assert.deepEqual(dismissed, [], "no overlay container ⇒ no clicks");
});
});
// ---------------------------------------------------------------------------
// irContentExtent — pure geometry (unclamp / pollution trigger).
// ---------------------------------------------------------------------------
describe("normalize: irContentExtent", () => {
const bbox = (x: number, y: number, w: number, h: number): BBox => ({ x, y, width: w, height: h });
const style = (over: Partial<StyleMap> = {}): StyleMap => ({ position: "static", ...over } as StyleMap);
const node = (over: Partial<IRNode>): IRNode => ({
id: "n", tag: "div", attrs: {},
visibleByVp: { 1280: true }, bboxByVp: { 1280: bbox(0, 0, 100, 100) },
computedByVp: { 1280: style() }, children: [], ...over,
});
it("returns the max in-flow descendant border-box bottom", () => {
const root = node({
tag: "body",
children: [
node({ bboxByVp: { 1280: bbox(0, 0, 1280, 800) } }),
node({ bboxByVp: { 1280: bbox(0, 800, 1280, 4398) } }), // footer bottom = 5198
],
});
assert.equal(irContentExtent(root, 1280), 5198);
});
it("excludes out-of-flow (fixed/absolute) and floated boxes", () => {
const root = node({
tag: "body",
children: [
node({ bboxByVp: { 1280: bbox(0, 0, 1280, 900) } }),
node({ computedByVp: { 1280: style({ position: "fixed" }) }, bboxByVp: { 1280: bbox(0, 0, 1280, 9999) } }),
node({ computedByVp: { 1280: style({ float: "left" } as Partial<StyleMap>) }, bboxByVp: { 1280: bbox(0, 0, 1280, 8888) } }),
],
});
assert.equal(irContentExtent(root, 1280), 900, "fixed + float extents ignored");
});
it("descends into nested in-flow subtrees", () => {
const root = node({
tag: "body",
children: [node({ children: [node({ bboxByVp: { 1280: bbox(0, 0, 500, 3000) } })] })],
});
assert.equal(irContentExtent(root, 1280), 3000);
});
it("ignores nodes invisible at the queried viewport", () => {
const root = node({
tag: "body",
children: [node({ visibleByVp: { 1280: false }, bboxByVp: { 1280: bbox(0, 0, 100, 5000) } })],
});
assert.equal(irContentExtent(root, 1280), 0);
});
});
// ---------------------------------------------------------------------------
// gatePollution — scroll-locked-capture contradiction.
// ---------------------------------------------------------------------------
describe("validate: pollution gate scroll-locked contradiction", () => {
const bbox = (x: number, y: number, w: number, h: number): BBox => ({ x, y, width: w, height: h });
const st = (over: Partial<StyleMap> = {}): StyleMap => ({ position: "static", ...over } as StyleMap);
// A body with plenty of text and a footer at y=5198 (content extent ~6.4 viewports @800).
const mkIr = (): IR => ({
doc: {
sourceUrl: "https://x.test", title: "t", lang: "en", charset: "UTF-8", metaViewport: "",
viewports: [1280], sampleViewports: [1280], canonicalViewport: 1280,
perViewport: { 1280: { scrollHeight: 800, scrollWidth: 1280, htmlBg: "", bodyBg: "", bodyColor: "", bodyFont: "" } },
nodeCount: 200, keyframes: [],
},
root: {
id: "n0", tag: "body", attrs: {}, visibleByVp: { 1280: true },
bboxByVp: { 1280: bbox(0, 0, 1280, 800) }, computedByVp: { 1280: st() },
children: [
{ id: "n1", tag: "section", attrs: {}, visibleByVp: { 1280: true }, bboxByVp: { 1280: bbox(0, 0, 1280, 800) }, computedByVp: { 1280: st() },
children: [{ text: "hero copy ".repeat(20) }] },
{ id: "n2", tag: "footer", attrs: {}, visibleByVp: { 1280: true }, bboxByVp: { 1280: bbox(0, 4074, 1280, 1124) }, computedByVp: { 1280: st() },
children: [{ text: "footer copy ".repeat(20) }] },
],
},
});
const mkCapture = (scrollHeight: number): CaptureResult => ({
perViewport: [
{ viewport: 375, height: 812, scrollHeight: 812, nodeCount: 200, truncated: false },
{ viewport: 1280, height: 800, scrollHeight, nodeCount: 200, truncated: false },
],
dismissal: { dismissed: [], overlaysRemaining: 0, removed: 0, videoStills: 0, blocking: false },
} as unknown as CaptureResult);
it("FAILS when scrollHeight is pinned to ~1 viewport at every width but IR content spans multiple viewports", () => {
const res = gatePollution(mkIr(), mkCapture(800), [1280]);
assert.equal(res.pass, false, "the scroll-locked contradiction is caught");
assert.ok(res.issues.some((i) => i.includes("scroll-locked capture")), `issue reported (got ${JSON.stringify(res.issues)})`);
});
it("PASSES when scrollHeight matches the tall content (a genuinely scrollable page)", () => {
const res = gatePollution(mkIr(), mkCapture(5198), [1280]);
assert.ok(!res.issues.some((i) => i.includes("scroll-locked capture")), "no false positive when the page really scrolled");
});
});