Section decomposition: recursive descent with semantic band detection
detectSections gave up on real pages: <body> swallowed everything, a single oversized-wrapper expansion pass required >=3 candidates, and a 62px navbar failed the 64px height bar - so nearly every clone shipped as one section and one monolithic component. Rewritten around a shared recursive core (detectSectionNodes): containers covering >50% of the page are wrappers to descend into; a split is accepted only when >=2 stacked bands tile >=80% of the container (side-by-side columns and overlays never split; truly single-band pages legally stay 1). Semantic tags get a 24px bar with nav-evidence detection for thin bars. The validator's section list and the emitted section components now consume the same root set, so gate-5 bands and sections/*.tsx files always agree. Naming is evidence-based and deterministic: navbar/header/hero (only when actually near the top), recipe names (logo-cloud, product-grid...), heading slugs, contact/media, positional fallback. Frozen-capture probes: 1 -> 6..17 sections across the reference runs, with the old whole-page "hero-section" monoliths splitting into honest bands. 231 tests pass (13 new), typecheck clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Samraaj Bath
co-authored by
Claude Fable 5
Claude Fable 5
parent
d3ec154bae
commit
6a930d57b1
+154
-87
@@ -4,10 +4,19 @@ import { round } from "../util/canonical.js";
|
||||
|
||||
/**
|
||||
* Deterministic section detection. Splits the page into visually coherent
|
||||
* top-level blocks using semantic tags and geometry (we do not rely on class
|
||||
* names — those are intentionally dropped from the IR). Sections are metadata:
|
||||
* stable ids + per-viewport bboxes for the layout/section gate. Role guesses are
|
||||
* advisory and never affect fidelity.
|
||||
* top-level bands using semantic tags and geometry (we do not rely on class
|
||||
* names — those are intentionally dropped from the IR).
|
||||
*
|
||||
* The core is a recursive descent: a container that covers most of the page
|
||||
* (<body> → <div id=root> → <main>) is a WRAPPER, not a band — descend into it
|
||||
* and re-collect among its children, repeating until every candidate is
|
||||
* band-like. A descent only replaces the container when its children actually
|
||||
* tile it (stacked full-width blocks with near-complete coverage), so a page
|
||||
* that truly is one tall band legally stays a single section.
|
||||
*
|
||||
* Sections are metadata: stable ids + per-viewport bboxes for the layout/section
|
||||
* gate, and the shared root set for section-per-file emission (sectionSplit).
|
||||
* Role guesses are advisory and never affect fidelity.
|
||||
*/
|
||||
|
||||
export type Section = {
|
||||
@@ -19,104 +28,162 @@ export type Section = {
|
||||
};
|
||||
|
||||
const SEMANTIC = new Set(["header", "nav", "main", "section", "article", "footer", "aside"]);
|
||||
const MIN_BAND_H = 64;
|
||||
const MIN_SEMANTIC_H = 24; // a 62px navbar is still the navbar
|
||||
const MIN_BAND_W_FRAC = 0.55; // a band spans most of the viewport
|
||||
const WRAPPER_FRAC = 0.5; // taller than this fraction of the page → wrapper, try descending
|
||||
const MIN_CHILD_COVERAGE = 0.8; // children must tile the container to replace it
|
||||
const MAX_STACK_OVERLAP = 0.3; // consecutive bands may overlap at most this much of the smaller
|
||||
const MAX_BANDS_PER_SPLIT = 20; // a split into more pieces than this is fragmentation, not bands
|
||||
const MAX_DEPTH = 12;
|
||||
|
||||
type Cand = { node: IRNode; path: string; y: number; width: number; height: number };
|
||||
type Cand = { node: IRNode; y: number; width: number; height: number };
|
||||
type Ctx = { cw: number; pageH: number };
|
||||
|
||||
/** The ordered section-root nodes of the page (top to bottom). Shared by the
|
||||
* validator (detectSections) and the generator's section splitter, so the gate's
|
||||
* section list and the emitted section components describe the same bands.
|
||||
* Falls back to `[ir.root]` when the page has no decomposable structure. */
|
||||
export function detectSectionNodes(ir: IR): IRNode[] {
|
||||
const cw = ir.doc.canonicalViewport;
|
||||
const ctx: Ctx = { cw, pageH: ir.doc.perViewport[cw]?.scrollHeight ?? 0 };
|
||||
const bands = bandsOf(ir.root, 0, ctx);
|
||||
// Stable sort by top edge; ties keep document order (a fixed nav stays before
|
||||
// the hero it overlays).
|
||||
bands.sort((a, b) => a.y - b.y);
|
||||
// Nested wrappers can repeat the same box from disjoint subtrees — keep the first.
|
||||
const final: Cand[] = [];
|
||||
for (const c of bands) {
|
||||
if (!final.some((f) => Math.abs(f.y - c.y) < 4 && Math.abs(f.height - c.height) < 4)) final.push(c);
|
||||
}
|
||||
if (final.length === 0) return [ir.root];
|
||||
return final.map((c) => c.node);
|
||||
}
|
||||
|
||||
export function detectSections(ir: IR): Section[] {
|
||||
const cw = ir.doc.canonicalViewport;
|
||||
const vpWidth = cw;
|
||||
|
||||
const candidates: Cand[] = [];
|
||||
const walk = (node: IRNode, path: string): void => {
|
||||
const bbox = node.bboxByVp[cw];
|
||||
const visible = node.visibleByVp[cw];
|
||||
if (bbox && visible) {
|
||||
const isSemantic = SEMANTIC.has(node.tag);
|
||||
const isBigBlock = bbox.width >= vpWidth * 0.55 && bbox.height >= 64;
|
||||
const display = node.computedByVp[cw]?.display ?? "";
|
||||
const blockish = !/^(inline|inline-block|none)$/.test(display);
|
||||
if ((isSemantic || isBigBlock) && blockish) {
|
||||
candidates.push({ node, path, y: bbox.y, width: bbox.width, height: bbox.height });
|
||||
}
|
||||
}
|
||||
for (const c of node.children) {
|
||||
if (isTextChild(c)) continue;
|
||||
walk(c, `${path}/${c.id}`);
|
||||
}
|
||||
};
|
||||
walk(ir.root, ir.root.id);
|
||||
|
||||
// Outermost wins: drop candidates nested inside another candidate.
|
||||
const byShallow = candidates.slice().sort((a, b) => a.path.split("/").length - b.path.split("/").length);
|
||||
let kept: Cand[] = [];
|
||||
for (const c of byShallow) {
|
||||
if (kept.some((k) => c.path.startsWith(k.path + "/"))) continue;
|
||||
kept.push(c);
|
||||
}
|
||||
|
||||
// If the only survivor is a giant wrapper, expand into its section-like children.
|
||||
const pageHeight = ir.doc.perViewport[cw]?.scrollHeight ?? 0;
|
||||
kept = expandOversized(kept, vpWidth, pageHeight, cw);
|
||||
|
||||
// Sort by Y, then assign ids/roles.
|
||||
kept.sort((a, b) => a.y - b.y || b.height - a.height);
|
||||
// Deduplicate near-identical y/height wrappers (keep the first/shallowest).
|
||||
const final: Cand[] = [];
|
||||
for (const c of kept) {
|
||||
const dup = final.find((f) => Math.abs(f.y - c.y) < 4 && Math.abs(f.height - c.height) < 4);
|
||||
if (!dup) final.push(c);
|
||||
}
|
||||
|
||||
return final.map((c, i) => ({
|
||||
const pageH = ir.doc.perViewport[cw]?.scrollHeight ?? 0;
|
||||
const nodes = detectSectionNodes(ir);
|
||||
const roles = guessRoles(nodes, cw, pageH);
|
||||
return nodes.map((node, i) => ({
|
||||
id: `section-${String(i + 1).padStart(3, "0")}`,
|
||||
nodeId: c.node.id,
|
||||
role: guessRole(c.node, i, final.length),
|
||||
nodeId: node.id,
|
||||
role: roles[i]!,
|
||||
order: i,
|
||||
bboxByVp: bboxesFor(c.node),
|
||||
bboxByVp: bboxesFor(node),
|
||||
}));
|
||||
}
|
||||
|
||||
function expandOversized(cands: Cand[], vpWidth: number, pageHeight: number, cw: number): Cand[] {
|
||||
if (cands.length > 2) return cands;
|
||||
const threshold = Math.max(pageHeight * 0.55, 1200);
|
||||
/** Landmark evidence that a bar is site chrome: an ARIA landmark role on the node
|
||||
* itself, or a <nav> within a shallow wrapper chain. Lets a thin fixed bar (often a
|
||||
* 62px styled <div> around the real <nav>) clear the semantic height bar. */
|
||||
export function navEvidence(node: IRNode): boolean {
|
||||
const role = node.attrs.role ?? "";
|
||||
if (role === "banner" || role === "navigation") return true;
|
||||
return subtreeHas(node, (n) => n.tag === "nav" || n.attrs.role === "navigation", 3);
|
||||
}
|
||||
|
||||
function candOf(node: IRNode, ctx: Ctx): Cand | null {
|
||||
const bbox = node.bboxByVp[ctx.cw];
|
||||
if (!bbox || !node.visibleByVp[ctx.cw]) return null;
|
||||
const display = node.computedByVp[ctx.cw]?.display ?? "";
|
||||
if (/^(inline|inline-block|none)$/.test(display)) return null;
|
||||
if (bbox.width < ctx.cw * MIN_BAND_W_FRAC) return null;
|
||||
const minH = SEMANTIC.has(node.tag) || navEvidence(node) ? MIN_SEMANTIC_H : MIN_BAND_H;
|
||||
if (bbox.height < minH) return null;
|
||||
return { node, y: bbox.y, width: bbox.width, height: bbox.height };
|
||||
}
|
||||
|
||||
/** Collect the band candidates among `container`'s children. A band-sized child is
|
||||
* kept; a page-covering child is descended into (its children replace it only when
|
||||
* they tile it — see acceptSplit); anything else is a transparent wrapper we look
|
||||
* through. Output is in document order. */
|
||||
function bandsOf(container: IRNode, depth: number, ctx: Ctx): Cand[] {
|
||||
if (depth > MAX_DEPTH) return [];
|
||||
const out: Cand[] = [];
|
||||
for (const c of cands) {
|
||||
if (c.height < threshold) { out.push(c); continue; }
|
||||
const inner: Cand[] = [];
|
||||
const collect = (node: IRNode, path: string, depth: number): void => {
|
||||
if (depth > 6) return;
|
||||
for (const ch of node.children) {
|
||||
if (isTextChild(ch)) continue;
|
||||
const bbox = ch.bboxByVp[cw];
|
||||
if (bbox && ch.visibleByVp[cw] && bbox.width >= vpWidth * 0.5 && bbox.height >= 64) {
|
||||
inner.push({ node: ch, path: `${path}/${ch.id}`, y: bbox.y, width: bbox.width, height: bbox.height });
|
||||
} else {
|
||||
collect(ch, `${path}/${ch.id}`, depth + 1);
|
||||
}
|
||||
}
|
||||
};
|
||||
collect(c.node, c.path, 0);
|
||||
// outermost wins within inner
|
||||
const byShallow = inner.slice().sort((a, b) => a.path.split("/").length - b.path.split("/").length);
|
||||
const keptInner: Cand[] = [];
|
||||
for (const ic of byShallow) {
|
||||
if (keptInner.some((k) => ic.path.startsWith(k.path + "/"))) continue;
|
||||
keptInner.push(ic);
|
||||
for (const child of container.children) {
|
||||
if (isTextChild(child)) continue;
|
||||
const cand = candOf(child, ctx);
|
||||
if (!cand) {
|
||||
out.push(...bandsOf(child, depth + 1, ctx));
|
||||
continue;
|
||||
}
|
||||
if (keptInner.length >= 3) out.push(...keptInner);
|
||||
else out.push(c);
|
||||
if (ctx.pageH > 0 && cand.height > ctx.pageH * WRAPPER_FRAC) {
|
||||
const inner = bandsOf(child, depth + 1, ctx);
|
||||
if (acceptSplit(inner, cand)) {
|
||||
out.push(...inner);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
out.push(cand);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function guessRole(node: IRNode, index: number, total: number): string {
|
||||
if (node.tag === "header") return "header";
|
||||
if (node.tag === "nav") return "nav";
|
||||
if (node.tag === "footer") return "footer";
|
||||
if (node.tag === "main") return "main";
|
||||
if (index === 0) return "hero";
|
||||
if (index === total - 1) return "footer";
|
||||
return "section";
|
||||
/** May `inner` replace its containing candidate? Only when it reads as a stack of
|
||||
* bands: at least two, not a fragmentation explosion, vertically stacked (side-by-side
|
||||
* columns or overlaid layers must not be split), and tiling the container with
|
||||
* near-complete coverage so no substantial content is silently dropped. */
|
||||
function acceptSplit(inner: Cand[], parent: Cand): boolean {
|
||||
if (inner.length < 2 || inner.length > MAX_BANDS_PER_SPLIT) return false;
|
||||
const sorted = inner.slice().sort((a, b) => a.y - b.y);
|
||||
for (let i = 1; i < sorted.length; i++) {
|
||||
const prev = sorted[i - 1]!, cur = sorted[i]!;
|
||||
const overlap = prev.y + prev.height - cur.y;
|
||||
if (overlap > Math.min(prev.height, cur.height) * MAX_STACK_OVERLAP) return false;
|
||||
}
|
||||
let covered = 0, cursor = -Infinity;
|
||||
for (const c of sorted) {
|
||||
const top = Math.max(c.y, cursor), bot = c.y + c.height;
|
||||
if (bot > top) covered += bot - top;
|
||||
cursor = Math.max(cursor, bot);
|
||||
}
|
||||
return covered >= parent.height * MIN_CHILD_COVERAGE;
|
||||
}
|
||||
|
||||
function subtreeHas(node: IRNode, pred: (n: IRNode) => boolean, depth = 6): boolean {
|
||||
if (depth < 0) return false;
|
||||
for (const c of node.children) {
|
||||
if (isTextChild(c)) continue;
|
||||
if (pred(c) || subtreeHas(c, pred, depth - 1)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** A <header> that carries the page's h1 and real height is the hero band, not
|
||||
* site chrome — "header → chrome" only holds for the thin bar variant. */
|
||||
export function heroLikeHeader(node: IRNode, cw: number): boolean {
|
||||
const h = node.bboxByVp[cw]?.height ?? 0;
|
||||
return h >= 200 && subtreeHas(node, (n) => n.tag === "h1");
|
||||
}
|
||||
|
||||
/** Advisory roles, one pass top-to-bottom: tag evidence first, then the first
|
||||
* near-top content band claims "hero" (once), then content evidence. */
|
||||
function guessRoles(nodes: IRNode[], cw: number, pageH: number): string[] {
|
||||
let heroClaimed = false;
|
||||
return nodes.map((node, index) => {
|
||||
if (node.tag === "nav") return "navbar";
|
||||
if (node.tag === "header") {
|
||||
if (!heroLikeHeader(node, cw)) return "header";
|
||||
heroClaimed = true;
|
||||
return "hero";
|
||||
}
|
||||
if (node.tag === "footer") return "footer";
|
||||
if (node.tag === "aside") return "aside";
|
||||
if (node.tag === "main") return "main";
|
||||
const bbox = node.bboxByVp[cw];
|
||||
const h = bbox?.height ?? 0, y = bbox?.y ?? 0;
|
||||
// thin top bar without the <nav> tag (first band, or a fixed bar with landmark evidence)
|
||||
if (h <= 140 && y <= 160 && (index === 0 || navEvidence(node))) return "navbar";
|
||||
if (!heroClaimed && (bbox?.y ?? 0) <= Math.max(900, pageH * 0.25)) {
|
||||
heroClaimed = true;
|
||||
return "hero";
|
||||
}
|
||||
if (index === nodes.length - 1) return "footer";
|
||||
if (subtreeHas(node, (n) => n.tag === "form")) return "contact";
|
||||
if (subtreeHas(node, (n) => n.tag === "video" || n.tag === "iframe")) return "media";
|
||||
return "section";
|
||||
});
|
||||
}
|
||||
|
||||
function bboxesFor(node: IRNode): Section["bboxByVp"] {
|
||||
|
||||
Reference in New Issue
Block a user