Initial commit
This commit is contained in:
Samraaj Bath
Executable
+913
@@ -0,0 +1,913 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
capture.py — Playwright-based site capture for the clone pipeline.
|
||||
|
||||
Produces a full capture bundle at --output. Layout is documented in
|
||||
.opencode/skills/site-manifest/SKILL.md and consumed by the analyze + generate
|
||||
agents.
|
||||
|
||||
Pipeline:
|
||||
Stage 0: Per-viewport scroll-loop capture — DOM + computed styles, screenshots,
|
||||
HAR, animation library dumps, fonts, css-vars, css-rules.
|
||||
Stage 1: Alt-height capture at the canonical desktop width (1280) for
|
||||
vh-relative detection.
|
||||
Stage 2: Per-section cropped screenshot pass — scroll each candidate section
|
||||
into view and crop to its bbox.
|
||||
Stage 3: Post-process — derive vh-relative flags, merge HAR + CSS asset URLs,
|
||||
write meta.json.
|
||||
|
||||
Replay mode: --replay skips stages 0-2 and only re-runs stage 3 against existing
|
||||
capture data. Use this when iterating on prompts/agents/skills without paying the
|
||||
~3-minute capture cost again.
|
||||
|
||||
Dependencies: playwright (pip install playwright && playwright install chromium)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout
|
||||
except ImportError:
|
||||
print(json.dumps({"error": "Missing dependencies. pip install playwright && playwright install chromium"}), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
THIRD_PARTY_HOST_RE = re.compile(
|
||||
r"(intercom\.io|driftt\.com|drift\.com|typeform\.com|calendly\.com|hsforms\.(net|com)|"
|
||||
r"cookielaw\.org|cookiebot\.com|osano\.com|googletagmanager\.com|google-analytics\.com|"
|
||||
r"segment\.(io|com)|hotjar\.com|fullstory\.com)"
|
||||
)
|
||||
|
||||
# Viewport that the analyze agent treats as canonical desktop. We capture a second
|
||||
# DOM dump at this width but a different height to detect vh-relative dimensions.
|
||||
CANONICAL_WIDTH = 1280
|
||||
CANONICAL_HEIGHT_PRIMARY = 720
|
||||
CANONICAL_HEIGHT_ALT = 1080
|
||||
|
||||
|
||||
def sha1_8(s: str) -> str:
|
||||
return hashlib.sha1(s.encode("utf-8")).hexdigest()[:8]
|
||||
|
||||
|
||||
def sanitize(name: str) -> str:
|
||||
name = name.lower()
|
||||
name = re.sub(r"\s+", "-", name)
|
||||
name = re.sub(r"[^a-z0-9.\-]", "", name)
|
||||
return name[:80] or "asset"
|
||||
|
||||
|
||||
def load_init_hooks(hooks_dir: Path) -> list[str]:
|
||||
sources: list[str] = []
|
||||
for path in sorted(hooks_dir.glob("*.js")):
|
||||
sources.append(path.read_text())
|
||||
return sources
|
||||
|
||||
|
||||
def asset_type_from_url(url: str, content_type: str | None) -> str | None:
|
||||
u = url.lower().split("?", 1)[0]
|
||||
ct = (content_type or "").lower()
|
||||
# SVG must be checked before generic image/* — content-type "image/svg+xml" matches both.
|
||||
if u.endswith(".svg") or ct == "image/svg+xml":
|
||||
return "svg"
|
||||
if any(u.endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif")) or ct.startswith("image/"):
|
||||
return "image"
|
||||
if any(u.endswith(ext) for ext in (".mp4", ".mov", ".webm")) or ct.startswith("video/"):
|
||||
return "video"
|
||||
if any(u.endswith(ext) for ext in (".woff2", ".woff", ".ttf", ".otf")) or ct in (
|
||||
"font/woff2", "font/woff", "font/ttf", "application/font-woff2", "application/font-woff", "application/x-font-ttf"
|
||||
):
|
||||
return "font"
|
||||
if u.endswith(".json") and ("lottie" in u or "animations" in u):
|
||||
return "lottie"
|
||||
return None
|
||||
|
||||
|
||||
def asset_type_from_url_only(url: str) -> str | None:
|
||||
return asset_type_from_url(url, None)
|
||||
|
||||
|
||||
def viewport_height_for(width: int) -> int:
|
||||
"""Default capture height — 16:9 ratio, but rounded sensibly."""
|
||||
return round(width * 9 / 16)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Browser helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_context(browser, viewport_w: int, viewport_h: int, har_path: Path | None, init_sources: list[str], skip_third_party: bool):
|
||||
ctx_kwargs = {"viewport": {"width": viewport_w, "height": viewport_h}}
|
||||
if har_path is not None:
|
||||
ctx_kwargs["record_har_path"] = str(har_path)
|
||||
ctx = browser.new_context(**ctx_kwargs)
|
||||
|
||||
if skip_third_party:
|
||||
def route_handler(route):
|
||||
if THIRD_PARTY_HOST_RE.search(route.request.url):
|
||||
route.abort()
|
||||
else:
|
||||
route.continue_()
|
||||
ctx.route("**/*", route_handler)
|
||||
|
||||
for src in init_sources:
|
||||
ctx.add_init_script(src)
|
||||
|
||||
return ctx
|
||||
|
||||
|
||||
def goto_and_settle(page, url: str, wait_strategy: str) -> None:
|
||||
try:
|
||||
page.goto(url, wait_until=wait_strategy, timeout=60_000)
|
||||
except PWTimeout:
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=60_000)
|
||||
|
||||
try:
|
||||
page.evaluate("() => document.fonts.ready")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
page.wait_for_function("window.__CLONE_READY__ === true", timeout=10_000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def dump_dom(page) -> dict:
|
||||
"""Use the rich __CLONE_DUMP_COMPUTED__ walker (which filters per-tag defaults
|
||||
and captures pseudo-elements). Falls back to a minimal walk if the hook
|
||||
didn't load."""
|
||||
try:
|
||||
out = page.evaluate("() => (typeof window.__CLONE_DUMP_COMPUTED__ === 'function') ? window.__CLONE_DUMP_COMPUTED__() : null")
|
||||
if out is not None:
|
||||
return out
|
||||
except Exception:
|
||||
pass
|
||||
return page.evaluate(
|
||||
"""() => {
|
||||
const serialize = (el) => {
|
||||
if (el.nodeType === Node.TEXT_NODE) {
|
||||
const t = el.textContent;
|
||||
return t && t.trim() ? { text: t } : null;
|
||||
}
|
||||
if (el.nodeType !== Node.ELEMENT_NODE) return null;
|
||||
const cs = getComputedStyle(el);
|
||||
const computed = {};
|
||||
for (const p of ['color','backgroundColor','backgroundImage','fontFamily','fontSize','fontWeight','lineHeight','display','width','height','padding','margin','position']) {
|
||||
const v = cs[p];
|
||||
if (v && v !== 'normal' && v !== 'none' && v !== 'auto' && v !== '0px' && v !== 'rgba(0, 0, 0, 0)') computed[p] = v;
|
||||
}
|
||||
const attrs = {};
|
||||
for (const a of el.attributes) attrs[a.name] = a.value;
|
||||
const r = el.getBoundingClientRect();
|
||||
const children = [];
|
||||
for (const c of el.childNodes) { const s = serialize(c); if (s) children.push(s); }
|
||||
return {
|
||||
tag: el.tagName.toLowerCase(),
|
||||
attrs,
|
||||
computed,
|
||||
bbox: { x: r.x, y: r.y + window.scrollY, width: r.width, height: r.height },
|
||||
children,
|
||||
};
|
||||
};
|
||||
return serialize(document.body);
|
||||
}"""
|
||||
)
|
||||
|
||||
|
||||
def list_sections(page) -> list[dict]:
|
||||
try:
|
||||
return page.evaluate("() => (typeof window.__CLONE_LIST_SECTIONS__ === 'function') ? window.__CLONE_LIST_SECTIONS__() : []") or []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def smart_scroll_targets(sections: list[dict], page_height: int, viewport_h: int) -> list[int]:
|
||||
"""Compute scroll y positions to visit. We always include 0 (top), then
|
||||
each section's y minus a small offset, deduped. Also fill any gap larger
|
||||
than 1.5x viewport_h with intermediate stops so we don't skip content
|
||||
between sections.
|
||||
"""
|
||||
raw = [0]
|
||||
for sec in sections:
|
||||
y = max(int(sec.get("y") or 0) - 80, 0)
|
||||
raw.append(y)
|
||||
raw.append(max(page_height - viewport_h, 0))
|
||||
|
||||
# Sort + dedupe + fill gaps
|
||||
raw = sorted(set(raw))
|
||||
filled: list[int] = []
|
||||
for i, y in enumerate(raw):
|
||||
if filled and y - filled[-1] > int(viewport_h * 1.5):
|
||||
mid = filled[-1] + viewport_h
|
||||
while mid < y - viewport_h:
|
||||
filled.append(mid)
|
||||
mid += viewport_h
|
||||
filled.append(y)
|
||||
# Cap to avoid pathological cases
|
||||
return filled[:50]
|
||||
|
||||
|
||||
def settle(page, max_ms: int = 1500) -> None:
|
||||
"""Wait for the layout to settle: networkidle attempt + a short fixed delay
|
||||
+ waitForFunction on document.fonts.ready. If a MutationObserver fires
|
||||
rapidly, we yield more time up to max_ms."""
|
||||
deadline = time.monotonic() + (max_ms / 1000.0)
|
||||
try:
|
||||
page.wait_for_load_state("networkidle", timeout=max(int(max_ms / 2), 500))
|
||||
except Exception:
|
||||
pass
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining > 0:
|
||||
# Watch for mutation activity; if quiet for 250ms, return
|
||||
try:
|
||||
page.evaluate(
|
||||
"""async (maxMs) => {
|
||||
return await new Promise((resolve) => {
|
||||
let lastMutation = performance.now();
|
||||
const start = lastMutation;
|
||||
const obs = new MutationObserver(() => { lastMutation = performance.now(); });
|
||||
obs.observe(document.body, { subtree: true, childList: true, attributes: true });
|
||||
const tick = () => {
|
||||
const now = performance.now();
|
||||
if (now - lastMutation > 250 || now - start > maxMs) { obs.disconnect(); resolve(true); }
|
||||
else requestAnimationFrame(tick);
|
||||
};
|
||||
tick();
|
||||
});
|
||||
}""",
|
||||
int(remaining * 1000),
|
||||
)
|
||||
except Exception:
|
||||
time.sleep(min(remaining, 0.5))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 0 — per-viewport scroll-loop capture
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def capture_viewport(
|
||||
browser,
|
||||
viewport_width: int,
|
||||
url: str,
|
||||
wait_strategy: str,
|
||||
init_sources: list[str],
|
||||
output_dir: Path,
|
||||
skip_third_party: bool,
|
||||
) -> dict:
|
||||
viewport_height = viewport_height_for(viewport_width)
|
||||
vp_dir = output_dir / "screenshots" / str(viewport_width)
|
||||
dom_dir = output_dir / "dom" / str(viewport_width)
|
||||
har_dir = output_dir / "har"
|
||||
for d in (vp_dir, dom_dir, har_dir):
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
har_path = har_dir / f"{viewport_width}.har"
|
||||
page_context = make_context(browser, viewport_width, viewport_height, har_path, init_sources, skip_third_party)
|
||||
|
||||
page = page_context.new_page()
|
||||
discovered_assets: dict[str, dict] = {}
|
||||
|
||||
def on_response(response):
|
||||
try:
|
||||
url_ = response.url
|
||||
content_type = response.headers.get("content-type")
|
||||
t = asset_type_from_url(url_, content_type)
|
||||
if t is None:
|
||||
return
|
||||
if url_ in discovered_assets:
|
||||
return
|
||||
discovered_assets[url_] = {
|
||||
"type": t,
|
||||
"source_url": url_,
|
||||
"content_type": content_type,
|
||||
"status": response.status,
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
page.on("response", on_response)
|
||||
goto_and_settle(page, url, wait_strategy)
|
||||
|
||||
sections = list_sections(page)
|
||||
page_height = page.evaluate("() => document.documentElement.scrollHeight") or 0
|
||||
targets = smart_scroll_targets(sections, page_height, viewport_height)
|
||||
|
||||
for step, y in enumerate(targets):
|
||||
page.evaluate(f"window.scrollTo({{ top: {y}, behavior: 'instant' }})")
|
||||
settle(page, max_ms=1200)
|
||||
page.screenshot(path=str(vp_dir / f"step-{step:02d}.png"), full_page=False)
|
||||
snapshot = dump_dom(page)
|
||||
(dom_dir / f"step-{step:02d}.json").write_text(json.dumps(snapshot, separators=(",", ":")))
|
||||
|
||||
# Pull animation/css-rules hook output
|
||||
hook_output = page.evaluate("() => window.__CLONE_CAPTURE__ || {}") or {}
|
||||
for kind in ("shaders", "gsap", "framer", "lottie", "threejs"):
|
||||
d = output_dir / kind
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
(d / f"{viewport_width}.json").write_text(json.dumps(hook_output.get(kind, []), indent=2))
|
||||
for kind, dirname in (("cssVars", "css-vars"), ("fonts", "fonts"), ("cssRules", "css-rules")):
|
||||
d = output_dir / dirname
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
(d / f"{viewport_width}.json").write_text(json.dumps(hook_output.get(kind, {}), indent=2))
|
||||
|
||||
# Dump candidate sections per viewport (analyze agent uses these)
|
||||
sec_dir = output_dir / "sections"
|
||||
sec_dir.mkdir(parents=True, exist_ok=True)
|
||||
(sec_dir / f"{viewport_width}.json").write_text(json.dumps(sections, indent=2))
|
||||
|
||||
# Hover capture
|
||||
try:
|
||||
hover_rects = page.evaluate(
|
||||
"""() => {
|
||||
const els = document.querySelectorAll('a, button, [role="button"], [data-cta]');
|
||||
const out = [];
|
||||
for (const el of els) {
|
||||
const r = el.getBoundingClientRect();
|
||||
if (r.width < 40 || r.height < 20) continue;
|
||||
out.push({ x: r.x + r.width/2, y: r.y + r.height/2, tag: el.tagName });
|
||||
if (out.length >= 10) break;
|
||||
}
|
||||
return out;
|
||||
}"""
|
||||
)
|
||||
hover_dir = output_dir / "screenshots" / str(viewport_width) / "hover"
|
||||
hover_dir.mkdir(parents=True, exist_ok=True)
|
||||
for idx, pos in enumerate(hover_rects):
|
||||
try:
|
||||
page.mouse.move(pos["x"], pos["y"])
|
||||
time.sleep(0.2)
|
||||
page.screenshot(path=str(hover_dir / f"hover-{idx:02d}.png"), full_page=False)
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
page.close()
|
||||
page_context.close()
|
||||
|
||||
return {
|
||||
"viewport": viewport_width,
|
||||
"height": viewport_height,
|
||||
"steps": len(targets),
|
||||
"section_candidates": len(sections),
|
||||
"hook_output_keys": [k for k in hook_output.keys()],
|
||||
"har": str(har_path.relative_to(output_dir)),
|
||||
"assets": discovered_assets,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 1 — alt-height capture at canonical width (vh detection)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def capture_alt_height(browser, url: str, wait_strategy: str, init_sources: list[str], output_dir: Path, skip_third_party: bool) -> dict:
|
||||
"""Capture only DOM at scroll 0 at (CANONICAL_WIDTH x CANONICAL_HEIGHT_ALT).
|
||||
The post-process compares this to the primary capture to flag vh-relative
|
||||
dimensions."""
|
||||
page_context = make_context(browser, CANONICAL_WIDTH, CANONICAL_HEIGHT_ALT, None, init_sources, skip_third_party)
|
||||
page = page_context.new_page()
|
||||
goto_and_settle(page, url, wait_strategy)
|
||||
settle(page, max_ms=1200)
|
||||
snapshot = dump_dom(page)
|
||||
sections = list_sections(page)
|
||||
|
||||
out_dir = output_dir / "dom-alt" / f"{CANONICAL_WIDTH}-{CANONICAL_HEIGHT_ALT}"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
(out_dir / "step-00.json").write_text(json.dumps(snapshot, separators=(",", ":")))
|
||||
(out_dir / "sections.json").write_text(json.dumps(sections, indent=2))
|
||||
|
||||
page.close()
|
||||
page_context.close()
|
||||
return {"viewport": CANONICAL_WIDTH, "alt_height": CANONICAL_HEIGHT_ALT, "section_candidates": len(sections)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 2 — per-section cropped screenshots
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def capture_section_screenshots(browser, url: str, wait_strategy: str, init_sources: list[str], output_dir: Path, skip_third_party: bool, viewport_widths: list[int]) -> int:
|
||||
"""For each viewport width, scroll each candidate section into view and
|
||||
save a screenshot cropped to its bbox. The analyze agent maps these to
|
||||
section ids; the validate agent uses them as the diff reference."""
|
||||
total = 0
|
||||
for vw in viewport_widths:
|
||||
sections_path = output_dir / "sections" / f"{vw}.json"
|
||||
if not sections_path.exists():
|
||||
continue
|
||||
sections = json.loads(sections_path.read_text())
|
||||
if not sections:
|
||||
continue
|
||||
vh = viewport_height_for(vw)
|
||||
page_context = make_context(browser, vw, vh, None, init_sources, skip_third_party)
|
||||
page = page_context.new_page()
|
||||
goto_and_settle(page, url, wait_strategy)
|
||||
out_dir = output_dir / "section-shots" / str(vw)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for idx, sec in enumerate(sections):
|
||||
try:
|
||||
# Scroll so the section's top is ~80px below the viewport top
|
||||
target_y = max(int(sec.get("y") or 0) - 80, 0)
|
||||
page.evaluate(f"window.scrollTo({{ top: {target_y}, behavior: 'instant' }})")
|
||||
settle(page, max_ms=1000)
|
||||
clip = {
|
||||
"x": max(int(sec.get("x") or 0), 0),
|
||||
"y": max(int(sec.get("y") or 0) - target_y, 0),
|
||||
"width": min(int(sec.get("width") or vw), vw),
|
||||
"height": min(int(sec.get("height") or vh), vh * 4),
|
||||
}
|
||||
# Playwright clip cannot exceed the page viewport vertically; if the section is
|
||||
# taller than the viewport, capture as-is and analyze handles the partial.
|
||||
clip["height"] = min(clip["height"], vh)
|
||||
page.screenshot(path=str(out_dir / f"section-{idx:02d}.png"), clip=clip)
|
||||
total += 1
|
||||
except Exception:
|
||||
continue
|
||||
page.close()
|
||||
page_context.close()
|
||||
return total
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 3 — post-process: build assets list, derive vh flags, write meta.json
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def merge_css_assets(output_dir: Path, har_assets: dict[str, dict]) -> dict[str, dict]:
|
||||
"""Walk css-rules/<vp>.json across viewports, append every url() reference
|
||||
not already in har_assets. Type is inferred from the URL alone since CSS
|
||||
references don't carry content-type."""
|
||||
merged = dict(har_assets)
|
||||
css_rules_dir = output_dir / "css-rules"
|
||||
if css_rules_dir.exists():
|
||||
for f in css_rules_dir.glob("*.json"):
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
except Exception:
|
||||
continue
|
||||
for asset in data.get("assets", []) or []:
|
||||
src = asset.get("source_url")
|
||||
if not src or src in merged:
|
||||
continue
|
||||
t = asset_type_from_url_only(src)
|
||||
if t is None:
|
||||
continue
|
||||
merged[src] = {
|
||||
"type": t,
|
||||
"source_url": src,
|
||||
"content_type": None,
|
||||
"status": None,
|
||||
"from_css": True,
|
||||
}
|
||||
return merged
|
||||
|
||||
|
||||
def build_assets_list(all_assets: dict[str, dict]) -> list[dict]:
|
||||
out = []
|
||||
for url, a in all_assets.items():
|
||||
name = sanitize(unquote(os.path.basename(urlparse(url).path)) or "asset")
|
||||
local_path = f"public/assets/cloned/{a['type']}s/{sha1_8(url)}-{name}"
|
||||
entry = {
|
||||
"type": a["type"],
|
||||
"source_url": url,
|
||||
"local_path": local_path,
|
||||
}
|
||||
if a.get("from_css"):
|
||||
entry["from_css"] = True
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
|
||||
def detect_vh_flags(output_dir: Path) -> list[dict]:
|
||||
"""Compare DOM at (1280x720) vs (1280x1080). For each element matchable by
|
||||
structural path, if `h_alt / h_primary ≈ alt_height / primary_height`, flag
|
||||
it as vh-relative and record the implied vh percentage.
|
||||
Output: a flat list of { path, primary_h, alt_h, vh } entries.
|
||||
"""
|
||||
primary = output_dir / "dom" / str(CANONICAL_WIDTH) / "step-00.json"
|
||||
alt = output_dir / "dom-alt" / f"{CANONICAL_WIDTH}-{CANONICAL_HEIGHT_ALT}" / "step-00.json"
|
||||
if not primary.exists() or not alt.exists():
|
||||
return []
|
||||
try:
|
||||
a = json.loads(primary.read_text())
|
||||
b = json.loads(alt.read_text())
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
expected_ratio = CANONICAL_HEIGHT_ALT / CANONICAL_HEIGHT_PRIMARY # 1.5
|
||||
tolerance = 0.06 # 6% — accounts for content-driven elements that happen to be near vh
|
||||
|
||||
flags: list[dict] = []
|
||||
|
||||
def walk(na, nb, path):
|
||||
if not isinstance(na, dict) or not isinstance(nb, dict):
|
||||
return
|
||||
if na.get("tag") != nb.get("tag"):
|
||||
return
|
||||
ha = (na.get("bbox") or {}).get("height") or 0
|
||||
hb = (nb.get("bbox") or {}).get("height") or 0
|
||||
if ha > 40 and hb > 40:
|
||||
ratio = hb / ha
|
||||
if abs(ratio - expected_ratio) / expected_ratio < tolerance:
|
||||
vh_pct = round((ha / CANONICAL_HEIGHT_PRIMARY) * 100)
|
||||
flags.append({
|
||||
"path": path,
|
||||
"tag": na.get("tag"),
|
||||
"id": (na.get("attrs") or {}).get("id") or None,
|
||||
"class": (na.get("attrs") or {}).get("class") or None,
|
||||
"primary_h": round(ha, 2),
|
||||
"alt_h": round(hb, 2),
|
||||
"ratio": round(ratio, 3),
|
||||
"vh": vh_pct,
|
||||
})
|
||||
ca = na.get("children") or []
|
||||
cb = nb.get("children") or []
|
||||
# Only walk matched element children; pair by tag-aware index
|
||||
ai = bi = 0
|
||||
idx = 0
|
||||
while ai < len(ca) and bi < len(cb):
|
||||
ea = ca[ai]
|
||||
eb = cb[bi]
|
||||
if not isinstance(ea, dict) or not isinstance(eb, dict) or ea.get("tag") != eb.get("tag"):
|
||||
# Skip text-only / mismatched
|
||||
if not isinstance(ea, dict) or "tag" not in ea:
|
||||
ai += 1
|
||||
continue
|
||||
if not isinstance(eb, dict) or "tag" not in eb:
|
||||
bi += 1
|
||||
continue
|
||||
# Different tags at same index — skip both
|
||||
ai += 1
|
||||
bi += 1
|
||||
continue
|
||||
walk(ea, eb, f"{path}>{ea.get('tag')}[{idx}]")
|
||||
ai += 1
|
||||
bi += 1
|
||||
idx += 1
|
||||
|
||||
walk(a, b, a.get("tag", "body"))
|
||||
return flags
|
||||
|
||||
|
||||
SECTION_LIKE_TAGS = {"section", "header", "footer", "nav", "main", "article", "aside"}
|
||||
SECTION_CLASS_RE = re.compile(r"(section|hero|banner|elementor-element|e-parent|e-con|footer|header|nav|main)", re.I)
|
||||
SECTION_ID_RE = re.compile(r"(hero|banner|section|footer|nav|header|main)", re.I)
|
||||
|
||||
|
||||
def _is_section_like(node: dict) -> bool:
|
||||
tag = (node.get("tag") or "").lower()
|
||||
if tag in SECTION_LIKE_TAGS:
|
||||
return True
|
||||
attrs = node.get("attrs") or {}
|
||||
eid = attrs.get("id") or ""
|
||||
if eid and SECTION_ID_RE.search(eid):
|
||||
return True
|
||||
cls = attrs.get("class") or ""
|
||||
if cls and SECTION_CLASS_RE.search(cls):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_visible(node: dict) -> bool:
|
||||
cs = node.get("computed") or {}
|
||||
if cs.get("display") == "none" or cs.get("visibility") == "hidden":
|
||||
return False
|
||||
op = cs.get("opacity")
|
||||
if op is not None and op != "":
|
||||
try:
|
||||
if float(op) == 0:
|
||||
return False
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def _build_selector(node: dict) -> str:
|
||||
attrs = node.get("attrs") or {}
|
||||
if attrs.get("id"):
|
||||
return f"#{attrs['id']}"
|
||||
cls = (attrs.get("class") or "").strip().split()
|
||||
if cls:
|
||||
# First class is usually the most distinctive (Elementor style)
|
||||
return f"{node.get('tag')}.{cls[0]}"
|
||||
return node.get("tag") or "?"
|
||||
|
||||
|
||||
def _walk_section_candidates(node: dict, vw: int, path: str, idx: int, out: list[dict]) -> None:
|
||||
if not isinstance(node, dict) or "tag" not in node:
|
||||
return
|
||||
cur_path = f"{path}>{node.get('tag')}[{idx}]" if path else (node.get("tag") or "?")
|
||||
bbox = node.get("bbox") or {}
|
||||
w = bbox.get("width") or 0
|
||||
h = bbox.get("height") or 0
|
||||
if _is_visible(node) and _is_section_like(node) and w >= vw * 0.5 and h >= 80:
|
||||
attrs = node.get("attrs") or {}
|
||||
out.append({
|
||||
"_path": cur_path,
|
||||
"_node": node,
|
||||
"selector": _build_selector(node),
|
||||
"tag": node.get("tag"),
|
||||
"id": attrs.get("id") or None,
|
||||
"className": (attrs.get("class") or "")[:200] or None,
|
||||
"x": round(bbox.get("x") or 0, 1),
|
||||
"y": round(bbox.get("y") or 0, 1),
|
||||
"width": round(w, 1),
|
||||
"height": round(h, 1),
|
||||
})
|
||||
for i, c in enumerate(node.get("children") or []):
|
||||
_walk_section_candidates(c, vw, cur_path, i, out)
|
||||
|
||||
|
||||
def _outermost_wins(candidates: list[dict]) -> list[dict]:
|
||||
# Sort by path depth — shallowest first; only keep candidates that are not
|
||||
# descendants of an already-kept candidate.
|
||||
by_depth = sorted(candidates, key=lambda c: c["_path"].count(">"))
|
||||
kept: list[dict] = []
|
||||
for c in by_depth:
|
||||
if any(c["_path"].startswith(k["_path"] + ">") for k in kept):
|
||||
continue
|
||||
kept.append(c)
|
||||
return kept
|
||||
|
||||
|
||||
def _expand_oversized(candidates: list[dict], vw: int, vh: int, doc_h: float) -> list[dict]:
|
||||
"""For wrapper-shaped candidates (>2x viewport_h with >=3 inner sections),
|
||||
replace the wrapper with its section-like children. Without this step,
|
||||
Elementor / WP layouts where everything is wrapped in a single <main id="main">
|
||||
or <div class="elementor-section-wrap"> collapse to a single huge candidate
|
||||
and the smart-scroll loop never visits the actual sections."""
|
||||
threshold = max(vh * 2, doc_h * 0.5)
|
||||
expanded: list[dict] = []
|
||||
for c in candidates:
|
||||
if c["height"] < threshold:
|
||||
expanded.append(c)
|
||||
continue
|
||||
inner: list[dict] = []
|
||||
for i, child in enumerate(c["_node"].get("children") or []):
|
||||
_walk_section_candidates(child, vw, c["_path"], i, inner)
|
||||
inner = _outermost_wins(inner)
|
||||
# Recurse: an inner section may itself be an oversized wrapper.
|
||||
inner = _expand_oversized(inner, vw, vh, doc_h)
|
||||
if len(inner) >= 3:
|
||||
expanded.extend(inner)
|
||||
else:
|
||||
expanded.append(c)
|
||||
return expanded
|
||||
|
||||
|
||||
def enrich_sections(output_dir: Path) -> int:
|
||||
"""Re-derive sections/<vp>.json from the captured dom/<vp>/step-00.json so
|
||||
the JS section-scan's outermost-wins rule is corrected for wrapper-shaped
|
||||
layouts. Idempotent — safe to run in both fresh-capture and replay paths.
|
||||
Returns the total candidate count across viewports."""
|
||||
dom_root = output_dir / "dom"
|
||||
sec_root = output_dir / "sections"
|
||||
if not dom_root.exists():
|
||||
return 0
|
||||
sec_root.mkdir(parents=True, exist_ok=True)
|
||||
total = 0
|
||||
for vp_dir in sorted(dom_root.iterdir()):
|
||||
if not vp_dir.is_dir() or not vp_dir.name.isdigit():
|
||||
continue
|
||||
vp = int(vp_dir.name)
|
||||
step0 = vp_dir / "step-00.json"
|
||||
if not step0.exists():
|
||||
continue
|
||||
try:
|
||||
root = json.loads(step0.read_text())
|
||||
except Exception:
|
||||
continue
|
||||
doc_h = (root.get("bbox") or {}).get("height") or 0
|
||||
vh = viewport_height_for(vp)
|
||||
candidates: list[dict] = []
|
||||
_walk_section_candidates(root, vp, "", 0, candidates)
|
||||
kept = _outermost_wins(candidates)
|
||||
expanded = _expand_oversized(kept, vp, vh, doc_h)
|
||||
# Strip private fields, sort by y
|
||||
public = [{k: v for k, v in c.items() if not k.startswith("_")} for c in expanded]
|
||||
public = [s for s in public if s.get("height", 0) > 80]
|
||||
public.sort(key=lambda s: s.get("y") or 0)
|
||||
(sec_root / f"{vp}.json").write_text(json.dumps(public, indent=2))
|
||||
total += len(public)
|
||||
return total
|
||||
|
||||
|
||||
def detect_libs(output_dir: Path) -> list[str]:
|
||||
libs_detected = set()
|
||||
for kind in ("gsap", "framer", "lottie", "three", "shaders"):
|
||||
dirname = "threejs" if kind == "three" else kind
|
||||
d = output_dir / dirname
|
||||
if not d.exists():
|
||||
continue
|
||||
for f in d.glob("*.json"):
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
if data:
|
||||
libs_detected.add(kind)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
return sorted(libs_detected)
|
||||
|
||||
|
||||
def collect_har_assets_from_disk(output_dir: Path) -> dict[str, dict]:
|
||||
"""Fallback for replay mode — read each har/<vp>.har and extract response URLs."""
|
||||
har_dir = output_dir / "har"
|
||||
out: dict[str, dict] = {}
|
||||
if not har_dir.exists():
|
||||
return out
|
||||
for f in har_dir.glob("*.har"):
|
||||
try:
|
||||
har = json.loads(f.read_text())
|
||||
except Exception:
|
||||
continue
|
||||
for entry in (har.get("log") or {}).get("entries", []):
|
||||
req = entry.get("request") or {}
|
||||
res = entry.get("response") or {}
|
||||
url = req.get("url")
|
||||
if not url:
|
||||
continue
|
||||
ct = None
|
||||
for h in (res.get("headers") or []):
|
||||
if (h.get("name") or "").lower() == "content-type":
|
||||
ct = h.get("value")
|
||||
break
|
||||
t = asset_type_from_url(url, ct)
|
||||
if t and url not in out:
|
||||
out[url] = {"type": t, "source_url": url, "content_type": ct, "status": res.get("status")}
|
||||
return out
|
||||
|
||||
|
||||
def write_meta(output_dir: Path, source_url: str, viewports: list[int], per_viewport_summary: list[dict], all_assets: dict[str, dict]) -> Path:
|
||||
merged = merge_css_assets(output_dir, all_assets)
|
||||
assets_list = build_assets_list(merged)
|
||||
vh_flags = detect_vh_flags(output_dir)
|
||||
# Emit vh-flags.json at the workspace level (parent of capture/) so it lives next to manifest.json.
|
||||
if vh_flags:
|
||||
workspace_root = output_dir.parent
|
||||
(workspace_root / "vh-flags.json").write_text(json.dumps(vh_flags, indent=2))
|
||||
# Re-derive sections/<vp>.json from the DOM dumps. Replaces the JS section-scan
|
||||
# output with a Python pass that recurses into oversized wrappers (the JS hook
|
||||
# also has the recursion now, but Python being authoritative makes the fix
|
||||
# available in --replay mode against existing capture data).
|
||||
enriched_section_total = enrich_sections(output_dir)
|
||||
|
||||
meta = {
|
||||
"source_url": source_url,
|
||||
"captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"viewports": viewports,
|
||||
"canonical_viewport": {"width": CANONICAL_WIDTH, "height": CANONICAL_HEIGHT_PRIMARY, "alt_height": CANONICAL_HEIGHT_ALT},
|
||||
"per_viewport": per_viewport_summary,
|
||||
"assets": assets_list,
|
||||
"asset_sources": {
|
||||
"har": sum(1 for a in assets_list if not a.get("from_css")),
|
||||
"css": sum(1 for a in assets_list if a.get("from_css")),
|
||||
},
|
||||
"libs_detected": detect_libs(output_dir),
|
||||
"dom_snapshots": sum(s.get("steps", 0) for s in per_viewport_summary),
|
||||
"screenshots": sum(s.get("steps", 0) for s in per_viewport_summary),
|
||||
"vh_relative_count": len(vh_flags),
|
||||
"section_shots": len(list((output_dir / "section-shots").rglob("*.png"))) if (output_dir / "section-shots").exists() else 0,
|
||||
"section_candidates_total": enriched_section_total,
|
||||
"canvas_regions": 0,
|
||||
}
|
||||
out_path = output_dir / "meta.json"
|
||||
out_path.write_text(json.dumps(meta, indent=2))
|
||||
return out_path
|
||||
|
||||
|
||||
def replay(output_dir: Path, source_url: str | None) -> Path:
|
||||
"""Replay mode: re-derive meta.json + vh-flags.json from existing capture data.
|
||||
Useful for iterating on the analyze/generate/validate agents without re-running
|
||||
the (slow) browser pipeline."""
|
||||
if not output_dir.exists():
|
||||
raise SystemExit(f"replay: output dir does not exist: {output_dir}")
|
||||
|
||||
# Try existing meta.json for source_url + viewports
|
||||
existing_meta_path = output_dir / "meta.json"
|
||||
existing_meta = {}
|
||||
if existing_meta_path.exists():
|
||||
try:
|
||||
existing_meta = json.loads(existing_meta_path.read_text())
|
||||
except Exception:
|
||||
existing_meta = {}
|
||||
if not source_url:
|
||||
source_url = existing_meta.get("source_url") or "unknown://replay"
|
||||
|
||||
# Infer viewports from existing dom/<vp>/ folders
|
||||
dom_root = output_dir / "dom"
|
||||
viewports: list[int] = []
|
||||
if dom_root.exists():
|
||||
for child in dom_root.iterdir():
|
||||
if child.is_dir() and child.name.isdigit():
|
||||
viewports.append(int(child.name))
|
||||
viewports.sort()
|
||||
|
||||
per_viewport_summary: list[dict] = []
|
||||
for vp in viewports:
|
||||
steps = len(list((dom_root / str(vp)).glob("step-*.json")))
|
||||
sections_path = output_dir / "sections" / f"{vp}.json"
|
||||
section_count = 0
|
||||
if sections_path.exists():
|
||||
try:
|
||||
section_count = len(json.loads(sections_path.read_text()))
|
||||
except Exception:
|
||||
pass
|
||||
per_viewport_summary.append({
|
||||
"viewport": vp,
|
||||
"height": viewport_height_for(vp),
|
||||
"steps": steps,
|
||||
"section_candidates": section_count,
|
||||
"har": f"har/{vp}.har" if (output_dir / "har" / f"{vp}.har").exists() else None,
|
||||
})
|
||||
|
||||
har_assets = collect_har_assets_from_disk(output_dir)
|
||||
return write_meta(output_dir, source_url, viewports, per_viewport_summary, har_assets)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", required=False, help="Required unless --replay is set")
|
||||
parser.add_argument("--viewports", required=False, default="375,768,1280,1920", help="csv of widths")
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--wait-strategy", default="networkidle")
|
||||
parser.add_argument("--scroll-step", type=int, default=900, help="(legacy; ignored — scroll is now section-driven)")
|
||||
parser.add_argument("--skip-third-party", action="store_true")
|
||||
parser.add_argument("--replay", action="store_true", help="Skip browser capture; only re-run post-process")
|
||||
parser.add_argument("--skip-alt-height", action="store_true", help="Skip the vh-detection alt-height pass")
|
||||
parser.add_argument("--skip-section-shots", action="store_true", help="Skip the per-section cropped screenshot pass")
|
||||
args = parser.parse_args()
|
||||
|
||||
output_dir = Path(args.output).resolve()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if args.replay:
|
||||
meta_path = replay(output_dir, args.url)
|
||||
print(json.dumps({"status": "ok", "mode": "replay", "meta": str(meta_path)}))
|
||||
return
|
||||
|
||||
if not args.url:
|
||||
raise SystemExit("--url is required (unless --replay)")
|
||||
|
||||
script_dir = Path(__file__).resolve().parent
|
||||
init_sources = load_init_hooks(script_dir / "init-hooks")
|
||||
|
||||
viewports = [int(v.strip()) for v in args.viewports.split(",") if v.strip()]
|
||||
|
||||
all_assets: dict[str, dict] = {}
|
||||
per_viewport_summary: list[dict] = []
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled"])
|
||||
|
||||
# Stage 0
|
||||
for vp in viewports:
|
||||
summary = capture_viewport(browser, vp, args.url, args.wait_strategy, init_sources, output_dir, args.skip_third_party)
|
||||
for url, a in summary.pop("assets", {}).items():
|
||||
if url not in all_assets:
|
||||
all_assets[url] = a
|
||||
per_viewport_summary.append(summary)
|
||||
|
||||
# Stage 1
|
||||
if not args.skip_alt_height and CANONICAL_WIDTH in viewports:
|
||||
try:
|
||||
alt_summary = capture_alt_height(browser, args.url, args.wait_strategy, init_sources, output_dir, args.skip_third_party)
|
||||
per_viewport_summary.append({"alt_pass": True, **alt_summary})
|
||||
except Exception as e:
|
||||
print(json.dumps({"warning": f"alt-height capture failed: {e}"}), file=sys.stderr)
|
||||
|
||||
# Stage 2
|
||||
if not args.skip_section_shots:
|
||||
try:
|
||||
shots = capture_section_screenshots(browser, args.url, args.wait_strategy, init_sources, output_dir, args.skip_third_party, viewports)
|
||||
per_viewport_summary.append({"section_shots_total": shots})
|
||||
except Exception as e:
|
||||
print(json.dumps({"warning": f"section-shots pass failed: {e}"}), file=sys.stderr)
|
||||
|
||||
browser.close()
|
||||
|
||||
# Stage 3
|
||||
meta_path = write_meta(output_dir, args.url, viewports, per_viewport_summary, all_assets)
|
||||
print(json.dumps({"status": "ok", "mode": "capture", "meta": str(meta_path)}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
diff.py — Pixel diff two PNGs, producing a highlighted diff image and stats.
|
||||
|
||||
Uses the `pixelmatch` Python port (pip install pixelmatch pillow). Returns JSON to stdout:
|
||||
{ diff_pct, diff_image_path, worst_regions: [ { x, y, width, height } ] }
|
||||
|
||||
Worst regions are computed via connected-component analysis on the diff mask, sorted by area desc.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
from pixelmatch.contrib.PIL import pixelmatch
|
||||
except ImportError:
|
||||
print(json.dumps({"error": "Missing dependencies. pip install pixelmatch pillow numpy"}), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
from scipy import ndimage
|
||||
HAVE_SCIPY = True
|
||||
except ImportError:
|
||||
HAVE_SCIPY = False
|
||||
|
||||
|
||||
def resize_to_match(a: Image.Image, b: Image.Image) -> tuple[Image.Image, Image.Image]:
|
||||
if a.size == b.size:
|
||||
return a, b
|
||||
target = (min(a.size[0], b.size[0]), min(a.size[1], b.size[1]))
|
||||
return a.resize(target), b.resize(target)
|
||||
|
||||
|
||||
def worst_regions_from_diff(diff_img: Image.Image, top_n: int = 5) -> list[dict]:
|
||||
if not HAVE_SCIPY:
|
||||
return []
|
||||
arr = np.array(diff_img)
|
||||
if arr.ndim == 3:
|
||||
mask = (arr[..., :3].sum(axis=-1) > 0).astype(np.uint8)
|
||||
else:
|
||||
mask = (arr > 0).astype(np.uint8)
|
||||
labels, n = ndimage.label(mask)
|
||||
if n == 0:
|
||||
return []
|
||||
regions = []
|
||||
for i in range(1, n + 1):
|
||||
ys, xs = np.where(labels == i)
|
||||
if len(xs) < 20:
|
||||
continue
|
||||
regions.append({
|
||||
"x": int(xs.min()),
|
||||
"y": int(ys.min()),
|
||||
"width": int(xs.max() - xs.min() + 1),
|
||||
"height": int(ys.max() - ys.min() + 1),
|
||||
"area": int(len(xs)),
|
||||
})
|
||||
regions.sort(key=lambda r: r["area"], reverse=True)
|
||||
out = []
|
||||
for r in regions[:top_n]:
|
||||
r.pop("area", None)
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--before", required=True)
|
||||
parser.add_argument("--after", required=True)
|
||||
parser.add_argument("--diff-out", required=True)
|
||||
parser.add_argument("--threshold", type=float, default=0.1)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
a = Image.open(args.before).convert("RGBA")
|
||||
b = Image.open(args.after).convert("RGBA")
|
||||
a, b = resize_to_match(a, b)
|
||||
diff = Image.new("RGBA", a.size, (0, 0, 0, 0))
|
||||
|
||||
mismatched = pixelmatch(a, b, diff, threshold=args.threshold, includeAA=False)
|
||||
total = a.size[0] * a.size[1]
|
||||
diff_pct = (mismatched / total) * 100.0 if total else 0.0
|
||||
|
||||
Path(args.diff_out).parent.mkdir(parents=True, exist_ok=True)
|
||||
diff.save(args.diff_out)
|
||||
|
||||
worst = worst_regions_from_diff(diff)
|
||||
|
||||
result = {
|
||||
"diff_pct": round(diff_pct, 3),
|
||||
"diff_image_path": args.diff_out,
|
||||
"worst_regions": worst,
|
||||
"size": {"width": a.size[0], "height": a.size[1]},
|
||||
}
|
||||
if args.json:
|
||||
print(json.dumps(result))
|
||||
else:
|
||||
print(f"diff_pct={result['diff_pct']}% diff_image={result['diff_image_path']} regions={len(worst)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,342 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
dom-diff.py — Structural diff between two DOM JSON snapshots produced by the
|
||||
__CLONE_DUMP_COMPUTED__ walker. Used by the validate agent to surface concrete,
|
||||
actionable issues to the generate agent (font sizes, missing elements, wrong
|
||||
dimensions) instead of relying solely on PNG pixel diffs.
|
||||
|
||||
Inputs: --captured <captured DOM JSON>, --rendered <rendered DOM JSON>.
|
||||
Optionally --root-selector to scope the comparison to a specific subtree
|
||||
(typically a section's id or class), and --max-depth to cap recursion.
|
||||
|
||||
Output (JSON to stdout): {
|
||||
matched: <int>,
|
||||
missing_in_rendered: [{ path, tag, id, class, expected_h, expected_w }],
|
||||
extra_in_rendered: [{ path, tag, id, class }],
|
||||
style_mismatches: [{ path, property, expected, actual, severity }],
|
||||
size_mismatches: [{ path, expected, actual, delta_pct }],
|
||||
issues: [string, string, ...] // top human-readable issues
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Properties we treat as load-bearing for visual fidelity. Reported in this
|
||||
# order in the human-readable `issues` list.
|
||||
COMPARE_STYLE_PROPS = [
|
||||
"fontSize",
|
||||
"fontFamily",
|
||||
"fontWeight",
|
||||
"lineHeight",
|
||||
"color",
|
||||
"backgroundColor",
|
||||
"backgroundImage",
|
||||
"padding",
|
||||
"margin",
|
||||
"borderRadius",
|
||||
"boxShadow",
|
||||
"display",
|
||||
"flexDirection",
|
||||
"justifyContent",
|
||||
"alignItems",
|
||||
"gridTemplateColumns",
|
||||
"gap",
|
||||
"textAlign",
|
||||
"letterSpacing",
|
||||
"textTransform",
|
||||
]
|
||||
|
||||
# How far apart two values can be before we flag them.
|
||||
NUMERIC_TOLERANCE_PX = 2
|
||||
SIZE_TOLERANCE_PCT = 5.0
|
||||
|
||||
|
||||
def find_subtree(node: dict, selector: str | None) -> dict | None:
|
||||
if not selector:
|
||||
return node
|
||||
target = selector.lstrip("#.")
|
||||
is_id = selector.startswith("#")
|
||||
is_class = selector.startswith(".")
|
||||
|
||||
def walk(n: dict) -> dict | None:
|
||||
if not isinstance(n, dict):
|
||||
return None
|
||||
attrs = n.get("attrs") or {}
|
||||
if is_id and attrs.get("id") == target:
|
||||
return n
|
||||
if is_class:
|
||||
cls = attrs.get("class") or ""
|
||||
if target in cls.split():
|
||||
return n
|
||||
if not is_id and not is_class:
|
||||
# Tag selector
|
||||
if n.get("tag") == target:
|
||||
return n
|
||||
for c in n.get("children") or []:
|
||||
r = walk(c)
|
||||
if r is not None:
|
||||
return r
|
||||
return None
|
||||
|
||||
return walk(node)
|
||||
|
||||
|
||||
def parse_px(v: str | None) -> float | None:
|
||||
if not v:
|
||||
return None
|
||||
m = re.match(r"^(-?\d+(?:\.\d+)?)px$", str(v).strip())
|
||||
if m:
|
||||
return float(m.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def normalize_style_value(v: str | None) -> str:
|
||||
if v is None:
|
||||
return ""
|
||||
s = str(v).strip()
|
||||
s = re.sub(r"\s+", " ", s)
|
||||
return s
|
||||
|
||||
|
||||
def style_close(prop: str, expected: str | None, actual: str | None) -> bool:
|
||||
if expected == actual:
|
||||
return True
|
||||
e_px = parse_px(expected)
|
||||
a_px = parse_px(actual)
|
||||
if e_px is not None and a_px is not None:
|
||||
return abs(e_px - a_px) <= NUMERIC_TOLERANCE_PX
|
||||
if normalize_style_value(expected) == normalize_style_value(actual):
|
||||
return True
|
||||
# backgroundImage: if both reference url(), only flag if the url paths differ
|
||||
if prop == "backgroundImage" and expected and actual:
|
||||
e_url = re.search(r"url\([^)]+\)", expected)
|
||||
a_url = re.search(r"url\([^)]+\)", actual)
|
||||
if e_url and a_url:
|
||||
return False # both have urls but different — flag
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def child_key(n: dict, idx: int) -> str:
|
||||
if not isinstance(n, dict):
|
||||
return f"text[{idx}]"
|
||||
tag = n.get("tag") or "?"
|
||||
attrs = n.get("attrs") or {}
|
||||
if attrs.get("id"):
|
||||
return f"{tag}#{attrs['id']}"
|
||||
return f"{tag}[{idx}]"
|
||||
|
||||
|
||||
def compare(
|
||||
a: dict | None,
|
||||
b: dict | None,
|
||||
path: str,
|
||||
issues: list[dict],
|
||||
counters: dict,
|
||||
max_depth: int,
|
||||
depth: int = 0,
|
||||
) -> None:
|
||||
if depth > max_depth:
|
||||
return
|
||||
|
||||
if a is None and b is None:
|
||||
return
|
||||
if a is None or not isinstance(a, dict):
|
||||
return # only flag when source had it
|
||||
if b is None or not isinstance(b, dict):
|
||||
# Element missing in rendered
|
||||
attrs = a.get("attrs") or {}
|
||||
bbox = a.get("bbox") or {}
|
||||
issues.append({
|
||||
"kind": "missing_in_rendered",
|
||||
"path": path,
|
||||
"tag": a.get("tag"),
|
||||
"id": attrs.get("id"),
|
||||
"class": attrs.get("class"),
|
||||
"expected_w": round(bbox.get("width") or 0, 1),
|
||||
"expected_h": round(bbox.get("height") or 0, 1),
|
||||
})
|
||||
counters["missing"] += 1
|
||||
return
|
||||
|
||||
# Tag mismatch
|
||||
if a.get("tag") != b.get("tag"):
|
||||
attrs_a = a.get("attrs") or {}
|
||||
attrs_b = b.get("attrs") or {}
|
||||
issues.append({
|
||||
"kind": "tag_mismatch",
|
||||
"path": path,
|
||||
"expected_tag": a.get("tag"),
|
||||
"actual_tag": b.get("tag"),
|
||||
"expected_id": attrs_a.get("id"),
|
||||
"actual_id": attrs_b.get("id"),
|
||||
})
|
||||
counters["tag_mismatch"] += 1
|
||||
return
|
||||
|
||||
counters["matched"] += 1
|
||||
|
||||
# Style comparison
|
||||
ca = (a.get("computed") or {})
|
||||
cb = (b.get("computed") or {})
|
||||
for prop in COMPARE_STYLE_PROPS:
|
||||
ev, av = ca.get(prop), cb.get(prop)
|
||||
if ev is None and av is None:
|
||||
continue
|
||||
# If property only exists on rendered, that's a divergence too — but
|
||||
# we only flag when source explicitly set it (prevents noise).
|
||||
if ev is None:
|
||||
continue
|
||||
if not style_close(prop, ev, av):
|
||||
issues.append({
|
||||
"kind": "style_mismatch",
|
||||
"path": path,
|
||||
"property": prop,
|
||||
"expected": ev,
|
||||
"actual": av,
|
||||
})
|
||||
counters["style"] += 1
|
||||
|
||||
# Size comparison
|
||||
ba = a.get("bbox") or {}
|
||||
bb = b.get("bbox") or {}
|
||||
ew, eh = ba.get("width") or 0, ba.get("height") or 0
|
||||
aw, ah = bb.get("width") or 0, bb.get("height") or 0
|
||||
if ew > 8 and eh > 8:
|
||||
wdelta = abs(ew - aw) / max(ew, 1) * 100
|
||||
hdelta = abs(eh - ah) / max(eh, 1) * 100
|
||||
if wdelta > SIZE_TOLERANCE_PCT or hdelta > SIZE_TOLERANCE_PCT:
|
||||
issues.append({
|
||||
"kind": "size_mismatch",
|
||||
"path": path,
|
||||
"expected": [round(ew, 1), round(eh, 1)],
|
||||
"actual": [round(aw, 1), round(ah, 1)],
|
||||
"delta_pct": round(max(wdelta, hdelta), 1),
|
||||
})
|
||||
counters["size"] += 1
|
||||
|
||||
# Children — match by (tag, id) tuple when ids exist, else positional
|
||||
ka = a.get("children") or []
|
||||
kb = b.get("children") or []
|
||||
# Filter to element children (skip pure text)
|
||||
ka_el = [(i, c) for i, c in enumerate(ka) if isinstance(c, dict) and "tag" in c]
|
||||
kb_el = [(i, c) for i, c in enumerate(kb) if isinstance(c, dict) and "tag" in c]
|
||||
|
||||
# Build a lookup of rendered children by id (cheap) and by (tag, idx) fallback
|
||||
used_b: set[int] = set()
|
||||
b_by_id: dict[str, int] = {}
|
||||
for j, (_, ch) in enumerate(kb_el):
|
||||
cid = (ch.get("attrs") or {}).get("id")
|
||||
if cid:
|
||||
b_by_id[cid] = j
|
||||
|
||||
for ai, (_, ch) in enumerate(ka_el):
|
||||
a_id = (ch.get("attrs") or {}).get("id")
|
||||
match_idx = None
|
||||
if a_id and a_id in b_by_id and b_by_id[a_id] not in used_b:
|
||||
match_idx = b_by_id[a_id]
|
||||
elif ai < len(kb_el) and ai not in used_b:
|
||||
cand = kb_el[ai][1]
|
||||
if cand.get("tag") == ch.get("tag"):
|
||||
match_idx = ai
|
||||
if match_idx is None:
|
||||
compare(ch, None, f"{path} > {child_key(ch, ai)}", issues, counters, max_depth, depth + 1)
|
||||
else:
|
||||
used_b.add(match_idx)
|
||||
compare(ch, kb_el[match_idx][1], f"{path} > {child_key(ch, ai)}", issues, counters, max_depth, depth + 1)
|
||||
|
||||
# Extra children in rendered (no source counterpart) — only flag at top levels
|
||||
# to avoid noise from minor wrappers
|
||||
if depth < 3:
|
||||
for j, (_, ch) in enumerate(kb_el):
|
||||
if j in used_b:
|
||||
continue
|
||||
attrs = ch.get("attrs") or {}
|
||||
issues.append({
|
||||
"kind": "extra_in_rendered",
|
||||
"path": f"{path} > {child_key(ch, j)}",
|
||||
"tag": ch.get("tag"),
|
||||
"id": attrs.get("id"),
|
||||
"class": attrs.get("class"),
|
||||
})
|
||||
counters["extra"] += 1
|
||||
|
||||
|
||||
def humanize(issues: list[dict], top_n: int = 12) -> list[str]:
|
||||
"""Convert structured issues to human-readable strings, ranked by impact."""
|
||||
# Rank: missing > tag_mismatch > size_mismatch > style_mismatch > extra
|
||||
weights = {"missing_in_rendered": 100, "tag_mismatch": 80, "size_mismatch": 50, "style_mismatch": 25, "extra_in_rendered": 10}
|
||||
style_weights = {"backgroundImage": 4, "fontSize": 3, "color": 2, "backgroundColor": 2, "fontFamily": 2}
|
||||
|
||||
def score(i: dict) -> float:
|
||||
base = weights.get(i["kind"], 0)
|
||||
if i["kind"] == "size_mismatch":
|
||||
base += min(i.get("delta_pct", 0), 50)
|
||||
if i["kind"] == "style_mismatch":
|
||||
base += style_weights.get(i.get("property", ""), 1)
|
||||
return base
|
||||
|
||||
issues_sorted = sorted(issues, key=score, reverse=True)
|
||||
out: list[str] = []
|
||||
for i in issues_sorted[:top_n]:
|
||||
path = i.get("path", "?")
|
||||
if i["kind"] == "missing_in_rendered":
|
||||
out.append(f"{path}: missing in rendered (expected {i.get('tag')}{' #' + i['id'] if i.get('id') else ''}, ~{i.get('expected_w')}x{i.get('expected_h')})")
|
||||
elif i["kind"] == "tag_mismatch":
|
||||
out.append(f"{path}: expected <{i.get('expected_tag')}>, rendered <{i.get('actual_tag')}>")
|
||||
elif i["kind"] == "size_mismatch":
|
||||
ew, eh = i.get("expected", [0, 0])
|
||||
aw, ah = i.get("actual", [0, 0])
|
||||
out.append(f"{path}: size {aw}x{ah}, should be {ew}x{eh} (Δ{i.get('delta_pct')}%)")
|
||||
elif i["kind"] == "style_mismatch":
|
||||
out.append(f"{path}: {i.get('property')} is {i.get('actual')}, should be {i.get('expected')}")
|
||||
elif i["kind"] == "extra_in_rendered":
|
||||
out.append(f"{path}: extra element in rendered (<{i.get('tag')}>{' #' + i['id'] if i.get('id') else ''}) — not in source")
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--captured", required=True)
|
||||
parser.add_argument("--rendered", required=True)
|
||||
parser.add_argument("--root-selector", default=None, help="e.g. #hero or .ecosystem to scope the diff to a section")
|
||||
parser.add_argument("--max-depth", type=int, default=8)
|
||||
parser.add_argument("--max-issues", type=int, default=200)
|
||||
args = parser.parse_args()
|
||||
|
||||
captured = json.loads(Path(args.captured).read_text())
|
||||
rendered = json.loads(Path(args.rendered).read_text())
|
||||
|
||||
a_root = find_subtree(captured, args.root_selector)
|
||||
b_root = find_subtree(rendered, args.root_selector)
|
||||
|
||||
if a_root is None:
|
||||
print(json.dumps({"error": f"root selector not found in captured: {args.root_selector}"}))
|
||||
sys.exit(2)
|
||||
if b_root is None:
|
||||
print(json.dumps({"error": f"root selector not found in rendered: {args.root_selector}"}))
|
||||
sys.exit(3)
|
||||
|
||||
issues: list[dict] = []
|
||||
counters = {"matched": 0, "missing": 0, "extra": 0, "tag_mismatch": 0, "style": 0, "size": 0}
|
||||
compare(a_root, b_root, args.root_selector or a_root.get("tag", "body"), issues, counters, args.max_depth)
|
||||
|
||||
issues = issues[: args.max_issues]
|
||||
humanized = humanize(issues, top_n=12)
|
||||
|
||||
print(json.dumps({
|
||||
"matched": counters["matched"],
|
||||
"counts": counters,
|
||||
"issues": humanized,
|
||||
"structured_issues": issues,
|
||||
}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
download-assets.py — Download all assets referenced in a capture meta.json (or manifest.json)
|
||||
into <project>/public/assets/cloned/ using hash-based filenames.
|
||||
|
||||
Input: JSON file with `assets[]` of shape { type, source_url, local_path }.
|
||||
Output (to --json stdout): { downloaded: [...], failed: [...], skipped: [...] }
|
||||
|
||||
Dependencies: httpx (pip install httpx).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import httpx
|
||||
except ImportError:
|
||||
print(json.dumps({"error": "Missing dependencies. pip install httpx"}), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
LICENSED_FONT_HOST_RE = re.compile(r"(use\.typekit\.net|fonts\.adobe\.com|fast\.fonts\.net|cloud\.typography\.com)")
|
||||
ROTATING_AUTH_RE = re.compile(r"[?&](token|signature|expires)=")
|
||||
USER_AGENT = "Mozilla/5.0 (compatible; IonCloneBot/1.0)"
|
||||
|
||||
|
||||
def download_one(client: httpx.Client, url: str, out_path: Path) -> str:
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with client.stream("GET", url, timeout=30, headers={"User-Agent": USER_AGENT}) as r:
|
||||
r.raise_for_status()
|
||||
with out_path.open("wb") as f:
|
||||
for chunk in r.iter_bytes():
|
||||
f.write(chunk)
|
||||
return "downloaded"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--manifest", required=True, help="Path to meta.json or manifest.json")
|
||||
parser.add_argument("--public-dir", required=True, help="Path to <project>/public/assets/cloned")
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest_path = Path(args.manifest)
|
||||
public_dir = Path(args.public_dir)
|
||||
public_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
assets = manifest.get("assets", [])
|
||||
|
||||
downloaded: list[str] = []
|
||||
failed: list[dict] = []
|
||||
skipped: list[dict] = []
|
||||
|
||||
with httpx.Client(follow_redirects=True) as client:
|
||||
for a in assets:
|
||||
url = a.get("source_url")
|
||||
local_path = a.get("local_path")
|
||||
atype = a.get("type")
|
||||
if not url or not local_path:
|
||||
failed.append({"url": url, "reason": "malformed_asset"})
|
||||
continue
|
||||
|
||||
if atype == "font" and LICENSED_FONT_HOST_RE.search(url):
|
||||
skipped.append({"url": url, "reason": "licensed_font_unclear"})
|
||||
continue
|
||||
|
||||
if ROTATING_AUTH_RE.search(url):
|
||||
skipped.append({"url": url, "reason": "rotating_auth_token"})
|
||||
continue
|
||||
|
||||
# local_path in the asset is relative to the project root; public-dir is
|
||||
# <project>/public/assets/cloned, so strip that prefix.
|
||||
rel = local_path
|
||||
for prefix in ("public/assets/cloned/", "assets/cloned/"):
|
||||
if rel.startswith(prefix):
|
||||
rel = rel[len(prefix):]
|
||||
break
|
||||
out_path = public_dir / rel
|
||||
|
||||
try:
|
||||
download_one(client, url, out_path)
|
||||
downloaded.append(str(out_path))
|
||||
except httpx.HTTPError as e:
|
||||
failed.append({"url": url, "reason": str(e)})
|
||||
except Exception as e:
|
||||
failed.append({"url": url, "reason": str(e)})
|
||||
|
||||
result = {"downloaded": downloaded, "failed": failed, "skipped": skipped}
|
||||
if args.json:
|
||||
print(json.dumps(result))
|
||||
else:
|
||||
print(f"downloaded={len(downloaded)} failed={len(failed)} skipped={len(skipped)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
dump-rendered.py — Open a URL (typically the local dev server) and dump its DOM
|
||||
using the same __CLONE_DUMP_COMPUTED__ walker the capture pipeline uses for the
|
||||
source. This produces a directly-comparable structural snapshot the validate
|
||||
agent can diff against the captured original.
|
||||
|
||||
Usage:
|
||||
dump-rendered.py --url http://localhost:3000/ --output workspace/rendered/1280-step-00.json \
|
||||
--viewport 1280 --scroll-y 0
|
||||
|
||||
Dependencies: playwright.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout
|
||||
except ImportError:
|
||||
print(json.dumps({"error": "Missing dependencies. pip install playwright && playwright install chromium"}), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def load_init_hooks(hooks_dir: Path) -> list[str]:
|
||||
sources: list[str] = []
|
||||
# We only need computed-styles + section-scan for rendered-DOM purposes.
|
||||
for name in ("bootstrap.js", "computed-styles.js", "section-scan.js"):
|
||||
p = hooks_dir / name
|
||||
if p.exists():
|
||||
sources.append(p.read_text())
|
||||
return sources
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", required=True)
|
||||
parser.add_argument("--output", required=True, help="Path to write the rendered DOM JSON")
|
||||
parser.add_argument("--viewport", type=int, default=1280)
|
||||
parser.add_argument("--viewport-height", type=int, default=None, help="Default: 16:9 of viewport width")
|
||||
parser.add_argument("--scroll-y", type=int, default=0)
|
||||
parser.add_argument("--reduce-motion", action="store_true")
|
||||
parser.add_argument("--settle-ms", type=int, default=800)
|
||||
args = parser.parse_args()
|
||||
|
||||
output = Path(args.output).resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
height = args.viewport_height or round(args.viewport * 9 / 16)
|
||||
script_dir = Path(__file__).resolve().parent
|
||||
init_sources = load_init_hooks(script_dir / "init-hooks")
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled"])
|
||||
ctx = browser.new_context(viewport={"width": args.viewport, "height": height})
|
||||
for src in init_sources:
|
||||
ctx.add_init_script(src)
|
||||
page = ctx.new_page()
|
||||
|
||||
try:
|
||||
page.goto(args.url, wait_until="networkidle", timeout=30_000)
|
||||
except PWTimeout:
|
||||
page.goto(args.url, wait_until="domcontentloaded", timeout=30_000)
|
||||
|
||||
if args.reduce_motion:
|
||||
try:
|
||||
page.evaluate("document.documentElement.classList.add('reduce-motion')")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
page.evaluate("() => document.fonts.ready")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
page.wait_for_function("window.__CLONE_READY__ === true", timeout=5_000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if args.scroll_y:
|
||||
page.evaluate(f"window.scrollTo({{ top: {args.scroll_y}, behavior: 'instant' }})")
|
||||
|
||||
time.sleep(args.settle_ms / 1000.0)
|
||||
|
||||
snapshot = page.evaluate(
|
||||
"() => (typeof window.__CLONE_DUMP_COMPUTED__ === 'function') ? window.__CLONE_DUMP_COMPUTED__() : null"
|
||||
)
|
||||
sections = page.evaluate(
|
||||
"() => (typeof window.__CLONE_LIST_SECTIONS__ === 'function') ? window.__CLONE_LIST_SECTIONS__() : []"
|
||||
) or []
|
||||
|
||||
if snapshot is None:
|
||||
raise SystemExit("dump-rendered: __CLONE_DUMP_COMPUTED__ unavailable — init hooks failed to load")
|
||||
|
||||
output.write_text(json.dumps(snapshot, separators=(",", ":")))
|
||||
sections_path = output.with_name(output.stem + "-sections.json")
|
||||
sections_path.write_text(json.dumps(sections, indent=2))
|
||||
|
||||
page.close()
|
||||
ctx.close()
|
||||
browser.close()
|
||||
|
||||
print(json.dumps({
|
||||
"status": "ok",
|
||||
"output": str(output),
|
||||
"sections_path": str(sections_path),
|
||||
"section_count": len(sections),
|
||||
}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+135
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
extract-fonts.py — Parse @font-face rules from captured CSS, resolve URLs, download,
|
||||
and write license_hint into the manifest.
|
||||
|
||||
Run after capture.py and before generation. Input: capture directory with fonts/*.json.
|
||||
Output: writes font files into <public_dir>/fonts/ and an updated JSON summary.
|
||||
|
||||
Dependencies: httpx, tinycss2 (pip install httpx tinycss2).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin
|
||||
|
||||
try:
|
||||
import httpx
|
||||
import tinycss2
|
||||
except ImportError:
|
||||
print(json.dumps({"error": "Missing deps. pip install httpx tinycss2"}), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
LICENSED_HOSTS = re.compile(r"(use\.typekit\.net|fonts\.adobe\.com|fast\.fonts\.net|cloud\.typography\.com)")
|
||||
OPEN_HOSTS = re.compile(r"(fonts\.googleapis\.com|fonts\.gstatic\.com)")
|
||||
|
||||
|
||||
def sha1_8(s: str) -> str:
|
||||
return hashlib.sha1(s.encode("utf-8")).hexdigest()[:8]
|
||||
|
||||
|
||||
def classify(url: str) -> str:
|
||||
if LICENSED_HOSTS.search(url):
|
||||
return "licensed"
|
||||
if OPEN_HOSTS.search(url):
|
||||
return "open"
|
||||
return "unclear"
|
||||
|
||||
|
||||
def parse_face_rules(css_text: str, base_url: str) -> list[dict]:
|
||||
out = []
|
||||
rules = tinycss2.parse_stylesheet(css_text, skip_comments=True, skip_whitespace=True)
|
||||
for rule in rules:
|
||||
if rule.type != "at-rule" or rule.lower_at_keyword != "font-face":
|
||||
continue
|
||||
block = tinycss2.parse_declaration_list(rule.content or [])
|
||||
face: dict = {}
|
||||
for decl in block:
|
||||
if decl.type != "declaration":
|
||||
continue
|
||||
name = decl.lower_name
|
||||
value = tinycss2.serialize(decl.value).strip().strip('"').strip("'")
|
||||
if name == "font-family":
|
||||
face["family"] = value
|
||||
elif name == "font-weight":
|
||||
face["weight"] = value
|
||||
elif name == "font-style":
|
||||
face["style"] = value
|
||||
elif name == "src":
|
||||
urls = re.findall(r"url\(\s*['\"]?([^'\")]+)['\"]?\s*\)", tinycss2.serialize(decl.value))
|
||||
face["urls"] = [urljoin(base_url, u) for u in urls]
|
||||
if face.get("family") and face.get("urls"):
|
||||
out.append(face)
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--capture-dir", required=True)
|
||||
parser.add_argument("--public-dir", required=True)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
capture_dir = Path(args.capture_dir)
|
||||
public_dir = Path(args.public_dir)
|
||||
fonts_out = public_dir / "fonts"
|
||||
fonts_out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
face_dumps = list((capture_dir / "fonts").glob("*.json"))
|
||||
|
||||
all_faces: list[dict] = []
|
||||
for dump in face_dumps:
|
||||
data = json.loads(dump.read_text())
|
||||
for entry in data if isinstance(data, list) else []:
|
||||
css = entry.get("cssText") or ""
|
||||
base = entry.get("baseUrl") or ""
|
||||
all_faces.extend(parse_face_rules(css, base))
|
||||
|
||||
downloaded: list[dict] = []
|
||||
skipped: list[dict] = []
|
||||
|
||||
with httpx.Client(follow_redirects=True) as client:
|
||||
for face in all_faces:
|
||||
family = face["family"]
|
||||
license_hint = "unclear"
|
||||
for url in face["urls"]:
|
||||
license_hint = classify(url)
|
||||
if license_hint == "licensed":
|
||||
skipped.append({"family": family, "url": url, "reason": "licensed"})
|
||||
break
|
||||
try:
|
||||
ext = url.rsplit(".", 1)[-1].split("?", 1)[0].lower()
|
||||
if ext not in ("woff2", "woff", "ttf", "otf"):
|
||||
ext = "woff2"
|
||||
fname = f"{sha1_8(url)}-{re.sub(r'[^a-z0-9]+', '-', family.lower())}.{ext}"
|
||||
out_path = fonts_out / fname
|
||||
r = client.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0 (compatible; IonCloneBot/1.0)"})
|
||||
r.raise_for_status()
|
||||
out_path.write_bytes(r.content)
|
||||
downloaded.append({
|
||||
"family": family,
|
||||
"weight": face.get("weight"),
|
||||
"style": face.get("style"),
|
||||
"source_url": url,
|
||||
"local_path": str(out_path),
|
||||
"license_hint": license_hint,
|
||||
})
|
||||
except Exception as e:
|
||||
skipped.append({"family": family, "url": url, "reason": str(e)})
|
||||
|
||||
result = {"downloaded": downloaded, "skipped": skipped}
|
||||
if args.json:
|
||||
print(json.dumps(result))
|
||||
else:
|
||||
print(f"downloaded={len(downloaded)} skipped={len(skipped)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
// Initializes the shared capture bucket that every other hook writes into.
|
||||
// Runs first because hooks load in lexicographic order.
|
||||
(() => {
|
||||
if (window.__CLONE_CAPTURE__) return;
|
||||
window.__CLONE_CAPTURE__ = {
|
||||
shaders: [],
|
||||
gsap: [],
|
||||
framer: [],
|
||||
lottie: [],
|
||||
threejs: [],
|
||||
cssVars: {},
|
||||
fonts: [],
|
||||
};
|
||||
// Sentinel the capture script waits on after DOMContentLoaded + a tick.
|
||||
// Hooks that dump state on demand (gsap, framer) set __CLONE_READY__ when they've run.
|
||||
window.__CLONE_READY__ = false;
|
||||
window.addEventListener('load', () => {
|
||||
setTimeout(() => {
|
||||
try {
|
||||
if (typeof window.__CLONE_FINALIZE__ === 'function') window.__CLONE_FINALIZE__();
|
||||
} catch {}
|
||||
window.__CLONE_READY__ = true;
|
||||
}, 800);
|
||||
});
|
||||
})();
|
||||
@@ -0,0 +1,193 @@
|
||||
// Exposes window.__CLONE_DUMP_COMPUTED__() which walks the DOM and returns a minimal tree
|
||||
// of resolved computed styles (only non-default values). The capture script calls this on demand
|
||||
// after each scroll step — cheap to call repeatedly, much richer than the inline per-step snapshot
|
||||
// the Playwright script does as a fallback.
|
||||
(() => {
|
||||
const PROPS = [
|
||||
'color',
|
||||
'backgroundColor',
|
||||
'backgroundImage',
|
||||
'backgroundSize',
|
||||
'backgroundPosition',
|
||||
'backgroundRepeat',
|
||||
'backgroundAttachment',
|
||||
'backgroundClip',
|
||||
'fontFamily',
|
||||
'fontSize',
|
||||
'fontWeight',
|
||||
'lineHeight',
|
||||
'letterSpacing',
|
||||
'textTransform',
|
||||
'textAlign',
|
||||
'textDecoration',
|
||||
'textShadow',
|
||||
'whiteSpace',
|
||||
'display',
|
||||
'flexDirection',
|
||||
'flexWrap',
|
||||
'justifyContent',
|
||||
'alignItems',
|
||||
'alignContent',
|
||||
'alignSelf',
|
||||
'gap',
|
||||
'rowGap',
|
||||
'columnGap',
|
||||
'gridTemplateColumns',
|
||||
'gridTemplateRows',
|
||||
'gridTemplateAreas',
|
||||
'gridColumn',
|
||||
'gridRow',
|
||||
'gridAutoFlow',
|
||||
'padding',
|
||||
'paddingTop',
|
||||
'paddingRight',
|
||||
'paddingBottom',
|
||||
'paddingLeft',
|
||||
'margin',
|
||||
'marginTop',
|
||||
'marginRight',
|
||||
'marginBottom',
|
||||
'marginLeft',
|
||||
'width',
|
||||
'height',
|
||||
'maxWidth',
|
||||
'maxHeight',
|
||||
'minWidth',
|
||||
'minHeight',
|
||||
'aspectRatio',
|
||||
'position',
|
||||
'top',
|
||||
'right',
|
||||
'bottom',
|
||||
'left',
|
||||
'zIndex',
|
||||
'inset',
|
||||
'border',
|
||||
'borderTop',
|
||||
'borderRight',
|
||||
'borderBottom',
|
||||
'borderLeft',
|
||||
'borderRadius',
|
||||
'borderColor',
|
||||
'borderWidth',
|
||||
'borderStyle',
|
||||
'boxShadow',
|
||||
'opacity',
|
||||
'transform',
|
||||
'transformOrigin',
|
||||
'filter',
|
||||
'backdropFilter',
|
||||
'clipPath',
|
||||
'mask',
|
||||
'maskImage',
|
||||
'mixBlendMode',
|
||||
'isolation',
|
||||
'overflow',
|
||||
'overflowX',
|
||||
'overflowY',
|
||||
'scrollBehavior',
|
||||
'scrollSnapType',
|
||||
'cursor',
|
||||
'pointerEvents',
|
||||
'userSelect',
|
||||
'willChange',
|
||||
'contain',
|
||||
];
|
||||
|
||||
// Compute defaults once per tag by creating a hidden iframe so we don't treat
|
||||
// inherited defaults as "used styles". The iframe is created lazily — at
|
||||
// init-script time, document.documentElement may still be null (Playwright
|
||||
// fires add_init_script before the HTML parser materialises <html>).
|
||||
let iframe = null;
|
||||
function ensureIframe() {
|
||||
if (iframe) return iframe;
|
||||
if (!document.documentElement) return null;
|
||||
try {
|
||||
iframe = document.createElement('iframe');
|
||||
iframe.style.cssText = 'position:absolute;width:0;height:0;border:0;opacity:0;pointer-events:none';
|
||||
iframe.srcdoc = '<!doctype html><html><body></body></html>';
|
||||
document.documentElement.appendChild(iframe);
|
||||
} catch {
|
||||
iframe = null;
|
||||
}
|
||||
return iframe;
|
||||
}
|
||||
|
||||
const defaultsByTag = new Map();
|
||||
function defaultsFor(tag) {
|
||||
if (defaultsByTag.has(tag)) return defaultsByTag.get(tag);
|
||||
const ifr = ensureIframe();
|
||||
const doc = ifr ? ifr.contentDocument : null;
|
||||
if (!doc || !doc.body) return {};
|
||||
const el = doc.createElement(tag);
|
||||
doc.body.appendChild(el);
|
||||
const cs = doc.defaultView.getComputedStyle(el);
|
||||
const d = {};
|
||||
for (const p of PROPS) d[p] = cs[p];
|
||||
doc.body.removeChild(el);
|
||||
defaultsByTag.set(tag, d);
|
||||
return d;
|
||||
}
|
||||
|
||||
function nonDefault(el) {
|
||||
const cs = getComputedStyle(el);
|
||||
const defaults = defaultsFor(el.tagName.toLowerCase());
|
||||
const out = {};
|
||||
for (const p of PROPS) {
|
||||
const v = cs[p];
|
||||
if (v == null || v === '') continue;
|
||||
if (v === defaults[p]) continue;
|
||||
out[p] = v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function pseudoStyles(el) {
|
||||
const out = {};
|
||||
for (const pseudo of ['::before', '::after']) {
|
||||
try {
|
||||
const cs = getComputedStyle(el, pseudo);
|
||||
const content = cs.content;
|
||||
if (content && content !== 'normal' && content !== 'none') {
|
||||
const block = {};
|
||||
for (const p of PROPS) {
|
||||
const v = cs[p];
|
||||
if (v && v !== 'none' && v !== 'normal' && v !== 'auto') block[p] = v;
|
||||
}
|
||||
block.content = content;
|
||||
out[pseudo] = block;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return Object.keys(out).length ? out : undefined;
|
||||
}
|
||||
|
||||
function serialize(el) {
|
||||
if (el.nodeType === Node.TEXT_NODE) {
|
||||
const t = el.textContent;
|
||||
return t && t.trim() ? { text: t } : null;
|
||||
}
|
||||
if (el.nodeType !== Node.ELEMENT_NODE) return null;
|
||||
if (el.tagName === 'SCRIPT' || el.tagName === 'STYLE' || el.tagName === 'NOSCRIPT') return null;
|
||||
const attrs = {};
|
||||
for (const a of el.attributes) attrs[a.name] = a.value;
|
||||
const computed = nonDefault(el);
|
||||
const pseudo = pseudoStyles(el);
|
||||
if (pseudo) computed.pseudo = pseudo;
|
||||
const r = el.getBoundingClientRect();
|
||||
const children = [];
|
||||
for (const c of el.childNodes) {
|
||||
const s = serialize(c);
|
||||
if (s) children.push(s);
|
||||
}
|
||||
return {
|
||||
tag: el.tagName.toLowerCase(),
|
||||
attrs,
|
||||
computed,
|
||||
bbox: { x: r.x, y: r.y + window.scrollY, width: r.width, height: r.height },
|
||||
children,
|
||||
};
|
||||
}
|
||||
|
||||
window.__CLONE_DUMP_COMPUTED__ = () => serialize(document.body);
|
||||
})();
|
||||
@@ -0,0 +1,121 @@
|
||||
// Dumps every same-origin CSS rule as { selector, cssText } into __CLONE_CAPTURE__.cssRules,
|
||||
// and extracts every url(...) asset reference (resolved to absolute URLs) into
|
||||
// __CLONE_CAPTURE__.cssAssets. Catches background-images set in stylesheets that the
|
||||
// response listener missed (e.g. @media-gated, ::before, mask-image, list-style-image, cursor).
|
||||
(() => {
|
||||
function isSameOrigin(href) {
|
||||
if (!href) return true;
|
||||
try {
|
||||
const u = new URL(href, location.href);
|
||||
return u.origin === location.origin;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveUrl(raw, baseHref) {
|
||||
if (!raw) return null;
|
||||
let s = raw.trim();
|
||||
if (s.startsWith('"') || s.startsWith("'")) s = s.slice(1, -1);
|
||||
if (s.startsWith('data:') || s.startsWith('#')) return null;
|
||||
try {
|
||||
return new URL(s, baseHref || location.href).href;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function collectFromRules(rules, baseHref, out) {
|
||||
if (!rules) return;
|
||||
for (const rule of rules) {
|
||||
const type = rule.constructor?.name || '';
|
||||
if (type === 'CSSStyleRule') {
|
||||
const text = rule.cssText || '';
|
||||
out.rules.push({ selector: rule.selectorText, cssText: text, baseHref });
|
||||
const urls = text.match(/url\(\s*([^)]+?)\s*\)/g) || [];
|
||||
for (const m of urls) {
|
||||
const inner = m
|
||||
.slice(4, -1)
|
||||
.trim()
|
||||
.replace(/^['"]|['"]$/g, '');
|
||||
const abs = resolveUrl(inner, baseHref);
|
||||
if (abs && !out.assetSet.has(abs)) {
|
||||
out.assetSet.add(abs);
|
||||
out.assets.push({ source_url: abs, selector: rule.selectorText });
|
||||
}
|
||||
}
|
||||
} else if (type === 'CSSMediaRule' || type === 'CSSSupportsRule' || type === 'CSSContainerRule') {
|
||||
const condition = rule.conditionText || rule.media?.mediaText;
|
||||
out.media.push({ condition, type });
|
||||
collectFromRules(rule.cssRules, baseHref, out);
|
||||
} else if (type === 'CSSImportRule') {
|
||||
// Recurse into imported stylesheet if same-origin and accessible
|
||||
try {
|
||||
const importedHref = rule.styleSheet?.href || baseHref;
|
||||
if (isSameOrigin(importedHref)) {
|
||||
collectFromRules(rule.styleSheet?.cssRules, importedHref, out);
|
||||
}
|
||||
} catch {}
|
||||
} else if (type === 'CSSFontFaceRule') {
|
||||
const text = rule.cssText || '';
|
||||
out.fontFaces.push({ cssText: text, baseHref });
|
||||
const urls = text.match(/url\(\s*([^)]+?)\s*\)/g) || [];
|
||||
for (const m of urls) {
|
||||
const inner = m
|
||||
.slice(4, -1)
|
||||
.trim()
|
||||
.replace(/^['"]|['"]$/g, '');
|
||||
const abs = resolveUrl(inner, baseHref);
|
||||
if (abs && !out.assetSet.has(abs)) {
|
||||
out.assetSet.add(abs);
|
||||
out.assets.push({ source_url: abs, selector: '@font-face' });
|
||||
}
|
||||
}
|
||||
} else if (type === 'CSSKeyframesRule') {
|
||||
const frames = [];
|
||||
for (const k of rule.cssRules || []) frames.push({ key: k.keyText, cssText: k.style?.cssText || '' });
|
||||
out.keyframes.push({ name: rule.name, frames, baseHref });
|
||||
} else if (rule.cssRules) {
|
||||
collectFromRules(rule.cssRules, baseHref, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function collect() {
|
||||
const out = {
|
||||
rules: [],
|
||||
media: [],
|
||||
fontFaces: [],
|
||||
keyframes: [],
|
||||
assets: [],
|
||||
assetSet: new Set(),
|
||||
crossOrigin: 0,
|
||||
};
|
||||
for (const sheet of document.styleSheets) {
|
||||
const baseHref = sheet.href || location.href;
|
||||
let rules;
|
||||
try {
|
||||
rules = sheet.cssRules;
|
||||
} catch {
|
||||
out.crossOrigin += 1;
|
||||
continue;
|
||||
}
|
||||
collectFromRules(rules, baseHref, out);
|
||||
}
|
||||
delete out.assetSet;
|
||||
window.__CLONE_CAPTURE__.cssRules = out;
|
||||
}
|
||||
|
||||
const prevFinalize = window.__CLONE_FINALIZE__;
|
||||
window.__CLONE_FINALIZE__ = function () {
|
||||
if (prevFinalize)
|
||||
try {
|
||||
prevFinalize();
|
||||
} catch {}
|
||||
try {
|
||||
collect();
|
||||
} catch (e) {
|
||||
window.__CLONE_CAPTURE__.cssRules = { error: String(e) };
|
||||
}
|
||||
};
|
||||
})();
|
||||
@@ -0,0 +1,51 @@
|
||||
// Dumps every :root --custom-property and every @property registration into
|
||||
// window.__CLONE_CAPTURE__.cssVars, keyed under { custom, registered }.
|
||||
(() => {
|
||||
function collect() {
|
||||
const custom = {};
|
||||
try {
|
||||
const cs = getComputedStyle(document.documentElement);
|
||||
for (let i = 0; i < cs.length; i++) {
|
||||
const name = cs[i];
|
||||
if (!name.startsWith('--')) continue;
|
||||
try {
|
||||
custom[name] = cs.getPropertyValue(name).trim();
|
||||
} catch {}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
const registered = [];
|
||||
try {
|
||||
for (const sheet of document.styleSheets) {
|
||||
let rules;
|
||||
try {
|
||||
rules = sheet.cssRules;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!rules) continue;
|
||||
for (const rule of rules) {
|
||||
if (rule.constructor?.name === 'CSSPropertyRule' || rule.type === 18 /* @property */) {
|
||||
registered.push({
|
||||
name: rule.name,
|
||||
syntax: rule.syntax,
|
||||
initialValue: rule.initialValue,
|
||||
inherits: rule.inherits,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
window.__CLONE_CAPTURE__.cssVars = { custom, registered };
|
||||
}
|
||||
|
||||
const prevFinalize = window.__CLONE_FINALIZE__;
|
||||
window.__CLONE_FINALIZE__ = function () {
|
||||
if (prevFinalize)
|
||||
try {
|
||||
prevFinalize();
|
||||
} catch {}
|
||||
collect();
|
||||
};
|
||||
})();
|
||||
@@ -0,0 +1,107 @@
|
||||
// Walks the React fiber tree (via __REACT_DEVTOOLS_GLOBAL_HOOK__) to find framer-motion
|
||||
// components and dump their animate/variants/transition/whileInView/whileHover props.
|
||||
// Called from __CLONE_FINALIZE__.
|
||||
(() => {
|
||||
function cssPath(el) {
|
||||
if (!el || el.nodeType !== 1) return null;
|
||||
if (el.id) return `#${el.id}`;
|
||||
const parts = [];
|
||||
while (el && el.nodeType === 1 && parts.length < 10) {
|
||||
let s = el.tagName.toLowerCase();
|
||||
const p = el.parentNode;
|
||||
if (p) {
|
||||
const sibs = Array.from(p.children).filter((c) => c.tagName === el.tagName);
|
||||
if (sibs.length > 1) s += `:nth-of-type(${sibs.indexOf(el) + 1})`;
|
||||
}
|
||||
parts.unshift(s);
|
||||
el = el.parentNode;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
}
|
||||
|
||||
function isMotionComponent(fiber) {
|
||||
const type = fiber?.type;
|
||||
if (!type) return false;
|
||||
if (type.$$typeof && type.render && type.render.displayName?.startsWith('motion.')) return true;
|
||||
const dn = type.displayName || type.name;
|
||||
if (typeof dn === 'string' && (dn.startsWith('motion.') || dn === 'MotionComponent')) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function serializeProp(v) {
|
||||
try {
|
||||
if (v == null) return v;
|
||||
if (typeof v === 'function') return null;
|
||||
if (typeof v === 'object') {
|
||||
const out = {};
|
||||
for (const k of Object.keys(v)) {
|
||||
const val = v[k];
|
||||
if (typeof val === 'function') continue;
|
||||
if (val && typeof val === 'object' && val.nodeType) continue;
|
||||
out[k] = val;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return v;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function findFibers(fiber, out) {
|
||||
if (!fiber) return;
|
||||
if (isMotionComponent(fiber)) {
|
||||
const props = fiber.memoizedProps || {};
|
||||
let node = fiber.stateNode;
|
||||
if (!(node instanceof Element)) {
|
||||
// Walk down to the first DOM host for selector extraction.
|
||||
let cursor = fiber.child;
|
||||
while (cursor) {
|
||||
if (cursor.stateNode instanceof Element) {
|
||||
node = cursor.stateNode;
|
||||
break;
|
||||
}
|
||||
cursor = cursor.child;
|
||||
}
|
||||
}
|
||||
out.push({
|
||||
selector: node instanceof Element ? cssPath(node) : null,
|
||||
initial: serializeProp(props.initial),
|
||||
animate: serializeProp(props.animate),
|
||||
exit: serializeProp(props.exit),
|
||||
variants: serializeProp(props.variants),
|
||||
transition: serializeProp(props.transition),
|
||||
whileHover: serializeProp(props.whileHover),
|
||||
whileTap: serializeProp(props.whileTap),
|
||||
whileInView: serializeProp(props.whileInView),
|
||||
viewport: serializeProp(props.viewport),
|
||||
});
|
||||
}
|
||||
if (fiber.child) findFibers(fiber.child, out);
|
||||
if (fiber.sibling) findFibers(fiber.sibling, out);
|
||||
}
|
||||
|
||||
const prevFinalize = window.__CLONE_FINALIZE__;
|
||||
window.__CLONE_FINALIZE__ = function () {
|
||||
if (prevFinalize)
|
||||
try {
|
||||
prevFinalize();
|
||||
} catch {}
|
||||
try {
|
||||
const hook = window.__REACT_DEVTOOLS_GLOBAL_HOOK__;
|
||||
if (!hook || !hook.renderers) return;
|
||||
for (const [, renderer] of hook.renderers) {
|
||||
if (!renderer) continue;
|
||||
const roots = renderer.findFiberByHostInstance ? null : renderer.roots || null;
|
||||
const scanFrom = [];
|
||||
if (hook.getFiberRoots) {
|
||||
const rootSet = hook.getFiberRoots(1) || new Set();
|
||||
for (const r of rootSet) scanFrom.push(r.current);
|
||||
}
|
||||
for (const fiber of scanFrom) findFibers(fiber, window.__CLONE_CAPTURE__.framer);
|
||||
}
|
||||
} catch (e) {
|
||||
window.__CLONE_CAPTURE__.framer.push({ error: String(e) });
|
||||
}
|
||||
};
|
||||
})();
|
||||
@@ -0,0 +1,103 @@
|
||||
// Dumps GSAP timelines + tweens into window.__CLONE_CAPTURE__.gsap after load.
|
||||
// Runs on __CLONE_FINALIZE__ so that page-load animations have had time to register.
|
||||
(() => {
|
||||
function cssPath(el) {
|
||||
if (!el || el.nodeType !== 1) return null;
|
||||
if (el.id) return `#${el.id}`;
|
||||
const parts = [];
|
||||
while (el && el.nodeType === 1 && parts.length < 10) {
|
||||
let s = el.tagName.toLowerCase();
|
||||
const p = el.parentNode;
|
||||
if (p) {
|
||||
const sibs = Array.from(p.children).filter((c) => c.tagName === el.tagName);
|
||||
if (sibs.length > 1) s += `:nth-of-type(${sibs.indexOf(el) + 1})`;
|
||||
}
|
||||
parts.unshift(s);
|
||||
el = el.parentNode;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
}
|
||||
|
||||
function targetsToSelectors(targets) {
|
||||
if (!targets) return [];
|
||||
if (targets instanceof Element) return [cssPath(targets)];
|
||||
if (Array.isArray(targets) || targets.length != null) {
|
||||
const out = [];
|
||||
for (let i = 0; i < targets.length; i++) {
|
||||
const t = targets[i];
|
||||
if (t instanceof Element) out.push(cssPath(t));
|
||||
else if (typeof t === 'string') out.push(t);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (typeof targets === 'string') return [targets];
|
||||
return [];
|
||||
}
|
||||
|
||||
function serializeEase(ease) {
|
||||
if (!ease) return null;
|
||||
if (typeof ease === 'string') return ease;
|
||||
if (typeof ease === 'function') return ease.name || ease.toString().slice(0, 40);
|
||||
return String(ease);
|
||||
}
|
||||
|
||||
function serializeTween(t) {
|
||||
const vars = {};
|
||||
if (t.vars) {
|
||||
for (const k of Object.keys(t.vars)) {
|
||||
if (['onComplete', 'onStart', 'onUpdate', 'onRepeat', 'scrollTrigger'].includes(k)) continue;
|
||||
try {
|
||||
const v = t.vars[k];
|
||||
if (typeof v === 'function') continue;
|
||||
if (v && typeof v === 'object' && v.nodeType) continue;
|
||||
vars[k] = v;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
return {
|
||||
type: 'tween',
|
||||
targets: targetsToSelectors(t._targets || t.targets?.()),
|
||||
duration: t.duration?.() ?? t._duration,
|
||||
ease: serializeEase(t.vars?.ease),
|
||||
vars,
|
||||
scrollTrigger: t.vars?.scrollTrigger
|
||||
? {
|
||||
trigger:
|
||||
t.vars.scrollTrigger.trigger instanceof Element
|
||||
? cssPath(t.vars.scrollTrigger.trigger)
|
||||
: t.vars.scrollTrigger.trigger,
|
||||
start: t.vars.scrollTrigger.start,
|
||||
end: t.vars.scrollTrigger.end,
|
||||
scrub: t.vars.scrollTrigger.scrub,
|
||||
pin: t.vars.scrollTrigger.pin,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function walk(tl) {
|
||||
const out = { type: 'timeline', duration: tl.duration?.(), children: [] };
|
||||
const kids = tl.getChildren ? tl.getChildren(true, true, true) : [];
|
||||
for (const c of kids) {
|
||||
if (c.getChildren) out.children.push(walk(c));
|
||||
else out.children.push(serializeTween(c));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const prevFinalize = window.__CLONE_FINALIZE__;
|
||||
window.__CLONE_FINALIZE__ = function () {
|
||||
if (prevFinalize)
|
||||
try {
|
||||
prevFinalize();
|
||||
} catch {}
|
||||
try {
|
||||
if (!window.gsap) return;
|
||||
const root = window.gsap.globalTimeline;
|
||||
if (!root) return;
|
||||
window.__CLONE_CAPTURE__.gsap.push(walk(root));
|
||||
} catch (e) {
|
||||
window.__CLONE_CAPTURE__.gsap.push({ error: String(e) });
|
||||
}
|
||||
};
|
||||
})();
|
||||
@@ -0,0 +1,47 @@
|
||||
// Pulls registered Lottie animations + their animationData into window.__CLONE_CAPTURE__.lottie.
|
||||
// Called from __CLONE_FINALIZE__. Supports lottie-web directly; lottie-react uses lottie-web internally.
|
||||
(() => {
|
||||
function cssPath(el) {
|
||||
if (!el || el.nodeType !== 1) return null;
|
||||
if (el.id) return `#${el.id}`;
|
||||
const parts = [];
|
||||
while (el && el.nodeType === 1 && parts.length < 10) {
|
||||
let s = el.tagName.toLowerCase();
|
||||
const p = el.parentNode;
|
||||
if (p) {
|
||||
const sibs = Array.from(p.children).filter((c) => c.tagName === el.tagName);
|
||||
if (sibs.length > 1) s += `:nth-of-type(${sibs.indexOf(el) + 1})`;
|
||||
}
|
||||
parts.unshift(s);
|
||||
el = el.parentNode;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
}
|
||||
|
||||
const prevFinalize = window.__CLONE_FINALIZE__;
|
||||
window.__CLONE_FINALIZE__ = function () {
|
||||
if (prevFinalize)
|
||||
try {
|
||||
prevFinalize();
|
||||
} catch {}
|
||||
try {
|
||||
const lottie = window.lottie || window.bodymovin;
|
||||
if (!lottie || typeof lottie.getRegisteredAnimations !== 'function') return;
|
||||
const anims = lottie.getRegisteredAnimations();
|
||||
for (const a of anims) {
|
||||
try {
|
||||
window.__CLONE_CAPTURE__.lottie.push({
|
||||
selector: a.wrapper instanceof Element ? cssPath(a.wrapper) : null,
|
||||
renderer: a.renderer?.name || 'unknown',
|
||||
animationData: a.animationData ? JSON.parse(JSON.stringify(a.animationData)) : null,
|
||||
loop: a.loop,
|
||||
autoplay: a.autoplay,
|
||||
name: a.name,
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
} catch (e) {
|
||||
window.__CLONE_CAPTURE__.lottie.push({ error: String(e) });
|
||||
}
|
||||
};
|
||||
})();
|
||||
@@ -0,0 +1,147 @@
|
||||
// Exposes window.__CLONE_LIST_SECTIONS__() which returns a list of candidate top-level
|
||||
// section bounding boxes for the capture script's scroll planner. We emit one entry per
|
||||
// large flex/block child of <body>, <main>, or any element classed as section/container/etc.
|
||||
// The capture script uses these to scroll-to-section instead of fixed-pixel stepping.
|
||||
(() => {
|
||||
const SECTIONLIKE_TAGS = new Set(['section', 'header', 'footer', 'nav', 'main', 'article', 'aside']);
|
||||
const SECTIONLIKE_CLASS_RE =
|
||||
/(^|\s)(section|hero|container|banner|elementor-section|elementor-element-[^\s]+|e-con|e-parent|wp-block-[^\s]+)(\s|$)/i;
|
||||
|
||||
function isViewportWide(el, vw) {
|
||||
const r = el.getBoundingClientRect();
|
||||
return r.width >= vw * 0.5 && r.height >= 80;
|
||||
}
|
||||
|
||||
function isVisible(el) {
|
||||
const cs = getComputedStyle(el);
|
||||
if (cs.display === 'none' || cs.visibility === 'hidden') return false;
|
||||
if (parseFloat(cs.opacity) === 0) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function looksLikeSection(el) {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (SECTIONLIKE_TAGS.has(tag)) return true;
|
||||
if (el.id && /(hero|banner|section|footer|nav|header)/i.test(el.id)) return true;
|
||||
if (typeof el.className === 'string' && SECTIONLIKE_CLASS_RE.test(el.className)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function cssPath(el) {
|
||||
if (!el || el.nodeType !== 1) return null;
|
||||
if (el.id) return `#${el.id}`;
|
||||
const parts = [];
|
||||
while (el && el.nodeType === 1 && parts.length < 8) {
|
||||
let s = el.tagName.toLowerCase();
|
||||
const p = el.parentNode;
|
||||
if (p && p.nodeType === 1) {
|
||||
const sibs = Array.from(p.children).filter((c) => c.tagName === el.tagName);
|
||||
if (sibs.length > 1) s += `:nth-of-type(${sibs.indexOf(el) + 1})`;
|
||||
}
|
||||
parts.unshift(s);
|
||||
el = el.parentNode;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
}
|
||||
|
||||
const SECTION_QUERY =
|
||||
'section, header, footer, nav, main, article, ' +
|
||||
'[class*="section"], [class*="elementor-element"], [class*="e-parent"], [class*="e-con"], ' +
|
||||
'[class*="hero"], [class*="banner"], [id*="section"], [id*="hero"], [id*="banner"]';
|
||||
|
||||
function snapshotEntry(el, vw) {
|
||||
const r = el.getBoundingClientRect();
|
||||
return {
|
||||
element: el,
|
||||
selector: cssPath(el),
|
||||
tag: el.tagName.toLowerCase(),
|
||||
id: el.id || null,
|
||||
className: typeof el.className === 'string' ? el.className.slice(0, 200) : null,
|
||||
x: r.x,
|
||||
y: r.y + window.scrollY,
|
||||
width: r.width,
|
||||
height: r.height,
|
||||
looksLikeSection: looksLikeSection(el),
|
||||
};
|
||||
}
|
||||
|
||||
function collectInScope(scope, vw, out, candidates) {
|
||||
for (const el of candidates) {
|
||||
if (!scope.contains(el) || el === scope) continue;
|
||||
if (!isVisible(el) || !isViewportWide(el, vw)) continue;
|
||||
// Outermost-wins inside this scope
|
||||
let skip = false;
|
||||
for (const o of out) {
|
||||
if (o.element.contains(el)) {
|
||||
skip = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (skip) continue;
|
||||
for (let i = out.length - 1; i >= 0; i--) {
|
||||
if (el.contains(out[i].element)) out.splice(i, 1);
|
||||
}
|
||||
out.push(snapshotEntry(el, vw));
|
||||
}
|
||||
}
|
||||
|
||||
window.__CLONE_LIST_SECTIONS__ = () => {
|
||||
const vw = window.innerWidth;
|
||||
const vh = window.innerHeight;
|
||||
const docHeight = Math.max(document.documentElement.scrollHeight, document.body ? document.body.scrollHeight : 0);
|
||||
|
||||
// Pass 1: known section-like elements
|
||||
const candidates = Array.from(document.querySelectorAll(SECTION_QUERY));
|
||||
|
||||
// Pass 2: large direct children of body/main as a fallback
|
||||
const root = document.querySelector('main') || document.body;
|
||||
if (root) {
|
||||
for (const child of root.children) {
|
||||
if (!candidates.includes(child)) candidates.push(child);
|
||||
}
|
||||
}
|
||||
|
||||
// First-pass dedup with outermost-wins
|
||||
const out = [];
|
||||
for (const el of candidates) {
|
||||
if (!isVisible(el) || !isViewportWide(el, vw)) continue;
|
||||
let skip = false;
|
||||
for (const o of out) {
|
||||
if (o.element.contains(el)) {
|
||||
skip = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (skip) continue;
|
||||
for (let i = out.length - 1; i >= 0; i--) {
|
||||
if (el.contains(out[i].element)) out.splice(i, 1);
|
||||
}
|
||||
out.push(snapshotEntry(el, vw));
|
||||
}
|
||||
|
||||
// Pass 3: oversized-wrapper expansion. WordPress / Elementor sites commonly
|
||||
// wrap the whole page in <main id="main">. Without this step the outer
|
||||
// wrapper wins and individual sections are dropped, so the smart-scroll
|
||||
// loop never visits them and lazy backgrounds never fire.
|
||||
const oversizedThreshold = Math.max(vh * 2, docHeight * 0.5);
|
||||
const expanded = [];
|
||||
for (const sec of out) {
|
||||
if (sec.height < oversizedThreshold) {
|
||||
expanded.push(sec);
|
||||
continue;
|
||||
}
|
||||
const inner = [];
|
||||
const innerCandidates = Array.from(sec.element.querySelectorAll(SECTION_QUERY));
|
||||
collectInScope(sec.element, vw, inner, innerCandidates);
|
||||
// Only replace the wrapper if recursion produced enough sections to be
|
||||
// meaningful — otherwise this is a genuine large section, not a wrapper.
|
||||
if (inner.length >= 3) expanded.push(...inner);
|
||||
else expanded.push(sec);
|
||||
}
|
||||
|
||||
return expanded
|
||||
.filter((s) => s.height > 80)
|
||||
.sort((a, b) => a.y - b.y)
|
||||
.map(({ element, ...rest }) => rest);
|
||||
};
|
||||
})();
|
||||
@@ -0,0 +1,97 @@
|
||||
// Extract GLSL from WebGL1 + WebGL2 by intercepting shaderSource / compileShader / linkProgram / useProgram.
|
||||
// Stashes into window.__CLONE_CAPTURE__.shaders as
|
||||
// { programId, vertex, fragment, attributes, uniforms, canvasSelector }
|
||||
(() => {
|
||||
const CAPTURE = () => window.__CLONE_CAPTURE__;
|
||||
|
||||
const shaderMap = new WeakMap(); // WebGLShader -> { type, source }
|
||||
const programMap = new WeakMap(); // WebGLProgram -> { vertex, fragment }
|
||||
let nextProgramId = 1;
|
||||
const contextCanvas = new WeakMap(); // WebGL(2)RenderingContext -> canvas
|
||||
|
||||
function patch(ctxProto, isGL2) {
|
||||
const origShaderSource = ctxProto.shaderSource;
|
||||
ctxProto.shaderSource = function (shader, source) {
|
||||
shaderMap.set(shader, { source, type: this.getShaderParameter?.(shader, this.SHADER_TYPE) });
|
||||
return origShaderSource.call(this, shader, source);
|
||||
};
|
||||
|
||||
const origAttach = ctxProto.attachShader;
|
||||
ctxProto.attachShader = function (program, shader) {
|
||||
const info = shaderMap.get(shader);
|
||||
const t = this.getShaderParameter(shader, this.SHADER_TYPE);
|
||||
const entry = programMap.get(program) || {};
|
||||
if (t === this.VERTEX_SHADER) entry.vertex = info?.source || '';
|
||||
else if (t === this.FRAGMENT_SHADER) entry.fragment = info?.source || '';
|
||||
programMap.set(program, entry);
|
||||
return origAttach.call(this, program, shader);
|
||||
};
|
||||
|
||||
const origLink = ctxProto.linkProgram;
|
||||
ctxProto.linkProgram = function (program) {
|
||||
const res = origLink.call(this, program);
|
||||
const entry = programMap.get(program) || {};
|
||||
if (!entry.id) entry.id = nextProgramId++;
|
||||
const canvas = contextCanvas.get(this);
|
||||
const sel = canvas ? cssPath(canvas) : null;
|
||||
const attributes = [];
|
||||
const uniforms = [];
|
||||
try {
|
||||
const aCount = this.getProgramParameter(program, this.ACTIVE_ATTRIBUTES);
|
||||
for (let i = 0; i < aCount; i++) {
|
||||
const info = this.getActiveAttrib(program, i);
|
||||
if (info) attributes.push({ name: info.name, type: info.type, size: info.size });
|
||||
}
|
||||
const uCount = this.getProgramParameter(program, this.ACTIVE_UNIFORMS);
|
||||
for (let i = 0; i < uCount; i++) {
|
||||
const info = this.getActiveUniform(program, i);
|
||||
if (info) uniforms.push({ name: info.name, type: info.type, size: info.size });
|
||||
}
|
||||
} catch {}
|
||||
CAPTURE().shaders.push({
|
||||
programId: entry.id,
|
||||
vertex: entry.vertex || '',
|
||||
fragment: entry.fragment || '',
|
||||
attributes,
|
||||
uniforms,
|
||||
canvasSelector: sel,
|
||||
isWebGL2: isGL2,
|
||||
});
|
||||
return res;
|
||||
};
|
||||
}
|
||||
|
||||
// Intercept getContext to remember which canvas owns which WebGL context.
|
||||
const origGetContext = HTMLCanvasElement.prototype.getContext;
|
||||
HTMLCanvasElement.prototype.getContext = function (type, ...rest) {
|
||||
const ctx = origGetContext.call(this, type, ...rest);
|
||||
if (ctx && (type === 'webgl' || type === 'webgl2' || type === 'experimental-webgl')) {
|
||||
contextCanvas.set(ctx, this);
|
||||
}
|
||||
return ctx;
|
||||
};
|
||||
|
||||
if (typeof WebGLRenderingContext !== 'undefined') patch(WebGLRenderingContext.prototype, false);
|
||||
if (typeof WebGL2RenderingContext !== 'undefined') patch(WebGL2RenderingContext.prototype, true);
|
||||
|
||||
function cssPath(el) {
|
||||
if (!el) return null;
|
||||
const parts = [];
|
||||
while (el && el.nodeType === 1 && parts.length < 10) {
|
||||
let s = el.tagName.toLowerCase();
|
||||
if (el.id) {
|
||||
s += '#' + el.id;
|
||||
parts.unshift(s);
|
||||
break;
|
||||
}
|
||||
const parent = el.parentNode;
|
||||
if (parent) {
|
||||
const sibs = Array.from(parent.children).filter((c) => c.tagName === el.tagName);
|
||||
if (sibs.length > 1) s += `:nth-of-type(${sibs.indexOf(el) + 1})`;
|
||||
}
|
||||
parts.unshift(s);
|
||||
el = el.parentNode;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,130 @@
|
||||
// Patches THREE.WebGLRenderer to track its canvas + scene root, then serializes the scene graph
|
||||
// at __CLONE_FINALIZE__. Records geometry types, material types + uniforms, lights, cameras,
|
||||
// and per-object position/rotation/scale.
|
||||
(() => {
|
||||
const renderers = new Set();
|
||||
const rendererToScene = new WeakMap();
|
||||
|
||||
function cssPath(el) {
|
||||
if (!el || el.nodeType !== 1) return null;
|
||||
if (el.id) return `#${el.id}`;
|
||||
const parts = [];
|
||||
while (el && el.nodeType === 1 && parts.length < 10) {
|
||||
let s = el.tagName.toLowerCase();
|
||||
const p = el.parentNode;
|
||||
if (p) {
|
||||
const sibs = Array.from(p.children).filter((c) => c.tagName === el.tagName);
|
||||
if (sibs.length > 1) s += `:nth-of-type(${sibs.indexOf(el) + 1})`;
|
||||
}
|
||||
parts.unshift(s);
|
||||
el = el.parentNode;
|
||||
}
|
||||
return parts.join(' > ');
|
||||
}
|
||||
|
||||
function patchWhenReady() {
|
||||
if (!window.THREE || !window.THREE.WebGLRenderer) return false;
|
||||
const WR = window.THREE.WebGLRenderer;
|
||||
const origRender = WR.prototype.render;
|
||||
WR.prototype.render = function (scene, camera) {
|
||||
renderers.add(this);
|
||||
if (scene) rendererToScene.set(this, { scene, camera });
|
||||
return origRender.call(this, scene, camera);
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!patchWhenReady()) {
|
||||
const timer = setInterval(() => {
|
||||
if (patchWhenReady()) clearInterval(timer);
|
||||
}, 200);
|
||||
setTimeout(() => clearInterval(timer), 10_000);
|
||||
}
|
||||
|
||||
function serializeObject(obj) {
|
||||
const out = {
|
||||
type: obj.type || obj.constructor?.name || 'Object3D',
|
||||
name: obj.name || undefined,
|
||||
position: obj.position ? [obj.position.x, obj.position.y, obj.position.z] : undefined,
|
||||
rotation: obj.rotation ? [obj.rotation.x, obj.rotation.y, obj.rotation.z] : undefined,
|
||||
scale: obj.scale ? [obj.scale.x, obj.scale.y, obj.scale.z] : undefined,
|
||||
visible: obj.visible,
|
||||
children: [],
|
||||
};
|
||||
if (obj.geometry) {
|
||||
out.geometry = {
|
||||
type: obj.geometry.type,
|
||||
parameters: obj.geometry.parameters ? { ...obj.geometry.parameters } : undefined,
|
||||
};
|
||||
}
|
||||
if (obj.material) {
|
||||
const m = Array.isArray(obj.material) ? obj.material[0] : obj.material;
|
||||
out.material = {
|
||||
type: m.type,
|
||||
color: m.color ? m.color.getHex?.().toString(16) : undefined,
|
||||
opacity: m.opacity,
|
||||
transparent: m.transparent,
|
||||
uniforms: m.uniforms ? summarizeUniforms(m.uniforms) : undefined,
|
||||
vertexShader: m.vertexShader?.slice(0, 4000),
|
||||
fragmentShader: m.fragmentShader?.slice(0, 4000),
|
||||
};
|
||||
}
|
||||
if (obj.isLight) {
|
||||
out.light = {
|
||||
type: obj.type,
|
||||
color: obj.color?.getHex?.()?.toString(16),
|
||||
intensity: obj.intensity,
|
||||
};
|
||||
}
|
||||
if (obj.isCamera) {
|
||||
out.camera = {
|
||||
type: obj.type,
|
||||
fov: obj.fov,
|
||||
aspect: obj.aspect,
|
||||
near: obj.near,
|
||||
far: obj.far,
|
||||
};
|
||||
}
|
||||
if (obj.children?.length) {
|
||||
out.children = obj.children.map(serializeObject);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function summarizeUniforms(u) {
|
||||
const out = {};
|
||||
for (const k of Object.keys(u)) {
|
||||
try {
|
||||
const v = u[k]?.value;
|
||||
if (v == null) continue;
|
||||
if (typeof v === 'number') out[k] = v;
|
||||
else if (Array.isArray(v)) out[k] = v.slice(0, 16);
|
||||
else if (v.x != null && v.y != null) out[k] = [v.x, v.y, v.z ?? 0, v.w ?? 0];
|
||||
else out[k] = String(v).slice(0, 80);
|
||||
} catch {}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const prevFinalize = window.__CLONE_FINALIZE__;
|
||||
window.__CLONE_FINALIZE__ = function () {
|
||||
if (prevFinalize)
|
||||
try {
|
||||
prevFinalize();
|
||||
} catch {}
|
||||
try {
|
||||
for (const r of renderers) {
|
||||
const { scene, camera } = rendererToScene.get(r) || {};
|
||||
const dom = r.domElement;
|
||||
window.__CLONE_CAPTURE__.threejs.push({
|
||||
canvasSelector: dom instanceof Element ? cssPath(dom) : null,
|
||||
size: dom ? [dom.width, dom.height] : null,
|
||||
scene: scene ? serializeObject(scene) : null,
|
||||
camera: camera ? serializeObject(camera) : null,
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
window.__CLONE_CAPTURE__.threejs.push({ error: String(e) });
|
||||
}
|
||||
};
|
||||
})();
|
||||
@@ -0,0 +1,7 @@
|
||||
playwright>=1.40
|
||||
httpx>=0.25
|
||||
tinycss2>=1.2
|
||||
pixelmatch>=0.3
|
||||
pillow>=10
|
||||
numpy>=1.26
|
||||
scipy>=1.11
|
||||
Reference in New Issue
Block a user