commit 66dfdcc58d1a69bcd05dc5d437afc2fd9222c784 Author: Samraaj Bath Date: Mon Jun 29 15:11:48 2026 -0700 Initial commit diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c4d8bff --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false + +[*.{json,yml,yaml}] +indent_size = 2 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e588ad7 --- /dev/null +++ b/.env.example @@ -0,0 +1,25 @@ +# ---- API ---- +PORT=8787 +# When DATABASE_URL is set the API runs in async (queue) mode; otherwise it runs +# clones inline in-memory (handy for a quick local single-page demo). +# CLONE_TTL_MS=1800000 # in-memory mode: how long results are retained + +# ---- Database + queue (Postgres; pg-boss uses the same DB) ---- +DATABASE_URL=postgresql://postgres:postgres@localhost:5432/ditto_site + +# ---- Worker ---- +ARTIFACTS_DIR=./local-data/artifacts +# Cache time-to-stale: how long a cached clone is served before re-capture. +# Duration units: ms/s/m/h/d. "0" disables caching. +CACHE_STALE_AFTER=24h +# VERIFY_TIER=stage2 # perceptual-gate tier when options.verify is set +# HARNESS_DIR= # per-worker Next build harness for verify (M5) + +# ---- Blob storage (S3 / R2 / MinIO) — used from M4 ---- +# S3_ENDPOINT=http://localhost:9000 # MinIO locally; omit for AWS S3 +# S3_REGION=auto +# S3_BUCKET=ditto-site-artifacts +# S3_ACCESS_KEY_ID=minioadmin +# S3_SECRET_ACCESS_KEY=minioadmin +# S3_FORCE_PATH_STYLE=true # required for MinIO +# S3_PUBLIC_URL= # optional CDN/base url for objects diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..bfa5f21 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,27 @@ +--- +name: Bug report +about: Report a problem with ditto.site CLI or service behavior +title: "" +labels: bug +--- + +**What happened** + + +**To reproduce** +- Surface: +- URL cloned (if applicable): +- Command or API/MCP call: +- Options (interactions/components/motion/multiPage/verify, …): + +**Expected behavior** + + +**Environment** +- OS + Node version (`node -v`): +- Commit / version: +- Chromium installed (`npx playwright install chromium`)? + +**Logs / artifacts** + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..61a5832 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Security vulnerability + url: https://github.com/ion-design/ditto.site/security/advisories/new + about: Please report security issues privately — do not open a public issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..58c1077 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,19 @@ +--- +name: Feature request +about: Suggest an improvement to ditto.site +title: "" +labels: enhancement +--- + +**Problem / motivation** + + +**Proposed solution** + + +**Alternatives considered** + + +**Scope notes** + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1db178f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: + # npm workspaces (single root lockfile). + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + groups: + dev-dependencies: + dependency-type: "development" + production-dependencies: + dependency-type: "production" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..f7fefac --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,23 @@ +## Summary + + + +## Changes + +- + +## Testing + +- [ ] `npm run typecheck` passes +- [ ] `npm test` passes +- [ ] Added/updated tests for the change + +## Compiler determinism + +- [ ] No change to the compiler's deterministic clone semantics, **or** the change + is intentional and validated against the benchmarks (`npm run bench`) with + results noted above. + +## Notes + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bba2f7e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: ditto_site + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + # DB/queue tests use this instead of an ephemeral instance. + TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/ditto_site + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - name: Install Chromium (for capture/browser-gated tests) + run: npx playwright install --with-deps chromium + - run: npm run typecheck + - run: npm test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..91699c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,52 @@ +# Dependencies +node_modules/ +compiler/node_modules/ + +# Build output +compiler/dist/ +*.tsbuildinfo + +# Run artifacts (immutable captures/generated apps/validation) — large, not committed +runs/ + +# Build harness transient inputs (deps reinstalled from package.json/lockfile) +compiler/.harness/src/ +compiler/.harness/public/ +compiler/.harness/out/ +compiler/.harness/.next/ +compiler/.harness/next.config.mjs +compiler/.harness/tsconfig.json +compiler/.harness/next-env.d.ts + +# Generated app build artifacts inside runs are covered by runs/, but ignore stray ones +.next/ +out/ + +# OS +.DS_Store + +# Logs +*.log +compiler/.harness2/src/ +compiler/.harness2/public/ +compiler/.harness2/out/ +compiler/.harness2/.next/ +compiler/.harness2/next.config.mjs +compiler/.harness2/tsconfig.json +compiler/.harness2/next-env.d.ts + +# Service layer (env + per-worker build harnesses + local artifacts) +.env +.env.* +!.env.example +packages/*/dist/ +packages/*/.turbo/ +.harnesses/ +local-data/ + +# Local clone outputs + capture bridge certs (this session) +compiler/output/ +compiler/output-final/ +compiler/.bridge/*.pem + +compiler/compiler/ diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..2bd5a0a --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +22 diff --git a/.opencode/.gitignore b/.opencode/.gitignore new file mode 100644 index 0000000..59df0c0 --- /dev/null +++ b/.opencode/.gitignore @@ -0,0 +1,3 @@ +node_modules +.clone-workspace +scripts/.venv diff --git a/.opencode/agents/clone-advanced.md b/.opencode/agents/clone-advanced.md new file mode 100644 index 0000000..9460070 --- /dev/null +++ b/.opencode/agents/clone-advanced.md @@ -0,0 +1,141 @@ +--- +description: Stage 4 only. Reconstructs Three.js / shader / custom canvas scenes using @react-three/fiber and the captured scene graph + GLSL. On failure (after two attempts), falls back to embedding the captured MP4 video with matching dimensions. Best-effort — fallbacks are a valid result, not a failure. +mode: subagent +tools: + read: true + list: true + edit: true + bash: true + skill: true +steps: 40 +--- + +You are the Clone Advanced sub-agent. + +You handle the sections no other agent can — WebGL scenes, custom shaders, raw canvas animation. You try twice to reconstruct cleanly. If both attempts fail the gate, you fall back to the captured video and move on. + +## Inputs + +```json +{ + "manifest_path": "/manifest.json", + "section_id": "hero-3d", + "capture_dir": "/capture", + "workspace_dir": "", + "attempt": 1 | 2 | "fallback" +} +``` + +Project root is always cwd. Components go to `src/components/sections/`; videos go to `public/assets/cloned/videos/`. + +## Flow + +### Attempt 1 & 2: Reconstruct + +1. Load the scene graph dump from `/threejs/.json`. Typical shape: + + ```json + { + "renderer": { "type": "WebGLRenderer", "size": [1920, 800] }, + "camera": { "type": "PerspectiveCamera", "fov": 50, "position": [0,0,5], ... }, + "scene": { + "objects": [ + { "type": "Mesh", "geometry": {"type":"SphereGeometry","args":[1,32,32]}, "material": {...}, "position": [...], ... } + ], + "lights": [...] + } + } + ``` + +2. Load GLSL from `/shaders/.json` (vertex + fragment source per program). + +3. Translate to `@react-three/fiber` JSX in a Client Component at `src/components/sections/.tsx`. Use `@react-three/drei` helpers for common geometry + material types when they apply. + +4. For custom shaders, use `` with the extracted vertex + fragment source and the captured uniforms. + +5. Wrap in `` with the captured size + DPR. Hide the `` until mounted on client to avoid hydration mismatch. + +6. Import into `src/app/page.tsx` at the correct order. + +### Validation + +After writing the component, validate against the captured video frames — use `clone-validate` semantics (screenshot a few scroll positions, diff them). The gate is `diff_pct < 15%` — looser than other stages because Three.js reconstructions are inherently approximate. + +### Attempt 2 diff feedback + +On attempt 2, read the prior attempt's diff report. Common failure modes and fixes: + +- Colors off → check tone mapping (`gl.toneMapping = ACESFilmicToneMapping`), color space (`gl.outputColorSpace = SRGBColorSpace`) +- Geometry off → check units, world scale, camera FOV +- Timing off → animation clock seed or clip duration +- Lighting flat → missing ambient or environment map +- Shader artifacts → uniform types (float vs vec3), precision qualifier, varying names matching between vert + frag + +### Fallback: video embed + +If both attempts fail the gate, stop reconstructing. Check for a captured MP4 at `/video/.mp4`. + +Copy it to `public/assets/cloned/videos/.mp4` and write a component: + +```tsx +export default function HeroScene() { + return ( +
/' }}> +
+ ); +} +``` + +If there is no captured video either, emit a transparent placeholder of the captured bounding box and flag the section to the orchestrator. Do not leave it broken. + +## Return + +On reconstruction success: + +```json +{ + "status": "success", + "mode": "reconstructed", + "section_id": "...", + "files_written": ["src/components/sections/HeroScene.tsx"] +} +``` + +On fallback: + +```json +{ + "status": "success", + "mode": "video_fallback", + "section_id": "...", + "files_written": ["src/components/sections/HeroScene.tsx", "public/assets/cloned/videos/.mp4"], + "notes": "Reconstruction attempted twice; diff % > 15% gate" +} +``` + +On total failure (no video to fall back to): + +```json +{ + "status": "skipped", + "mode": "placeholder", + "section_id": "...", + "notes": "No viable reconstruction or video capture" +} +``` + +## Rules + +- Two attempts, then fallback. Do not loop forever on WebGL. +- Never ship a broken scene. If the reconstruction looks wrong, fall back to video. +- Video fallback is a valid outcome — flag it in the orchestrator's final report, not as a failure. +- Do not touch other sections. Only write the files for this one section. diff --git a/.opencode/agents/clone-analyze.md b/.opencode/agents/clone-analyze.md new file mode 100644 index 0000000..c361fb5 --- /dev/null +++ b/.opencode/agents/clone-analyze.md @@ -0,0 +1,164 @@ +--- +description: Reads a capture bundle and produces manifest.json — the blueprint the generate agent reads to build the clone. Extracts design tokens, decomposes the page into sections with bounding boxes, identifies repeated component candidates, detects animation libraries, and classifies each section by the maximum stage required (pure CSS vs needs Framer vs needs WebGL). +mode: subagent +tools: + read: true + list: true + grep: true + bash: true + skill: true + edit: true +steps: 30 +--- + +You are the Clone Analyze sub-agent. + +You turn a raw capture bundle into a clean, machine-readable manifest that the generate agent can consume section-by-section. + +## Inputs + +- `capture_dir` — e.g. `.clone-workspace/-/capture` +- `workspace_dir` — e.g. `.clone-workspace/-` (manifest.json goes here) +- `source_url` — original URL + +## Read the spec + +Load `skills/site-manifest/SKILL.md` — it defines the exact schema for `manifest.json`. Do not drift from it. Downstream agents depend on the shape being stable. + +## Flow + +### 1. Read the capture + +- `meta.json` — high-level index (viewports, asset list, detected libs, vh_relative_count, asset_sources) +- `dom//.json` — DOM + computed styles per scroll step (richer walker, filters per-tag defaults, includes pseudo-elements) +- `dom-alt/1280-1080/step-00.json` — DOM at canonical width but alt height (used by capture to derive vh-flags; you don't usually need to read this directly) +- `screenshots//.png` — per-scroll-step viewport screenshots +- `section-shots//section-NN.png` — pre-cropped per-section screenshots; map these to manifest sections by index when you produce `screenshot_paths` +- `sections/.json` — `__CLONE_LIST_SECTIONS__` candidates with bbox + selector. Use these as scaffolding for section decomposition; the smarter scroll loop already used them so each candidate has a clean DOM dump +- `css-vars/*.json` — `:root` custom properties +- `css-rules/*.json` — every same-origin CSS rule + url() asset refs. Use this to recover source intent (`min-height: 100vh`, `width: 100%`, `aspect-ratio: 16/9`) that resolved computed styles erase +- `fonts/*.json` — `@font-face` declarations + resolved URLs +- `/vh-flags.json` (next to manifest.json) — pre-computed vh-relative element list. Cross-reference against your sections to populate `section.vh_relative` + `section.vh_value` + +Read the 1280px DOM first — that is the canonical desktop layout. Skim `sections/1280.json` next to anchor your section list. + +### 2. Extract design tokens + +Walk the computed-styles dump to derive a token set: + +- **Colors**: histogram all color values used (background, color, border, fill, stroke). Promote any value used 3+ times to a semantic name (`bg-primary`, `text-primary`, `accent-1`, ...). Also copy every `--*` custom property from `:root`. +- **Typography**: every `font-family` + `font-size` + `font-weight` + `line-height` + `letter-spacing` combination used on text nodes. Cluster into a type scale. +- **Spacing**: histogram of margin + padding + gap values. Identify the implicit scale (4/8/12/16... or whatever the site actually uses). +- **Radii**: `border-radius` values used. +- **Shadows**: `box-shadow` + `filter: drop-shadow()` values. +- **Breakpoints**: inspect the CSS for `@media` queries; the min-widths used are the breakpoints. + +### 3. Decompose into sections + +A "section" is a visually and semantically coherent top-level block of the page. Typical shape: nav, hero, logo bar, feature grid, testimonials, pricing, CTA banner, footer. + +For each section: + +- `id` — stable slug (`hero`, `feature-grid`, `testimonials-0`, ...) +- `name` — human-readable name +- `dom_path` — relative path to the captured DOM fragment +- `screenshot_paths` — per viewport, prefer `section-shots//section-NN.png` when the candidate index matches; else closest scroll step +- `section_anchor` — CSS selector for the section root (`#id` or `.class`). The validate agent passes this to `dom-diff --root-selector` +- `bounding_box` — `{x, y, width, height}` at 1280×720 capture viewport +- `vh_relative` — true if `vh-flags.json` flags this section's height as viewport-derived +- `vh_value` — implied vh percentage (e.g. 88 for ~88vh) when vh_relative is true +- `full_width` — true when the section spans the full viewport width AND the source CSS suggests full-bleed (use `w-full` not literal px) +- `max_stage_required` — see classification below + +### 4. Classify each section by max_stage_required + +| Content | max_stage_required | +| ---------------------------------------------------------------------------- | ------------------ | +| Static HTML + CSS, no transitions | 1 | +| CSS gradients, filters, keyframes, hover/focus | 2 | +| IntersectionObserver reveals, scroll-linked animations, Framer, Lottie, GSAP | 3 | +| Three.js / custom WebGL / raw canvas animation | 4 | + +Be conservative — it is better to defer a stage than to pull a section too far forward. + +If a section is mostly static but has one Lottie icon, mark `max_stage_required: 3` and note the Lottie in `detected_libs`. The generate agent will implement the static part in stages 1-2 and only wire up the Lottie in stage 3. + +### 5. Identify repeated patterns + +Within and across sections, look for DOM structures that repeat 3+ times with similar shape — feature cards, testimonial cards, logo tiles, nav items. For each, propose: + +```json +{ + "selector": "...", + "count": 6, + "candidate_component": { + "name": "FeatureCard", + "props": ["title", "description", "icon"] + } +} +``` + +Attach to the section where the pattern lives. If the pattern spans sections, attach it to the first. + +### 6. Detect animation libraries + +From the capture hook dumps: + +- `gsap/` non-empty → add `gsap` to `detected_libs` +- `framer/` non-empty → add `framer-motion` +- `lottie/` non-empty → add `lottie-react` +- `threejs/` non-empty → add `three` +- `shaders/` non-empty → flag sections containing the shader's canvas as stage 4 + +### 7. Flag third-party widgets + +Scan the HAR + DOM for known widget signatures: + +- Intercom: `widget.intercom.io`, `
` +- Drift: `js.driftt.com`, ` + + + +
+

flex chips (inline-block, control for layout)

+
+ Planning + Issues + Roadmaps + Insights +
+
+ +
Polish fixture · deterministic local test
+ + diff --git a/compiler/fixtures/robots.txt b/compiler/fixtures/robots.txt new file mode 100644 index 0000000..b27b184 --- /dev/null +++ b/compiler/fixtures/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Allow: / +Sitemap: /sitemap.xml diff --git a/compiler/fixtures/seo-icon.png b/compiler/fixtures/seo-icon.png new file mode 100644 index 0000000..1a3a506 Binary files /dev/null and b/compiler/fixtures/seo-icon.png differ diff --git a/compiler/fixtures/seo-rich.html b/compiler/fixtures/seo-rich.html new file mode 100644 index 0000000..486e52a --- /dev/null +++ b/compiler/fixtures/seo-rich.html @@ -0,0 +1,58 @@ + + + + + + SEO Rich Fixture + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

SEO Rich Fixture

+

This page exercises generator-level SEO inventory, metadata emission, icons, manifests, structured data, and llms.txt preservation.

+
+

Captured content

+

The generated llms fallback can summarize useful page and route content when the source does not expose its own llms.txt.

+
+
+ + diff --git a/compiler/fixtures/site.webmanifest b/compiler/fixtures/site.webmanifest new file mode 100644 index 0000000..a095e31 --- /dev/null +++ b/compiler/fixtures/site.webmanifest @@ -0,0 +1,19 @@ +{ + "name": "SEO Rich Fixture", + "short_name": "SEOFixture", + "icons": [ + { + "src": "/manifest-icon-192.png", + "sizes": "192x192", + "type": "image/png" + }, + { + "src": "/brand.svg", + "sizes": "any", + "type": "image/svg+xml" + } + ], + "theme_color": "#123456", + "background_color": "#ffffff", + "display": "standalone" +} diff --git a/compiler/fixtures/site/blog.html b/compiler/fixtures/site/blog.html new file mode 100644 index 0000000..a1cd422 --- /dev/null +++ b/compiler/fixtures/site/blog.html @@ -0,0 +1,75 @@ + + + + + + Acme — Blog + + + +
+ +
+
+

From the blog

+ +
+
© 2026 Acme, Inc. All rights reserved.
+ + diff --git a/compiler/fixtures/site/faq.html b/compiler/fixtures/site/faq.html new file mode 100644 index 0000000..8df3120 --- /dev/null +++ b/compiler/fixtures/site/faq.html @@ -0,0 +1,55 @@ + + + + + + Acme — FAQ + + + +
+ +
+
+

Frequently asked questions

+
+
+ +

Sign up for the Starter plan and follow the onboarding guide.

+
+
+ + +
+
+ + +
+
+
+
© 2026 Acme, Inc. All rights reserved.
+ + + diff --git a/compiler/fixtures/site/index.html b/compiler/fixtures/site/index.html new file mode 100644 index 0000000..38128ca --- /dev/null +++ b/compiler/fixtures/site/index.html @@ -0,0 +1,71 @@ + + + + + + Acme — Home + + + +
+ +
+
+

Build faster with Acme

+

Everything your team needs to ship, in one place.

+
+
+
+

Fast by default

+

Sensible defaults and zero-config builds get you to production quickly.

+
+
+
+

Collaborative

+

Share previews and gather feedback without leaving your editor.

+
+
+
+

Secure

+

SSO, audit logs, and granular permissions for every workspace.

+
+
+
+

Observable

+

Built-in metrics and tracing so you always know what shipped.

+
+
+
+

Extensible

+

A plugin API and webhooks connect Acme to the rest of your stack.

+
+
+
+

Supported

+

Priority support and a responsive community keep you unblocked.

+
+
+
+
© 2026 Acme, Inc. All rights reserved.
+ + diff --git a/compiler/fixtures/sitemap.xml b/compiler/fixtures/sitemap.xml new file mode 100644 index 0000000..dd11246 --- /dev/null +++ b/compiler/fixtures/sitemap.xml @@ -0,0 +1,4 @@ + + + /seo-rich.html + diff --git a/compiler/fixtures/twitter-image.png b/compiler/fixtures/twitter-image.png new file mode 100644 index 0000000..fed7cb9 Binary files /dev/null and b/compiler/fixtures/twitter-image.png differ diff --git a/compiler/package-lock.json b/compiler/package-lock.json new file mode 100644 index 0000000..ad5ac31 --- /dev/null +++ b/compiler/package-lock.json @@ -0,0 +1,670 @@ +{ + "name": "clone-static", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "clone-static", + "version": "0.1.0", + "dependencies": { + "pixelmatch": "5.3.0", + "playwright": "1.56.1", + "pngjs": "7.0.0" + }, + "bin": { + "clone-static": "src/cli.ts" + }, + "devDependencies": { + "@types/node": "22.10.5", + "@types/pixelmatch": "5.2.6", + "@types/pngjs": "6.0.5", + "tsx": "4.22.4", + "typescript": "5.7.3" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "22.10.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.5.tgz", + "integrity": "sha512-F8Q+SeGimwOo86fiovQh8qiXfFEh2/ocYv7tU5pJ3EXMSSxk1Joj5wefpFK2fHTf/N6HKGSxIDBT9f3gCxXPkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.20.0" + } + }, + "node_modules/@types/pixelmatch": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", + "integrity": "sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/pngjs": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/@types/pngjs/-/pngjs-6.0.5.tgz", + "integrity": "sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/pixelmatch": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.3.0.tgz", + "integrity": "sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==", + "license": "ISC", + "dependencies": { + "pngjs": "^6.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, + "node_modules/pixelmatch/node_modules/pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "license": "MIT", + "engines": { + "node": ">=12.13.0" + } + }, + "node_modules/playwright": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.56.1.tgz", + "integrity": "sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.56.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.56.1.tgz", + "integrity": "sha512-hutraynyn31F+Bifme+Ps9Vq59hKuUCz7H1kDOcBs+2oGguKkWTU50bBWrtz34OUWmIwpBTWDxaRPXrIXkgvmQ==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "license": "MIT", + "engines": { + "node": ">=14.19.0" + } + }, + "node_modules/tsx": { + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz", + "integrity": "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/typescript": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz", + "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/compiler/package.json b/compiler/package.json new file mode 100644 index 0000000..093c9e1 --- /dev/null +++ b/compiler/package.json @@ -0,0 +1,38 @@ +{ + "name": "clone-static", + "version": "0.1.0", + "private": true, + "type": "module", + "bin": { + "clone-static": "./src/cli.ts" + }, + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + }, + "scripts": { + "clone": "tsx src/cli.ts", + "clone-site": "tsx src/site/cloneSite.ts", + "validate-site": "tsx src/site/validateSite.ts", + "bench": "tsx src/runner/benchmark.ts", + "bench-site": "tsx src/runner/siteBenchmark.ts", + "validate-one": "tsx src/runner/validateOne.ts", + "quality": "tsx src/runner/qualityScore.ts", + "audit": "tsx src/runner/codeAudit.ts", + "regen": "tsx src/runner/regen.ts", + "test": "node --import tsx --test test/*.test.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "playwright": "1.56.1", + "pixelmatch": "5.3.0", + "pngjs": "7.0.0" + }, + "devDependencies": { + "tsx": "4.22.4", + "typescript": "5.7.3", + "@types/node": "22.10.5", + "@types/pixelmatch": "5.2.6", + "@types/pngjs": "6.0.5" + } +} diff --git a/compiler/scripts/analyze-dense.py b/compiler/scripts/analyze-dense.py new file mode 100644 index 0000000..84b3ef3 --- /dev/null +++ b/compiler/scripts/analyze-dense.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Classify width-varying nodes in a DENSE capture by their sizing law, using the full +sample curve. Run after a dense clone: python3 scripts/analyze-dense.py """ +import json, sys, statistics, glob, os +run = sys.argv[1] if len(sys.argv) > 1 else "output-dense/sample/.clone" +caps = {} +for f in sorted(glob.glob(os.path.join(run, "source/capture/dom-*.json"))): + vp = int(os.path.basename(f)[4:-5]) + caps[vp] = json.load(open(f))["root"] +VPS = sorted(caps) +print("sample widths:", VPS) + +def flat(n, a): + if isinstance(n, dict): + a.append(n) + for c in (n.get("children") or []): flat(c, a) +F = {vp: [] for vp in VPS} +for vp in VPS: flat(caps[vp], F[vp]) +def wp(n, p, m): + m[id(n)] = p + for c in (n.get("children") or []): wp(c, n, m) +PM = {vp: {} for vp in VPS} +for vp in VPS: wp(caps[vp], None, PM[vp]) +def pf(v): + try: return float(str(v).replace("px", "")) + except: return 0.0 + +n = min(len(F[vp]) for vp in VPS) +cats = {"fills_container": 0, "prop_container": 0, "prop_viewport": 0, "clamped_maxw": 0, + "shrink_recoverable": 0, "real_breakpoint": 0, "fixed": 0, "unknown": 0} +for i in range(n): + nd = {vp: F[vp][i] for vp in VPS} + if not all(nd[vp].get("visible") for vp in VPS): continue + w = {vp: (nd[vp].get("bbox") or {}).get("width", 0) for vp in VPS} + if max(w.values()) - min(w.values()) <= 2: continue # constant → no band + # container content width per vp + cw = {} + for vp in VPS: + p = PM[vp].get(id(nd[vp])); pb = (p or {}).get("bbox"); pcs = (p or {}).get("computed") or {} + base = pb["width"] if pb else vp + cw[vp] = base - pf(pcs.get("paddingLeft")) - pf(pcs.get("paddingRight")) + rc = [w[vp] / cw[vp] for vp in VPS if cw[vp] > 0] + rv = [w[vp] / vp for vp in VPS] + def cv(xs): + m = statistics.mean(xs); return statistics.pstdev(xs) / m if m else 9 + wl = [w[vp] for vp in VPS] + if rc and cv(rc) < 0.02 and abs(statistics.mean(rc) - 1) < 0.02: cats["fills_container"] += 1 + elif rc and cv(rc) < 0.03: cats["prop_container"] += 1 + elif cv(rv) < 0.03: cats["prop_viewport"] += 1 + elif wl[-1] == wl[-2] and wl[0] < wl[-1]: # plateau at the widest → clamp or recoverable shrink + # constant above a knee, smaller below + cats["clamped_maxw"] += 1 + elif wl[-1] < wl[-2]: # still shrinking even at the widest → natural beyond range + cats["shrink_recoverable"] += 1 + else: + # piecewise? count distinct plateaus + cats["unknown"] += 1 +print("\nwidth-varying nodes by inferred sizing law (dense):") +for k, v in cats.items(): print(f" {k:20} {v}") +print(f" TOTAL varying: {sum(cats.values())}") diff --git a/compiler/scripts/classify-width-bands.py b/compiler/scripts/classify-width-bands.py new file mode 100644 index 0000000..da83f9a --- /dev/null +++ b/compiler/scripts/classify-width-bands.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""For every node whose width VARIES across viewports (=> would emit a width band), +classify the underlying law: hairline / parent-fill / fraction-of-parent / step. +Tells us how many bands each recovery law would eliminate.""" +import json, glob, os, sys, collections +run = sys.argv[1] if len(sys.argv) > 1 else "compiler/output/sample/.clone" +caps = {} +for f in sorted(glob.glob(os.path.join(run, "source/capture/dom-*.json"))): + caps[int(os.path.basename(f)[4:-5])] = json.load(open(f))["root"] +VPS = sorted(caps) +# flatten each viewport in pre-order, aligned by index +F = {vp: [] for vp in VPS} +PARENT = [] +def flat(n, a, parent_idx, store_parent): + idx = len(a); a.append(n) + if store_parent: PARENT.append(parent_idx) + for c in (n.get("children") or []): + flat(c, a, idx, store_parent) +for vp in VPS: + flat(caps[vp], F[vp], -1, vp == VPS[0]) +N = len(F[VPS[0]]) +def pf(v): + try: return float(str(v).replace("px","")) + except: return 0.0 +def cs(i, vp): return F[vp][i].get("computed") or {} +def bb(i, vp): return F[vp][i].get("bbox") or {} +def vis(i, vp): + if not F[vp][i].get("visible"): return False + if (cs(i,vp).get("display")=="none"): return False + return True +def width(i, vp): return bb(i, vp).get("width", 0) +def parent_content_w(i, vp): + p = PARENT[i] + if p < 0: return bb(i,vp).get("width",0) + pc = cs(p, vp) + return bb(p,vp).get("width",0) - pf(pc.get("paddingLeft")) - pf(pc.get("paddingRight")) - pf(pc.get("borderLeftWidth")) - pf(pc.get("borderRightWidth")) + +FRACTIONS = {1/2:"1/2",1/3:"1/3",2/3:"2/3",1/4:"1/4",3/4:"3/4",1/5:"1/5",2/5:"2/5",3/5:"3/5",4/5:"4/5",1/6:"1/6",5/6:"5/6"} +cat = collections.Counter() +band_cat = collections.Counter() +examples = collections.defaultdict(list) +for i in range(N): + ws = {vp: width(i,vp) for vp in VPS if vis(i,vp)} + if len(ws) < 2: continue + wv = list(ws.values()) + spread = max(wv) - min(wv) + if spread <= 1.0: continue # constant => no band + # how many band variants would this node emit? (# distinct rounded widths across the 4 band vps) - 1 + distinct = len(set(round(w) for w in wv)) + bands = distinct - 1 + if bands < 1: continue + # --- classify --- + if max(wv) < 2.0: + c = "hairline(<2px)" + else: + ratios = [] + ok_parent = True + for vp, w in ws.items(): + pcw = parent_content_w(i, vp) + if pcw <= 0: ok_parent = False; break + ratios.append(w / pcw) + if not ok_parent: + c = "no-parent" + elif max(ratios) - min(ratios) < 0.03 and abs(sum(ratios)/len(ratios) - 1.0) < 0.03: + c = "parent-fill(100%)" + else: + avg = sum(ratios)/len(ratios) + # nearest simple fraction + best = min(FRACTIONS, key=lambda f: abs(f-avg)) + if max(ratios)-min(ratios) < 0.03 and abs(best-avg) < 0.015: + c = f"fraction~{FRACTIONS[best]}" + elif max(ratios)-min(ratios) < 0.04: + c = "const-%(other)" + else: + c = "step/other" + cat[c] += 1 + band_cat[c] += bands + if len(examples[c]) < 4: + examples[c].append((round(min(wv),1), round(max(wv),1), [round(width(i,vp)) for vp in VPS])) + +print(f"nodes with width-bands: {sum(cat.values())} total band-variants: {sum(band_cat.values())}") +print(f"{'category':22} {'nodes':>6} {'bands':>6}") +for c,_ in band_cat.most_common(): + print(f"{c:22} {cat[c]:>6} {band_cat[c]:>6} eg {examples[c][:3]}") diff --git a/compiler/scripts/diag-flex.py b/compiler/scripts/diag-flex.py new file mode 100644 index 0000000..0bfb2d4 --- /dev/null +++ b/compiler/scripts/diag-flex.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Replicate the (updated) reconstructFlexRow and report success + bail reasons + band impact.""" +import json, glob, os, sys, collections +run = sys.argv[1] if len(sys.argv) > 1 else "compiler/output-dense/sample/.clone" +caps = {} +for f in sorted(glob.glob(os.path.join(run, "source/capture/dom-*.json"))): + caps[int(os.path.basename(f)[4:-5])] = json.load(open(f))["root"] +VPS = sorted(caps); BAND_VPS = [v for v in (375, 768, 1280, 1920) if v in VPS] +acc = [] +def walkidx(n): + n["_i"] = len(acc); acc.append(n); n["_kids"] = [] + for c in (n.get("children") or []): + n["_kids"].append(len(acc)); walkidx(c) +walkidx(caps[VPS[0]]) +N = len(acc) +# per-vp flat lists aligned by index +F = {vp: [] for vp in VPS} +def flat(n, a): + a.append(n) + for c in (n.get("children") or []): flat(c, a) +for vp in VPS: flat(caps[vp], F[vp]) +def pf(v): + try: return float(str(v).replace("px", "")) + except: return 0.0 +def CS(i, vp): return F[vp][i].get("computed") or {} +def BB(i, vp): return F[vp][i].get("bbox") or {} +def VIS(i, vp): return F[vp][i].get("visible") +reasons = collections.Counter(); succ = 0; succ_items = 0; bands_removed = 0 +for i in range(N): + c0 = CS(i, VPS[0]); disp = c0.get("display", "") + if "flex" not in disp: continue + if c0.get("flexDirection", "row") not in ("row", "row-reverse"): reasons["not_row"] += 1; continue + if (c0.get("flexWrap") or "nowrap") != "nowrap": reasons["wrap"] += 1; continue + info = []; bail = None + for k in acc[i]["_kids"]: + if not acc[k].get("tag"): continue + ck = CS(k, VPS[0]); pos = ck.get("position", "static") + if pos in ("absolute", "fixed"): continue + if (ck.get("float") or "none") != "none": continue + if ck.get("marginLeft") == "auto" or ck.get("marginRight") == "auto": bail = "auto_margin"; break + if (ck.get("boxSizing") or "border-box") == "content-box": bail = "content_box"; break + if pf(ck.get("flexGrow")) > 0: bail = "grow"; break + basis = ck.get("flexBasis") or "auto" + if basis != "auto" and not basis.endswith("px"): bail = "basis_pct"; break + info.append((k, pf(ck.get("flexShrink")), pf(basis) if basis.endswith("px") else None, + pf(ck.get("minWidth")) if (ck.get("minWidth") or "").endswith("px") else 0, + pf(ck.get("maxWidth")) if (ck.get("maxWidth") or "").endswith("px") else float("inf"))) + if bail: reasons[bail] += 1; continue + if not info: continue + used = {}; margin = {}; cw = {}; gap = {} + for vp in VPS: + pcs = CS(i, vp); pb = BB(i, vp) + cw[vp] = pb.get("width", 0) - pf(pcs.get("paddingLeft")) - pf(pcs.get("paddingRight")) + gap[vp] = pf(pcs.get("columnGap") if pcs.get("columnGap") not in (None, "normal") else pcs.get("gap")) + u = []; m = [] + for (k, sh, bp, mn, mx) in info: + ck = CS(k, vp); hidden = (not VIS(k, vp)) or ck.get("display") == "none" + u.append(None if hidden else BB(k, vp).get("width", 0)); m.append(0 if hidden else pf(ck.get("marginLeft")) + pf(ck.get("marginRight"))) + used[vp] = u; margin[vp] = m + def slack(vp): + vis = [j for j in range(len(info)) if used[vp][j] is not None] + return cw[vp] - sum(used[vp][j] for j in vis) - (gap[vp] * max(0, len(vis) - 1) + sum(margin[vp][j] for j in vis)) + bases = []; ok = True + for j, (k, sh, bp, mn, mx) in enumerate(info): + if bp is not None: bases.append(bp); continue + base = None + for v in range(len(VPS) - 1, -1, -1): + vp = VPS[v] + if used[vp][j] is None: continue + if slack(vp) >= -0.5: base = used[vp][j]; break + if base is None: ok = False; break + bases.append(base) + if not ok: reasons["no_base"] += 1; continue + bad = False + for vp in VPS: + vis = [j for j in range(len(info)) if used[vp][j] is not None] + if not vis: continue + hyp = [min(max(bases[j], info[j][3]), info[j][4]) for j in vis] + gaps = gap[vp] * max(0, len(vis) - 1) + sum(margin[vp][j] for j in vis) + free = cw[vp] - sum(hyp) - gaps + sim = list(hyp) + if free < -0.5: + ts = sum(info[vis[k]][1] * hyp[k] for k in range(len(vis))) + if ts > 0: + for k in range(len(vis)): sim[k] = max(hyp[k] - abs(free) * (info[vis[k]][1] * hyp[k]) / ts, info[vis[k]][3]) + for k in range(len(vis)): + if abs(sim[k] - used[vp][vis[k]]) > 1: bad = True + if bad: reasons["sim_mismatch"] += 1; continue + varies = [j for j in range(len(info)) if len([used[vp][j] for vp in VPS if used[vp][j] is not None]) >= 2 and max(x for vp in VPS if (x:=used[vp][j]) is not None) - min(x for vp in VPS if (x:=used[vp][j]) is not None) > 8] + if not varies: reasons["no_variation"] += 1; continue + succ += 1; succ_items += len(info) + # estimate band variants removed: for each varying item, count distinct band-vp widths != base-vp + for j in varies: + bandw = set(round(used[vp][j]) for vp in BAND_VPS if used[vp][j] is not None) + bands_removed += max(0, len(bandw) - 1) +print(f"SUCCESS lines: {succ} items: {succ_items} est. width-band variants removed: ~{bands_removed}") +for k, v in reasons.most_common(): print(f" bail {k:16} {v}") diff --git a/compiler/scripts/diag-hydration.mjs b/compiler/scripts/diag-hydration.mjs new file mode 100644 index 0000000..9729b53 --- /dev/null +++ b/compiler/scripts/diag-hydration.mjs @@ -0,0 +1,43 @@ +// Run a generated app in Next DEV mode (unminified React) and print hydration errors. +import { spawn } from "node:child_process"; +import { cpSync, rmSync, existsSync } from "node:fs"; +import { chromium } from "playwright"; + +const appDir = process.argv[2]; +const harness = new URL("../.harness", import.meta.url).pathname; +const PORT = 3987; + +for (const s of ["src", "public", "next.config.mjs", "tsconfig.json", "next-env.d.ts", ".next", "out"]) { + const p = `${harness}/${s}`; + if (existsSync(p)) rmSync(p, { recursive: true, force: true }); +} +cpSync(`${appDir}/src`, `${harness}/src`, { recursive: true }); +if (existsSync(`${appDir}/public`)) cpSync(`${appDir}/public`, `${harness}/public`, { recursive: true }); +for (const f of ["next.config.mjs", "tsconfig.json", "next-env.d.ts"]) { + if (existsSync(`${appDir}/${f}`)) cpSync(`${appDir}/${f}`, `${harness}/${f}`); +} + +const dev = spawn("./node_modules/.bin/next", ["dev", "-p", String(PORT)], { + cwd: harness, env: { ...process.env, NEXT_TELEMETRY_DISABLED: "1" }, +}); +let ready = false; +dev.stdout.on("data", (d) => { if (/Ready in|Local:|started server/i.test(d.toString())) ready = true; }); +dev.stderr.on("data", () => {}); + +const t0 = Date.now(); +while (!ready && Date.now() - t0 < 90000) await new Promise((r) => setTimeout(r, 500)); +await new Promise((r) => setTimeout(r, 4000)); + +const browser = await chromium.launch(); +const page = await browser.newPage(); +const msgs = []; +page.on("console", (m) => { msgs.push(m.text()); }); +page.on("pageerror", (e) => msgs.push("PAGEERROR: " + e.message)); +try { await page.goto(`http://localhost:${PORT}/`, { waitUntil: "networkidle", timeout: 60000 }); } catch (e) { msgs.push("goto: " + e.message); } +await new Promise((r) => setTimeout(r, 3000)); + +console.log("=== ALL CONSOLE MESSAGES (" + msgs.length + ") ==="); +for (const e of msgs.slice(0, 12)) console.log("\n---\n" + e.slice(0, 1500)); +await browser.close(); +dev.kill("SIGKILL"); +process.exit(0); diff --git a/compiler/scripts/gen_results.mjs b/compiler/scripts/gen_results.mjs new file mode 100644 index 0000000..7b8efa4 --- /dev/null +++ b/compiler/scripts/gen_results.mjs @@ -0,0 +1,50 @@ +// Generate examples//RESULTS.md from the latest run per site. +import { readFileSync, readdirSync, existsSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const ROOT = new URL("../..", import.meta.url).pathname; +const RUNS = join(ROOT, "runs"); +const tier = process.argv[2] || "stage2"; +const sites = JSON.parse(readFileSync(join(ROOT, "compiler", "benchmarks", `${tier}.json`), "utf8")); + +function siteId(url) { + const u = new URL(url); + const host = u.hostname.replace(/^www\./, ""); + const path = u.pathname.replace(/\/+$/, "").replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-|-$/g, ""); + return (host + (path ? "-" + path : "")).replace(/[^a-zA-Z0-9.-]/g, "-").slice(0, 80); +} +function latest(host) { + const dir = join(RUNS, host); + if (!existsSync(dir)) return null; + const runs = readdirSync(dir).filter((d) => /^\d/.test(d)).sort(); + for (let i = runs.length - 1; i >= 0; i--) { + if (existsSync(join(dir, runs[i], "validation", "report.json"))) return join(dir, runs[i]); + } + return null; +} + +let g06 = 0, s2 = 0, sum = 0, n = 0; +const rows = []; +for (const s of sites) { + const run = latest(siteId(s.url)); + if (!run) { rows.push(`| ${s.id} | ${s.url} | — | — | no run |`); continue; } + const r = JSON.parse(readFileSync(join(run, "validation", "report.json"), "utf8")); + n++; sum += r.scorecard.total; if (r.gates0to6Pass) g06++; if (r.stage2Pass) s2++; + const fails = Object.entries(r.gates).filter(([, gg]) => !gg.pass).map(([k]) => k).join(","); + const host = new URL(s.url).hostname.replace(/^www\./, "") + new URL(s.url).pathname.replace(/\/$/, ""); + rows.push(`| ${s.id} | ${host} | ${r.scorecard.total} | ${r.gates0to6Pass ? "PASS" : "FAIL"} | ${r.stage2Pass ? "PASS" : "FAIL"} | ${fails || ""} |`); +} +const avg = Math.round((sum / n) * 10) / 10; +const md = `# Stage 2 — results (capture-state correctness) + +**${g06}/${n} pass gates 0-6; ${s2}/${n} pass the stricter stage-2 bar** (gates 0-6 + non-degenerate capture + perceptually-close render), average ${avg}. +Stage 2 = popup/video/animation pages where the captured frame must be the settled, unobstructed state. Stage-2 gates: **pollution** (degenerate/wall/blocking-modal) and **perceptual** (tier-thresholded screenshot diff). + +| id | site | score | gates0-6 | stage2 | failing | +|----|------|------:|:--------:|:------:|---------| +${rows.join("\n")} + +Documented residuals (limitations, not defects): wix (heavy-JS site-builder — custom-element carousel positioned by script we don't run); warbyparker/squarespace (dynamic/autoplay hero — perceptual-only, frame-to-frame non-deterministic). +`; +writeFileSync(join(ROOT, "examples", tier, "RESULTS.md"), md); +console.log(`wrote examples/${tier}/RESULTS.md — gates0-6 ${g06}/${n}, stage2 ${s2}/${n}, avg ${avg}`); diff --git a/compiler/scripts/human-tells.py b/compiler/scripts/human-tells.py new file mode 100644 index 0000000..e61525a --- /dev/null +++ b/compiler/scripts/human-tells.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Human-tell diagnostic: count patterns a human would (almost) never hand-write. + +Separate from codeAudit.ts — this is the "would a human write this?" lens, focused on +specific robotic constructs, scanned over the .tsx/.css of each tree. Prints a column per tree. +Usage: human-tells.py ... +""" +import re, sys, os + +def read_tree(d): + tsx, css = [], [] + for root, dirs, files in os.walk(d): + dirs[:] = [x for x in dirs if x not in ("node_modules", ".next", "out") and not x.startswith(".")] + for f in files: + p = os.path.join(root, f) + try: + s = open(p, encoding="utf8").read() + except Exception: + continue + if f.endswith((".tsx", ".jsx", ".ts")): + tsx.append(s) + elif f.endswith(".css"): + css.append(s) + return "\n".join(tsx), "\n".join(css) + +def count(pat, s): + return len(re.findall(pat, s)) + +TELLS = { + # noise / redundancy — almost never hand-written + "noop relative inset (top-0 right-0 bottom-0 left-0)": (lambda tsx, css: count(r'\btop-0 right-0 bottom-0 left-0\b', tsx), True), + "redundant gap+gap-x+gap-y (same n)": (lambda tsx, css: len([m for m in re.finditer(r'\bgap-(\d+(?:\.\d+)?) gap-y-(\d+(?:\.\d+)?) gap-x-(\d+(?:\.\d+)?)', tsx) if m.group(1)==m.group(2)==m.group(3)]), True), + "4-side longhand pad (pt-x pr-y pb-x pl-y)": (lambda tsx, css: count(r'\bpt-\S+ pr-\S+ pb-\S+ pl-\S+', tsx), True), + "per-side border arbitrary [border-*]": (lambda tsx, css: count(r'\[border-(?:top|right|bottom|left)-(?:style|color|width):', tsx+css), True), + "baked runtime id (radix/_R_/:r)": (lambda tsx, css: count(r'id=\"(?:radix-|[^\"]*_R_|:r)', tsx), True), + "[text-align:inherit] override": (lambda tsx, css: count(r'\[text-align:inherit\]', tsx), True), + "data-cid shipped (jsx)": (lambda tsx, css: count(r'data-cid', tsx), True), + "data-cid selectors in ditto.css": (lambda tsx, css: count(r'data-cid', css), True), + # font-size arbitrary that maps to a named tailwind size (should be text-xs..text-9xl) + "font-size arbitrary mappable to named": (lambda tsx, css: len([m for m in re.finditer(r'text-\[(\d+(?:\.\d+)?)(px|rem)\]', tsx) if round((float(m.group(1))*(16 if m.group(2)=='rem' else 1)))/16 in NAMED_REM]), True), + # decimal arbitrary in JSX classes (sub-pixel frozen measurement) + "decimal arbitrary [N.NNpx/rem] (jsx)": (lambda tsx, css: len([m for m in re.finditer(r'\[(-?\d+\.?\d*)(px|rem)\]', tsx) if abs((v:=float(m.group(1))*(16 if m.group(2)=='rem' else 1))-round(v))>0.02]), True), + # decimal px in ditto.css (uncounted by codeAudit) + "decimal px in ditto.css (Npx)": (lambda tsx, css: len([m for m in re.finditer(r'(-?\d+\.\d+)px', css) if abs((v:=float(m.group(1)))-round(v))>0.02]), True), + "pseudo-element rules in ditto.css": (lambda tsx, css: count(r'::(?:before|after)\b', css), True), +} +# named tailwind text sizes in rem +NAMED_REM = {0.75, 0.875, 1.0, 1.125, 1.25, 1.5, 1.875, 2.25, 3.0, 3.75, 4.5, 6.0, 8.0} + +def main(): + trees = [] + for a in sys.argv[1:]: + label, d = a.split("=", 1) + trees.append((label, read_tree(d))) + w0 = max(len(k) for k in TELLS) + cw = max(11, *(len(l) for l, _ in trees)) + print("\n" + "metric".ljust(w0) + " " + "".join(l.ljust(cw) for l, _ in trees)) + print("-" * (w0 + 2 + cw * len(trees))) + for k, (fn, _) in TELLS.items(): + cells = "".join(str(fn(tsx, css)).ljust(cw) for _, (tsx, css) in trees) + print(k.ljust(w0) + " " + cells) + print() + +main() diff --git a/compiler/scripts/make_compare.py b/compiler/scripts/make_compare.py new file mode 100644 index 0000000..ea324e0 --- /dev/null +++ b/compiler/scripts/make_compare.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Build before/after (source vs clone) screenshot composites for a tier. + +For each site: a single PNG with a desktop row (1280px) and a mobile row (375px), +each showing SOURCE on the left and CLONE on the right, with labels. +""" +import json, os, sys, glob, re +from PIL import Image, ImageDraw, ImageFont + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +RUNS = os.path.join(ROOT, "runs") +TIER = sys.argv[1] if len(sys.argv) > 1 else "easy" +OUT = os.path.join(RUNS, f"compare-{TIER}") +os.makedirs(OUT, exist_ok=True) + +def font(sz): + for p in ["/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"]: + if os.path.exists(p): + return ImageFont.truetype(p, sz) + return ImageFont.load_default() + +F_TITLE = font(30) +F_LABEL = font(22) + +def host_of(url): + # Mirror siteIdFromUrl() in cli.ts: host (minus www) + slugified path. + from urllib.parse import urlparse + u = urlparse(url) + host = re.sub(r"^www\.", "", u.hostname or "") + path = re.sub(r"^-|-$", "", re.sub(r"[^a-zA-Z0-9]+", "-", u.path.rstrip("/"))) + sid = host + (("-" + path) if path else "") + return re.sub(r"[^a-zA-Z0-9.-]", "-", sid)[:80] + +def latest_run(host): + dirs = sorted(glob.glob(os.path.join(RUNS, host, "*")), reverse=True) + for d in dirs: + if (os.path.exists(os.path.join(d, "source", "screenshots")) and + os.path.exists(os.path.join(d, "rendered", "screenshots"))): + return d + return None + +def load(path, scale, max_h): + if not os.path.exists(path): + return None + im = Image.open(path).convert("RGB") + w = int(im.width * scale) + h = int(im.height * scale) + im = im.resize((w, h), Image.LANCZOS) + if im.height > max_h: # cap very tall pages + im = im.crop((0, 0, im.width, max_h)) + return im + +PAD = 16 +HEADER = 46 # per-row label strip +TITLE = 56 # site title strip +GAP = 10 # gap between source and clone +BG = (250, 250, 250) +INK = (20, 20, 20) +RULE = (210, 210, 210) + +def row(src_path, clone_path, scale, max_h, label): + """One labeled row: [SOURCE | CLONE] side by side, top-aligned.""" + s = load(src_path, scale, max_h) + c = load(clone_path, scale, max_h) + if s is None and c is None: + return None + sw = s.width if s else (c.width if c else 0) + cw = c.width if c else sw + sh = s.height if s else 0 + ch = c.height if c else 0 + body_h = max(sh, ch) + W = sw + GAP + cw + H = HEADER + body_h + img = Image.new("RGB", (W, H), BG) + d = ImageDraw.Draw(img) + d.text((4, 10), label, font=F_LABEL, fill=INK) + d.text((4, HEADER - 4), "SOURCE", font=F_LABEL, fill=(90, 90, 90)) + d.text((sw + GAP + 4, HEADER - 4), "CLONE", font=F_LABEL, fill=(90, 90, 90)) + if s: img.paste(s, (0, HEADER)) + if c: img.paste(c, (sw + GAP, HEADER)) + # divider line between the pair + d.line([(sw + GAP // 2, HEADER), (sw + GAP // 2, H)], fill=RULE, width=2) + return img + +def stack(title, rows): + rows = [r for r in rows if r is not None] + if not rows: + return None + W = max(r.width for r in rows) + PAD * 2 + H = TITLE + sum(r.height for r in rows) + PAD * (len(rows) + 1) + img = Image.new("RGB", (W, H), BG) + d = ImageDraw.Draw(img) + d.text((PAD, 14), title, font=F_TITLE, fill=INK) + d.line([(0, TITLE - 4), (W, TITLE - 4)], fill=RULE, width=2) + y = TITLE + PAD + for r in rows: + img.paste(r, (PAD, y)) + y += r.height + PAD + return img + +def main(): + sites = json.load(open(os.path.join(ROOT, "compiler", "benchmarks", f"{TIER}.json"))) + made = [] + for site in sites: + host = host_of(site["url"]) + run = latest_run(host) + if not run: + print(f"SKIP {site['id']} {host}: no run with screenshots") + continue + ss = os.path.join(run, "source", "screenshots") + rs = os.path.join(run, "rendered", "screenshots") + title = f"{site['id']} — {host}" + desktop = row(os.path.join(ss, "1280.png"), os.path.join(rs, "1280.png"), + 0.5, 1600, "Desktop · 1280px viewport (shown at 50%)") + mobile = row(os.path.join(ss, "375.png"), os.path.join(rs, "375.png"), + 1.0, 1800, "Mobile · 375px viewport (100%)") + img = stack(title, [desktop, mobile]) + if img is None: + print(f"SKIP {site['id']}: no images") + continue + out = os.path.join(OUT, f"{site['id']}-{host}.png") + img.save(out, optimize=True) + made.append(out) + print(f"OK {out} ({img.width}x{img.height})") + print(f"\n{len(made)} composites -> {OUT}") + +if __name__ == "__main__": + main() diff --git a/compiler/scripts/make_site_compare.py b/compiler/scripts/make_site_compare.py new file mode 100644 index 0000000..1bd2654 --- /dev/null +++ b/compiler/scripts/make_site_compare.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Build before/after (source vs clone) composites for the multi-page `sites` benchmark. + +For each site: a stacked PNG of a few key routes (entry + listings/representatives), +each a labeled SOURCE|CLONE row at the desktop (1280px) viewport, titled with the +site's per-route pass summary. Mirrors make_compare.py but reads the site run layout +(runs/site-//routes//{source,rendered}/screenshots/.png). +""" +import json, os, sys, glob, re +from PIL import Image, ImageDraw, ImageFont + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +RUNS = os.path.join(ROOT, "runs") +OUT = os.path.join(RUNS, "compare-sites") +os.makedirs(OUT, exist_ok=True) +MAX_ROUTES = int(os.environ.get("MAX_ROUTES", "4")) + +def font(sz): + for p in ["/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"]: + if os.path.exists(p): + return ImageFont.truetype(p, sz) + return ImageFont.load_default() + +F_TITLE, F_LABEL = font(30), font(20) +PAD, HEADER, TITLE, GAP = 16, 42, 60, 10 +BG, INK, RULE = (250, 250, 250), (20, 20, 20), (210, 210, 210) + +def host_of(url): + from urllib.parse import urlparse + u = urlparse(url) + host = re.sub(r"^www\.", "", u.hostname or "") + path = re.sub(r"^-|-$", "", re.sub(r"[^a-zA-Z0-9]+", "-", u.path.rstrip("/"))) + sid = host + (("-" + path) if path else "") + return "site-" + re.sub(r"[^a-zA-Z0-9.-]", "-", sid)[:80] + +def sanitize_seg(s): + out = re.sub(r"-+", "-", re.sub(r"[^A-Za-z0-9._-]", "-", s)).strip("-") + if out == "" or re.match(r"^[_(@.]", out): + out = "r-" + re.sub(r"^[_(@.]+", "", out) + return out or "r" + +def route_key(path): + if path == "/": + return "home" + segs = [sanitize_seg(s) for s in path.split("/") if s] + return "__".join(segs) or "home" + +def latest_run(host): + for d in sorted(glob.glob(os.path.join(RUNS, host, "*")), reverse=True): + if os.path.exists(os.path.join(d, "site-manifest.json")): + return d + return None + +def load(path, scale, max_h): + if not os.path.exists(path): + return None + im = Image.open(path).convert("RGB") + im = im.resize((int(im.width * scale), int(im.height * scale)), Image.LANCZOS) + if im.height > max_h: + im = im.crop((0, 0, im.width, max_h)) + return im + +def row(src_path, clone_path, scale, max_h, label): + s, c = load(src_path, scale, max_h), load(clone_path, scale, max_h) + if s is None and c is None: + return None + sw = s.width if s else (c.width if c else 0) + cw = c.width if c else sw + body_h = max(s.height if s else 0, c.height if c else 0) + img = Image.new("RGB", (sw + GAP + cw, HEADER + body_h), BG) + d = ImageDraw.Draw(img) + d.text((4, 8), label, font=F_LABEL, fill=INK) + d.text((4, HEADER - 6), "SOURCE", font=F_LABEL, fill=(90, 90, 90)) + d.text((sw + GAP + 4, HEADER - 6), "CLONE", font=F_LABEL, fill=(90, 90, 90)) + if s: img.paste(s, (0, HEADER)) + if c: img.paste(c, (sw + GAP, HEADER)) + d.line([(sw + GAP // 2, HEADER), (sw + GAP // 2, img.height)], fill=RULE, width=2) + return img + +def stack(title, rows): + rows = [r for r in rows if r is not None] + if not rows: + return None + W = max(r.width for r in rows) + PAD * 2 + H = TITLE + sum(r.height for r in rows) + PAD * (len(rows) + 1) + img = Image.new("RGB", (W, H), BG) + d = ImageDraw.Draw(img) + d.text((PAD, 16), title, font=F_TITLE, fill=INK) + d.line([(0, TITLE - 4), (W, TITLE - 4)], fill=RULE, width=2) + y = TITLE + PAD + for r in rows: + img.paste(r, (PAD, y)) + y += r.height + PAD + return img + +def main(): + sites = json.load(open(os.path.join(ROOT, "compiler", "benchmarks", "sites.json"))) + made = [] + for site in sites: + host = host_of(site["url"]) + run = latest_run(host) + if not run: + print(f"SKIP {site['id']}: no run"); continue + man = json.load(open(os.path.join(run, "site-manifest.json"))) + report = None + rp = os.path.join(run, "validation", "site-report.json") + if os.path.exists(rp): + report = json.load(open(rp)) + summ = "" + if report: + summ = f" — {report['routesGates0to6']}/{report['routesTotal']} routes pass gates 0–6" + routes = man.get("routes", [])[:MAX_ROUTES] + rows = [] + for r in routes: + key = route_key(r["routePath"]) + ss = os.path.join(run, "routes", key, "source", "screenshots", "1280.png") + rs = os.path.join(run, "routes", key, "rendered", "screenshots", "1280.png") + rows.append(row(ss, rs, 0.5, 1400, f"{r.get('role','page')} · {r['href']}")) + img = stack(f"{site['id']} ({host}){summ}", rows) + if img is None: + print(f"SKIP {site['id']}: no screenshots"); continue + out = os.path.join(OUT, f"{site['id']}.png") + img.save(out, optimize=True) + made.append(out) + print(f"OK {out} ({img.width}x{img.height})") + print(f"\n{len(made)} composites -> {OUT}") + +if __name__ == "__main__": + main() diff --git a/compiler/scripts/probe-highlights.mjs b/compiler/scripts/probe-highlights.mjs new file mode 100644 index 0000000..b7eec60 --- /dev/null +++ b/compiler/scripts/probe-highlights.mjs @@ -0,0 +1,35 @@ +// Probe the recent-highlights container width vs its parent across desktop widths — confirm it +// FILLS (no gutter/snap) instead of freezing at a baked w-310. +import { chromium } from "playwright"; +const url = process.argv[2] || "http://127.0.0.1:8139"; +const widths = [1280, 1400, 1536, 1700, 1920]; +const browser = await chromium.launch(); +const page = await browser.newPage(); +await page.goto(url, { waitUntil: "networkidle", timeout: 60000 }); + +// the grid that holds the highlight cards: a grid with grid-cols and >=3 children +const ok = await page.evaluate(() => { + const g = [...document.querySelectorAll("div")].find( + (d) => /grid/.test(getComputedStyle(d).display) && d.querySelectorAll(":scope > div > article, :scope > div > a, :scope > a").length >= 3 + && d.closest("section")?.textContent?.includes("Recent highlights") + ); + if (!g) return false; + g.setAttribute("data-hl", "1"); + return true; +}); +if (!ok) { console.log("recent-highlights grid not found"); await browser.close(); process.exit(0); } + +console.log(`\n=== ${url} (recent-highlights) ===`); +for (const w of widths) { + await page.setViewportSize({ width: w, height: 1000 }); + await page.waitForTimeout(200); + const d = await page.evaluate(() => { + const g = document.querySelector("[data-hl]"); + const p = g.parentElement; + const gw = g.getBoundingClientRect().width; + const pw = p.getBoundingClientRect().width; + return { gw: Math.round(gw), pw: Math.round(pw), gutter: Math.round(pw - gw) }; + }); + console.log(`w=${String(w).padEnd(5)} grid=${d.gw} parent=${d.pw} gutter=${d.gutter}px ${d.gutter > 4 ? "**GUTTER**" : "fills"}`); +} +await browser.close(); diff --git a/compiler/scripts/setup.sh b/compiler/scripts/setup.sh new file mode 100644 index 0000000..3fff316 --- /dev/null +++ b/compiler/scripts/setup.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Reproduce the compiler environment from a clean checkout. +# - installs compiler deps +# - installs the Chromium build Playwright needs +# - installs the shared build harness deps (Next/Vite/React) used to build & render +# every generated app during validation +# Safe to re-run; each step is idempotent. +set -euo pipefail +cd "$(dirname "$0")/.." + +echo "[setup] installing compiler deps" +npm install + +echo "[setup] installing Playwright chromium" +npx playwright install chromium + +echo "[setup] installing build harness deps" +( cd .harness && npm install ) + +echo "[setup] done. Try: npm run clone -- https://example.com/" diff --git a/compiler/scripts/test-lineclamp.mjs b/compiler/scripts/test-lineclamp.mjs new file mode 100644 index 0000000..63d4afa --- /dev/null +++ b/compiler/scripts/test-lineclamp.mjs @@ -0,0 +1,51 @@ +// Hypothesis test: does adding -webkit-line-clamp:5 to quote cards make their +// heights uniform? Inject the clamp at runtime and re-measure heights. +import { chromium } from "playwright"; + +const url = process.argv[2] || "http://127.0.0.1:8139"; +const widths = [1024, 1100, 1200, 1280, 1440]; +const browser = await chromium.launch(); +const page = await browser.newPage(); +await page.goto(url, { waitUntil: "networkidle", timeout: 60000 }); + +const gridSel = await page.evaluate(() => { + const grids = [...document.querySelectorAll("div")].filter( + (d) => getComputedStyle(d).display === "grid" && d.querySelector("blockquote") + ); + if (!grids[0]) return null; + grids[0].setAttribute("data-probe-grid", "1"); + return true; +}); +if (!gridSel) { console.log("no grid"); await browser.close(); process.exit(0); } + +const measure = async (label) => { + console.log(`\n--- ${label} ---`); + for (const w of widths) { + await page.setViewportSize({ width: w, height: 1000 }); + await page.waitForTimeout(200); + const d = await page.evaluate(() => { + const grid = document.querySelector('[data-probe-grid]'); + const vis = [...grid.children].filter((c) => getComputedStyle(c).display !== "none"); + const H = vis.map((c) => Math.round((c.querySelector(":scope > div") || c).getBoundingClientRect().height * 10) / 10); + return { cols: getComputedStyle(grid).gridTemplateColumns.split(" ").length, H, uniq: [...new Set(H)].length }; + }); + console.log(`w=${String(w).padEnd(5)} cols=${d.cols} H=${JSON.stringify(d.H)} ${d.uniq === 1 ? "UNIFORM" : "**NONUNIFORM " + d.uniq + "**"}`); + } +}; + +await measure("BEFORE (current clone)"); + +// (0) h-full ONLY (no line-clamp) — does filling the definite grid cell alone equalize within rows? +await page.addStyleTag({ + content: `[data-probe-grid] > div > div { height:100% !important; } + [data-probe-grid] figure { height:100% !important; }`, +}); +await measure("AFTER-0 (card/figure h-full ONLY, no line-clamp)"); + +// (1) add line-clamp:5 on top. +await page.addStyleTag({ + content: `[data-probe-grid] blockquote p { display:-webkit-box !important; -webkit-box-orient:vertical !important; -webkit-line-clamp:5 !important; overflow:hidden !important; }`, +}); +await measure("AFTER-1 (h-full + line-clamp:5)"); + +await browser.close(); diff --git a/compiler/scripts/test-select.ts b/compiler/scripts/test-select.ts new file mode 100644 index 0000000..802190e --- /dev/null +++ b/compiler/scripts/test-select.ts @@ -0,0 +1,19 @@ +// Dev-only: replay selectRoutes on a site's discovered crawl paths (from crawl.json) to +// inspect/verify the route plan WITHOUT re-capturing. Usage: tsx scripts/test-select.ts [maxRoutes] +import { selectRoutes } from "../src/crawl/routeTemplates.js"; +import { readFileSync } from "node:fs"; + +const sites: [string, string][] = [ + ["cropin", "output/cropin"], +]; +const maxRoutes = Number(process.argv[2] ?? 25); +for (const [site, dir] of sites) { + let c: { entryPath?: string; depthByPath?: Record }; + try { c = JSON.parse(readFileSync(`${dir}/.clone/crawl.json`, "utf8")); } catch { console.log(`\n### ${site} (no crawl.json)`); continue; } + const paths = Object.keys(c.depthByPath ?? {}); + const plan = selectRoutes({ entryPath: c.entryPath ?? "/", paths, maxRoutes }); + console.log(`\n### ${site} (discovered ${paths.length}, maxRoutes ${maxRoutes}) ###`); + console.log(" collections:", plan.collections.map((x) => `${x.template}(${x.instanceCount})`).join(" ") || "none"); + console.log(" selected:"); + for (const r of plan.selected) console.log(` ${r.role.padEnd(14)} ${r.path}`); +} diff --git a/compiler/scripts/triage.mjs b/compiler/scripts/triage.mjs new file mode 100644 index 0000000..25979a7 --- /dev/null +++ b/compiler/scripts/triage.mjs @@ -0,0 +1,45 @@ +// Triage the latest run per stage-2 site: pass state + the key metric per failure. +import { readFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +const ROOT = new URL("../..", import.meta.url).pathname; +const RUNS = join(ROOT, "runs"); +const tier = process.argv[2] || "stage2"; +const sites = JSON.parse(readFileSync(join(ROOT, "compiler", "benchmarks", `${tier}.json`), "utf8")); + +function siteId(url) { + const u = new URL(url); + const host = u.hostname.replace(/^www\./, ""); + const path = u.pathname.replace(/\/+$/, "").replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-|-$/g, ""); + return (host + (path ? "-" + path : "")).replace(/[^a-zA-Z0-9.-]/g, "-").slice(0, 80); +} +function latest(host) { + const dir = join(RUNS, host); + if (!existsSync(dir)) return null; + const runs = readdirSync(dir).filter((d) => /^\d/.test(d)).sort(); + for (let i = runs.length - 1; i >= 0; i--) { + const r = join(dir, runs[i], "validation", "report.json"); + if (existsSync(r)) return join(dir, runs[i]); + } + return null; +} + +let pass = 0, g06 = 0; +const lines = []; +for (const s of sites) { + const run = latest(siteId(s.url)); + if (!run) { lines.push(`${s.id} NO RUN`); continue; } + const r = JSON.parse(readFileSync(join(run, "validation", "report.json"), "utf8")); + if (r.gates0to6Pass) g06++; + if (r.stage2Pass) pass++; + const fails = Object.entries(r.gates).filter(([, g]) => !g.pass).map(([k]) => k); + let detail = ""; + if (fails.includes("perceptual")) detail += ` perc=${JSON.stringify(r.gates.perceptual.metrics.perViewport)}`; + if (fails.includes("layout")) { const pv = r.gates.layout.metrics.perViewport; detail += " layout=" + Object.entries(pv).filter(([, m]) => (m.heightDeltaPct > 0.05 || m.leafMedianDelta > 8 || m.sectionsBboxOkPct < 0.9)).map(([vp, m]) => `${vp}:h${m.heightDeltaPct},leaf${m.leafMedianDelta}`).join(","); } + if (fails.includes("pollution")) detail += " poll=" + JSON.stringify(r.gates.pollution.issues); + if (fails.includes("style")) detail += " style=" + JSON.stringify(r.gates.style.metrics.topFailingProps); + if (fails.includes("build")) detail += " build=" + (r.gates.build.metrics.runtimeErrorSample || []).length + "errs"; + lines.push(`${r.stage2Pass ? "✅" : (r.gates0to6Pass ? "🟡" : "❌")} ${s.id} ${r.scorecard.total} [${fails.join(",") || "PASS"}]${detail}`); +} +console.log(lines.join("\n")); +console.log(`\n${tier}: gates0-6 ${g06}/${sites.length}, stage2 ${pass}/${sites.length}`); diff --git a/compiler/scripts/tw-diff.ts b/compiler/scripts/tw-diff.ts new file mode 100644 index 0000000..d1d8dba --- /dev/null +++ b/compiler/scripts/tw-diff.ts @@ -0,0 +1,283 @@ +/** + * Source-vs-clone Tailwind diff for the cloner (Stage 4/5 instrument). + * + * For a Tailwind-built source, the live `class` attribute is the AUTHORED fluid intent — + * the exact utility our generator should have inferred. This tool rebuilds the IR (which now carries + * the source `class` as `srcClass`) and our emitted classes (`buildTailwind().classOf`) IN-PROCESS, + * both keyed by node id, so the per-node diff is perfectly aligned with zero live-probe flakiness. + * + * It then NORMALIZES away faithful, unavoidable differences (colors, design-system spacing tokens, + * data-*, no-preflight arbitrary transforms) and focuses the diff on the LAYOUT/SIZING utility + * families — where the class name itself is the signal — to surface the real misses: places we baked + * per-viewport px where the source expressed ONE fluid rule. Misses are clustered by axis + source + * utility and ranked by frequency, so one law fixes many nodes. + * + * npx tsx scripts/tw-diff.ts [runDir=output/sample/.clone] [--axis=width] [--samples=N] [--json=path] + * + * Run from compiler/. + */ +import { join, resolve } from "node:path"; +import { writeFileSync } from "node:fs"; +import { buildIR, isTextChild, type IR, type IRNode } from "../src/normalize/ir.js"; +import { buildTailwind } from "../src/generate/tailwind.js"; +import { buildColorPalette } from "../src/infer/semanticTokens.js"; +import { buildAssetGraph, type AssetGraph } from "../src/infer/assets.js"; +import { readJSON } from "../src/util/fsx.js"; +import type { CaptureResult } from "../src/capture/capture.js"; + +function buildAssetMap(assetGraph: AssetGraph): Map { + const m = new Map(); + for (const e of assetGraph.entries) { + if (e.classification === "downloaded" && e.localPath && e.type !== "css") m.set(e.sourceUrl, e.localPath); + } + return m; +} + +// ---- token parsing ------------------------------------------------------- + +type Tok = { raw: string; variant: string; core: string }; + +/** Split a className string into tokens, separating the responsive/state variant prefix + * (`md:`, `lg:`, `max-md:`, `hover:`, `2xl:` …) from the core utility. Arbitrary values can + * themselves contain `:` inside `[...]`, so only split on `:` that are OUTSIDE brackets. */ +function parseClass(cls: string): Tok[] { + const out: Tok[] = []; + for (const raw of cls.split(/\s+/).filter(Boolean)) { + // find the last variant-colon that's outside brackets/parens + let depth = 0, lastColon = -1; + for (let i = 0; i < raw.length; i++) { + const ch = raw[i]!; + if (ch === "[" || ch === "(") depth++; + else if (ch === "]" || ch === ")") depth--; + else if (ch === ":" && depth === 0) lastColon = i; + } + const variant = lastColon >= 0 ? raw.slice(0, lastColon) : ""; + const core = lastColon >= 0 ? raw.slice(lastColon + 1) : raw; + out.push({ raw, variant, core }); + } + return out; +} + +// ---- sizing-axis classification ----------------------------------------- + +type Axis = "width" | "height" | "grid-cols" | "grid-rows" | "flex" | "aspect" | "line-clamp"; + +/** Does an arbitrary value `[…]` (already unwrapped) denote a BAKED fixed length (px/rem)? */ +function isBakedLen(v: string): boolean { + return /^-?[\d.]+(px|rem|em)$/.test(v); +} +/** Does an arbitrary value denote a FLUID length (%, vw/vh, fr, auto/min/max-content, calc with %/vw)? */ +function isFluidLen(v: string): boolean { + if (/^-?[\d.]+(%|vw|vh|svh|lvh|dvh|svw|lvw|dvw|fr)$/.test(v)) return true; + if (/^(auto|min-content|max-content|fit-content|stretch)$/.test(v)) return true; + if (/(%|vw|vh|fr|min-content|max-content|fit-content)/.test(v) && /^(calc|min|max|clamp|repeat|fit-content)\(/.test(v)) return true; + return false; +} + +/** Spacing-scale numbers (`w-24`, `h-2.5`, `w-93`) are FIXED px. Fractions (`w-1/2`) are fluid. */ +function spacingIsFixed(rest: string): boolean { + return /^-?[\d.]+$/.test(rest); // pure number → fixed scale step; a fraction has a '/' +} + +type AxisVerdict = { fluid?: string; baked?: string }; + +/** Classify a single core utility onto an axis with a fluid|baked verdict. + * Returns null for utilities that are not on a sizing axis. */ +function classify(core: string): { axis: Axis; v: AxisVerdict } | null { + // grid templates + let m = core.match(/^grid-(cols|rows)-(.+)$/); + if (m) { + const axis: Axis = m[1] === "cols" ? "grid-cols" : "grid-rows"; + const val = m[2]!; + if (/^\d+$/.test(val)) return { axis, v: { fluid: `grid-${m[1]}-N` } }; // fixed-count fluid grid + if (val === "none" || val === "subgrid") return { axis, v: { fluid: val } }; + if (val.startsWith("[")) { + const inner = val.slice(1, -1); + // baked iff it contains an explicit px/rem track and NO fr/%/min/max/auto + const hasFixed = /[\d.]+(px|rem)/.test(inner); + const hasFluid = /(fr|%|min-content|max-content|auto|minmax|repeat)/.test(inner); + if (hasFixed && !hasFluid) return { axis, v: { baked: `grid-${m[1]}-[px]` } }; + if (hasFixed && hasFluid) return { axis, v: { baked: `grid-${m[1]}-[mixed-px]` } }; // partly baked + return { axis, v: { fluid: `grid-${m[1]}-[fr]` } }; + } + return { axis, v: { fluid: `grid-${m[1]}-${val}` } }; + } + // aspect-ratio + m = core.match(/^aspect-(.+)$/); + if (m) { + const val = m[1]!; + if (val === "square" || val === "video" || val === "auto") return { axis: "aspect", v: { fluid: `aspect-${val}` } }; + if (val.startsWith("[")) { + const inner = val.slice(1, -1); + // clean ratio like [16/9] is fluid intent; [auto_42_/_42] baked-from-px + if (/^\d+\s*\/\s*\d+$/.test(inner.replace(/_/g, " "))) return { axis: "aspect", v: { fluid: "aspect-[ratio]" } }; + return { axis: "aspect", v: { baked: "aspect-[baked]" } }; + } + return { axis: "aspect", v: { fluid: `aspect-${val}` } }; + } + // line-clamp + m = core.match(/^line-clamp-(.+)$/); + if (m) return { axis: "line-clamp", v: { fluid: /^\d+$/.test(m[1]!) ? "line-clamp-N" : `line-clamp-${m[1]}` } }; + // flex shorthand / grow / shrink / basis (all width-flow on the main axis) + if (/^(flex-1|flex-auto|flex-initial|grow|grow-0|shrink|shrink-0)$/.test(core)) return { axis: "flex", v: { fluid: core } }; + m = core.match(/^basis-(.+)$/); + if (m) { + const val = m[1]!; + if (val === "full" || val === "auto" || /\d+\/\d+/.test(val)) return { axis: "flex", v: { fluid: `basis-${val.includes("/") ? "frac" : val}` } }; + if (val.startsWith("[")) { const inner = val.slice(1, -1); return isBakedLen(inner) ? { axis: "flex", v: { baked: "basis-[px]" } } : { axis: "flex", v: { fluid: "basis-[fluid]" } }; } + return { axis: "flex", v: { baked: "basis-px" } }; // basis-24 etc + } + // width / height (incl. min-/max-) + m = core.match(/^(min-|max-)?([wh])-(.+)$/); + if (m) { + const axis: Axis = m[2] === "w" ? "width" : "height"; + const val = m[3]!; + const fam = `${m[1] ?? ""}${m[2]}`; // w, min-w, max-w, h, … + if (/^(full|screen|auto|fit|min|max|svh|lvh|dvh|svw|lvw|dvw|prose)$/.test(val)) return { axis, v: { fluid: `${fam}-${val}` } }; + if (/^\d+\/\d+$/.test(val)) return { axis, v: { fluid: `${fam}-frac` } }; // w-1/2 + if (/^(screen-|3xl|2xl|xl|lg|md|sm|xs|7xl|6xl|5xl|4xl)/.test(val) && fam === "max-w") return { axis, v: { fluid: `max-w-${val}` } }; // named cap + if (val.startsWith("[")) { + const inner = val.slice(1, -1); + if (inner.startsWith("var(")) return { axis, v: { fluid: `${fam}-[var]` } }; // token-driven → fluid-ish + if (isFluidLen(inner)) return { axis, v: { fluid: `${fam}-[fluid]` } }; + if (isBakedLen(inner)) return { axis, v: { baked: `${fam}-[px]` } }; + return { axis, v: {} }; + } + if (spacingIsFixed(val)) return { axis, v: { baked: `${fam}-num` } }; // w-24, h-93 → fixed scale + return { axis, v: {} }; + } + return null; +} + +type NodeAxisInfo = Record; srcBaked: Set; cloFluid: Set; cloBaked: Set }>; + +function emptyAxisInfo(): NodeAxisInfo { + const mk = () => ({ srcFluid: new Set(), srcBaked: new Set(), cloFluid: new Set(), cloBaked: new Set() }); + return { width: mk(), height: mk(), "grid-cols": mk(), "grid-rows": mk(), flex: mk(), aspect: mk(), "line-clamp": mk() }; +} + +function collectAxis(cls: string, info: NodeAxisInfo, side: "src" | "clo"): void { + for (const t of parseClass(cls)) { + const c = classify(t.core); + if (!c) continue; + const bucket = info[c.axis]; + if (c.v.fluid) (side === "src" ? bucket.srcFluid : bucket.cloFluid).add(c.v.fluid); + if (c.v.baked) (side === "src" ? bucket.srcBaked : bucket.cloBaked).add(c.v.baked); + } +} + +// ---- main ---------------------------------------------------------------- + +const argv = process.argv.slice(2); +const runDir = resolve(argv.find((a) => !a.startsWith("--")) ?? "output/sample/.clone"); +const axisFilter = (argv.find((a) => a.startsWith("--axis="))?.split("=")[1] ?? "") as Axis | ""; +const sampleN = parseInt(argv.find((a) => a.startsWith("--samples="))?.split("=")[1] ?? "20", 10); +const jsonOut = argv.find((a) => a.startsWith("--json="))?.split("=")[1]; + +const inspectIds = (argv.find((a) => a.startsWith("--ids="))?.split("=")[1] ?? "").split(",").filter(Boolean); + +const sourceDir = join(runDir, "source"); +const input = readJSON<{ url: string; viewports: number[] }>(join(runDir, "input.json")); +const capture = readJSON(join(sourceDir, "capture", "capture-result.json")); +const cloneOpts = readJSON<{ reflow?: boolean }>(join(sourceDir, "clone-options.json")); + +const ir: IR = buildIR(sourceDir, capture.viewports, { motion: !!capture.motion, bandViewports: input.viewports }); +const assetGraph = buildAssetGraph(capture); +const palette = buildColorPalette(ir); +const tw = buildTailwind(ir, buildAssetMap(assetGraph), palette.varForColor, { interaction: capture.interaction, reflow: !!cloneOpts.reflow }); +const classOf = tw.classOf; + +// --ids inspect mode: dump src/clone class + per-vp height geometry for specific nodes, then exit. +if (inspectIds.length) { + const idx = new Map(); + const idxWalk = (n: IRNode): void => { idx.set(n.id, n); for (const c of n.children) if (!isTextChild(c)) idxWalk(c); }; + idxWalk(ir.root); + for (const id of inspectIds) { + const n = idx.get(id); + if (!n) { console.log(`${id}: NOT FOUND`); continue; } + const vps = ir.doc.sampleViewports; + console.log(`\n${id} <${n.tag}>`); + console.log(` src: ${n.srcClass ?? "(none)"}`); + console.log(` clone: ${classOf.get(id) ?? "(none)"}`); + console.log(` bboxH: ${vps.map((vp) => n.bboxByVp[vp]?.height ?? "·").join("/")}`); + } + process.exit(0); +} + +// Per-axis miss tally + per (axis, srcUtil) cluster + sample rows. +type MissRow = { id: string; tag: string; text: string; axis: Axis; srcUtil: string; cloBaked: string; source: string; clone: string }; +const misses: MissRow[] = []; +const clusterCount = new Map(); // "axis · srcUtil" → count +const axisMiss = new Map(); +let nNodes = 0, nSrc = 0, nClone = 0, nBoth = 0; + +function textOf(n: IRNode): string { + for (const c of n.children) if (isTextChild(c) && c.text.trim()) return c.text.trim().slice(0, 40); + return ""; +} + +function walk(n: IRNode): void { + nNodes++; + const src = n.srcClass ?? ""; + const clo = classOf.get(n.id) ?? ""; + if (src) nSrc++; + if (clo) nClone++; + if (src && clo) { + nBoth++; + const info = emptyAxisInfo(); + collectAxis(src, info, "src"); + collectAxis(clo, info, "clo"); + for (const axis of Object.keys(info) as Axis[]) { + const b = info[axis]; + // MISS = source expressed a fluid rule on this axis, clone baked px (or token absent) and + // produced NO fluid utility of its own on that axis. + const cloneHasFluid = b.cloFluid.size > 0; + if (b.srcFluid.size > 0 && !cloneHasFluid && (b.cloBaked.size > 0 || (b.srcBaked.size === 0 && axisExpectedInClone(axis, clo)))) { + const srcUtil = [...b.srcFluid].sort().join("+"); + const cloBaked = [...b.cloBaked].sort().join("+") || "(absent)"; + misses.push({ id: n.id, tag: n.tag, text: textOf(n), axis, srcUtil, cloBaked, source: src, clone: clo }); + const key = `${axis} · ${srcUtil}`; + clusterCount.set(key, (clusterCount.get(key) ?? 0) + 1); + axisMiss.set(axis, (axisMiss.get(axis) ?? 0) + 1); + } + } + } + for (const c of n.children) if (!isTextChild(c)) walk(c); +} + +/** For line-clamp/aspect a missing clone utility is itself the miss; for width/height/grid we only + * count a miss when the clone actually baked something (handled by cloBaked.size>0). */ +function axisExpectedInClone(axis: Axis, _clone: string): boolean { + return axis === "line-clamp" || axis === "aspect"; +} + +walk(ir.root); + +// ---- report -------------------------------------------------------------- + +console.log(`\n=== tw-diff: ${runDir} ===`); +console.log(`nodes=${nNodes} withSrcClass=${nSrc} withCloneClass=${nClone} both=${nBoth}`); +console.log(`\n--- real misses by axis (source fluid, clone baked/absent) ---`); +const totalMiss = misses.length; +for (const [axis, n] of [...axisMiss.entries()].sort((a, b) => b[1] - a[1])) { + console.log(` ${axis.padEnd(11)} ${n}`); +} +console.log(` ${"TOTAL".padEnd(11)} ${totalMiss}`); + +console.log(`\n--- top miss clusters (axis · source-util → count) ---`); +for (const [key, n] of [...clusterCount.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20)) { + console.log(` ${String(n).padStart(4)} ${key}`); +} + +const shown = axisFilter ? misses.filter((m) => m.axis === axisFilter) : misses; +console.log(`\n--- sample miss rows${axisFilter ? ` [axis=${axisFilter}]` : ""} (${Math.min(sampleN, shown.length)} of ${shown.length}) ---`); +for (const m of shown.slice(0, sampleN)) { + console.log(`\n ${m.id} <${m.tag}> ${m.text ? `“${m.text}”` : ""} [${m.axis}: src ${m.srcUtil} → clone ${m.cloBaked}]`); + console.log(` src: ${m.source}`); + console.log(` clone: ${m.clone}`); +} + +if (jsonOut) { + writeFileSync(resolve(jsonOut), JSON.stringify({ stats: { nNodes, nSrc, nClone, nBoth, totalMiss }, axisMiss: Object.fromEntries(axisMiss), clusters: Object.fromEntries([...clusterCount.entries()].sort((a, b) => b[1] - a[1])), misses }, null, 2)); + console.log(`\nwrote ${jsonOut}`); +} diff --git a/compiler/src/capture/breakpoints.ts b/compiler/src/capture/breakpoints.ts new file mode 100644 index 0000000..db4baa6 --- /dev/null +++ b/compiler/src/capture/breakpoints.ts @@ -0,0 +1,63 @@ +/** + * Breakpoint discovery using the browser as the layout oracle. + * + * We capture at a few fixed widths and force the responsive bands onto Tailwind's default + * breakpoints — which is wrong for any site whose real cut points differ, and can spawn extra bands + * when our sample widths happen to straddle a breakpoint that isn't really there. Instead, ASK the + * page: sweep the viewport width and find the exact widths where the layout actually changes. + * + * Media queries cause DISCRETE jumps in structural properties (display, flex-direction, flex-wrap, + * grid track count, position, float, text-align, visibility) — those flip only at a breakpoint, + * whereas widths/heights scale smoothly. So we hash that discrete signature across a coarse width + * scan, then binary-search each interval where the hash changed down to 1px. The result is the + * site's REAL breakpoint set, to drive both the capture sample widths and the emitted band edges. + */ +import type { Page } from "playwright"; + +/** In-page: a hash of every element's discrete (media-query-toggled) layout properties. Pure + * structural — excludes widths/heights/font-size, which vary continuously and would mask the steps. */ +const DISCRETE_SIGNATURE = (): string => { + let s = ""; + const walk = (el: Element): void => { + const cs = getComputedStyle(el); + const cols = (cs.gridTemplateColumns || "none") === "none" ? 0 : cs.gridTemplateColumns.split(" ").filter(Boolean).length; + const painted = (el as HTMLElement).offsetParent !== null || cs.position === "fixed" ? 1 : 0; + s += `${cs.display}|${cs.flexDirection}|${cs.flexWrap}|${cols}|${cs.position}|${cs.float}|${cs.textAlign}|${cs.visibility}|${painted};`; + for (let i = 0; i < el.children.length; i++) walk(el.children[i]!); + }; + walk(document.body); + let h = 0; + for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0; + return `${s.length}:${h}`; +}; + +export async function discoverBreakpoints( + page: Page, + opts?: { min?: number; max?: number; coarseStep?: number; height?: number }, +): Promise { + const min = opts?.min ?? 320; + const max = opts?.max ?? 1920; + const step = opts?.coarseStep ?? 16; + const height = opts?.height ?? 1200; + const sig = async (w: number): Promise => { + await page.setViewportSize({ width: w, height }); + return page.evaluate(DISCRETE_SIGNATURE); + }; + + // Coarse scan: record where the signature changes between adjacent samples. + const coarse: { w: number; sig: string }[] = []; + for (let w = min; w <= max; w += step) coarse.push({ w, sig: await sig(w) }); + + // Binary-search each changed interval down to 1px — the edge where the NEW layout begins. + const edges: number[] = []; + for (let i = 1; i < coarse.length; i++) { + if (coarse[i]!.sig === coarse[i - 1]!.sig) continue; + let lo = coarse[i - 1]!.w, hi = coarse[i]!.w; const loSig = coarse[i - 1]!.sig; + while (hi - lo > 1) { + const mid = (lo + hi) >> 1; + if ((await sig(mid)) === loSig) lo = mid; else hi = mid; + } + edges.push(hi); + } + return edges; +} diff --git a/compiler/src/capture/capture.ts b/compiler/src/capture/capture.ts new file mode 100644 index 0000000..50f9fb9 --- /dev/null +++ b/compiler/src/capture/capture.ts @@ -0,0 +1,996 @@ +import { chromium, type Browser, type BrowserContext } from "playwright"; +import { join } from "node:path"; +import { collectPage, type PageSnapshot, type FontFace } from "./walker.js"; +import { tagElements, captureInteractions, type InteractionCapture } from "./interactions.js"; +import { captureMotion, probeReveals, type MotionCapture } from "./motion.js"; +import { discoverBreakpoints } from "./breakpoints.js"; +import { writeJSON, writeJSONCompact, writeBytes, ensureDir } from "../util/fsx.js"; +import { sha1_12, round } from "../util/canonical.js"; + +export const REQUIRED_VIEWPORTS = [375, 768, 1280, 1920] as const; +// The dense width set captured for SIZE INFERENCE: a node sampled at 9 widths reveals its sizing +// law (constant / proportional / clamped / flex-distributed) far better than 4 — enough to drop a +// baked px for a relative construct with confidence (and to surface a shrunk item's natural size, +// which needs widths beyond 1920). REQUIRED_VIEWPORTS ⊂ this; only those 4 carry responsive bands. +export const SAMPLE_VIEWPORTS = [375, 480, 640, 768, 1024, 1280, 1536, 1920, 2560] as const; + +const VIEWPORT_HEIGHTS: Record = { + 375: 812, + 480: 854, + 640: 960, + 768: 1024, + 1024: 768, + 1280: 800, + 1536: 864, + 1920: 1080, + 2560: 1440, +}; + +const DESKTOP_UA = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; + +// Defines esbuild/tsx runtime helpers in the page so serialized evaluate +// callbacks that reference them don't throw ReferenceError. +const ESBUILD_SHIM = + "globalThis.__name = globalThis.__name || ((fn) => fn);" + + "globalThis.__defProp = globalThis.__defProp || Object.defineProperty;"; + +export type DiscoveredAsset = { + url: string; + type: string; // image|svg|video|font|lottie|css|manifest|other + contentType: string | null; + status: number | null; + storedAs: string | null; // sha1-named file in assets-store, or null if not downloaded + bytes: number; // size of stored file + via: string[]; // discovery sources +}; + +export type SeoResource = { + kind: "robots" | "sitemap" | "llms" | "llms-full"; + url: string; + status: number | null; + contentType: string | null; + text?: string; +}; + +export type CaptureResult = { + sourceUrl: string; + capturedAt: string; + viewports: number[]; + // Browser-as-oracle: the widths where the SOURCE layout actually restructures (display / + // flex-direction / wrap / grid-track-count / position / visibility flips), found by sweeping the + // viewport and binary-searching each discrete-signature change. Ground truth for the real + // responsive band edges — distinct from REQUIRED_VIEWPORTS (our fixed capture widths). Absent if + // discovery was disabled or the sweep failed. See breakpoints.ts. + breakpoints?: number[]; + perViewport: Array<{ + viewport: number; + height: number; + scrollHeight: number; + nodeCount: number; + truncated: boolean; + overlaysRemaining?: number; + blocking?: boolean; + quiescent?: boolean; + }>; + assets: DiscoveredAsset[]; + seoResources?: SeoResource[]; + fontFaces: FontFace[]; + cssTexts: string[]; // sha1 names of stored css files + // Stage 2: overlay/popup dismissal audit (union of actions across viewports). + dismissal?: { dismissed: string[]; overlaysRemaining: number; removed: number; videoStills: number; blocking: boolean }; + // Stage 4: optional interaction capture (hover/focus + recognized patterns). + interaction?: InteractionCapture; + // Stage 5: optional motion capture (WAAPI animations + rotating text). CSS @keyframes + // motion is reconstructed from the IR, so it isn't re-captured here. + motion?: MotionCapture; +}; + +function viewportHeight(width: number): number { + return VIEWPORT_HEIGHTS[width] ?? Math.round(width * 0.66); +} + +function extFromUrl(url: string): string { + try { + const p = new URL(url).pathname; + const dot = p.lastIndexOf("."); + if (dot >= 0 && dot > p.lastIndexOf("/")) { + const ext = p.slice(dot + 1).toLowerCase().slice(0, 5); + if (/^[a-z0-9]+$/.test(ext)) return ext; + } + } catch { /* ignore */ } + return ""; +} + +const CONTENT_TYPE_EXT: Record = { + "image/webp": "webp", "image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg", + "image/avif": "avif", "image/gif": "gif", "image/svg+xml": "svg", + "image/x-icon": "ico", "image/vnd.microsoft.icon": "ico", "image/bmp": "bmp", + "video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", "video/ogg": "ogv", + "font/woff2": "woff2", "font/woff": "woff", "font/ttf": "ttf", "font/otf": "otf", + "application/font-woff2": "woff2", "application/font-woff": "woff", + "application/x-font-ttf": "ttf", "application/vnd.ms-fontobject": "eot", + "text/css": "css", "application/json": "json", "application/manifest+json": "webmanifest", +}; + +function extFromContentType(contentType: string | null): string { + if (!contentType) return ""; + const ct = contentType.split(";")[0]!.trim().toLowerCase(); + return CONTENT_TYPE_EXT[ct] ?? ""; +} + +function classifyAsset(url: string, contentType: string | null): string | null { + const u = url.toLowerCase().split("?")[0]!; + const ct = (contentType || "").toLowerCase(); + if (u.endsWith(".svg") || ct === "image/svg+xml") return "svg"; + if (/\.(jpg|jpeg|png|webp|avif|gif|ico|bmp)$/.test(u) || ct.startsWith("image/")) return "image"; + if (/\.(mp4|mov|webm|m4v|ogv)$/.test(u) || ct.startsWith("video/")) return "video"; + if (/\.(woff2|woff|ttf|otf|eot)$/.test(u) || ct.startsWith("font/") || + ct.includes("font-woff") || ct.includes("x-font")) return "font"; + if (u.endsWith(".json") && (u.includes("lottie") || u.includes("animation"))) return "lottie"; + if (u.endsWith(".webmanifest") || /(?:^|\/)manifest\.json$/.test(u) || ct.includes("manifest+json")) return "manifest"; + if (u.endsWith(".css") || ct.startsWith("text/css")) return "css"; + return null; +} + +function isCss(url: string, contentType: string | null): boolean { + return classifyAsset(url, contentType) === "css"; +} + +async function autoScroll(page: import("playwright").Page, vpHeight: number): Promise { + // Scroll through the page to trigger lazy-loaded images/backgrounds, then return + // to the top so document-coordinate bboxes are measured from a settled layout. + await page.evaluate(async (step: number) => { + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + const maxScroll = () => document.documentElement.scrollHeight - window.innerHeight; + let y = 0; + let guard = 0; + while (y < maxScroll() && guard < 100) { + y += step; + window.scrollTo(0, y); + await sleep(60); + guard++; + } + window.scrollTo(0, 0); + await sleep(120); + }, Math.max(Math.round(vpHeight * 0.8), 300)); +} + +async function settle(page: import("playwright").Page, maxMs = 2500): Promise { + try { + await page.waitForLoadState("networkidle", { timeout: Math.max(maxMs / 2, 800) }); + } catch { /* ignore */ } + try { + // document.fonts.ready can never resolve when a font request hangs (heavy SaaS + // pages), and page.evaluate has no default timeout — so bound it explicitly. + await Promise.race([ + page.evaluate(() => (document as Document).fonts?.ready as unknown as Promise), + new Promise((r) => setTimeout(r, 6000)), + ]); + } catch { /* ignore */ } + await page.waitForTimeout(250); +} + +export type DismissResult = { dismissed: string[]; overlaysRemaining: number; removed: number; blocking: boolean }; + +/** + * Stage 2 — overlay/popup dismissal, phase 1: click the accept/close affordance. + * Cookie-consent walls, newsletter modals, region/age/app-install interstitials + * cover the real page; the capture must see the state a returning user sees. + * Deterministic + replayable: click the same known/accept controls in DOM order. + * Removal of a stuck overlay happens later (finalizeOverlays) AFTER a settle, so a + * just-clicked dialog has time to close and unlock scrolling before we judge it. + */ +async function clickDismiss(page: import("playwright").Page): Promise { + try { + return await Promise.race([ + page.evaluate(() => { + const dismissed: string[] = []; + const vis = (el: Element): boolean => { + const cs = getComputedStyle(el); + if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) return false; + const r = (el as HTMLElement).getBoundingClientRect(); + return r.width > 0 && r.height > 0; + }; + const click = (el: Element): void => { try { (el as HTMLElement).click(); } catch { /* ignore */ } }; + + // 1) Known consent-framework / generic close affordances, in priority order. + const KNOWN = [ + "#onetrust-accept-btn-handler", "#accept-recommended-btn-handler", ".onetrust-close-btn-handler", + "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", "#CybotCookiebotDialogBodyButtonAccept", + "#truste-consent-button", ".osano-cm-accept-all", ".osano-cm-dialog__close", + "[data-testid='uc-accept-all-button']", "[data-testid='uc-deny-all-button']", + "#didomi-notice-agree-button", ".didomi-continue-without-agreeing", + ".qc-cmp2-summary-buttons button[mode='primary']", "button[aria-label='Consent']", + ".cc-allow", ".cookie-consent-accept", "#hs-eu-confirmation-button", "#gdpr-consent-tool-wrapper button", + ]; + for (const sel of KNOWN) { + try { + for (const el of Array.from(document.querySelectorAll(sel))) { + if (vis(el)) { click(el); dismissed.push(sel); break; } + } + } catch { /* invalid selector in this browser */ } + } + + // 2) Scoped text-button pass: only inside overlay-ish containers (dialogs, + // or id/class naming cookie/consent/modal/popup/newsletter) so we never + // click an ordinary page button. + const ACCEPT = new Set([ + "accept", "accept all", "accept all cookies", "accept cookies", "accept & close", + "i accept", "i agree", "agree", "agree and continue", "allow all", "allow cookies", + "allow all cookies", "got it", "ok", "okay", "continue", "no thanks", "no, thanks", + "dismiss", "close", "got it!", "understood", "yes, i agree", + ]); + const containerSel = + "[role='dialog'],[aria-modal='true'],[id*='cookie' i],[class*='cookie' i],[id*='consent' i]," + + "[class*='consent' i],[class*='gdpr' i],[id*='gdpr' i],[class*='modal' i],[class*='popup' i]," + + "[class*='newsletter' i],[class*='interstitial' i]"; + let containers: Element[] = []; + try { containers = Array.from(document.querySelectorAll(containerSel)); } catch { /* ignore */ } + for (const c of containers) { + if (!vis(c)) continue; + const btns = Array.from(c.querySelectorAll("button,[role='button'],a,input[type='button'],input[type='submit']")); + for (const b of btns) { + const t = (b.textContent || (b as HTMLInputElement).value || "").replace(/\s+/g, " ").trim().toLowerCase(); + if (t && ACCEPT.has(t) && vis(b)) { click(b); dismissed.push("text:" + t); break; } + } + } + return dismissed; + }), + new Promise((res) => setTimeout(() => res([]), 6000)), + ]); + } catch { + return []; + } +} + +/** + * Stage 2 — overlay/popup dismissal, phase 2 (run after a settle). Detect any + * full-viewport, high-z, fixed/sticky overlay still present. A fixed nav is wide + * but short; a sticky sidebar is tall but narrow — both excluded; a consent wall / + * modal backdrop covers most of both axes. If the page is *still scroll-locked* + * (the modal is genuinely blocking) remove the overlay — but ONLY when its id/class + * looks like a consent/modal layer, never a header/nav/main/footer, so legitimate + * sticky chrome is never stripped. Reports `blocking` = a scroll-locking overlay we + * could not clear (the pollution gate keys off this, not mere overlay presence). + */ +async function finalizeOverlays(page: import("playwright").Page): Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }> { + try { + return await Promise.race([ + page.evaluate(() => { + const vw = window.innerWidth, vh = window.innerHeight; + const bigOverlays = (): HTMLElement[] => { + const out: HTMLElement[] = []; + for (const el of Array.from(document.body.querySelectorAll("*"))) { + const cs = getComputedStyle(el); + if (cs.position !== "fixed" && cs.position !== "sticky") continue; + if (cs.display === "none" || cs.visibility === "hidden" || parseFloat(cs.opacity || "1") === 0) continue; + const r = (el as HTMLElement).getBoundingClientRect(); + const z = parseInt(cs.zIndex || "0", 10) || 0; + const area = (r.width * r.height) / (vw * vh); + if (area >= 0.5 && z >= 100 && r.width >= vw * 0.7 && r.height >= vh * 0.5) out.push(el as HTMLElement); + } + return out.filter((el) => !out.some((o) => o !== el && o.contains(el))); + }; + const isLocked = (): boolean => { + const b = document.body, h = document.documentElement; + return getComputedStyle(b).overflow === "hidden" || getComputedStyle(h).overflow === "hidden" || + getComputedStyle(b).position === "fixed"; + }; + // A scroll-locked page behind a full-viewport overlay IS a blocking modal by + // definition (legit pages don't scroll-lock). So remove ANY such overlay that + // isn't page chrome — many modals/drawers carry no consent/modal keyword and + // an icon-only close, so a keyword/aria allowlist misses them (ruggable's + // z-[1001] drawer). PROTECTED guards real chrome (header/nav/footer) only. + const PROTECTED = /header|navbar|nav-|site-nav|topbar|masthead|footer/i; + const sig = (el: HTMLElement): string => `${el.id} ${el.className}`.toString(); + + const removedLabels: string[] = []; + let removed = 0; + let remaining = bigOverlays(); + if (remaining.length && isLocked()) { + for (const el of remaining) { + const s = sig(el); + const z = parseInt(getComputedStyle(el).zIndex || "0", 10) || 0; + // Scroll-locked + full-viewport ⇒ blocking modal; remove unless it's page + // chrome. Always remove iframes (cross-origin close, unclickable) and the + // max-z-index popup trick. + const removable = !PROTECTED.test(s) || el.getAttribute("aria-modal") === "true" || + el.tagName === "IFRAME" || z >= 2_000_000_000; + if (removable) { el.remove(); removed++; removedLabels.push((el.id || el.className || el.tagName).toString().slice(0, 40)); } + } + if (removed) { document.body.style.overflow = "visible"; document.documentElement.style.overflow = "visible"; document.body.style.position = "static"; } + remaining = bigOverlays(); + } + return { overlaysRemaining: remaining.length, removed, blocking: remaining.length > 0 && isLocked(), removedLabels }; + }), + new Promise<{ overlaysRemaining: number; removed: number; blocking: boolean; removedLabels: string[] }>((res) => setTimeout(() => res({ overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }), 6000)), + ]); + } catch { + return { overlaysRemaining: 0, removed: 0, blocking: false, removedLabels: [] }; + } +} + +/** + * Stage 2 — animation settling. Wait until layout stops moving (entrance/scroll + * reveals finished) before measuring, so geometry isn't sampled mid-transition + * ("sizes off due to animations in progress"). Samples large-box geometry across + * frames; resolves when stable for several windows or the bound elapses. + */ +async function waitForQuiescence(page: import("playwright").Page, maxMs = 4000): Promise { + try { + return await Promise.race([ + page.evaluate(async (budget: number) => { + const sample = (): string => { + const els = Array.from(document.body.querySelectorAll("*")).filter((e) => { + const r = (e as HTMLElement).getBoundingClientRect(); + return r.width > 80 && r.height > 40; + }).slice(0, 240); + return els.map((e) => { + const r = (e as HTMLElement).getBoundingClientRect(); + return `${Math.round(r.x)},${Math.round(r.y)},${Math.round(r.width)},${Math.round(r.height)}`; + }).join("|"); + }; + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + const deadline = Date.now() + budget; + let prev = sample(); + let stable = 0; + while (Date.now() < deadline && stable < 3) { + await sleep(130); + const cur = sample(); + if (cur === prev) stable++; else { stable = 0; prev = cur; } + } + return stable >= 3; + }, maxMs), + new Promise((res) => setTimeout(() => res(false), maxMs + 1500)), + ]); + } catch { + return false; + } +} + +/** + * Stage 2 — dynamic-media first frame. A streamed `