Initial commit

packages/core/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@cloner/core",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "node --import tsx --test test/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "clone-static": "*"
+  },
+  "devDependencies": {
+    "@cloner/test-utils": "*",
+    "tsx": "4.22.4",
+    "typescript": "5.7.3",
+    "@types/node": "22.10.5"
+  }
+}

packages/core/src/cacheKey.ts

@@ -0,0 +1,62 @@
+import { createHash } from "node:crypto";
+import type { CloneOptions } from "./types.js";
+import { resolveCloneOptions } from "./options.js";
+
+/** Normalize a URL so trivially-different spellings of the same page share a cache
+ *  entry: lowercase scheme+host, drop default ports, drop the fragment, collapse a
+ *  bare/trailing-slash path. Query is preserved (it can select a different page) but
+ *  its params are sorted for stability. Falsy/invalid input is returned trimmed. */
+export function normalizeUrl(raw: string): string {
+  let u: URL;
+  try {
+    u = new URL(raw.trim());
+  } catch {
+    return raw.trim();
+  }
+  u.protocol = u.protocol.toLowerCase();
+  u.hostname = u.hostname.toLowerCase().replace(/\.$/, "");
+  if (
+    (u.protocol === "http:" && u.port === "80") ||
+    (u.protocol === "https:" && u.port === "443")
+  ) {
+    u.port = "";
+  }
+  u.hash = "";
+  // Collapse trailing slashes on the path (but keep "/" for root).
+  u.pathname = u.pathname.replace(/\/+$/, "") || "/";
+  // Sort query params for stability.
+  u.searchParams.sort();
+  return u.toString();
+}
+
+/** The subset of options that change the generated output, serialized canonically.
+ *  `noCache` is intentionally excluded (it's a request-time switch, not an output
+ *  determinant). `verify`/`asyncVerify` are included so a verified result isn't
+ *  served for an unverified request and vice-versa. Viewports are sorted; booleans
+ *  normalized. */
+export function canonicalOptions(options: CloneOptions = {}): string {
+  const resolved = resolveCloneOptions(options);
+  const norm = {
+    mode: resolved.mode,
+    styling: resolved.styling,
+    framework: resolved.framework,
+    viewports: resolved.viewports ? [...resolved.viewports].sort((a, b) => a - b) : null,
+    interactions: resolved.interactions,
+    components: resolved.components,
+    motion: resolved.motion,
+    verify: !!resolved.verify,
+    asyncVerify: !!resolved.asyncVerify,
+    maxRoutes: resolved.maxRoutes ?? null,
+    maxCollection: resolved.maxCollection ?? null,
+  };
+  return JSON.stringify(norm);
+}
+
+/** cacheKey = sha256(normalizedUrl + canonicalOptions + compilerVersion).
+ *  A compilerVersion bump invalidates everything (the output changed). The cache is
+ *  freshness-bounded by the caller (CACHE_STALE_AFTER), because two *captures* of a
+ *  live site can differ even though generation from one capture is byte-stable. */
+export function cacheKey(url: string, options: CloneOptions | undefined, compilerVersion: string): string {
+  const payload = [normalizeUrl(url), canonicalOptions(options), compilerVersion].join("\n");
+  return createHash("sha256").update(payload).digest("hex");
+}

packages/core/src/collectFileMap.ts

@@ -0,0 +1,67 @@
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync, statSync, existsSync } from "node:fs";
+import { join, extname, basename, relative, sep } from "node:path";
+import type { CollectedFile, FileMap } from "./types.js";
+
+/** Extensions whose bytes are returned inline as UTF-8 text (the code a consumer
+ *  reads). Everything else under public/assets (images/fonts/video) is binary and
+ *  returned by reference. */
+const TEXT_EXTS = new Set([
+  ".tsx", ".ts", ".jsx", ".js", ".mjs", ".cjs", ".css", ".json",
+  ".html", ".xml", ".txt", ".md", ".map", ".d.ts",
+]);
+
+function isTextFile(path: string): boolean {
+  const base = basename(path);
+  if (base === ".gitignore" || base === "next-env.d.ts") return true;
+  return TEXT_EXTS.has(extname(path).toLowerCase());
+}
+
+function* walk(dir: string): Generator<string> {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      yield* walk(full);
+    } else if (entry.isFile()) {
+      yield full;
+    }
+  }
+}
+
+const toPosix = (p: string): string => (sep === "/" ? p : p.split(sep).join("/"));
+
+/** Collect the generated app (`<runDir>/generated/app/`) into a file map keyed
+ *  by app-relative POSIX path. Text files inline their content; binaries carry a
+ *  local path + sha256 for the storage layer to upload and presign. Keys are sorted
+ *  so the map is deterministic for the same generated app (golden-file friendly). */
+export function collectFileMap(runDir: string): FileMap {
+  const appDir = join(runDir, "generated", "app");
+  if (!existsSync(appDir)) {
+    throw new Error(`collectFileMap: no generated app at ${appDir}`);
+  }
+  const files: CollectedFile[] = [];
+  for (const abs of walk(appDir)) {
+    const rel = toPosix(relative(appDir, abs));
+    const buf = readFileSync(abs);
+    const sha256 = createHash("sha256").update(buf).digest("hex");
+    const bytes = statSync(abs).size;
+    if (isTextFile(rel)) {
+      files.push({ path: rel, kind: "text", bytes, sha256, content: buf.toString("utf8"), absPath: abs });
+    } else {
+      files.push({ path: rel, kind: "binary", bytes, sha256, absPath: abs });
+    }
+  }
+  files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const map: FileMap = {};
+  for (const f of files) map[f.path] = f;
+  return map;
+}
+
+/** Total bytes + file count for a quick overview (the cheap metadata an MCP
+ *  `get_clone_result` returns before any file content is read). */
+export function fileMapStats(files: FileMap): { fileCount: number; totalBytes: number } {
+  let totalBytes = 0;
+  const vals = Object.values(files);
+  for (const f of vals) totalBytes += f.bytes;
+  return { fileCount: vals.length, totalBytes };
+}

packages/core/src/index.ts

@@ -0,0 +1,26 @@
+/**
+ * @cloner/core — the only package that imports the deterministic compiler.
+ * The api / worker / mcp layers depend on this, never on compiler internals.
+ */
+export { runCloneJob, verifyCloneJobResult } from "./runCloneJob.js";
+export { collectFileMap, fileMapStats } from "./collectFileMap.js";
+export { cacheKey, normalizeUrl, canonicalOptions } from "./cacheKey.js";
+export {
+  normalizeCloneRequestOptions,
+  resolveCloneMode,
+  resolveCloneOptions,
+  resolveCloneStyling,
+} from "./options.js";
+export { COMPILER_VERSION } from "clone-static";
+export type {
+  CloneMode,
+  CloneOptions,
+  CloneStyling,
+  CollectedFile,
+  FileMap,
+  CaptureSanity,
+  CloneTimings,
+  RouteInfo,
+  CloneJobResult,
+  RunCloneJobInput,
+} from "./types.js";

packages/core/src/options.ts

@@ -0,0 +1,57 @@
+import type { CloneFramework, CloneMode, CloneOptions, CloneStyling } from "./types.js";
+
+export type ResolvedCloneOptions = CloneOptions & {
+  mode: CloneMode;
+  styling: CloneStyling;
+  framework: CloneFramework;
+  multiPage: boolean;
+  humanizeMode: CloneStyling;
+  interactions: boolean;
+  components: boolean;
+  motion: boolean;
+};
+
+export function resolveCloneMode(options: CloneOptions = {}): CloneMode {
+  return options.mode ?? (options.multiPage ? "multi" : "single");
+}
+
+export function resolveCloneStyling(options: CloneOptions = {}): CloneStyling {
+  return options.styling ?? options.humanizeMode ?? "tailwind";
+}
+
+export function resolveCloneFramework(options: CloneOptions = {}): CloneFramework {
+  return options.framework ?? "next";
+}
+
+/** Normalize the request-facing shape. Deprecated aliases are consumed but not
+ * echoed, so REST/MCP results present the product-level option names. */
+export function normalizeCloneRequestOptions(options: CloneOptions = {}): CloneOptions {
+  const normalized: CloneOptions = {
+    ...options,
+    mode: resolveCloneMode(options),
+    styling: resolveCloneStyling(options),
+    framework: resolveCloneFramework(options),
+  };
+  delete normalized.multiPage;
+  delete normalized.humanizeMode;
+  return normalized;
+}
+
+/** Resolve options for the compiler adapter. This is where automatic internal
+ * defaults live; callers should not need to choose these in normal use. */
+export function resolveCloneOptions(options: CloneOptions = {}): ResolvedCloneOptions {
+  const mode = resolveCloneMode(options);
+  const styling = resolveCloneStyling(options);
+  const framework = resolveCloneFramework(options);
+  return {
+    ...options,
+    mode,
+    styling,
+    framework,
+    multiPage: mode === "multi",
+    humanizeMode: styling,
+    interactions: options.interactions ?? true,
+    components: options.components ?? true,
+    motion: options.motion ?? true,
+  };
+}

packages/core/src/runCloneJob.ts

@@ -0,0 +1,203 @@
+import { mkdtempSync, rmSync, cpSync, existsSync, mkdirSync, statSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import {
+  runClone,
+  runCloneSite,
+  validateRun,
+  validateSite,
+  buildIR,
+  gatePollution,
+  readJSON,
+  siteIdFromUrl,
+  COMPILER_VERSION,
+  type CaptureResult,
+  type CloneResult as CompilerCloneResult,
+  type CloneSiteResult,
+} from "clone-static";
+import { collectFileMap } from "./collectFileMap.js";
+import type { CaptureSanity, CloneJobResult, CloneOptions, RouteInfo, RunCloneJobInput } from "./types.js";
+import { normalizeCloneRequestOptions, resolveCloneOptions } from "./options.js";
+
+/** Compute the cheap capture-sanity audit (no build): node count + whether the
+ *  pollution gate flags the capture as degenerate, and whether bot/egress-wall text
+ *  was seen. Falls back to safe defaults if the source artifacts are missing. */
+function captureSanity(sourceDir: string, viewports: number[]): CaptureSanity {
+  try {
+    const capture = readJSON<CaptureResult>(join(sourceDir, "capture", "capture-result.json"));
+    const vps = capture.viewports?.length ? capture.viewports : viewports;
+    const ir = buildIR(sourceDir, vps);
+    const p = gatePollution(ir, capture, vps);
+    return { nodeCount: ir.doc.nodeCount, pollution: !p.pass, blocked: !!p.metrics.wallTextDetected };
+  } catch {
+    return { nodeCount: 0, pollution: true, blocked: false };
+  }
+}
+
+/** Persistent entry-capture cache path for a URL. Key = the compiler's full host+path
+ *  site id (collision-safe, unlike the bare folder name), so only the SAME page reuses. */
+function entryCacheSource(cacheDir: string, url: string): string {
+  return join(cacheDir, siteIdFromUrl(url), "source");
+}
+/** Whether a cached capture exists and is fresh enough to reuse (ttlMs 0/undefined =
+ *  no expiry). Staleness is measured from the capture artifact's mtime. */
+function freshCapture(dir: string, ttlMs?: number): boolean {
+  const f = join(dir, "capture", "capture-result.json");
+  if (!existsSync(f)) return false;
+  if (!ttlMs) return true;
+  try { return Date.now() - statSync(f).mtimeMs < ttlMs; } catch { return false; }
+}
+/** Copy a fresh capture into the cache (atomic-ish: clear then copy). No-op if the
+ *  source has no capture artifact. */
+function persistCapture(srcDir: string | undefined, dest: string): void {
+  if (!srcDir || !existsSync(join(srcDir, "capture", "capture-result.json"))) return;
+  mkdirSync(dirname(dest), { recursive: true });
+  rmSync(dest, { recursive: true, force: true });
+  cpSync(srcDir, dest, { recursive: true });
+}
+
+/**
+ * The single seam the service depends on: run one clone end-to-end into a temp
+ * dir, collect its generated app into a file map, optionally verify it, then return
+ * a typed result. The caller (worker/api) uploads artifacts + persists, then the
+ * temp dir is removed in `finally`. No compiler behavior is changed — this only
+ * orchestrates the existing entry points with a parameterized output dir.
+ *
+ * Incremental speed path: when `captureCacheDir` is set, a single-page job stashes its
+ * entry capture there; a later multi-page job for the same URL reuses it as the entry
+ * route (skips re-capturing page 1) and regenerates the whole site on top of it.
+ */
+export async function runCloneJob(input: RunCloneJobInput): Promise<CloneJobResult> {
+  const requestOptions: CloneOptions = normalizeCloneRequestOptions(input.options ?? {});
+  const options = resolveCloneOptions(requestOptions);
+  const syncVerify = !!options.verify && !options.asyncVerify;
+  const captureValidationArtifacts = !!(options.verify || options.asyncVerify);
+  const log = input.log ?? (() => {});
+  const ownsTemp = !input.runsDir;
+  const runsDir = input.runsDir ?? mkdtempSync(join(tmpdir(), "clone-job-"));
+  const kind: "clone" | "clone_site" = options.mode === "multi" ? "clone_site" : "clone";
+
+  // Best-by-default (matches the CLI): interactions/components/motion ON unless the
+  // caller explicitly disables them. (runClone itself defaults them OFF; the CLI is
+  // what turns them on — so the service does the same here.)
+  const interactions = options.interactions;
+  const components = options.components;
+  const motion = options.motion;
+
+  try {
+    const t0 = Date.now();
+    let runDir: string;
+    let routes: RouteInfo[] | undefined;
+    let sanity: CaptureSanity;
+    let captureReused = false;
+    // Persistent entry-capture cache (the single→multi speed path), keyed by URL.
+    const cacheEntry = input.captureCacheDir ? entryCacheSource(input.captureCacheDir, input.url) : undefined;
+
+    if (kind === "clone_site") {
+      // Reuse a prior single-page capture for the entry route when the cache holds a fresh one.
+      const reuseEntrySource = cacheEntry && freshCapture(cacheEntry, input.captureCacheTtlMs) ? cacheEntry : undefined;
+      captureReused = !!reuseEntrySource;
+      const res: CloneSiteResult = await runCloneSite({
+        url: input.url,
+        runsDir,
+        validate: false,
+        interactions,
+        components,
+        humanizeMode: options.styling,
+        framework: options.framework,
+        reuseEntrySource,
+        maxRoutes: options.maxRoutes,
+        maxCollectionInstances: options.maxCollection,
+        captureConcurrency: options.captureConcurrency,
+        validationConcurrency: options.validationConcurrency,
+        viewportConcurrency: options.viewportConcurrency,
+        screenshots: captureValidationArtifacts,
+        log,
+      });
+      runDir = res.runDir;
+      // Reproduced routes + the collapsed-collection map (representativeOf).
+      routes = res.routes.map((r) => ({ route: r.routePath }));
+      for (const c of res.plan.collections) {
+        const rep = res.routes.find((r) => r.routePath === c.representative);
+        if (rep) {
+          for (const inst of c.instances) {
+            if (inst !== c.representative) routes!.push({ route: inst, representativeOf: c.representative });
+          }
+        }
+      }
+      const entry = res.routes.find((r) => r.routePath === res.plan.entry) ?? res.routes[0];
+      sanity = entry
+        ? captureSanity(entry.sourceDir, entry.ir.doc.viewports)
+        : { nodeCount: 0, pollution: true, blocked: false };
+      // Refresh the cache with the entry capture (seeds it for a cold multi-page run).
+      if (cacheEntry && entry) persistCapture(entry.sourceDir, cacheEntry);
+    } else {
+      const res: CompilerCloneResult = await runClone({
+        url: input.url,
+        runsDir,
+        viewports: options.viewports,
+        interactions,
+        components,
+        motion,
+        humanizeMode: options.styling,
+        framework: options.framework,
+        screenshots: captureValidationArtifacts,
+        log,
+      });
+      runDir = res.runDir;
+      sanity = captureSanity(res.sourceDir, options.viewports ?? [375, 768, 1280, 1920]);
+      // Stash this page's capture so a later multi-page job can expand on it (speed path).
+      if (cacheEntry) persistCapture(res.sourceDir, cacheEntry);
+    }
+    const captureMs = Date.now() - t0;
+
+    // Optional verify (build + serve + re-render + grade). The single isolation-
+    // sensitive step: pass a per-worker harnessDir so concurrent builds don't collide.
+    let verify: unknown;
+    let verifyMs: number | undefined;
+    if (syncVerify) {
+      const done = await verifyCloneJobResult({ kind, runDir }, { harnessDir: input.harnessDir, tier: input.tier ?? "stage2", validationConcurrency: options.validationConcurrency, viewportConcurrency: options.viewportConcurrency, log });
+      verify = done.verify;
+      verifyMs = done.verifyMs;
+    }
+
+    const files = collectFileMap(runDir);
+
+    return {
+      url: input.url,
+      kind,
+      options: requestOptions,
+      status: "succeeded",
+      compilerVersion: COMPILER_VERSION,
+      timings: { captureMs, generateMs: 0, ...(verifyMs !== undefined ? { verifyMs } : {}) },
+      routes,
+      files,
+      capture: sanity,
+      captureReused,
+      verify,
+      runDir,
+    };
+  } finally {
+    if (ownsTemp && !input.keepTemp) {
+      rmSync(runsDir, { recursive: true, force: true });
+    }
+  }
+}
+
+export async function verifyCloneJobResult(
+  result: Pick<CloneJobResult, "kind" | "runDir">,
+  opts?: {
+    harnessDir?: string;
+    tier?: string;
+    validationConcurrency?: number;
+    viewportConcurrency?: number;
+    log?: (e: Record<string, unknown>) => void;
+  },
+): Promise<{ verify: unknown; verifyMs: number }> {
+  const t0 = Date.now();
+  const tier = opts?.tier ?? "stage2";
+  const verify = result.kind === "clone_site"
+    ? await validateSite(result.runDir, { harnessDir: opts?.harnessDir, tier, routeConcurrency: opts?.validationConcurrency, viewportConcurrency: opts?.viewportConcurrency, log: opts?.log })
+    : await validateRun(result.runDir, { harnessDir: opts?.harnessDir, tier, log: opts?.log });
+  return { verify, verifyMs: Date.now() - t0 };
+}

packages/core/src/types.ts

@@ -0,0 +1,123 @@
+/**
+ * Service-layer types. These are the public wire shapes for clone options,
+ * clone results, and the eager output file map, plus the core's internal
+ * collected-file shape.
+ *
+ * The core layer is the ONLY place that imports the compiler. Everything above
+ * it (api, worker, mcp) speaks these types.
+ */
+
+export type CloneMode = "single" | "multi";
+export type CloneStyling = "tailwind" | "css";
+export type CloneFramework = "next" | "vite";
+
+/** Clone options accepted by the service/core boundary.
+ *
+ * Normal callers should use only the product choices:
+ * - `mode`: single page or multi-page site clone.
+ * - `styling`: Tailwind v4 or plain CSS output.
+ * - `framework`: Next.js App Router or Vite React output.
+ *
+ * The remaining fields are operational/dev controls or deprecated compatibility
+ * aliases. Internal pipeline features such as probing, recipes, component
+ * extraction, interactions, and motion stay automatic by default.
+ */
+export type CloneOptions = {
+  mode?: CloneMode;
+  styling?: CloneStyling;
+  framework?: CloneFramework;
+  verify?: boolean;
+  /** DB/worker mode: persist the clone first, then attach verify in the background. */
+  asyncVerify?: boolean;
+  maxRoutes?: number;
+  maxCollection?: number;
+  captureConcurrency?: number;
+  validationConcurrency?: number;
+  viewportConcurrency?: number;
+  /** Service-level: bypass the cache on read AND write (does not affect output). */
+  noCache?: boolean;
+
+  /** @deprecated Use `mode: "multi"` instead. */
+  multiPage?: boolean;
+  /** @deprecated Use `styling` instead. */
+  humanizeMode?: CloneStyling;
+
+  /** Dev-only/output-affecting escape hatches. Not part of the normal product surface. */
+  viewports?: number[];
+  interactions?: boolean;
+  components?: boolean;
+  motion?: boolean;
+};
+
+/** A single file in a clone's output, as collected from `generated/app/`.
+ *  Text files carry their content inline; binaries carry only a local path +
+ *  hash (the storage layer turns `absPath` into a presigned URL after upload). */
+export type CollectedFile = {
+  /** app-relative POSIX path, e.g. "src/app/page.tsx" or "public/assets/cloned/images/ab.png" */
+  path: string;
+  kind: "text" | "binary";
+  bytes: number;
+  /** sha256 of the raw file bytes (content-addressing for storage + cache integrity). */
+  sha256: string;
+  /** present for text files only */
+  content?: string;
+  /** local filesystem path (ephemeral — used by the storage layer to upload/stream). */
+  absPath: string;
+};
+
+export type FileMap = Record<string, CollectedFile>;
+
+/** Capture-sanity audit — surfaced so a structurally-perfect clone of the WRONG
+ *  page (bot wall, empty shell) is flagged rather than sold as success. */
+export type CaptureSanity = {
+  nodeCount: number;
+  pollution: boolean; // the pollution gate failed (degenerate capture)
+  blocked: boolean; // bot/egress wall text detected
+};
+
+export type CloneTimings = { captureMs: number; generateMs: number; verifyMs?: number };
+
+export type RouteInfo = { route: string; representativeOf?: string };
+
+/** What `runCloneJob` returns — the full result for one clone, BEFORE storage
+ *  upload assigns URLs to binaries. The worker/api persists + uploads from this. */
+export type CloneJobResult = {
+  url: string;
+  kind: "clone" | "clone_site";
+  options: CloneOptions;
+  status: "succeeded";
+  compilerVersion: string;
+  timings: CloneTimings;
+  routes?: RouteInfo[];
+  files: FileMap;
+  capture: CaptureSanity;
+  /** true when a multi-page job reused a prior single-page entry capture (no re-capture
+   *  of the entry route) — the "single page first, then expand" speed path. */
+  captureReused?: boolean;
+  /** present only when options.verify === true */
+  verify?: unknown; // compiler Report — kept opaque here to avoid leaking gate internals upward
+  /** the ephemeral local run dir (already collected into `files`); caller cleans up
+   *  unless `keepTemp` was set. */
+  runDir: string;
+};
+
+export type RunCloneJobInput = {
+  url: string;
+  options?: CloneOptions;
+  /** override the temp base dir (tests). When set, the dir is NOT auto-removed. */
+  runsDir?: string;
+  /** keep the run dir after collecting the file map (debug/tests). */
+  keepTemp?: boolean;
+  /** Persistent entry-capture cache base dir, keyed by URL. A single-page job copies its
+   *  capture here; a later multi-page job for the same URL REUSES it as the entry route
+   *  (no re-capture) — the "single page first, then expand to the full site" speed path. */
+  captureCacheDir?: string;
+  /** Max age (ms) of a cached entry capture that may be reused; older ⇒ re-capture.
+   *  Undefined/0 ⇒ no expiry (reuse whenever present). Bounds staleness for a service. */
+  captureCacheTtlMs?: number;
+  /** isolated build harness dir for verify (per-worker; defaults to the compiler's). */
+  harnessDir?: string;
+  /** tier threshold for the perceptual gate when verifying (default "stage2"). */
+  tier?: string;
+  log?: (e: Record<string, unknown>) => void;
+};

packages/core/test/cacheKey.test.ts

@@ -0,0 +1,41 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { cacheKey, normalizeUrl, canonicalOptions } from "../src/cacheKey.js";
+
+test("normalizeUrl: canonicalizes scheme/host/port/trailing-slash/fragment/query", () => {
+  assert.equal(normalizeUrl("HTTPS://Example.com:443/foo/#frag"), "https://example.com/foo");
+  assert.equal(normalizeUrl("http://example.com:80/"), "http://example.com/");
+  assert.equal(normalizeUrl("https://x.com"), "https://x.com/");
+  assert.equal(normalizeUrl("https://example.com/a?b=2&a=1"), "https://example.com/a?a=1&b=2");
+  assert.equal(normalizeUrl("not a url"), "not a url");
+});
+
+test("cacheKey: stable for equivalent input, varies by options + compilerVersion", () => {
+  const k1 = cacheKey("https://x.com/", { mode: "single", styling: "tailwind" }, "0.1.0");
+  const k2 = cacheKey("https://x.com", { mode: "single", styling: "tailwind" }, "0.1.0");
+  assert.equal(k1, k2, "trailing slash is normalized away");
+
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "multi", styling: "tailwind" }, "0.1.0"), "mode changes the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "css" }, "0.1.0"), "styling changes the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", framework: "vite" }, "0.1.0"), "framework changes the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind" }, "0.2.0"), "version bump invalidates");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", verify: true }, "0.1.0"), "verify is part of the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", asyncVerify: true }, "0.1.0"), "asyncVerify is part of the key");
+
+  // noCache is a request-time switch, not an output determinant — must not change the key.
+  assert.equal(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", noCache: true }, "0.1.0"));
+});
+
+test("canonicalOptions: deprecated aliases normalize to product options", () => {
+  assert.equal(
+    canonicalOptions({ mode: "multi", styling: "css" }),
+    canonicalOptions({ multiPage: true, humanizeMode: "css" }),
+  );
+});
+
+test("canonicalOptions: sorts viewports and is stable", () => {
+  assert.equal(
+    canonicalOptions({ viewports: [1920, 375, 768] }),
+    canonicalOptions({ viewports: [375, 768, 1920] }),
+  );
+});

packages/core/test/collectFileMap.test.ts

@@ -0,0 +1,54 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createHash } from "node:crypto";
+import { collectFileMap, fileMapStats } from "../src/collectFileMap.js";
+
+test("collectFileMap: inlines text, references binaries, sorts keys, hashes bytes", () => {
+  const root = mkdtempSync(join(tmpdir(), "cfm-"));
+  try {
+    const app = join(root, "generated", "app");
+    mkdirSync(join(app, "src", "app", "_clone"), { recursive: true });
+    mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+    writeFileSync(join(app, "package.json"), '{"name":"x"}\n');
+    writeFileSync(join(app, ".gitignore"), "node_modules\n");
+    writeFileSync(join(app, "src", "app", "page.tsx"), "export default function Page(){return null}\n");
+    writeFileSync(join(app, "src", "app", "globals.css"), "body{margin:0}\n");
+    const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]);
+    writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), png);
+
+    const map = collectFileMap(root);
+    const keys = Object.keys(map);
+    assert.deepEqual(keys, [...keys].sort(), "keys are sorted (deterministic)");
+
+    assert.equal(map["package.json"]!.kind, "text");
+    assert.equal(map[".gitignore"]!.kind, "text", ".gitignore is treated as text");
+    const page = map["src/app/page.tsx"]!;
+    assert.equal(page.kind, "text");
+    assert.ok(page.content!.includes("export default"));
+    assert.equal(page.sha256, createHash("sha256").update("export default function Page(){return null}\n").digest("hex"));
+
+    const bin = map["public/assets/cloned/images/a.png"]!;
+    assert.equal(bin.kind, "binary");
+    assert.equal(bin.content, undefined, "binaries are by reference, not inlined");
+    assert.equal(bin.sha256, createHash("sha256").update(png).digest("hex"));
+    assert.equal(bin.bytes, png.length);
+
+    const stats = fileMapStats(map);
+    assert.equal(stats.fileCount, 5);
+    assert.ok(stats.totalBytes > 0);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("collectFileMap: throws when there is no generated app", () => {
+  const root = mkdtempSync(join(tmpdir(), "cfm-empty-"));
+  try {
+    assert.throws(() => collectFileMap(root), /no generated app/);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});

packages/core/test/incremental.fixture.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runCloneJob } from "../src/runCloneJob.js";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+
+// The "single page first, then expand" speed path: a single-page clone stashes its
+// entry capture in captureCacheDir; a later multi-page clone of the SAME url reuses it
+// as the entry route (no re-capture) and regenerates the whole site on top of it.
+describe("runCloneJob incremental (single → multi reuse)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  let cacheDir: string;
+  before(async () => {
+    server = await serveDir(join(FIXTURES_DIR, "site"));
+    cacheDir = mkdtempSync(join(tmpdir(), "capture-cache-"));
+  });
+  after(async () => {
+    await server.close();
+    rmSync(cacheDir, { recursive: true, force: true });
+  });
+
+  it("reuses the single-page entry capture when expanding to the full site", async () => {
+    const url = server.url + "/";
+
+    // 1. Single page first — fast, returns one app; seeds the capture cache.
+    const single = await runCloneJob({ url, options: {}, captureCacheDir: cacheDir });
+    assert.equal(single.status, "succeeded");
+    assert.equal(single.kind, "clone");
+    assert.equal(single.captureReused, false, "first (single) job captures fresh");
+    assert.ok(single.files["src/app/page.tsx"], "single-page app emitted");
+
+    // 2. Expand to the full multi-route site — reuses page 1's capture (no re-capture),
+    //    captures the rest, regenerates ALL routes together.
+    const multi = await runCloneJob({ url, options: { mode: "multi" }, captureCacheDir: cacheDir });
+    assert.equal(multi.status, "succeeded");
+    assert.equal(multi.kind, "clone_site");
+    assert.equal(multi.captureReused, true, "multi-page job reused the cached entry capture");
+    assert.ok((multi.routes?.length ?? 0) >= 2, `expected >=2 routes, got ${multi.routes?.length}`);
+    assert.ok(multi.files["src/app/layout.tsx"], "shared layout emitted");
+    const subRoutePages = Object.keys(multi.files).filter((p) => /^src\/app\/.+\/page\.tsx$/.test(p));
+    assert.ok(subRoutePages.length >= 1, "at least one sub-route page beyond the entry");
+
+    // Tailwind is the default styling output end-to-end (service path included).
+    assert.ok(multi.files["postcss.config.mjs"], "Tailwind toolchain present");
+    assert.ok((multi.files["src/app/globals.css"]!.content ?? "").includes("tailwindcss"), "Tailwind globals");
+  });
+});

packages/core/test/multipage.fixture.test.ts

@@ -0,0 +1,34 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { join } from "node:path";
+import { runCloneJob } from "../src/runCloneJob.js";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+
+// Multi-page clone job: crawl a served fixture site (index → blog → faq) and clone
+// all routes into one Next app. Skipped without Chromium.
+describe("runCloneJob multi-page (served fixture site)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  before(async () => {
+    server = await serveDir(join(FIXTURES_DIR, "site"));
+  });
+  after(async () => {
+    await server.close();
+  });
+
+  it("crawls + clones multiple routes into one app (clone_site)", async () => {
+    const res = await runCloneJob({ url: server.url + "/", options: { mode: "multi", interactions: false, components: false } });
+    assert.equal(res.kind, "clone_site");
+    assert.ok(res.routes && res.routes.length >= 2, `expected >=2 routes, got ${res.routes?.length}`);
+    assert.ok(res.files["src/app/layout.tsx"], "shared layout emitted once");
+    assert.ok(res.files["src/app/page.tsx"], "home route page");
+    assert.ok(res.files["AGENTS.md"], "generated AGENTS.md emitted");
+    assert.ok(res.files["ARCHITECTURE.md"], "generated ARCHITECTURE.md emitted");
+    assert.ok(res.files["src/app/robots.ts"], "robots route emitted");
+    assert.ok(res.files["src/app/sitemap.ts"], "sitemap route emitted");
+    assert.ok(res.files["src/app/llms.txt/route.ts"], "llms route emitted");
+    assert.ok((res.files["src/app/llms.txt/route.ts"]!.content ?? "").includes("Build faster with Acme"), "generated llms includes captured route content");
+    const subRoutePages = Object.keys(res.files).filter((p) => /^src\/app\/.+\/page\.tsx$/.test(p));
+    assert.ok(subRoutePages.length >= 1, "at least one sub-route page");
+    assert.ok(res.capture.nodeCount > 0);
+  });
+});

packages/core/test/runCloneJob.fixture.test.ts

@@ -0,0 +1,170 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runClone } from "clone-static";
+import { runCloneJob } from "../src/runCloneJob.js";
+import { collectFileMap } from "../src/collectFileMap.js";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+
+const CHROMIUM = hasChromium();
+
+// End-to-end clone of a served fixture (zero external network). Skipped when no
+// Playwright Chromium is installed (run `npx playwright install chromium` first).
+describe("runCloneJob (served fixture)", { skip: CHROMIUM ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  before(async () => {
+    server = await serveDir(FIXTURES_DIR);
+  });
+  after(async () => {
+    await server.close();
+  });
+
+  it("produces the file-map contract for a single-page clone", async () => {
+    const url = server.url + "/components.html";
+    const res = await runCloneJob({
+      url,
+      options: { interactions: false, components: true, motion: false },
+    });
+
+    assert.equal(res.status, "succeeded");
+    assert.equal(res.kind, "clone");
+    assert.ok(res.compilerVersion);
+
+    // The essential scaffold is present and typed correctly.
+    for (const k of [
+      "package.json",
+      "tsconfig.json",
+      "next.config.mjs",
+      "src/app/layout.tsx",
+      "src/app/page.tsx",
+      "src/app/ditto.css",
+      "src/app/globals.css",
+    ]) {
+      assert.ok(res.files[k], `expected file ${k}`);
+      assert.equal(res.files[k]!.kind, "text");
+    }
+    assert.ok(res.files["src/app/page.tsx"]!.content!.includes("export default function Page"));
+
+    // Extracted components are split into their own files (compiler change); the
+    // service collects them as text automatically.
+    const componentFiles = Object.keys(res.files).filter((k) => k.startsWith("src/app/components/"));
+    assert.ok(componentFiles.length > 0, "extracted components emitted as separate files");
+    for (const k of componentFiles) {
+      assert.equal(res.files[k]!.kind, "text");
+      assert.ok((res.files[k]!.content ?? "").length > 0, `${k} has content`);
+    }
+
+    // Capture sanity: a real fixture is not degenerate / bot-walled.
+    assert.ok(res.capture.nodeCount > 0, "nodeCount > 0");
+    assert.equal(res.capture.blocked, false);
+
+    // temp dir is cleaned up by default.
+    const { existsSync } = await import("node:fs");
+    assert.equal(existsSync(res.runDir), false, "temp run dir removed");
+  });
+
+  it("can generate a Vite React app instead of Next", async () => {
+    const url = server.url + "/components.html";
+    const res = await runCloneJob({
+      url,
+      options: { framework: "vite", interactions: false, components: false, motion: false },
+    });
+
+    assert.equal(res.status, "succeeded");
+    assert.equal(res.options.framework, "vite");
+    for (const k of [
+      "index.html",
+      "vite.config.ts",
+      "src/main.tsx",
+      "src/page.tsx",
+      "src/ditto.css",
+      "src/globals.css",
+      "public/robots.txt",
+      "public/sitemap.xml",
+      "public/llms.txt",
+    ]) {
+      assert.ok(res.files[k], `expected file ${k}`);
+      assert.equal(res.files[k]!.kind, "text");
+    }
+    assert.ok(!res.files["next.config.mjs"], "Vite output should not include next.config.mjs");
+    assert.ok(!res.files["src/app/layout.tsx"], "Vite output should not include App Router layout");
+    assert.ok(res.files["package.json"]!.content!.includes('"dev": "vite"'));
+    assert.ok(res.files["src/main.tsx"]!.content!.includes('from "react-dom/client"'));
+    assert.ok((res.files["src/globals.css"]!.content ?? "").includes("#root { display: contents; }"));
+  });
+
+  it("names sections with valid JS identifiers even when a heading starts with a number", async () => {
+    // Repro: a section whose heading reads "0019 Iterate Faster" previously became the
+    // identifier `0019IterateSection` → `import 0019IterateSection` is a syntax error.
+    const url = server.url + "/numeric-section.html";
+    const res = await runCloneJob({ url, options: { interactions: false, components: false, motion: false } });
+    assert.equal(res.status, "succeeded");
+
+    const page = res.files["src/app/page.tsx"]!.content ?? "";
+    const imports = [...page.matchAll(/^import\s+([A-Za-z0-9_$]+)\s+from/gm)].map((m) => m[1]!);
+    assert.ok(imports.length > 0, "page imports section modules");
+    for (const id of imports) {
+      assert.match(id, /^[A-Za-z_$]/, `import identifier "${id}" must not start with a digit`);
+    }
+    // The numeric "0019" layer noise is dropped → a clean, valid name.
+    assert.ok(imports.includes("IterateFasterSection"), `expected IterateFasterSection, got ${imports.join(", ")}`);
+  });
+
+  it("preserves generated SEO metadata, icons, JSON-LD, llms, and docs", async () => {
+    const url = server.url + "/seo-rich.html";
+    const res = await runCloneJob({
+      url,
+      options: { interactions: false, components: false, motion: false },
+    });
+    assert.equal(res.status, "succeeded");
+
+    const layout = res.files["src/app/layout.tsx"]!.content ?? "";
+    assert.ok(layout.includes("SEO Rich Fixture"));
+    assert.ok(layout.includes("Open Graph description from the source page."));
+    assert.ok(layout.includes("summary_large_image"));
+    assert.ok(layout.includes("themeColor"));
+    assert.ok(layout.includes("colorScheme"));
+    assert.ok(layout.includes("application/ld+json"));
+
+    assert.equal(res.files["src/app/favicon.ico"]?.kind, "binary");
+    assert.equal(res.files["src/app/icon.png"]?.kind, "binary");
+    assert.equal(res.files["src/app/apple-icon.png"]?.kind, "binary");
+    assert.ok(Object.keys(res.files).some((k) => k.startsWith("public/assets/cloned/manifest/")), "web manifest materialized");
+    assert.ok(Object.keys(res.files).some((k) => k.startsWith("public/assets/cloned/images/")), "manifest/icon image assets materialized");
+
+    const llms = res.files["src/app/llms.txt/route.ts"]!.content ?? "";
+    assert.ok(llms.includes("Source LLMS"), "source llms.txt preserved");
+    const llmsFull = res.files["src/app/llms-full.txt/route.ts"]!.content ?? "";
+    assert.ok(llmsFull.includes("Source LLMS Full"), "source llms-full.txt preserved");
+
+    assert.ok(res.files["AGENTS.md"]!.content!.includes("generated ditto.site clone app"));
+    assert.ok(res.files["ARCHITECTURE.md"]!.content!.includes("data-ditto-id"));
+  });
+
+  it("generates byte-identical output from one frozen capture (golden / Gate 6)", async () => {
+    const url = server.url + "/components.html";
+    const a = mkdtempSync(join(tmpdir(), "golden-a-"));
+    const b = mkdtempSync(join(tmpdir(), "golden-b-"));
+    try {
+      const opts = { interactions: false, components: true, motion: false } as const;
+      const r1 = await runClone({ url, runsDir: a, ...opts });
+      // Regenerate from the SAME capture (no re-capture) → must be byte-identical.
+      const r2 = await runClone({ url, runsDir: b, reuseSource: r1.sourceDir, ...opts });
+
+      const m1 = collectFileMap(r1.runDir);
+      const m2 = collectFileMap(r2.runDir);
+      // Same file set + byte-identical contents across regenerations — covers the
+      // scaffold AND the split components/*.tsx, all from one frozen capture.
+      assert.deepEqual(Object.keys(m1).sort(), Object.keys(m2).sort(), "identical file set across regenerations");
+      for (const f of Object.keys(m1)) {
+        assert.equal(m1[f]!.sha256, m2[f]!.sha256, `${f} is byte-identical across regenerations`);
+      }
+      assert.ok(Object.keys(m1).some((k) => k.startsWith("src/app/components/")), "components are split into files");
+    } finally {
+      rmSync(a, { recursive: true, force: true });
+      rmSync(b, { recursive: true, force: true });
+    }
+  });
+});

packages/core/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"]
+}