Initial commit

packages/core/src/cacheKey.ts

@@ -0,0 +1,62 @@
+import { createHash } from "node:crypto";
+import type { CloneOptions } from "./types.js";
+import { resolveCloneOptions } from "./options.js";
+
+/** Normalize a URL so trivially-different spellings of the same page share a cache
+ *  entry: lowercase scheme+host, drop default ports, drop the fragment, collapse a
+ *  bare/trailing-slash path. Query is preserved (it can select a different page) but
+ *  its params are sorted for stability. Falsy/invalid input is returned trimmed. */
+export function normalizeUrl(raw: string): string {
+  let u: URL;
+  try {
+    u = new URL(raw.trim());
+  } catch {
+    return raw.trim();
+  }
+  u.protocol = u.protocol.toLowerCase();
+  u.hostname = u.hostname.toLowerCase().replace(/\.$/, "");
+  if (
+    (u.protocol === "http:" && u.port === "80") ||
+    (u.protocol === "https:" && u.port === "443")
+  ) {
+    u.port = "";
+  }
+  u.hash = "";
+  // Collapse trailing slashes on the path (but keep "/" for root).
+  u.pathname = u.pathname.replace(/\/+$/, "") || "/";
+  // Sort query params for stability.
+  u.searchParams.sort();
+  return u.toString();
+}
+
+/** The subset of options that change the generated output, serialized canonically.
+ *  `noCache` is intentionally excluded (it's a request-time switch, not an output
+ *  determinant). `verify`/`asyncVerify` are included so a verified result isn't
+ *  served for an unverified request and vice-versa. Viewports are sorted; booleans
+ *  normalized. */
+export function canonicalOptions(options: CloneOptions = {}): string {
+  const resolved = resolveCloneOptions(options);
+  const norm = {
+    mode: resolved.mode,
+    styling: resolved.styling,
+    framework: resolved.framework,
+    viewports: resolved.viewports ? [...resolved.viewports].sort((a, b) => a - b) : null,
+    interactions: resolved.interactions,
+    components: resolved.components,
+    motion: resolved.motion,
+    verify: !!resolved.verify,
+    asyncVerify: !!resolved.asyncVerify,
+    maxRoutes: resolved.maxRoutes ?? null,
+    maxCollection: resolved.maxCollection ?? null,
+  };
+  return JSON.stringify(norm);
+}
+
+/** cacheKey = sha256(normalizedUrl + canonicalOptions + compilerVersion).
+ *  A compilerVersion bump invalidates everything (the output changed). The cache is
+ *  freshness-bounded by the caller (CACHE_STALE_AFTER), because two *captures* of a
+ *  live site can differ even though generation from one capture is byte-stable. */
+export function cacheKey(url: string, options: CloneOptions | undefined, compilerVersion: string): string {
+  const payload = [normalizeUrl(url), canonicalOptions(options), compilerVersion].join("\n");
+  return createHash("sha256").update(payload).digest("hex");
+}

packages/core/src/collectFileMap.ts

@@ -0,0 +1,67 @@
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync, statSync, existsSync } from "node:fs";
+import { join, extname, basename, relative, sep } from "node:path";
+import type { CollectedFile, FileMap } from "./types.js";
+
+/** Extensions whose bytes are returned inline as UTF-8 text (the code a consumer
+ *  reads). Everything else under public/assets (images/fonts/video) is binary and
+ *  returned by reference. */
+const TEXT_EXTS = new Set([
+  ".tsx", ".ts", ".jsx", ".js", ".mjs", ".cjs", ".css", ".json",
+  ".html", ".xml", ".txt", ".md", ".map", ".d.ts",
+]);
+
+function isTextFile(path: string): boolean {
+  const base = basename(path);
+  if (base === ".gitignore" || base === "next-env.d.ts") return true;
+  return TEXT_EXTS.has(extname(path).toLowerCase());
+}
+
+function* walk(dir: string): Generator<string> {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      yield* walk(full);
+    } else if (entry.isFile()) {
+      yield full;
+    }
+  }
+}
+
+const toPosix = (p: string): string => (sep === "/" ? p : p.split(sep).join("/"));
+
+/** Collect the generated app (`<runDir>/generated/app/`) into a file map keyed
+ *  by app-relative POSIX path. Text files inline their content; binaries carry a
+ *  local path + sha256 for the storage layer to upload and presign. Keys are sorted
+ *  so the map is deterministic for the same generated app (golden-file friendly). */
+export function collectFileMap(runDir: string): FileMap {
+  const appDir = join(runDir, "generated", "app");
+  if (!existsSync(appDir)) {
+    throw new Error(`collectFileMap: no generated app at ${appDir}`);
+  }
+  const files: CollectedFile[] = [];
+  for (const abs of walk(appDir)) {
+    const rel = toPosix(relative(appDir, abs));
+    const buf = readFileSync(abs);
+    const sha256 = createHash("sha256").update(buf).digest("hex");
+    const bytes = statSync(abs).size;
+    if (isTextFile(rel)) {
+      files.push({ path: rel, kind: "text", bytes, sha256, content: buf.toString("utf8"), absPath: abs });
+    } else {
+      files.push({ path: rel, kind: "binary", bytes, sha256, absPath: abs });
+    }
+  }
+  files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const map: FileMap = {};
+  for (const f of files) map[f.path] = f;
+  return map;
+}
+
+/** Total bytes + file count for a quick overview (the cheap metadata an MCP
+ *  `get_clone_result` returns before any file content is read). */
+export function fileMapStats(files: FileMap): { fileCount: number; totalBytes: number } {
+  let totalBytes = 0;
+  const vals = Object.values(files);
+  for (const f of vals) totalBytes += f.bytes;
+  return { fileCount: vals.length, totalBytes };
+}

packages/core/src/index.ts

@@ -0,0 +1,26 @@
+/**
+ * @cloner/core — the only package that imports the deterministic compiler.
+ * The api / worker / mcp layers depend on this, never on compiler internals.
+ */
+export { runCloneJob, verifyCloneJobResult } from "./runCloneJob.js";
+export { collectFileMap, fileMapStats } from "./collectFileMap.js";
+export { cacheKey, normalizeUrl, canonicalOptions } from "./cacheKey.js";
+export {
+  normalizeCloneRequestOptions,
+  resolveCloneMode,
+  resolveCloneOptions,
+  resolveCloneStyling,
+} from "./options.js";
+export { COMPILER_VERSION } from "clone-static";
+export type {
+  CloneMode,
+  CloneOptions,
+  CloneStyling,
+  CollectedFile,
+  FileMap,
+  CaptureSanity,
+  CloneTimings,
+  RouteInfo,
+  CloneJobResult,
+  RunCloneJobInput,
+} from "./types.js";

packages/core/src/options.ts

@@ -0,0 +1,57 @@
+import type { CloneFramework, CloneMode, CloneOptions, CloneStyling } from "./types.js";
+
+export type ResolvedCloneOptions = CloneOptions & {
+  mode: CloneMode;
+  styling: CloneStyling;
+  framework: CloneFramework;
+  multiPage: boolean;
+  humanizeMode: CloneStyling;
+  interactions: boolean;
+  components: boolean;
+  motion: boolean;
+};
+
+export function resolveCloneMode(options: CloneOptions = {}): CloneMode {
+  return options.mode ?? (options.multiPage ? "multi" : "single");
+}
+
+export function resolveCloneStyling(options: CloneOptions = {}): CloneStyling {
+  return options.styling ?? options.humanizeMode ?? "tailwind";
+}
+
+export function resolveCloneFramework(options: CloneOptions = {}): CloneFramework {
+  return options.framework ?? "next";
+}
+
+/** Normalize the request-facing shape. Deprecated aliases are consumed but not
+ * echoed, so REST/MCP results present the product-level option names. */
+export function normalizeCloneRequestOptions(options: CloneOptions = {}): CloneOptions {
+  const normalized: CloneOptions = {
+    ...options,
+    mode: resolveCloneMode(options),
+    styling: resolveCloneStyling(options),
+    framework: resolveCloneFramework(options),
+  };
+  delete normalized.multiPage;
+  delete normalized.humanizeMode;
+  return normalized;
+}
+
+/** Resolve options for the compiler adapter. This is where automatic internal
+ * defaults live; callers should not need to choose these in normal use. */
+export function resolveCloneOptions(options: CloneOptions = {}): ResolvedCloneOptions {
+  const mode = resolveCloneMode(options);
+  const styling = resolveCloneStyling(options);
+  const framework = resolveCloneFramework(options);
+  return {
+    ...options,
+    mode,
+    styling,
+    framework,
+    multiPage: mode === "multi",
+    humanizeMode: styling,
+    interactions: options.interactions ?? true,
+    components: options.components ?? true,
+    motion: options.motion ?? true,
+  };
+}

packages/core/src/runCloneJob.ts

@@ -0,0 +1,203 @@
+import { mkdtempSync, rmSync, cpSync, existsSync, mkdirSync, statSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import {
+  runClone,
+  runCloneSite,
+  validateRun,
+  validateSite,
+  buildIR,
+  gatePollution,
+  readJSON,
+  siteIdFromUrl,
+  COMPILER_VERSION,
+  type CaptureResult,
+  type CloneResult as CompilerCloneResult,
+  type CloneSiteResult,
+} from "clone-static";
+import { collectFileMap } from "./collectFileMap.js";
+import type { CaptureSanity, CloneJobResult, CloneOptions, RouteInfo, RunCloneJobInput } from "./types.js";
+import { normalizeCloneRequestOptions, resolveCloneOptions } from "./options.js";
+
+/** Compute the cheap capture-sanity audit (no build): node count + whether the
+ *  pollution gate flags the capture as degenerate, and whether bot/egress-wall text
+ *  was seen. Falls back to safe defaults if the source artifacts are missing. */
+function captureSanity(sourceDir: string, viewports: number[]): CaptureSanity {
+  try {
+    const capture = readJSON<CaptureResult>(join(sourceDir, "capture", "capture-result.json"));
+    const vps = capture.viewports?.length ? capture.viewports : viewports;
+    const ir = buildIR(sourceDir, vps);
+    const p = gatePollution(ir, capture, vps);
+    return { nodeCount: ir.doc.nodeCount, pollution: !p.pass, blocked: !!p.metrics.wallTextDetected };
+  } catch {
+    return { nodeCount: 0, pollution: true, blocked: false };
+  }
+}
+
+/** Persistent entry-capture cache path for a URL. Key = the compiler's full host+path
+ *  site id (collision-safe, unlike the bare folder name), so only the SAME page reuses. */
+function entryCacheSource(cacheDir: string, url: string): string {
+  return join(cacheDir, siteIdFromUrl(url), "source");
+}
+/** Whether a cached capture exists and is fresh enough to reuse (ttlMs 0/undefined =
+ *  no expiry). Staleness is measured from the capture artifact's mtime. */
+function freshCapture(dir: string, ttlMs?: number): boolean {
+  const f = join(dir, "capture", "capture-result.json");
+  if (!existsSync(f)) return false;
+  if (!ttlMs) return true;
+  try { return Date.now() - statSync(f).mtimeMs < ttlMs; } catch { return false; }
+}
+/** Copy a fresh capture into the cache (atomic-ish: clear then copy). No-op if the
+ *  source has no capture artifact. */
+function persistCapture(srcDir: string | undefined, dest: string): void {
+  if (!srcDir || !existsSync(join(srcDir, "capture", "capture-result.json"))) return;
+  mkdirSync(dirname(dest), { recursive: true });
+  rmSync(dest, { recursive: true, force: true });
+  cpSync(srcDir, dest, { recursive: true });
+}
+
+/**
+ * The single seam the service depends on: run one clone end-to-end into a temp
+ * dir, collect its generated app into a file map, optionally verify it, then return
+ * a typed result. The caller (worker/api) uploads artifacts + persists, then the
+ * temp dir is removed in `finally`. No compiler behavior is changed — this only
+ * orchestrates the existing entry points with a parameterized output dir.
+ *
+ * Incremental speed path: when `captureCacheDir` is set, a single-page job stashes its
+ * entry capture there; a later multi-page job for the same URL reuses it as the entry
+ * route (skips re-capturing page 1) and regenerates the whole site on top of it.
+ */
+export async function runCloneJob(input: RunCloneJobInput): Promise<CloneJobResult> {
+  const requestOptions: CloneOptions = normalizeCloneRequestOptions(input.options ?? {});
+  const options = resolveCloneOptions(requestOptions);
+  const syncVerify = !!options.verify && !options.asyncVerify;
+  const captureValidationArtifacts = !!(options.verify || options.asyncVerify);
+  const log = input.log ?? (() => {});
+  const ownsTemp = !input.runsDir;
+  const runsDir = input.runsDir ?? mkdtempSync(join(tmpdir(), "clone-job-"));
+  const kind: "clone" | "clone_site" = options.mode === "multi" ? "clone_site" : "clone";
+
+  // Best-by-default (matches the CLI): interactions/components/motion ON unless the
+  // caller explicitly disables them. (runClone itself defaults them OFF; the CLI is
+  // what turns them on — so the service does the same here.)
+  const interactions = options.interactions;
+  const components = options.components;
+  const motion = options.motion;
+
+  try {
+    const t0 = Date.now();
+    let runDir: string;
+    let routes: RouteInfo[] | undefined;
+    let sanity: CaptureSanity;
+    let captureReused = false;
+    // Persistent entry-capture cache (the single→multi speed path), keyed by URL.
+    const cacheEntry = input.captureCacheDir ? entryCacheSource(input.captureCacheDir, input.url) : undefined;
+
+    if (kind === "clone_site") {
+      // Reuse a prior single-page capture for the entry route when the cache holds a fresh one.
+      const reuseEntrySource = cacheEntry && freshCapture(cacheEntry, input.captureCacheTtlMs) ? cacheEntry : undefined;
+      captureReused = !!reuseEntrySource;
+      const res: CloneSiteResult = await runCloneSite({
+        url: input.url,
+        runsDir,
+        validate: false,
+        interactions,
+        components,
+        humanizeMode: options.styling,
+        framework: options.framework,
+        reuseEntrySource,
+        maxRoutes: options.maxRoutes,
+        maxCollectionInstances: options.maxCollection,
+        captureConcurrency: options.captureConcurrency,
+        validationConcurrency: options.validationConcurrency,
+        viewportConcurrency: options.viewportConcurrency,
+        screenshots: captureValidationArtifacts,
+        log,
+      });
+      runDir = res.runDir;
+      // Reproduced routes + the collapsed-collection map (representativeOf).
+      routes = res.routes.map((r) => ({ route: r.routePath }));
+      for (const c of res.plan.collections) {
+        const rep = res.routes.find((r) => r.routePath === c.representative);
+        if (rep) {
+          for (const inst of c.instances) {
+            if (inst !== c.representative) routes!.push({ route: inst, representativeOf: c.representative });
+          }
+        }
+      }
+      const entry = res.routes.find((r) => r.routePath === res.plan.entry) ?? res.routes[0];
+      sanity = entry
+        ? captureSanity(entry.sourceDir, entry.ir.doc.viewports)
+        : { nodeCount: 0, pollution: true, blocked: false };
+      // Refresh the cache with the entry capture (seeds it for a cold multi-page run).
+      if (cacheEntry && entry) persistCapture(entry.sourceDir, cacheEntry);
+    } else {
+      const res: CompilerCloneResult = await runClone({
+        url: input.url,
+        runsDir,
+        viewports: options.viewports,
+        interactions,
+        components,
+        motion,
+        humanizeMode: options.styling,
+        framework: options.framework,
+        screenshots: captureValidationArtifacts,
+        log,
+      });
+      runDir = res.runDir;
+      sanity = captureSanity(res.sourceDir, options.viewports ?? [375, 768, 1280, 1920]);
+      // Stash this page's capture so a later multi-page job can expand on it (speed path).
+      if (cacheEntry) persistCapture(res.sourceDir, cacheEntry);
+    }
+    const captureMs = Date.now() - t0;
+
+    // Optional verify (build + serve + re-render + grade). The single isolation-
+    // sensitive step: pass a per-worker harnessDir so concurrent builds don't collide.
+    let verify: unknown;
+    let verifyMs: number | undefined;
+    if (syncVerify) {
+      const done = await verifyCloneJobResult({ kind, runDir }, { harnessDir: input.harnessDir, tier: input.tier ?? "stage2", validationConcurrency: options.validationConcurrency, viewportConcurrency: options.viewportConcurrency, log });
+      verify = done.verify;
+      verifyMs = done.verifyMs;
+    }
+
+    const files = collectFileMap(runDir);
+
+    return {
+      url: input.url,
+      kind,
+      options: requestOptions,
+      status: "succeeded",
+      compilerVersion: COMPILER_VERSION,
+      timings: { captureMs, generateMs: 0, ...(verifyMs !== undefined ? { verifyMs } : {}) },
+      routes,
+      files,
+      capture: sanity,
+      captureReused,
+      verify,
+      runDir,
+    };
+  } finally {
+    if (ownsTemp && !input.keepTemp) {
+      rmSync(runsDir, { recursive: true, force: true });
+    }
+  }
+}
+
+export async function verifyCloneJobResult(
+  result: Pick<CloneJobResult, "kind" | "runDir">,
+  opts?: {
+    harnessDir?: string;
+    tier?: string;
+    validationConcurrency?: number;
+    viewportConcurrency?: number;
+    log?: (e: Record<string, unknown>) => void;
+  },
+): Promise<{ verify: unknown; verifyMs: number }> {
+  const t0 = Date.now();
+  const tier = opts?.tier ?? "stage2";
+  const verify = result.kind === "clone_site"
+    ? await validateSite(result.runDir, { harnessDir: opts?.harnessDir, tier, routeConcurrency: opts?.validationConcurrency, viewportConcurrency: opts?.viewportConcurrency, log: opts?.log })
+    : await validateRun(result.runDir, { harnessDir: opts?.harnessDir, tier, log: opts?.log });
+  return { verify, verifyMs: Date.now() - t0 };
+}

packages/core/src/types.ts

@@ -0,0 +1,123 @@
+/**
+ * Service-layer types. These are the public wire shapes for clone options,
+ * clone results, and the eager output file map, plus the core's internal
+ * collected-file shape.
+ *
+ * The core layer is the ONLY place that imports the compiler. Everything above
+ * it (api, worker, mcp) speaks these types.
+ */
+
+export type CloneMode = "single" | "multi";
+export type CloneStyling = "tailwind" | "css";
+export type CloneFramework = "next" | "vite";
+
+/** Clone options accepted by the service/core boundary.
+ *
+ * Normal callers should use only the product choices:
+ * - `mode`: single page or multi-page site clone.
+ * - `styling`: Tailwind v4 or plain CSS output.
+ * - `framework`: Next.js App Router or Vite React output.
+ *
+ * The remaining fields are operational/dev controls or deprecated compatibility
+ * aliases. Internal pipeline features such as probing, recipes, component
+ * extraction, interactions, and motion stay automatic by default.
+ */
+export type CloneOptions = {
+  mode?: CloneMode;
+  styling?: CloneStyling;
+  framework?: CloneFramework;
+  verify?: boolean;
+  /** DB/worker mode: persist the clone first, then attach verify in the background. */
+  asyncVerify?: boolean;
+  maxRoutes?: number;
+  maxCollection?: number;
+  captureConcurrency?: number;
+  validationConcurrency?: number;
+  viewportConcurrency?: number;
+  /** Service-level: bypass the cache on read AND write (does not affect output). */
+  noCache?: boolean;
+
+  /** @deprecated Use `mode: "multi"` instead. */
+  multiPage?: boolean;
+  /** @deprecated Use `styling` instead. */
+  humanizeMode?: CloneStyling;
+
+  /** Dev-only/output-affecting escape hatches. Not part of the normal product surface. */
+  viewports?: number[];
+  interactions?: boolean;
+  components?: boolean;
+  motion?: boolean;
+};
+
+/** A single file in a clone's output, as collected from `generated/app/`.
+ *  Text files carry their content inline; binaries carry only a local path +
+ *  hash (the storage layer turns `absPath` into a presigned URL after upload). */
+export type CollectedFile = {
+  /** app-relative POSIX path, e.g. "src/app/page.tsx" or "public/assets/cloned/images/ab.png" */
+  path: string;
+  kind: "text" | "binary";
+  bytes: number;
+  /** sha256 of the raw file bytes (content-addressing for storage + cache integrity). */
+  sha256: string;
+  /** present for text files only */
+  content?: string;
+  /** local filesystem path (ephemeral — used by the storage layer to upload/stream). */
+  absPath: string;
+};
+
+export type FileMap = Record<string, CollectedFile>;
+
+/** Capture-sanity audit — surfaced so a structurally-perfect clone of the WRONG
+ *  page (bot wall, empty shell) is flagged rather than sold as success. */
+export type CaptureSanity = {
+  nodeCount: number;
+  pollution: boolean; // the pollution gate failed (degenerate capture)
+  blocked: boolean; // bot/egress wall text detected
+};
+
+export type CloneTimings = { captureMs: number; generateMs: number; verifyMs?: number };
+
+export type RouteInfo = { route: string; representativeOf?: string };
+
+/** What `runCloneJob` returns — the full result for one clone, BEFORE storage
+ *  upload assigns URLs to binaries. The worker/api persists + uploads from this. */
+export type CloneJobResult = {
+  url: string;
+  kind: "clone" | "clone_site";
+  options: CloneOptions;
+  status: "succeeded";
+  compilerVersion: string;
+  timings: CloneTimings;
+  routes?: RouteInfo[];
+  files: FileMap;
+  capture: CaptureSanity;
+  /** true when a multi-page job reused a prior single-page entry capture (no re-capture
+   *  of the entry route) — the "single page first, then expand" speed path. */
+  captureReused?: boolean;
+  /** present only when options.verify === true */
+  verify?: unknown; // compiler Report — kept opaque here to avoid leaking gate internals upward
+  /** the ephemeral local run dir (already collected into `files`); caller cleans up
+   *  unless `keepTemp` was set. */
+  runDir: string;
+};
+
+export type RunCloneJobInput = {
+  url: string;
+  options?: CloneOptions;
+  /** override the temp base dir (tests). When set, the dir is NOT auto-removed. */
+  runsDir?: string;
+  /** keep the run dir after collecting the file map (debug/tests). */
+  keepTemp?: boolean;
+  /** Persistent entry-capture cache base dir, keyed by URL. A single-page job copies its
+   *  capture here; a later multi-page job for the same URL REUSES it as the entry route
+   *  (no re-capture) — the "single page first, then expand to the full site" speed path. */
+  captureCacheDir?: string;
+  /** Max age (ms) of a cached entry capture that may be reused; older ⇒ re-capture.
+   *  Undefined/0 ⇒ no expiry (reuse whenever present). Bounds staleness for a service. */
+  captureCacheTtlMs?: number;
+  /** isolated build harness dir for verify (per-worker; defaults to the compiler's). */
+  harnessDir?: string;
+  /** tier threshold for the perceptual gate when verifying (default "stage2"). */
+  tier?: string;
+  log?: (e: Record<string, unknown>) => void;
+};