Initial commit

packages/api/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@cloner/api",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "dev": "tsx watch src/server.ts",
+    "start": "tsx src/server.ts",
+    "test": "node --import tsx --test test/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@cloner/core": "*",
+    "@cloner/db": "*",
+    "@cloner/storage": "*",
+    "@hono/node-server": "^1.13.7",
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "hono": "^4.6.14",
+    "zod": "^3.24.1"
+  },
+  "devDependencies": {
+    "@cloner/test-utils": "*",
+    "@types/node": "22.10.5",
+    "tsx": "4.22.4",
+    "typescript": "5.7.3"
+  }
+}

packages/api/src/app.ts

@@ -0,0 +1,173 @@
+import { Hono } from "hono";
+import { z } from "zod";
+import { RESPONSE_ALREADY_SENT } from "@hono/node-server/utils/response";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { normalizeCloneRequestOptions } from "@cloner/core";
+import type { Backend } from "./backend.js";
+import { createMcpServer } from "./mcp.js";
+import { apiKeyAuth, rateLimit, type AuthConfig } from "./auth.js";
+
+const OptionsSchema = z
+  .object({
+    mode: z.enum(["single", "multi"]).optional(),
+    styling: z.enum(["tailwind", "css"]).optional(),
+    framework: z.enum(["next", "vite"]).optional(),
+    verify: z.boolean().optional(),
+    asyncVerify: z.boolean().optional(),
+    maxRoutes: z.number().int().positive().optional(),
+    maxCollection: z.number().int().positive().optional(),
+    captureConcurrency: z.number().int().positive().optional(),
+    validationConcurrency: z.number().int().positive().optional(),
+    viewportConcurrency: z.number().int().positive().optional(),
+
+    // Deprecated compatibility aliases and dev-only escape hatches.
+    multiPage: z.boolean().optional(),
+    humanizeMode: z.enum(["tailwind", "css"]).optional(),
+    viewports: z.array(z.number().int().positive()).min(1).optional(),
+    interactions: z.boolean().optional(),
+    components: z.boolean().optional(),
+    motion: z.boolean().optional(),
+    noCache: z.boolean().optional(),
+  })
+  .strict();
+
+const CloneRequest = z.object({
+  url: z.string().url(),
+  options: OptionsSchema.optional(),
+});
+
+export type AppDeps = {
+  backend: Backend;
+  /** absolute base URL used in MCP-returned references (binary/bundle URLs). */
+  baseUrl?: string;
+  /** mount the MCP Streamable-HTTP endpoint at /mcp (default true). */
+  mcp?: boolean;
+  /** require an API key on /v1/* and /mcp (omit = open). */
+  auth?: AuthConfig;
+  /** per-window request cap on /v1/* and /mcp (omit = unlimited). */
+  rateLimitPerMinute?: number;
+  /** SSRF guard run on submit (omit = no check — set in production). Throws to reject. */
+  assertUrl?: (url: string) => Promise<void>;
+};
+
+/** Build the Hono app over a Backend. The in-memory backend (M1) runs clones inline
+ *  (POST → 200 + file map); the DB backend (M2) enqueues (POST → 202) and the worker
+ *  fills the result (poll via GET). The HTTP surface is identical either way. */
+export function createApp(deps: AppDeps): Hono {
+  const { backend } = deps;
+  const app = new Hono();
+
+  app.get("/healthz", (c) => c.json({ ok: true }));
+
+  // Protect the API + MCP surfaces (not /healthz). Auth before rate-limit so the
+  // limiter can key by API key.
+  if (deps.auth) {
+    const mw = apiKeyAuth(deps.auth);
+    app.use("/v1/*", mw);
+    app.use("/mcp", mw);
+  }
+  if (deps.rateLimitPerMinute && deps.rateLimitPerMinute > 0) {
+    const mw = rateLimit({ perMinute: deps.rateLimitPerMinute });
+    app.use("/v1/*", mw);
+    app.use("/mcp", mw);
+  }
+
+  app.post("/v1/clones", async (c) => {
+    const body = await c.req.json().catch(() => null);
+    const parsed = CloneRequest.safeParse(body);
+    if (!parsed.success) {
+      return c.json({ error: "invalid request", details: parsed.error.flatten() }, 400);
+    }
+    const { url, options } = parsed.data;
+    if (!/^https?:\/\//i.test(url)) {
+      return c.json({ error: "url must be http(s)" }, 400);
+    }
+    // SSRF guard (production): block private/link-local/metadata targets.
+    if (deps.assertUrl) {
+      try {
+        await deps.assertUrl(url);
+      } catch (e) {
+        return c.json({ error: "url not allowed", reason: String((e as Error).message ?? e) }, 400);
+      }
+    }
+    // Header alias for the per-request cache bypass.
+    const noCacheHeader = (c.req.header("cache-control") ?? "").toLowerCase().includes("no-cache");
+    const normalizedOptions = normalizeCloneRequestOptions(options ?? {});
+    const opts = noCacheHeader ? { ...normalizedOptions, noCache: true } : normalizedOptions;
+
+    try {
+      const out = await backend.submit(url, opts);
+      if (out.status === "queued") return c.json({ jobId: out.jobId, status: "queued" }, 202);
+      return c.json(out.result, 200);
+    } catch (e) {
+      return c.json({ status: "failed", error: String(e).slice(0, 500) }, 500);
+    }
+  });
+
+  app.get("/v1/clones", async (c) => {
+    return c.json({ clones: await backend.list() });
+  });
+
+  app.get("/v1/clones/:id", async (c) => {
+    const view = await backend.status(c.req.param("id"));
+    if (!view) return c.json({ error: "not found" }, 404);
+    return c.json(view, 200);
+  });
+
+  app.get("/v1/clones/:id/result", async (c) => {
+    const out = await backend.result(c.req.param("id"));
+    if (!out) return c.json({ error: "not found" }, 404);
+    if (!out.ready) return c.json({ jobId: c.req.param("id"), status: out.status, error: out.error }, 409);
+    return c.json(out.result, 200);
+  });
+
+  app.get("/v1/clones/:id/bundle", async (c) => {
+    const fmt = c.req.query("format") === "zip" ? "zip" : "tgz";
+    const b = await backend.bundle(c.req.param("id"), fmt);
+    if (!b) return c.json({ error: "not found or not ready" }, 404);
+    if (b.url) return c.redirect(b.url, 302); // S3: hand off to the presigned URL
+    c.header("content-type", fmt === "zip" ? "application/zip" : "application/gzip");
+    c.header("content-disposition", `attachment; filename="clone-${c.req.param("id")}.${fmt}"`);
+    c.header("content-length", String(b.bytes.length));
+    c.header("x-content-sha256", b.sha256);
+    return c.body(b.bytes);
+  });
+
+  app.get("/v1/clones/:id/files/:path{.+}", async (c) => {
+    const file = await backend.file(c.req.param("id"), c.req.param("path"));
+    if (!file) return c.json({ error: "file not found" }, 404);
+    c.header("content-type", file.contentType);
+    c.header("content-length", String(file.bytes.length));
+    return c.body(file.bytes);
+  });
+
+  app.delete("/v1/clones/:id", async (c) => {
+    const ok = await backend.remove(c.req.param("id"));
+    return c.json({ deleted: ok }, ok ? 200 : 404);
+  });
+
+  // MCP over Streamable-HTTP (stateless): a fresh server+transport per request.
+  // Requires the Node http req/res from @hono/node-server (not available under
+  // app.request — MCP is exercised in tests via the in-memory transport instead).
+  if (deps.mcp !== false) {
+    app.all("/mcp", async (c) => {
+      const env = c.env as { incoming?: IncomingMessage; outgoing?: ServerResponse };
+      if (!env?.incoming || !env?.outgoing) {
+        return c.json({ error: "MCP requires the Node HTTP server (run via @hono/node-server)" }, 501);
+      }
+      const server = createMcpServer(backend, { baseUrl: deps.baseUrl });
+      const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
+      env.outgoing.on("close", () => {
+        transport.close();
+        server.close();
+      });
+      await server.connect(transport);
+      const body = c.req.method === "POST" ? await c.req.json().catch(() => undefined) : undefined;
+      await transport.handleRequest(env.incoming, env.outgoing, body);
+      return RESPONSE_ALREADY_SENT;
+    });
+  }
+
+  return app;
+}

packages/api/src/auth.ts

@@ -0,0 +1,76 @@
+import { createHash } from "node:crypto";
+import type { MiddlewareHandler } from "hono";
+
+export function hashApiKey(key: string): string {
+  return createHash("sha256").update(key).digest("hex");
+}
+
+export type AuthConfig = {
+  /** sha256 hashes of accepted keys (from env). */
+  keyHashes: Set<string>;
+  /** optional async lookup (e.g. the DB apiKeys table) by key hash. */
+  lookup?: (keyHash: string) => Promise<boolean>;
+};
+
+function extractKey(authHeader: string | undefined, xApiKey: string | undefined): string | undefined {
+  if (xApiKey) return xApiKey;
+  if (authHeader) {
+    const m = /^Bearer\s+(.+)$/i.exec(authHeader);
+    return m ? m[1]! : authHeader;
+  }
+  return undefined;
+}
+
+/** Require a valid API key (Authorization: Bearer <key> or x-api-key). */
+export function apiKeyAuth(cfg: AuthConfig): MiddlewareHandler {
+  return async (c, next) => {
+    const provided = extractKey(c.req.header("authorization"), c.req.header("x-api-key"));
+    if (!provided) return c.json({ error: "missing API key" }, 401);
+    const h = hashApiKey(provided);
+    if (cfg.keyHashes.has(h)) return next();
+    if (cfg.lookup && (await cfg.lookup(h))) return next();
+    return c.json({ error: "invalid API key" }, 401);
+  };
+}
+
+export type RateLimitConfig = {
+  perMinute: number;
+  /** key the limit by (default: API-key hash, else client IP). */
+  keyFn?: (c: Parameters<MiddlewareHandler>[0]) => string;
+};
+
+/** Fixed-window in-memory rate limiter. Sufficient for a single API instance; a
+ *  shared store (Redis/PG) would be needed across replicas. Keyed by API key when
+ *  present, else client IP. */
+export function rateLimit(cfg: RateLimitConfig): MiddlewareHandler {
+  const windowMs = 60_000;
+  const buckets = new Map<string, { count: number; reset: number }>();
+  const keyFn =
+    cfg.keyFn ??
+    ((c) => {
+      const provided = extractKey(c.req.header("authorization"), c.req.header("x-api-key"));
+      if (provided) return "k:" + hashApiKey(provided);
+      const xff = c.req.header("x-forwarded-for");
+      const ip = (xff ? xff.split(",")[0]!.trim() : undefined) ?? c.req.header("x-real-ip") ?? "anon";
+      return "ip:" + ip;
+    });
+
+  return async (c, next) => {
+    const now = Date.now();
+    const key = keyFn(c);
+    let b = buckets.get(key);
+    if (!b || now > b.reset) {
+      b = { count: 0, reset: now + windowMs };
+      buckets.set(key, b);
+    }
+    b.count++;
+    const remaining = Math.max(0, cfg.perMinute - b.count);
+    c.header("x-ratelimit-limit", String(cfg.perMinute));
+    c.header("x-ratelimit-remaining", String(remaining));
+    if (b.count > cfg.perMinute) {
+      c.header("retry-after", String(Math.ceil((b.reset - now) / 1000)));
+      return c.json({ error: "rate limit exceeded" }, 429);
+    }
+    return next();
+  };
+}

packages/api/src/backend.ts

@@ -0,0 +1,59 @@
+import type { CloneOptions, RouteInfo } from "@cloner/core";
+import type { RestCloneResult } from "./rest.js";
+
+export type JobStatus = "queued" | "running" | "succeeded" | "failed" | "cached";
+
+/** The cheap status/list view (no file contents). */
+export type JobView = {
+  jobId: string;
+  url: string;
+  kind: "clone" | "clone_site";
+  status: JobStatus;
+  options: CloneOptions;
+  compilerVersion?: string;
+  timings?: unknown;
+  routes?: RouteInfo[];
+  capture?: { nodeCount: number; pollution: boolean; blocked: boolean };
+  verify?: unknown;
+  fileCount?: number;
+  totalBytes?: number;
+  bundleUrl?: string;
+  error?: string;
+};
+
+export type SubmitOutcome =
+  | { jobId: string; status: "succeeded" | "cached"; httpStatus: 200; result: RestCloneResult }
+  | { jobId: string; status: "queued"; httpStatus: 202 };
+
+export type ResultOutcome =
+  | { ready: true; result: RestCloneResult }
+  | { ready: false; status: JobStatus; error?: string };
+
+/** A file with a content/url accessor — the substrate for MCP list/read (so the
+ *  filtering + size-budget logic lives once, regardless of backend). */
+export type FileFacet = {
+  path: string;
+  kind: "text" | "binary";
+  bytes: number;
+  sha256: string;
+  content?: string; // text only
+  binaryUrl?: () => Promise<string>; // binary only
+};
+
+export type BundleFormat = "tgz" | "zip";
+export type CloneBundle = { bytes: Buffer; sha256: string; format: BundleFormat; url?: string };
+
+/** The HTTP routes talk to this; concrete backends are the in-memory sync runner
+ *  (M1) and the DB+queue async backend (M2). */
+export interface Backend {
+  submit(url: string, options: CloneOptions | undefined): Promise<SubmitOutcome>;
+  status(jobId: string): Promise<JobView | null>;
+  result(jobId: string): Promise<ResultOutcome | null>;
+  file(jobId: string, path: string): Promise<{ bytes: Buffer; contentType: string } | null>;
+  list(): Promise<JobView[]>;
+  remove(jobId: string): Promise<boolean>;
+  /** All files with content/url accessors (null if job not ready/found). */
+  facets(jobId: string): Promise<FileFacet[] | null>;
+  /** The whole app as one compressed archive (null if not ready/found). */
+  bundle(jobId: string, format?: BundleFormat): Promise<CloneBundle | null>;
+}

packages/api/src/backends/db.ts

@@ -0,0 +1,145 @@
+import { cacheKey, COMPILER_VERSION, resolveCloneMode, type CloneOptions, type RouteInfo } from "@cloner/core";
+import { repo, enqueueClone, type Db, type PgBoss } from "@cloner/db";
+import { makeTarGz, makeZip, sha256hex, type ArtifactStore, type StoredFile } from "@cloner/storage";
+import type { Backend, BundleFormat, CloneBundle, FileFacet, JobView, JobStatus, ResultOutcome, SubmitOutcome } from "../backend.js";
+import { contentTypeFor, restResultFromStored } from "../rest.js";
+
+/** The jsonb envelope persisted in clones.fileManifest: text files inline + binary
+ *  keys (from the ArtifactStore) plus the reproduced-route map. */
+export type StoredEnvelope = { files: StoredFile[]; routes?: RouteInfo[]; bundleKey?: string };
+
+/** Async, DB+queue backend (M2): submit enqueues a job and returns 202; the worker
+ *  processes it and writes the result. Reads come from Postgres + the ArtifactStore. */
+export class DbBackend implements Backend {
+  constructor(private deps: { db: Db; boss: PgBoss; store: ArtifactStore }) {}
+
+  async submit(url: string, options: CloneOptions | undefined): Promise<SubmitOutcome> {
+    const key = cacheKey(url, options, COMPILER_VERSION);
+    const kind: "clone" | "clone_site" = resolveCloneMode(options) === "multi" ? "clone_site" : "clone";
+
+    if (!options?.noCache) {
+      const hit = await repo.cacheGetFresh(this.deps.db, key, COMPILER_VERSION);
+      if (hit?.jobId) {
+        const r = await this.result(hit.jobId);
+        if (r && r.ready) {
+          return { jobId: hit.jobId, status: "cached", httpStatus: 200, result: { ...r.result, status: "cached" } };
+        }
+      }
+    }
+
+    const job = await repo.createJob(this.deps.db, { kind, url, options: options ?? {}, status: "queued", cacheKey: key });
+    await enqueueClone(this.deps.boss, job.id);
+    return { jobId: job.id, status: "queued", httpStatus: 202 };
+  }
+
+  async status(jobId: string): Promise<JobView | null> {
+    const job = await repo.getJob(this.deps.db, jobId);
+    if (!job) return null;
+    const base: JobView = {
+      jobId: job.id,
+      url: job.url,
+      kind: job.kind as "clone" | "clone_site",
+      status: job.status as JobStatus,
+      options: job.options as CloneOptions,
+      compilerVersion: job.compilerVersion ?? undefined,
+      timings: job.timings ?? undefined,
+      error: job.error ?? undefined,
+    };
+    if (job.status === "succeeded") {
+      const clone = await repo.getClone(this.deps.db, jobId);
+      if (clone) {
+        const env = clone.fileManifest as StoredEnvelope;
+        let totalBytes = 0;
+        for (const f of env.files) totalBytes += f.bytes;
+        return { ...base, capture: clone.captureMeta as JobView["capture"], verify: clone.verify ?? undefined, routes: env.routes, fileCount: env.files.length, totalBytes };
+      }
+    }
+    return base;
+  }
+
+  async result(jobId: string): Promise<ResultOutcome | null> {
+    const job = await repo.getJob(this.deps.db, jobId);
+    if (!job) return null;
+    if (job.status !== "succeeded") return { ready: false, status: job.status as JobStatus, error: job.error ?? undefined };
+    const clone = await repo.getClone(this.deps.db, jobId);
+    if (!clone) return { ready: false, status: "running" };
+    const env = clone.fileManifest as StoredEnvelope;
+    const result = await restResultFromStored(jobId, {
+      url: job.url,
+      kind: job.kind as "clone" | "clone_site",
+      options: job.options as CloneOptions,
+      compilerVersion: job.compilerVersion ?? COMPILER_VERSION,
+      timings: (job.timings as { captureMs: number; generateMs: number }) ?? { captureMs: 0, generateMs: 0 },
+      capture: clone.captureMeta as { nodeCount: number; pollution: boolean; blocked: boolean },
+      routes: env.routes,
+      verify: clone.verify ?? undefined,
+      files: env.files,
+      binaryUrl: (p) => this.deps.store.binaryUrl(jobId, p),
+    });
+    return { ready: true, result };
+  }
+
+  async file(jobId: string, path: string): Promise<{ bytes: Buffer; contentType: string } | null> {
+    // Text lives inline in the manifest (not in blob storage); binaries come from
+    // the store. This keeps /files/* working in S3 mode where text isn't uploaded.
+    const env = await this.envelope(jobId);
+    const meta = env?.files.find((f) => f.path === path);
+    if (meta?.kind === "text") return { bytes: Buffer.from(meta.content, "utf8"), contentType: contentTypeFor(path) };
+    const got = await this.deps.store.getFile(jobId, path);
+    if (!got) return null;
+    return { bytes: got.bytes, contentType: contentTypeFor(path) };
+  }
+
+  async list(): Promise<JobView[]> {
+    const rows = await repo.listJobs(this.deps.db, 50);
+    return rows.map((job) => ({
+      jobId: job.id,
+      url: job.url,
+      kind: job.kind as "clone" | "clone_site",
+      status: job.status as JobStatus,
+      options: job.options as CloneOptions,
+      compilerVersion: job.compilerVersion ?? undefined,
+      timings: job.timings ?? undefined,
+      error: job.error ?? undefined,
+    }));
+  }
+
+  async remove(jobId: string): Promise<boolean> {
+    await this.deps.store.remove(jobId).catch(() => {});
+    return repo.deleteJob(this.deps.db, jobId);
+  }
+
+  private async envelope(jobId: string): Promise<StoredEnvelope | null> {
+    const clone = await repo.getClone(this.deps.db, jobId);
+    if (!clone) return null;
+    return clone.fileManifest as StoredEnvelope;
+  }
+
+  async facets(jobId: string): Promise<FileFacet[] | null> {
+    const env = await this.envelope(jobId);
+    if (!env) return null;
+    return env.files.map((f) =>
+      f.kind === "text"
+        ? { path: f.path, kind: "text", bytes: f.bytes, sha256: f.sha256, content: f.content }
+        : { path: f.path, kind: "binary", bytes: f.bytes, sha256: f.sha256, binaryUrl: () => this.deps.store.binaryUrl(jobId, f.path) },
+    );
+  }
+
+  async bundle(jobId: string, format: BundleFormat = "tgz"): Promise<CloneBundle | null> {
+    const env = await this.envelope(jobId);
+    if (!env) return null;
+    const entries: Array<{ path: string; bytes: Buffer }> = [];
+    for (const f of env.files) {
+      if (f.kind === "text") {
+        entries.push({ path: f.path, bytes: Buffer.from(f.content, "utf8") });
+      } else {
+        const got = await this.deps.store.getFile(jobId, f.path);
+        if (got) entries.push({ path: f.path, bytes: got.bytes });
+      }
+    }
+    const bytes = format === "zip" ? makeZip(entries) : makeTarGz(entries);
+    // S3 store: upload + presign so the client downloads directly; local: served by the API.
+    const url = this.deps.store.uploadBundle ? await this.deps.store.uploadBundle(jobId, format, bytes) : undefined;
+    return { bytes, sha256: sha256hex(bytes), format, url };
+  }
+}

packages/api/src/backends/inMemory.ts

@@ -0,0 +1,105 @@
+import { mkdtempSync, rmSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import { resolveCloneMode, verifyCloneJobResult, type CloneJobResult, type CloneOptions, type RunCloneJobInput } from "@cloner/core";
+import { makeTarGz, makeZip, sha256hex } from "@cloner/storage";
+import { InMemoryStore } from "../store.js";
+import { buildRestResult, buildRestSummary, contentTypeFor } from "../rest.js";
+import type { Backend, BundleFormat, CloneBundle, FileFacet, JobView, ResultOutcome, SubmitOutcome } from "../backend.js";
+
+export type RunJob = (input: RunCloneJobInput) => Promise<CloneJobResult>;
+
+/** Sync, in-memory backend (M1): runs the clone INLINE on submit and holds results
+ *  in memory. POST returns the file map immediately (200). */
+export class InMemoryBackend implements Backend {
+  constructor(private deps: { store: InMemoryStore; runJob: RunJob; makeTempBase?: () => string; captureCacheDir?: string }) {}
+
+  private makeBase(): string {
+    return (this.deps.makeTempBase ?? (() => mkdtempSync(join(tmpdir(), "api-clone-"))))();
+  }
+
+  async submit(url: string, options: CloneOptions | undefined): Promise<SubmitOutcome> {
+    const id = randomUUID();
+    const base = this.makeBase();
+    try {
+      // captureCacheDir (persistent, shared across submits) powers the single→multi
+      // reuse: a single-page submit stashes its capture; a later multi-page submit for
+      // the same URL reuses it as the entry route (no re-capture) and expands the site.
+      const result = await this.deps.runJob({ url, options, runsDir: base, captureCacheDir: this.deps.captureCacheDir });
+      this.deps.store.put({ id, status: "succeeded", url, kind: result.kind, options: result.options, createdAt: Date.now(), result, base });
+      if (result.options.asyncVerify) {
+        void verifyCloneJobResult(result, {
+          validationConcurrency: result.options.validationConcurrency,
+          viewportConcurrency: result.options.viewportConcurrency,
+        }).then((done) => {
+          result.verify = done.verify;
+          result.timings = { ...result.timings, verifyMs: done.verifyMs };
+        }).catch((e) => {
+          result.verify = { error: String(e).slice(0, 500), async: true };
+        });
+      }
+      return { jobId: id, status: "succeeded", httpStatus: 200, result: buildRestResult(id, result, `/v1/clones/${id}/files`) };
+    } catch (e) {
+      try {
+        rmSync(base, { recursive: true, force: true });
+      } catch {
+        /* best effort */
+      }
+      this.deps.store.put({ id, status: "failed", url, kind: resolveCloneMode(options) === "multi" ? "clone_site" : "clone", options: options ?? {}, createdAt: Date.now(), error: String(e) });
+      throw e;
+    }
+  }
+
+  async status(jobId: string): Promise<JobView | null> {
+    const rec = this.deps.store.get(jobId);
+    if (!rec) return null;
+    if (rec.status === "succeeded" && rec.result) return { ...buildRestSummary(jobId, rec.result), verify: rec.result.verify };
+    return { jobId, url: rec.url, kind: rec.kind, status: rec.status, options: rec.options, error: rec.error };
+  }
+
+  async result(jobId: string): Promise<ResultOutcome | null> {
+    const rec = this.deps.store.get(jobId);
+    if (!rec) return null;
+    if (rec.status === "succeeded" && rec.result) return { ready: true, result: buildRestResult(jobId, rec.result, `/v1/clones/${jobId}/files`) };
+    return { ready: false, status: rec.status, error: rec.error };
+  }
+
+  async file(jobId: string, path: string): Promise<{ bytes: Buffer; contentType: string } | null> {
+    const rec = this.deps.store.get(jobId);
+    if (!rec?.result) return null;
+    const entry = rec.result.files[path];
+    if (!entry) return null;
+    return { bytes: readFileSync(entry.absPath), contentType: contentTypeFor(path) };
+  }
+
+  async list(): Promise<JobView[]> {
+    return this.deps.store.list().map((rec) =>
+      rec.status === "succeeded" && rec.result
+        ? buildRestSummary(rec.id, rec.result)
+        : { jobId: rec.id, url: rec.url, kind: rec.kind, status: rec.status, options: rec.options, error: rec.error },
+    );
+  }
+
+  async remove(jobId: string): Promise<boolean> {
+    return this.deps.store.remove(jobId);
+  }
+
+  async facets(jobId: string): Promise<FileFacet[] | null> {
+    const rec = this.deps.store.get(jobId);
+    if (!rec?.result) return null;
+    return Object.entries(rec.result.files).map(([path, f]) =>
+      f.kind === "text"
+        ? { path, kind: "text", bytes: f.bytes, sha256: f.sha256, content: f.content ?? "" }
+        : { path, kind: "binary", bytes: f.bytes, sha256: f.sha256, binaryUrl: async () => `/v1/clones/${jobId}/files/${path}` },
+    );
+  }
+
+  async bundle(jobId: string, format: BundleFormat = "tgz"): Promise<CloneBundle | null> {
+    const rec = this.deps.store.get(jobId);
+    if (!rec?.result) return null;
+    const entries = Object.entries(rec.result.files).map(([path, f]) => ({ path, bytes: readFileSync(f.absPath) }));
+    const bytes = format === "zip" ? makeZip(entries) : makeTarGz(entries);
+    return { bytes, sha256: sha256hex(bytes), format };
+  }
+}

packages/api/src/env.ts

@@ -0,0 +1,40 @@
+import { join } from "node:path";
+
+/** Service configuration from the environment. */
+export type ApiEnv = {
+  port: number;
+  /** in-memory mode only: retention before sweeping completed clones. */
+  cloneTtlMs: number;
+  /** when set, the API runs in async DB+queue mode; otherwise sync in-memory. */
+  databaseUrl?: string;
+  /** local blob root (until S3 in M4). */
+  artifactsDir: string;
+  /** in-memory mode: persistent entry-capture cache for the single→multi reuse path
+   *  ("" disables). */
+  captureCacheDir: string;
+  /** absolute base URL for MCP-returned references (binary/bundle URLs). */
+  publicBaseUrl?: string;
+  /** accepted API keys (raw, from API_KEYS=comma,separated). Empty = open. */
+  apiKeys: string[];
+  /** per-minute request cap on /v1/* and /mcp (0 = unlimited). */
+  rateLimitPerMinute: number;
+  /** SSRF guard (default on). */
+  ssrfEnabled: boolean;
+  /** allow loopback targets through SSRF (local dev cloning of localhost). */
+  ssrfAllowLoopback: boolean;
+};
+
+export function loadEnv(): ApiEnv {
+  return {
+    port: parseInt(process.env.PORT ?? "8787", 10),
+    cloneTtlMs: parseInt(process.env.CLONE_TTL_MS ?? String(30 * 60 * 1000), 10),
+    databaseUrl: process.env.DATABASE_URL,
+    artifactsDir: process.env.ARTIFACTS_DIR ?? join(process.cwd(), "local-data", "artifacts"),
+    captureCacheDir: process.env.CAPTURE_CACHE_DIR ?? join(process.cwd(), "local-data", "capture-cache"),
+    publicBaseUrl: process.env.PUBLIC_BASE_URL,
+    apiKeys: (process.env.API_KEYS ?? "").split(",").map((s) => s.trim()).filter(Boolean),
+    rateLimitPerMinute: parseInt(process.env.RATE_LIMIT_PER_MINUTE ?? "0", 10),
+    ssrfEnabled: process.env.SSRF_DISABLE !== "true",
+    ssrfAllowLoopback: process.env.SSRF_ALLOW_LOOPBACK === "true",
+  };
+}

packages/api/src/files.ts

@@ -0,0 +1,97 @@
+import type { FileFacet } from "./backend.js";
+
+export type FileMeta = { path: string; type: "text" | "binary"; bytes: number; sha256: string };
+
+/** Minimal glob → RegExp: `**` = any (incl. `/`), `*` = any except `/`, `?` = one
+ *  non-`/`. Sufficient for `**​/*.tsx`, `src/app/*.css`, etc. */
+export function globToRegExp(glob: string): RegExp {
+  let re = "";
+  for (let i = 0; i < glob.length; i++) {
+    const c = glob[i]!;
+    if (c === "*") {
+      if (glob[i + 1] === "*") {
+        re += ".*";
+        i++;
+      } else {
+        re += "[^/]*";
+      }
+    } else if (c === "?") {
+      re += "[^/]";
+    } else {
+      re += c.replace(/[.+^${}()|[\]\\]/g, "\\$&");
+    }
+  }
+  return new RegExp("^" + re + "$");
+}
+
+export function metaOf(facets: FileFacet[]): FileMeta[] {
+  return facets.map((f) => ({ path: f.path, type: f.kind, bytes: f.bytes, sha256: f.sha256 }));
+}
+
+/** Filter file metadata by glob and/or route (Next = `src/app/<seg>`, Vite = `src/routes/<key>`). */
+export function filterMetas(metas: FileMeta[], opts: { glob?: string; route?: string }): FileMeta[] {
+  let out = metas;
+  if (opts.glob) {
+    const re = globToRegExp(opts.glob);
+    out = out.filter((m) => re.test(m.path));
+  }
+  if (opts.route && opts.route !== "/") {
+    const seg = opts.route.replace(/^\/+/, "").replace(/\/+$/, "");
+    const viteKey = seg.replace(/\//g, "__") || "home";
+    out = out.filter((m) =>
+      m.path === `src/app/${seg}` || m.path.startsWith(`src/app/${seg}/`) ||
+      m.path === `src/routes/${viteKey}` || m.path.startsWith(`src/routes/${viteKey}/`)
+    );
+  }
+  return out;
+}
+
+/** Opaque numeric-offset pagination over a sorted list. */
+export function paginate<T>(arr: T[], cursor: string | undefined, limit: number): { items: T[]; nextCursor?: string } {
+  const offset = cursor ? Math.max(0, parseInt(cursor, 10) || 0) : 0;
+  const items = arr.slice(offset, offset + limit);
+  const next = offset + limit;
+  return next < arr.length ? { items, nextCursor: String(next) } : { items };
+}
+
+export type ReadFileEntry =
+  | { path: string; type: "text"; bytes: number; sha256: string; content: string }
+  | { path: string; type: "text"; bytes: number; sha256: string; skipped: true; reason: string }
+  | { path: string; type: "binary"; bytes: number; sha256: string; url: string }
+  | { path: string; error: string };
+
+/** Read the requested paths with a per-call text-size budget. Binaries are always
+ *  returned as URLs (never bytes); text over budget is flagged skipped, so a careless
+ *  `["**"]` (no exact match anyway) can't flood the consumer's context. */
+export async function readFiles(
+  facets: FileFacet[],
+  paths: string[],
+  opts: { maxBytes?: number; resolveUrl?: (u: string) => string },
+): Promise<{ files: ReadFileEntry[]; totalBytes: number; truncated: boolean }> {
+  const budget = opts.maxBytes ?? 256 * 1024;
+  const byPath = new Map(facets.map((f) => [f.path, f]));
+  const out: ReadFileEntry[] = [];
+  let used = 0;
+  let truncated = false;
+  for (const p of paths) {
+    const f = byPath.get(p);
+    if (!f) {
+      out.push({ path: p, error: "not found" });
+      continue;
+    }
+    if (f.kind === "binary") {
+      let url = f.binaryUrl ? await f.binaryUrl() : "";
+      if (opts.resolveUrl) url = opts.resolveUrl(url);
+      out.push({ path: p, type: "binary", bytes: f.bytes, sha256: f.sha256, url });
+      continue;
+    }
+    if (used + f.bytes > budget) {
+      out.push({ path: p, type: "text", bytes: f.bytes, sha256: f.sha256, skipped: true, reason: "per-call size budget exceeded" });
+      truncated = true;
+      continue;
+    }
+    used += f.bytes;
+    out.push({ path: p, type: "text", bytes: f.bytes, sha256: f.sha256, content: f.content ?? "" });
+  }
+  return { files: out, totalBytes: used, truncated };
+}

packages/api/src/index.ts

@@ -0,0 +1,19 @@
+export { createApp, type AppDeps } from "./app.js";
+export { InMemoryStore, type JobRecord } from "./store.js";
+export { InMemoryBackend, type RunJob } from "./backends/inMemory.js";
+export { DbBackend, type StoredEnvelope } from "./backends/db.js";
+export type { Backend, JobView, JobStatus, SubmitOutcome, ResultOutcome } from "./backend.js";
+export {
+  type RestCloneResult,
+  type RestCloneSummary,
+  type RestFileEntry,
+  buildRestResult,
+  buildRestSummary,
+  restResultFromStored,
+  contentTypeFor,
+} from "./rest.js";
+export { loadEnv, type ApiEnv } from "./env.js";
+export { createMcpServer } from "./mcp.js";
+export { globToRegExp, filterMetas, paginate, readFiles, metaOf, type FileMeta, type ReadFileEntry } from "./files.js";
+export { apiKeyAuth, rateLimit, hashApiKey, type AuthConfig, type RateLimitConfig } from "./auth.js";
+export { assertPublicUrl, isBlockedIp, SsrfError, type DnsResolver } from "./ssrf.js";

packages/api/src/mcp.ts

@@ -0,0 +1,154 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { normalizeCloneRequestOptions } from "@cloner/core";
+import type { Backend } from "./backend.js";
+import { filterMetas, metaOf, paginate, readFiles } from "./files.js";
+
+const optionsShape = {
+  mode: z.enum(["single", "multi"]).optional(),
+  styling: z.enum(["tailwind", "css"]).optional(),
+  framework: z.enum(["next", "vite"]).optional(),
+  verify: z.boolean().optional(),
+  asyncVerify: z.boolean().optional(),
+  maxRoutes: z.number().int().positive().optional(),
+  maxCollection: z.number().int().positive().optional(),
+  captureConcurrency: z.number().int().positive().optional(),
+  validationConcurrency: z.number().int().positive().optional(),
+  viewportConcurrency: z.number().int().positive().optional(),
+  multiPage: z.boolean().optional(),
+  humanizeMode: z.enum(["tailwind", "css"]).optional(),
+  viewports: z.array(z.number().int().positive()).optional(),
+  interactions: z.boolean().optional(),
+  components: z.boolean().optional(),
+  motion: z.boolean().optional(),
+  noCache: z.boolean().optional(),
+};
+
+const json = (data: unknown, isError = false) => ({
+  content: [{ type: "text" as const, text: JSON.stringify(data, null, 2) }],
+  ...(isError ? { isError: true } : {}),
+});
+
+/**
+ * MCP server exposing the same service functions as the REST API. The key
+ * principle: never flood the agent's context. Tools return references +
+ * manifests; the agent pulls only the files it needs (list-then-read), and
+ * binaries/bundles are always URLs, never bytes.
+ */
+export function createMcpServer(backend: Backend, opts?: { baseUrl?: string }): McpServer {
+  const baseUrl = opts?.baseUrl ?? "";
+  const abs = (u: string) => (u.startsWith("/") && baseUrl ? baseUrl + u : u);
+  const server = new McpServer({ name: "ditto.site", version: "0.1.0" });
+
+  // clone_website → start a job; returns immediately, never blocks.
+  server.registerTool(
+    "clone_website",
+    {
+      description: "Clone a website by URL. Returns { jobId, status } immediately — poll get_clone_status, then browse with list_clone_files / read_clone_files. Never returns file contents.",
+      inputSchema: { url: z.string().url(), options: z.object(optionsShape).optional() },
+    },
+    async ({ url, options }) => {
+      if (!/^https?:\/\//i.test(url)) return json({ error: "url must be http(s)" }, true);
+      const out = await backend.submit(url, normalizeCloneRequestOptions(options ?? {}));
+      return json({ jobId: out.jobId, status: out.status });
+    },
+  );
+
+  server.registerTool(
+    "get_clone_status",
+    { description: "Poll a clone job's status.", inputSchema: { jobId: z.string() } },
+    async ({ jobId }) => {
+      const v = await backend.status(jobId);
+      if (!v) return json({ error: "not found", jobId }, true);
+      return json({ jobId, status: v.status, timings: v.timings, capture: v.capture, error: v.error });
+    },
+  );
+
+  // get_clone_result → METADATA ONLY (no file contents): the cheap overview.
+  server.registerTool(
+    "get_clone_result",
+    { description: "Get a clone's result metadata (status, timings, routes, verify summary, capture sanity, fileCount, totalBytes, bundleUrl) — NO file contents.", inputSchema: { jobId: z.string() } },
+    async ({ jobId }) => {
+      const v = await backend.status(jobId);
+      if (!v) return json({ error: "not found", jobId }, true);
+      if (v.status !== "succeeded" && v.status !== "cached") return json({ jobId, status: v.status, error: v.error });
+      return json({
+        jobId,
+        status: v.status,
+        url: v.url,
+        kind: v.kind,
+        timings: v.timings,
+        routes: v.routes,
+        capture: v.capture,
+        verify: v.verify,
+        fileCount: v.fileCount,
+        totalBytes: v.totalBytes,
+        bundleUrl: abs(`/v1/clones/${jobId}/bundle?format=tgz`),
+      });
+    },
+  );
+
+  // list_clone_files → the manifest (paths only, no content), filterable + paginated.
+  server.registerTool(
+    "list_clone_files",
+    {
+      description: "List a clone's files as a manifest [{ path, type, bytes, sha256 }] with NO content. Filter by glob (e.g. \"**/*.tsx\") or route; paginated via cursor.",
+      inputSchema: { jobId: z.string(), glob: z.string().optional(), route: z.string().optional(), cursor: z.string().optional(), limit: z.number().int().positive().max(1000).optional() },
+    },
+    async ({ jobId, glob, route, cursor, limit }) => {
+      const facets = await backend.facets(jobId);
+      if (!facets) return json({ error: "not found or not ready", jobId }, true);
+      const metas = filterMetas(metaOf(facets), { glob, route }).sort((a, b) => (a.path < b.path ? -1 : 1));
+      const page = paginate(metas, cursor, limit ?? 200);
+      return json({ jobId, files: page.items, nextCursor: page.nextCursor, total: metas.length });
+    },
+  );
+
+  // read_clone_files → contents for SPECIFIC files (text inline, binaries as URLs),
+  // with a per-call size budget so a careless request can't blow the window.
+  server.registerTool(
+    "read_clone_files",
+    {
+      description: "Read specific files by exact path. Text inline; binaries as URLs (never bytes). Enforces a per-call size budget (default 256KB); oversized text is flagged skipped.",
+      inputSchema: { jobId: z.string(), paths: z.array(z.string()).min(1), maxBytes: z.number().int().positive().optional() },
+    },
+    async ({ jobId, paths, maxBytes }) => {
+      const facets = await backend.facets(jobId);
+      if (!facets) return json({ error: "not found or not ready", jobId }, true);
+      const res = await readFiles(facets, paths, { maxBytes, resolveUrl: abs });
+      return json({ jobId, ...res });
+    },
+  );
+
+  // get_clone_bundle → a DOWNLOAD REFERENCE to the whole app (URL, not bytes).
+  server.registerTool(
+    "get_clone_bundle",
+    { description: "Get a download reference to the whole clone as one compressed archive: { url, format, bytes, sha256 } — a URL, not the bytes. Fetch it out-of-band and expand to a runnable generated app.", inputSchema: { jobId: z.string(), format: z.enum(["tgz", "zip"]).optional() } },
+    async ({ jobId, format }) => {
+      const b = await backend.bundle(jobId, format ?? "tgz");
+      if (!b) return json({ error: "not found or not ready", jobId }, true);
+      const url = b.url ?? abs(`/v1/clones/${jobId}/bundle?format=${b.format}`);
+      return json({ jobId, url, format: b.format, bytes: b.bytes.length, sha256: b.sha256 });
+    },
+  );
+
+  server.registerTool(
+    "list_clones",
+    { description: "List recent clone jobs (metadata only).", inputSchema: {} },
+    async () => {
+      const clones = await backend.list();
+      return json({ clones: clones.map((c) => ({ jobId: c.jobId, url: c.url, kind: c.kind, status: c.status })) });
+    },
+  );
+
+  server.registerTool(
+    "cancel_clone",
+    { description: "Cancel/purge a clone job and its artifacts.", inputSchema: { jobId: z.string() } },
+    async ({ jobId }) => {
+      const ok = await backend.remove(jobId);
+      return json({ jobId, cancelled: ok });
+    },
+  );
+
+  return server;
+}

packages/api/src/rest.ts

@@ -0,0 +1,152 @@
+import { extname } from "node:path";
+import type { CloneJobResult, CloneOptions, CloneTimings, FileMap, RouteInfo } from "@cloner/core";
+import type { StoredFile } from "@cloner/storage";
+
+/** The eager REST file entry. Text is inline; binaries are by reference
+ *  (a URL the client fetches out-of-band). */
+export type RestFileEntry =
+  | { type: "text"; content: string; bytes: number; sha256: string }
+  | { type: "binary"; url: string; bytes: number; sha256: string };
+
+export type RestCloneResult = {
+  jobId: string;
+  url: string;
+  kind: "clone" | "clone_site";
+  options: CloneOptions;
+  status: "succeeded" | "failed" | "cached";
+  compilerVersion: string;
+  timings: CloneTimings;
+  routes?: RouteInfo[];
+  files: Record<string, RestFileEntry>;
+  bundleUrl?: string; // added in M4 (storage); omitted in the sync/in-memory build
+  verify?: unknown;
+  capture: { nodeCount: number; pollution: boolean; blocked: boolean };
+  /** true when a multi-page job reused a prior single-page entry capture (speed path). */
+  captureReused?: boolean;
+};
+
+/** The cheap overview (no file contents) for status polling / list. */
+export type RestCloneSummary = {
+  jobId: string;
+  url: string;
+  kind: "clone" | "clone_site";
+  status: "succeeded" | "failed" | "cached";
+  options: CloneOptions;
+  compilerVersion: string;
+  timings: CloneTimings;
+  routes?: RouteInfo[];
+  capture: { nodeCount: number; pollution: boolean; blocked: boolean };
+  captureReused?: boolean;
+  fileCount: number;
+  totalBytes: number;
+  bundleUrl?: string;
+};
+
+/** Map a collected FileMap to the eager REST shape. `filesBaseUrl` is the per-file
+ *  access prefix (e.g. "/v1/clones/<id>/files") used to reference binaries. */
+export function toRestFiles(files: FileMap, filesBaseUrl: string): Record<string, RestFileEntry> {
+  const out: Record<string, RestFileEntry> = {};
+  for (const [path, f] of Object.entries(files)) {
+    if (f.kind === "text") {
+      out[path] = { type: "text", content: f.content ?? "", bytes: f.bytes, sha256: f.sha256 };
+    } else {
+      out[path] = { type: "binary", url: `${filesBaseUrl}/${path}`, bytes: f.bytes, sha256: f.sha256 };
+    }
+  }
+  return out;
+}
+
+export function buildRestResult(jobId: string, result: CloneJobResult, filesBaseUrl: string): RestCloneResult {
+  return {
+    jobId,
+    url: result.url,
+    kind: result.kind,
+    options: result.options,
+    status: "succeeded",
+    compilerVersion: result.compilerVersion,
+    timings: result.timings,
+    routes: result.routes,
+    files: toRestFiles(result.files, filesBaseUrl),
+    verify: result.verify,
+    capture: result.capture,
+    captureReused: result.captureReused,
+  };
+}
+
+export function buildRestSummary(jobId: string, result: CloneJobResult): RestCloneSummary {
+  let totalBytes = 0;
+  const vals = Object.values(result.files);
+  for (const f of vals) totalBytes += f.bytes;
+  return {
+    jobId,
+    url: result.url,
+    kind: result.kind,
+    status: "succeeded",
+    options: result.options,
+    compilerVersion: result.compilerVersion,
+    timings: result.timings,
+    routes: result.routes,
+    capture: result.capture,
+    captureReused: result.captureReused,
+    fileCount: vals.length,
+    totalBytes,
+  };
+}
+
+/** Reconstruct the eager REST result from persisted (DB + storage) data — the
+ *  read path for the async DB backend. Text comes inline from the manifest;
+ *  binary URLs are resolved via the store (local API route or presigned S3 URL). */
+export async function restResultFromStored(
+  jobId: string,
+  args: {
+    url: string;
+    kind: "clone" | "clone_site";
+    options: CloneOptions;
+    compilerVersion: string;
+    timings: CloneTimings;
+    capture: { nodeCount: number; pollution: boolean; blocked: boolean };
+    routes?: RouteInfo[];
+    verify?: unknown;
+    files: StoredFile[];
+    bundleUrl?: string;
+    binaryUrl: (path: string) => Promise<string>;
+  },
+): Promise<RestCloneResult> {
+  const files: Record<string, RestFileEntry> = {};
+  for (const f of args.files) {
+    if (f.kind === "text") {
+      files[f.path] = { type: "text", content: f.content, bytes: f.bytes, sha256: f.sha256 };
+    } else {
+      files[f.path] = { type: "binary", url: await args.binaryUrl(f.path), bytes: f.bytes, sha256: f.sha256 };
+    }
+  }
+  return {
+    jobId,
+    url: args.url,
+    kind: args.kind,
+    options: args.options,
+    status: "succeeded",
+    compilerVersion: args.compilerVersion,
+    timings: args.timings,
+    routes: args.routes,
+    files,
+    bundleUrl: args.bundleUrl,
+    verify: args.verify,
+    capture: args.capture,
+  };
+}
+
+const CONTENT_TYPES: Record<string, string> = {
+  ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp",
+  ".avif": "image/avif", ".gif": "image/gif", ".svg": "image/svg+xml", ".ico": "image/x-icon",
+  ".woff2": "font/woff2", ".woff": "font/woff", ".ttf": "font/ttf", ".otf": "font/otf",
+  ".mp4": "video/mp4", ".webm": "video/webm",
+  ".tsx": "text/plain; charset=utf-8", ".ts": "text/plain; charset=utf-8",
+  ".css": "text/css; charset=utf-8", ".json": "application/json; charset=utf-8",
+  ".js": "text/javascript; charset=utf-8", ".mjs": "text/javascript; charset=utf-8",
+  ".html": "text/html; charset=utf-8", ".txt": "text/plain; charset=utf-8",
+};
+
+export function contentTypeFor(path: string): string {
+  return CONTENT_TYPES[extname(path).toLowerCase()] ?? "application/octet-stream";
+}

packages/api/src/server.ts

@@ -0,0 +1,71 @@
+import { serve } from "@hono/node-server";
+import { runCloneJob } from "@cloner/core";
+import { createDb, createBoss, repo, type Db } from "@cloner/db";
+import { artifactStoreFromEnv } from "@cloner/storage";
+import { createApp } from "./app.js";
+import { InMemoryStore } from "./store.js";
+import { InMemoryBackend } from "./backends/inMemory.js";
+import { DbBackend } from "./backends/db.js";
+import type { Backend } from "./backend.js";
+import { hashApiKey, type AuthConfig } from "./auth.js";
+import { assertPublicUrl } from "./ssrf.js";
+import { loadEnv, type ApiEnv } from "./env.js";
+
+function buildAuth(env: ApiEnv, db?: Db): AuthConfig | undefined {
+  const keyHashes = new Set(env.apiKeys.map(hashApiKey));
+  // DB-backed keys (apiKeys table) are honored too when a DB is present.
+  const lookup = db
+    ? async (h: string): Promise<boolean> => {
+        const k = await repo.getApiKeyByHash(db, h);
+        return !!k && !k.revokedAt;
+      }
+    : undefined;
+  if (keyHashes.size === 0 && !lookup) return undefined;
+  return { keyHashes, lookup };
+}
+
+async function main(): Promise<void> {
+  const env = loadEnv();
+  let backend: Backend;
+  let db: Db | undefined;
+
+  if (env.databaseUrl) {
+    const h = createDb(env.databaseUrl);
+    db = h.db;
+    const boss = await createBoss(env.databaseUrl);
+    const store = artifactStoreFromEnv();
+    backend = new DbBackend({ db, boss, store });
+    console.log(JSON.stringify({ event: "api_mode", mode: "db+queue" }));
+  } else {
+    const store = new InMemoryStore(env.cloneTtlMs);
+    store.startSweeper();
+    backend = new InMemoryBackend({ store, runJob: runCloneJob, captureCacheDir: env.captureCacheDir || undefined });
+    console.log(JSON.stringify({ event: "api_mode", mode: "in-memory" }));
+  }
+
+  const auth = buildAuth(env, db);
+  const app = createApp({
+    backend,
+    baseUrl: env.publicBaseUrl,
+    auth,
+    rateLimitPerMinute: env.rateLimitPerMinute,
+    assertUrl: env.ssrfEnabled ? async (url) => void (await assertPublicUrl(url, { allowLoopback: env.ssrfAllowLoopback })) : undefined,
+  });
+
+  serve({ fetch: app.fetch, port: env.port }, (info) => {
+    console.log(
+      JSON.stringify({
+        event: "api_listening",
+        port: info.port,
+        auth: !!auth,
+        rateLimitPerMinute: env.rateLimitPerMinute || null,
+        ssrf: env.ssrfEnabled,
+      }),
+    );
+  });
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

packages/api/src/ssrf.ts

@@ -0,0 +1,102 @@
+import { BlockList, isIP } from "node:net";
+import { lookup } from "node:dns/promises";
+
+export class SsrfError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "SsrfError";
+  }
+}
+
+/** Private / loopback / link-local / metadata / reserved ranges that a public
+ *  "fetch any URL" service must never reach (the cloud metadata endpoint
+ *  169.254.169.254 is inside link-local 169.254/16). */
+function buildBlockList(): BlockList {
+  const b = new BlockList();
+  // IPv4
+  b.addSubnet("0.0.0.0", 8, "ipv4");
+  b.addSubnet("10.0.0.0", 8, "ipv4");
+  b.addSubnet("100.64.0.0", 10, "ipv4"); // CGNAT
+  b.addSubnet("127.0.0.0", 8, "ipv4"); // loopback
+  b.addSubnet("169.254.0.0", 16, "ipv4"); // link-local incl. 169.254.169.254
+  b.addSubnet("172.16.0.0", 12, "ipv4");
+  b.addSubnet("192.0.0.0", 24, "ipv4");
+  b.addSubnet("192.168.0.0", 16, "ipv4");
+  b.addSubnet("198.18.0.0", 15, "ipv4"); // benchmarking
+  b.addSubnet("224.0.0.0", 4, "ipv4"); // multicast
+  b.addSubnet("240.0.0.0", 4, "ipv4"); // reserved
+  b.addAddress("255.255.255.255", "ipv4");
+  // IPv6
+  b.addAddress("::1", "ipv6"); // loopback
+  b.addAddress("::", "ipv6"); // unspecified
+  b.addSubnet("fc00::", 7, "ipv6"); // unique local
+  b.addSubnet("fe80::", 10, "ipv6"); // link-local
+  b.addSubnet("ff00::", 8, "ipv6"); // multicast
+  return b;
+}
+
+const BLOCK = buildBlockList();
+const LOOPBACK = (() => {
+  const b = new BlockList();
+  b.addSubnet("127.0.0.0", 8, "ipv4");
+  b.addAddress("::1", "ipv6");
+  return b;
+})();
+
+/** True if `ip` is in a blocked range. v4-mapped IPv6 (::ffff:a.b.c.d) is unwrapped
+ *  and checked as IPv4. When `allowLoopback`, loopback is permitted (local dev). */
+export function isBlockedIp(ip: string, allowLoopback = false): boolean {
+  let addr = ip;
+  let fam = isIP(addr);
+  if (fam === 6 && addr.includes(".")) {
+    // v4-mapped (e.g. ::ffff:127.0.0.1) — check the embedded v4.
+    const v4 = addr.slice(addr.lastIndexOf(":") + 1);
+    if (isIP(v4) === 4) {
+      addr = v4;
+      fam = 4;
+    }
+  }
+  if (fam === 0) return true; // not a valid IP → treat as blocked
+  const type = fam === 4 ? "ipv4" : "ipv6";
+  if (allowLoopback && LOOPBACK.check(addr, type)) return false;
+  return BLOCK.check(addr, type);
+}
+
+export type DnsResolver = (hostname: string) => Promise<string[]>;
+const defaultResolver: DnsResolver = async (hostname) => (await lookup(hostname, { all: true })).map((a) => a.address);
+
+const BLOCKED_HOST_RE = /^(localhost|.*\.localhost|.*\.local|.*\.internal|.*\.localdomain)$/i;
+
+/**
+ * Validate a target URL is safe to fetch: http(s) only, and every IP the host
+ * resolves to is public. Run at submit time so the service can't be used as an
+ * open proxy into the private network / cloud metadata. Returns the parsed URL.
+ */
+export async function assertPublicUrl(raw: string, opts?: { resolver?: DnsResolver; allowLoopback?: boolean }): Promise<URL> {
+  let u: URL;
+  try {
+    u = new URL(raw);
+  } catch {
+    throw new SsrfError("invalid URL");
+  }
+  if (u.protocol !== "http:" && u.protocol !== "https:") throw new SsrfError("only http(s) URLs are allowed");
+
+  const host = u.hostname.replace(/^\[|\]$/g, ""); // strip IPv6 brackets
+  if (!opts?.allowLoopback && BLOCKED_HOST_RE.test(host)) throw new SsrfError(`blocked host: ${host}`);
+
+  const literal = isIP(host) !== 0;
+  const ips = literal ? [host] : await safeResolve(opts?.resolver ?? defaultResolver, host);
+  if (ips.length === 0) throw new SsrfError(`host does not resolve: ${host}`);
+  for (const ip of ips) {
+    if (isBlockedIp(ip, opts?.allowLoopback)) throw new SsrfError(`blocked address for ${host}: ${ip}`);
+  }
+  return u;
+}
+
+async function safeResolve(resolver: DnsResolver, host: string): Promise<string[]> {
+  try {
+    return await resolver(host);
+  } catch {
+    throw new SsrfError(`DNS resolution failed: ${host}`);
+  }
+}

packages/api/src/store.ts

@@ -0,0 +1,76 @@
+import { rmSync } from "node:fs";
+import type { CloneJobResult, CloneOptions } from "@cloner/core";
+
+/** A completed (or failed) clone held in memory for the M1 sync API. The `base` is
+ *  the temp dir holding the run's files (kept alive so /files/* can stream them);
+ *  it is removed on eviction. M2/M4 replace this with DB rows + S3 objects. */
+export type JobRecord = {
+  id: string;
+  status: "succeeded" | "failed";
+  url: string;
+  kind: "clone" | "clone_site";
+  options: CloneOptions;
+  createdAt: number;
+  result?: CloneJobResult;
+  base?: string;
+  error?: string;
+};
+
+/** In-memory result store with TTL eviction. Swapped for a DB-backed store in M2. */
+export class InMemoryStore {
+  private jobs = new Map<string, JobRecord>();
+  private timer: NodeJS.Timeout | null = null;
+
+  constructor(private ttlMs: number) {}
+
+  put(rec: JobRecord): void {
+    this.jobs.set(rec.id, rec);
+  }
+
+  get(id: string): JobRecord | undefined {
+    return this.jobs.get(id);
+  }
+
+  list(): JobRecord[] {
+    return [...this.jobs.values()].sort((a, b) => b.createdAt - a.createdAt);
+  }
+
+  remove(id: string): boolean {
+    const rec = this.jobs.get(id);
+    if (!rec) return false;
+    if (rec.base) {
+      try {
+        rmSync(rec.base, { recursive: true, force: true });
+      } catch {
+        /* best effort */
+      }
+    }
+    return this.jobs.delete(id);
+  }
+
+  sweep(): void {
+    const now = Date.now();
+    for (const rec of [...this.jobs.values()]) {
+      if (now - rec.createdAt > this.ttlMs) this.remove(rec.id);
+    }
+  }
+
+  /** Start a periodic sweep (unref'd so it never holds the process open). */
+  startSweeper(intervalMs = 60_000): void {
+    if (this.timer) return;
+    this.timer = setInterval(() => this.sweep(), intervalMs);
+    this.timer.unref?.();
+  }
+
+  stopSweeper(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+  }
+
+  /** Evict everything (used by tests / shutdown). */
+  clear(): void {
+    for (const id of [...this.jobs.keys()]) this.remove(id);
+  }
+}

packages/api/test/app.fixture.test.ts

@@ -0,0 +1,38 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { runCloneJob } from "@cloner/core";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+import { createApp } from "../src/app.js";
+import { InMemoryStore } from "../src/store.js";
+import { InMemoryBackend } from "../src/backends/inMemory.js";
+
+// End-to-end: real clone of a served fixture through the HTTP layer. Skipped when
+// no Chromium is installed.
+describe("POST /v1/clones (real clone, served fixture)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  const store = new InMemoryStore(60_000);
+  before(async () => {
+    server = await serveDir(FIXTURES_DIR);
+  });
+  after(async () => {
+    store.clear();
+    await server.close();
+  });
+
+  it("clones a fixture and returns the real generated app file map", async () => {
+    const app = createApp({ backend: new InMemoryBackend({ store, runJob: runCloneJob }) });
+    const res = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url: server.url + "/components.html", options: { interactions: false, components: true, motion: false } }),
+    });
+    assert.equal(res.status, 200);
+    const body = await res.json();
+    assert.equal(body.status, "succeeded");
+    assert.ok(body.files["src/app/page.tsx"], "has page.tsx");
+    assert.equal(body.files["src/app/page.tsx"].type, "text");
+    assert.ok(body.files["package.json"], "has package.json");
+    assert.ok(body.capture.nodeCount > 0);
+    assert.equal(body.capture.blocked, false);
+  });
+});

packages/api/test/app.test.ts

@@ -0,0 +1,143 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { collectFileMap, type CloneJobResult } from "@cloner/core";
+import { createApp } from "../src/app.js";
+import { InMemoryStore } from "../src/store.js";
+import { InMemoryBackend, type RunJob } from "../src/backends/inMemory.js";
+
+/** A browser-free fake clone: writes a tiny generated app under the provided temp
+ *  base, then returns a real CloneJobResult via collectFileMap. Proves the REST
+ *  file-map contract (text inline + binary by reference + per-file streaming)
+ *  without launching Chromium. */
+const fakeRunJob: RunJob = async (input) => {
+  const base = input.runsDir!;
+  const app = join(base, "generated", "app");
+  mkdirSync(join(app, "src", "app"), { recursive: true });
+  mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+  writeFileSync(join(app, "package.json"), '{"name":"cloned-app"}\n');
+  writeFileSync(join(app, "src", "app", "page.tsx"), "export default function Page(){return <div/>}\n");
+  const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]);
+  writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), png);
+  const files = collectFileMap(base);
+  return {
+    url: input.url,
+    kind: "clone",
+    options: input.options ?? {},
+    status: "succeeded",
+    compilerVersion: "test-0",
+    timings: { captureMs: 5, generateMs: 0 },
+    files,
+    capture: { nodeCount: 42, pollution: false, blocked: false },
+    runDir: base,
+  } satisfies CloneJobResult;
+};
+
+test("POST /v1/clones returns the eager file map; binaries by reference; streaming + lifecycle", async () => {
+  const store = new InMemoryStore(60_000);
+  const app = createApp({ backend: new InMemoryBackend({ store, runJob: fakeRunJob }) });
+  try {
+    const res = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url: "https://example.com/", options: {} }),
+    });
+    assert.equal(res.status, 200);
+    const body = await res.json();
+    assert.ok(body.jobId);
+    assert.equal(body.status, "succeeded");
+    assert.equal(body.kind, "clone");
+
+    // Text inline.
+    const page = body.files["src/app/page.tsx"];
+    assert.equal(page.type, "text");
+    assert.ok(page.content.includes("Page"));
+    assert.ok(page.sha256);
+
+    // Binary by reference (URL, not bytes).
+    const bin = body.files["public/assets/cloned/images/a.png"];
+    assert.equal(bin.type, "binary");
+    assert.equal(bin.content, undefined);
+    assert.equal(bin.url, `/v1/clones/${body.jobId}/files/public/assets/cloned/images/a.png`);
+
+    // Per-file streaming returns the actual bytes.
+    const fileRes = await app.request(bin.url);
+    assert.equal(fileRes.status, 200);
+    assert.equal(fileRes.headers.get("content-type"), "image/png");
+    const bytes = Buffer.from(await fileRes.arrayBuffer());
+    assert.equal(bytes.length, 7);
+
+    // Cheap metadata overview (no file contents).
+    const meta = await (await app.request(`/v1/clones/${body.jobId}`)).json();
+    assert.equal(meta.fileCount, 3);
+    assert.equal(meta.capture.nodeCount, 42);
+    assert.equal(meta.totalBytes > 0, true);
+
+    // Full result fetch.
+    const result = await app.request(`/v1/clones/${body.jobId}/result`);
+    assert.equal(result.status, 200);
+
+    // List.
+    const list = await (await app.request("/v1/clones")).json();
+    assert.equal(list.clones.length, 1);
+
+    // Delete purges; subsequent fetch 404s.
+    const del = await app.request(`/v1/clones/${body.jobId}`, { method: "DELETE" });
+    assert.equal((await del.json()).deleted, true);
+    assert.equal((await app.request(`/v1/clones/${body.jobId}`)).status, 404);
+  } finally {
+    store.clear();
+  }
+});
+
+test("POST /v1/clones validates the body", async () => {
+  const store = new InMemoryStore(60_000);
+  const app = createApp({ backend: new InMemoryBackend({ store, runJob: fakeRunJob }) });
+  try {
+    assert.equal((await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status, 400);
+    assert.equal(
+      (await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ url: "ftp://x" }) })).status,
+      400,
+    );
+    assert.equal(
+      (await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ url: "https://x.com", options: { bogus: 1 } }) })).status,
+      400,
+    );
+  } finally {
+    store.clear();
+  }
+});
+
+test("POST /v1/clones normalizes product options and legacy aliases", async () => {
+  const store = new InMemoryStore(60_000);
+  const app = createApp({ backend: new InMemoryBackend({ store, runJob: fakeRunJob }) });
+  try {
+    const product = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url: "https://example.com/", options: { mode: "multi", styling: "css", framework: "vite" } }),
+    });
+    assert.equal(product.status, 200);
+    const productBody = await product.json();
+    assert.deepEqual(productBody.options, { mode: "multi", styling: "css", framework: "vite" });
+
+    const legacy = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url: "https://example.com/", options: { multiPage: true, humanizeMode: "css" } }),
+    });
+    assert.equal(legacy.status, 200);
+    const legacyBody = await legacy.json();
+    assert.deepEqual(legacyBody.options, { mode: "multi", styling: "css", framework: "next" });
+  } finally {
+    store.clear();
+  }
+});
+
+test("GET /healthz", async () => {
+  const app = createApp({ backend: new InMemoryBackend({ store: new InMemoryStore(1000), runJob: fakeRunJob }) });
+  const res = await app.request("/healthz");
+  assert.equal(res.status, 200);
+  assert.deepEqual(await res.json(), { ok: true });
+});

packages/api/test/auth.test.ts

@@ -0,0 +1,48 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { createApp } from "../src/app.js";
+import { InMemoryStore } from "../src/store.js";
+import { InMemoryBackend, type RunJob } from "../src/backends/inMemory.js";
+import { hashApiKey } from "../src/auth.js";
+
+const noopRun: RunJob = async () => {
+  throw new Error("clone should not run in these tests");
+};
+
+function appWith(opts: Partial<Parameters<typeof createApp>[0]>) {
+  return createApp({ backend: new InMemoryBackend({ store: new InMemoryStore(1000), runJob: noopRun }), ...opts });
+}
+
+test("auth: 401 without key / with wrong key, 200 with a valid key; /healthz stays open", async () => {
+  const app = appWith({ auth: { keyHashes: new Set([hashApiKey("s3cret")]) } });
+  assert.equal((await app.request("/v1/clones")).status, 401);
+  assert.equal((await app.request("/v1/clones", { headers: { authorization: "Bearer wrong" } })).status, 401);
+  assert.equal((await app.request("/v1/clones", { headers: { "x-api-key": "s3cret" } })).status, 200);
+  assert.equal((await app.request("/v1/clones", { headers: { authorization: "Bearer s3cret" } })).status, 200);
+  assert.equal((await app.request("/healthz")).status, 200, "health check is unauthenticated");
+});
+
+test("rate limit: 429 once the per-minute cap is exceeded", async () => {
+  const app = appWith({ rateLimitPerMinute: 2 });
+  const headers = { "x-forwarded-for": "9.9.9.9" };
+  assert.equal((await app.request("/v1/clones", { headers })).status, 200);
+  assert.equal((await app.request("/v1/clones", { headers })).status, 200);
+  const limited = await app.request("/v1/clones", { headers });
+  assert.equal(limited.status, 429);
+  assert.ok(limited.headers.get("retry-after"));
+});
+
+test("ssrf: a submit is rejected (400) when the URL guard throws", async () => {
+  const app = appWith({
+    assertUrl: async (u) => {
+      if (u.includes("169.254")) throw new Error("blocked address");
+    },
+  });
+  const res = await app.request("/v1/clones", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ url: "http://169.254.169.254/" }),
+  });
+  assert.equal(res.status, 400);
+  assert.equal((await res.json()).error, "url not allowed");
+});

packages/api/test/mcp.test.ts

@@ -0,0 +1,107 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { collectFileMap, type CloneJobResult } from "@cloner/core";
+import { createMcpServer } from "../src/mcp.js";
+import { InMemoryStore } from "../src/store.js";
+import { InMemoryBackend, type RunJob } from "../src/backends/inMemory.js";
+
+const fakeRunJob: RunJob = async (input) => {
+  const base = input.runsDir!;
+  const app = join(base, "generated", "app");
+  mkdirSync(join(app, "src", "app"), { recursive: true });
+  mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+  writeFileSync(join(app, "package.json"), '{"name":"cloned-app"}\n');
+  writeFileSync(join(app, "src", "app", "page.tsx"), "export default function Page(){return <div/>}\n");
+  writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), Buffer.from([1, 2, 3, 4]));
+  return {
+    url: input.url,
+    kind: "clone",
+    options: input.options ?? {},
+    status: "succeeded",
+    compilerVersion: "test-0",
+    timings: { captureMs: 1, generateMs: 0 },
+    files: collectFileMap(base),
+    capture: { nodeCount: 9, pollution: false, blocked: false },
+    runDir: base,
+  } satisfies CloneJobResult;
+};
+
+async function connect(backend: InMemoryBackend) {
+  const server = createMcpServer(backend, { baseUrl: "http://test" });
+  const [clientT, serverT] = InMemoryTransport.createLinkedPair();
+  const client = new Client({ name: "test-client", version: "0.0.0" });
+  await Promise.all([server.connect(serverT), client.connect(clientT)]);
+  return client;
+}
+const parse = (res: unknown): any => JSON.parse((res as { content: { text: string }[] }).content[0]!.text);
+
+test("MCP: list-then-read + bundle contract (never floods context)", async () => {
+  const store = new InMemoryStore(60_000);
+  const backend = new InMemoryBackend({ store, runJob: fakeRunJob });
+  const client = await connect(backend);
+  try {
+    const tools = (await client.listTools()).tools.map((t) => t.name);
+    for (const n of ["clone_website", "get_clone_status", "get_clone_result", "list_clone_files", "read_clone_files", "get_clone_bundle", "cancel_clone"]) {
+      assert.ok(tools.includes(n), `tool ${n} registered`);
+    }
+
+    // clone_website → jobId + status only (no files).
+    const cw = parse(await client.callTool({ name: "clone_website", arguments: { url: "https://example.com/", options: {} } }));
+    assert.ok(cw.jobId);
+    assert.equal(cw.status, "succeeded");
+    assert.ok(!("files" in cw));
+    const jobId = cw.jobId;
+
+    // status
+    const status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
+    assert.equal(status.status, "succeeded");
+
+    // get_clone_result → metadata only, NO file contents.
+    const meta = parse(await client.callTool({ name: "get_clone_result", arguments: { jobId } }));
+    assert.equal(meta.fileCount, 3);
+    assert.equal(meta.capture.nodeCount, 9);
+    assert.ok(!("files" in meta), "result metadata must not include file contents");
+    assert.ok(meta.bundleUrl.includes("/bundle"));
+
+    // list_clone_files (glob) → manifest only, no content.
+    const list = parse(await client.callTool({ name: "list_clone_files", arguments: { jobId, glob: "**/*.tsx" } }));
+    assert.ok(list.files.length >= 1);
+    assert.ok(list.files.every((f: any) => f.path.endsWith(".tsx")));
+    assert.ok(list.files.some((f: any) => f.path === "src/app/page.tsx"));
+    assert.ok(list.files.every((f: any) => !("content" in f)), "list must not include content");
+
+    // read_clone_files → text inline, binary by URL.
+    const read = parse(await client.callTool({ name: "read_clone_files", arguments: { jobId, paths: ["src/app/page.tsx", "public/assets/cloned/images/a.png"] } }));
+    const page = read.files.find((f: any) => f.path === "src/app/page.tsx");
+    assert.equal(page.type, "text");
+    assert.ok(page.content.includes("Page"));
+    const bin = read.files.find((f: any) => f.path.endsWith("a.png"));
+    assert.equal(bin.type, "binary");
+    assert.ok(bin.url.startsWith("http://test/"), "binary returned as absolute URL");
+    assert.ok(!("content" in bin), "binary must not inline bytes");
+
+    // read size budget → oversized text flagged skipped, not dumped.
+    const tiny = parse(await client.callTool({ name: "read_clone_files", arguments: { jobId, paths: ["src/app/page.tsx"], maxBytes: 1 } }));
+    assert.equal(tiny.files[0].skipped, true);
+    assert.equal(tiny.truncated, true);
+
+    // get_clone_bundle → a download reference, not bytes.
+    const bundle = parse(await client.callTool({ name: "get_clone_bundle", arguments: { jobId } }));
+    assert.equal(bundle.format, "tgz");
+    assert.ok(bundle.bytes > 0);
+    assert.ok(/^[0-9a-f]{64}$/.test(bundle.sha256));
+    assert.ok(bundle.url.includes(`/v1/clones/${jobId}/bundle`));
+
+    // cancel_clone → purges.
+    const cancel = parse(await client.callTool({ name: "cancel_clone", arguments: { jobId } }));
+    assert.equal(cancel.cancelled, true);
+    const after = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
+    assert.ok(after.error, "status after cancel reports not found");
+  } finally {
+    store.clear();
+  }
+});

packages/api/test/ssrf.test.ts

@@ -0,0 +1,35 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { assertPublicUrl, isBlockedIp, SsrfError } from "../src/ssrf.js";
+
+test("isBlockedIp: private / loopback / link-local / metadata / ULA are blocked", () => {
+  for (const ip of ["10.0.0.1", "127.0.0.1", "169.254.169.254", "192.168.1.1", "172.16.0.1", "100.64.0.1", "::1", "fe80::1", "fc00::1", "::ffff:127.0.0.1"]) {
+    assert.equal(isBlockedIp(ip), true, `${ip} should be blocked`);
+  }
+});
+
+test("isBlockedIp: public addresses are allowed", () => {
+  for (const ip of ["8.8.8.8", "1.1.1.1", "93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"]) {
+    assert.equal(isBlockedIp(ip), false, `${ip} should be allowed`);
+  }
+  assert.equal(isBlockedIp("127.0.0.1", true), false, "loopback allowed when opted in");
+});
+
+test("assertPublicUrl: rejects non-http, IP literals in blocked ranges, localhost", async () => {
+  await assert.rejects(() => assertPublicUrl("ftp://example.com/"), SsrfError);
+  await assert.rejects(() => assertPublicUrl("http://169.254.169.254/latest/meta-data/"), SsrfError);
+  await assert.rejects(() => assertPublicUrl("http://10.1.2.3/"), SsrfError);
+  await assert.rejects(() => assertPublicUrl("http://localhost:3000/"), SsrfError);
+});
+
+test("assertPublicUrl: blocks hostnames that resolve to private IPs (DNS rebinding)", async () => {
+  await assert.rejects(() => assertPublicUrl("http://rebind.evil.test/", { resolver: async () => ["10.0.0.5"] }), SsrfError);
+  await assert.rejects(() => assertPublicUrl("http://nope.test/", { resolver: async () => [] }), SsrfError);
+});
+
+test("assertPublicUrl: allows public hostnames + opt-in loopback", async () => {
+  const u = await assertPublicUrl("https://example.test/path", { resolver: async () => ["93.184.216.34"] });
+  assert.equal(u.hostname, "example.test");
+  const u2 = await assertPublicUrl("http://127.0.0.1:8080/", { allowLoopback: true });
+  assert.equal(u2.port, "8080");
+});

packages/api/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"]
+}

packages/core/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@cloner/core",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "node --import tsx --test test/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "clone-static": "*"
+  },
+  "devDependencies": {
+    "@cloner/test-utils": "*",
+    "tsx": "4.22.4",
+    "typescript": "5.7.3",
+    "@types/node": "22.10.5"
+  }
+}

packages/core/src/cacheKey.ts

@@ -0,0 +1,62 @@
+import { createHash } from "node:crypto";
+import type { CloneOptions } from "./types.js";
+import { resolveCloneOptions } from "./options.js";
+
+/** Normalize a URL so trivially-different spellings of the same page share a cache
+ *  entry: lowercase scheme+host, drop default ports, drop the fragment, collapse a
+ *  bare/trailing-slash path. Query is preserved (it can select a different page) but
+ *  its params are sorted for stability. Falsy/invalid input is returned trimmed. */
+export function normalizeUrl(raw: string): string {
+  let u: URL;
+  try {
+    u = new URL(raw.trim());
+  } catch {
+    return raw.trim();
+  }
+  u.protocol = u.protocol.toLowerCase();
+  u.hostname = u.hostname.toLowerCase().replace(/\.$/, "");
+  if (
+    (u.protocol === "http:" && u.port === "80") ||
+    (u.protocol === "https:" && u.port === "443")
+  ) {
+    u.port = "";
+  }
+  u.hash = "";
+  // Collapse trailing slashes on the path (but keep "/" for root).
+  u.pathname = u.pathname.replace(/\/+$/, "") || "/";
+  // Sort query params for stability.
+  u.searchParams.sort();
+  return u.toString();
+}
+
+/** The subset of options that change the generated output, serialized canonically.
+ *  `noCache` is intentionally excluded (it's a request-time switch, not an output
+ *  determinant). `verify`/`asyncVerify` are included so a verified result isn't
+ *  served for an unverified request and vice-versa. Viewports are sorted; booleans
+ *  normalized. */
+export function canonicalOptions(options: CloneOptions = {}): string {
+  const resolved = resolveCloneOptions(options);
+  const norm = {
+    mode: resolved.mode,
+    styling: resolved.styling,
+    framework: resolved.framework,
+    viewports: resolved.viewports ? [...resolved.viewports].sort((a, b) => a - b) : null,
+    interactions: resolved.interactions,
+    components: resolved.components,
+    motion: resolved.motion,
+    verify: !!resolved.verify,
+    asyncVerify: !!resolved.asyncVerify,
+    maxRoutes: resolved.maxRoutes ?? null,
+    maxCollection: resolved.maxCollection ?? null,
+  };
+  return JSON.stringify(norm);
+}
+
+/** cacheKey = sha256(normalizedUrl + canonicalOptions + compilerVersion).
+ *  A compilerVersion bump invalidates everything (the output changed). The cache is
+ *  freshness-bounded by the caller (CACHE_STALE_AFTER), because two *captures* of a
+ *  live site can differ even though generation from one capture is byte-stable. */
+export function cacheKey(url: string, options: CloneOptions | undefined, compilerVersion: string): string {
+  const payload = [normalizeUrl(url), canonicalOptions(options), compilerVersion].join("\n");
+  return createHash("sha256").update(payload).digest("hex");
+}

packages/core/src/collectFileMap.ts

@@ -0,0 +1,67 @@
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync, statSync, existsSync } from "node:fs";
+import { join, extname, basename, relative, sep } from "node:path";
+import type { CollectedFile, FileMap } from "./types.js";
+
+/** Extensions whose bytes are returned inline as UTF-8 text (the code a consumer
+ *  reads). Everything else under public/assets (images/fonts/video) is binary and
+ *  returned by reference. */
+const TEXT_EXTS = new Set([
+  ".tsx", ".ts", ".jsx", ".js", ".mjs", ".cjs", ".css", ".json",
+  ".html", ".xml", ".txt", ".md", ".map", ".d.ts",
+]);
+
+function isTextFile(path: string): boolean {
+  const base = basename(path);
+  if (base === ".gitignore" || base === "next-env.d.ts") return true;
+  return TEXT_EXTS.has(extname(path).toLowerCase());
+}
+
+function* walk(dir: string): Generator<string> {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      yield* walk(full);
+    } else if (entry.isFile()) {
+      yield full;
+    }
+  }
+}
+
+const toPosix = (p: string): string => (sep === "/" ? p : p.split(sep).join("/"));
+
+/** Collect the generated app (`<runDir>/generated/app/`) into a file map keyed
+ *  by app-relative POSIX path. Text files inline their content; binaries carry a
+ *  local path + sha256 for the storage layer to upload and presign. Keys are sorted
+ *  so the map is deterministic for the same generated app (golden-file friendly). */
+export function collectFileMap(runDir: string): FileMap {
+  const appDir = join(runDir, "generated", "app");
+  if (!existsSync(appDir)) {
+    throw new Error(`collectFileMap: no generated app at ${appDir}`);
+  }
+  const files: CollectedFile[] = [];
+  for (const abs of walk(appDir)) {
+    const rel = toPosix(relative(appDir, abs));
+    const buf = readFileSync(abs);
+    const sha256 = createHash("sha256").update(buf).digest("hex");
+    const bytes = statSync(abs).size;
+    if (isTextFile(rel)) {
+      files.push({ path: rel, kind: "text", bytes, sha256, content: buf.toString("utf8"), absPath: abs });
+    } else {
+      files.push({ path: rel, kind: "binary", bytes, sha256, absPath: abs });
+    }
+  }
+  files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const map: FileMap = {};
+  for (const f of files) map[f.path] = f;
+  return map;
+}
+
+/** Total bytes + file count for a quick overview (the cheap metadata an MCP
+ *  `get_clone_result` returns before any file content is read). */
+export function fileMapStats(files: FileMap): { fileCount: number; totalBytes: number } {
+  let totalBytes = 0;
+  const vals = Object.values(files);
+  for (const f of vals) totalBytes += f.bytes;
+  return { fileCount: vals.length, totalBytes };
+}

packages/core/src/index.ts

@@ -0,0 +1,26 @@
+/**
+ * @cloner/core — the only package that imports the deterministic compiler.
+ * The api / worker / mcp layers depend on this, never on compiler internals.
+ */
+export { runCloneJob, verifyCloneJobResult } from "./runCloneJob.js";
+export { collectFileMap, fileMapStats } from "./collectFileMap.js";
+export { cacheKey, normalizeUrl, canonicalOptions } from "./cacheKey.js";
+export {
+  normalizeCloneRequestOptions,
+  resolveCloneMode,
+  resolveCloneOptions,
+  resolveCloneStyling,
+} from "./options.js";
+export { COMPILER_VERSION } from "clone-static";
+export type {
+  CloneMode,
+  CloneOptions,
+  CloneStyling,
+  CollectedFile,
+  FileMap,
+  CaptureSanity,
+  CloneTimings,
+  RouteInfo,
+  CloneJobResult,
+  RunCloneJobInput,
+} from "./types.js";

packages/core/src/options.ts

@@ -0,0 +1,57 @@
+import type { CloneFramework, CloneMode, CloneOptions, CloneStyling } from "./types.js";
+
+export type ResolvedCloneOptions = CloneOptions & {
+  mode: CloneMode;
+  styling: CloneStyling;
+  framework: CloneFramework;
+  multiPage: boolean;
+  humanizeMode: CloneStyling;
+  interactions: boolean;
+  components: boolean;
+  motion: boolean;
+};
+
+export function resolveCloneMode(options: CloneOptions = {}): CloneMode {
+  return options.mode ?? (options.multiPage ? "multi" : "single");
+}
+
+export function resolveCloneStyling(options: CloneOptions = {}): CloneStyling {
+  return options.styling ?? options.humanizeMode ?? "tailwind";
+}
+
+export function resolveCloneFramework(options: CloneOptions = {}): CloneFramework {
+  return options.framework ?? "next";
+}
+
+/** Normalize the request-facing shape. Deprecated aliases are consumed but not
+ * echoed, so REST/MCP results present the product-level option names. */
+export function normalizeCloneRequestOptions(options: CloneOptions = {}): CloneOptions {
+  const normalized: CloneOptions = {
+    ...options,
+    mode: resolveCloneMode(options),
+    styling: resolveCloneStyling(options),
+    framework: resolveCloneFramework(options),
+  };
+  delete normalized.multiPage;
+  delete normalized.humanizeMode;
+  return normalized;
+}
+
+/** Resolve options for the compiler adapter. This is where automatic internal
+ * defaults live; callers should not need to choose these in normal use. */
+export function resolveCloneOptions(options: CloneOptions = {}): ResolvedCloneOptions {
+  const mode = resolveCloneMode(options);
+  const styling = resolveCloneStyling(options);
+  const framework = resolveCloneFramework(options);
+  return {
+    ...options,
+    mode,
+    styling,
+    framework,
+    multiPage: mode === "multi",
+    humanizeMode: styling,
+    interactions: options.interactions ?? true,
+    components: options.components ?? true,
+    motion: options.motion ?? true,
+  };
+}

packages/core/src/runCloneJob.ts

@@ -0,0 +1,203 @@
+import { mkdtempSync, rmSync, cpSync, existsSync, mkdirSync, statSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import {
+  runClone,
+  runCloneSite,
+  validateRun,
+  validateSite,
+  buildIR,
+  gatePollution,
+  readJSON,
+  siteIdFromUrl,
+  COMPILER_VERSION,
+  type CaptureResult,
+  type CloneResult as CompilerCloneResult,
+  type CloneSiteResult,
+} from "clone-static";
+import { collectFileMap } from "./collectFileMap.js";
+import type { CaptureSanity, CloneJobResult, CloneOptions, RouteInfo, RunCloneJobInput } from "./types.js";
+import { normalizeCloneRequestOptions, resolveCloneOptions } from "./options.js";
+
+/** Compute the cheap capture-sanity audit (no build): node count + whether the
+ *  pollution gate flags the capture as degenerate, and whether bot/egress-wall text
+ *  was seen. Falls back to safe defaults if the source artifacts are missing. */
+function captureSanity(sourceDir: string, viewports: number[]): CaptureSanity {
+  try {
+    const capture = readJSON<CaptureResult>(join(sourceDir, "capture", "capture-result.json"));
+    const vps = capture.viewports?.length ? capture.viewports : viewports;
+    const ir = buildIR(sourceDir, vps);
+    const p = gatePollution(ir, capture, vps);
+    return { nodeCount: ir.doc.nodeCount, pollution: !p.pass, blocked: !!p.metrics.wallTextDetected };
+  } catch {
+    return { nodeCount: 0, pollution: true, blocked: false };
+  }
+}
+
+/** Persistent entry-capture cache path for a URL. Key = the compiler's full host+path
+ *  site id (collision-safe, unlike the bare folder name), so only the SAME page reuses. */
+function entryCacheSource(cacheDir: string, url: string): string {
+  return join(cacheDir, siteIdFromUrl(url), "source");
+}
+/** Whether a cached capture exists and is fresh enough to reuse (ttlMs 0/undefined =
+ *  no expiry). Staleness is measured from the capture artifact's mtime. */
+function freshCapture(dir: string, ttlMs?: number): boolean {
+  const f = join(dir, "capture", "capture-result.json");
+  if (!existsSync(f)) return false;
+  if (!ttlMs) return true;
+  try { return Date.now() - statSync(f).mtimeMs < ttlMs; } catch { return false; }
+}
+/** Copy a fresh capture into the cache (atomic-ish: clear then copy). No-op if the
+ *  source has no capture artifact. */
+function persistCapture(srcDir: string | undefined, dest: string): void {
+  if (!srcDir || !existsSync(join(srcDir, "capture", "capture-result.json"))) return;
+  mkdirSync(dirname(dest), { recursive: true });
+  rmSync(dest, { recursive: true, force: true });
+  cpSync(srcDir, dest, { recursive: true });
+}
+
+/**
+ * The single seam the service depends on: run one clone end-to-end into a temp
+ * dir, collect its generated app into a file map, optionally verify it, then return
+ * a typed result. The caller (worker/api) uploads artifacts + persists, then the
+ * temp dir is removed in `finally`. No compiler behavior is changed — this only
+ * orchestrates the existing entry points with a parameterized output dir.
+ *
+ * Incremental speed path: when `captureCacheDir` is set, a single-page job stashes its
+ * entry capture there; a later multi-page job for the same URL reuses it as the entry
+ * route (skips re-capturing page 1) and regenerates the whole site on top of it.
+ */
+export async function runCloneJob(input: RunCloneJobInput): Promise<CloneJobResult> {
+  const requestOptions: CloneOptions = normalizeCloneRequestOptions(input.options ?? {});
+  const options = resolveCloneOptions(requestOptions);
+  const syncVerify = !!options.verify && !options.asyncVerify;
+  const captureValidationArtifacts = !!(options.verify || options.asyncVerify);
+  const log = input.log ?? (() => {});
+  const ownsTemp = !input.runsDir;
+  const runsDir = input.runsDir ?? mkdtempSync(join(tmpdir(), "clone-job-"));
+  const kind: "clone" | "clone_site" = options.mode === "multi" ? "clone_site" : "clone";
+
+  // Best-by-default (matches the CLI): interactions/components/motion ON unless the
+  // caller explicitly disables them. (runClone itself defaults them OFF; the CLI is
+  // what turns them on — so the service does the same here.)
+  const interactions = options.interactions;
+  const components = options.components;
+  const motion = options.motion;
+
+  try {
+    const t0 = Date.now();
+    let runDir: string;
+    let routes: RouteInfo[] | undefined;
+    let sanity: CaptureSanity;
+    let captureReused = false;
+    // Persistent entry-capture cache (the single→multi speed path), keyed by URL.
+    const cacheEntry = input.captureCacheDir ? entryCacheSource(input.captureCacheDir, input.url) : undefined;
+
+    if (kind === "clone_site") {
+      // Reuse a prior single-page capture for the entry route when the cache holds a fresh one.
+      const reuseEntrySource = cacheEntry && freshCapture(cacheEntry, input.captureCacheTtlMs) ? cacheEntry : undefined;
+      captureReused = !!reuseEntrySource;
+      const res: CloneSiteResult = await runCloneSite({
+        url: input.url,
+        runsDir,
+        validate: false,
+        interactions,
+        components,
+        humanizeMode: options.styling,
+        framework: options.framework,
+        reuseEntrySource,
+        maxRoutes: options.maxRoutes,
+        maxCollectionInstances: options.maxCollection,
+        captureConcurrency: options.captureConcurrency,
+        validationConcurrency: options.validationConcurrency,
+        viewportConcurrency: options.viewportConcurrency,
+        screenshots: captureValidationArtifacts,
+        log,
+      });
+      runDir = res.runDir;
+      // Reproduced routes + the collapsed-collection map (representativeOf).
+      routes = res.routes.map((r) => ({ route: r.routePath }));
+      for (const c of res.plan.collections) {
+        const rep = res.routes.find((r) => r.routePath === c.representative);
+        if (rep) {
+          for (const inst of c.instances) {
+            if (inst !== c.representative) routes!.push({ route: inst, representativeOf: c.representative });
+          }
+        }
+      }
+      const entry = res.routes.find((r) => r.routePath === res.plan.entry) ?? res.routes[0];
+      sanity = entry
+        ? captureSanity(entry.sourceDir, entry.ir.doc.viewports)
+        : { nodeCount: 0, pollution: true, blocked: false };
+      // Refresh the cache with the entry capture (seeds it for a cold multi-page run).
+      if (cacheEntry && entry) persistCapture(entry.sourceDir, cacheEntry);
+    } else {
+      const res: CompilerCloneResult = await runClone({
+        url: input.url,
+        runsDir,
+        viewports: options.viewports,
+        interactions,
+        components,
+        motion,
+        humanizeMode: options.styling,
+        framework: options.framework,
+        screenshots: captureValidationArtifacts,
+        log,
+      });
+      runDir = res.runDir;
+      sanity = captureSanity(res.sourceDir, options.viewports ?? [375, 768, 1280, 1920]);
+      // Stash this page's capture so a later multi-page job can expand on it (speed path).
+      if (cacheEntry) persistCapture(res.sourceDir, cacheEntry);
+    }
+    const captureMs = Date.now() - t0;
+
+    // Optional verify (build + serve + re-render + grade). The single isolation-
+    // sensitive step: pass a per-worker harnessDir so concurrent builds don't collide.
+    let verify: unknown;
+    let verifyMs: number | undefined;
+    if (syncVerify) {
+      const done = await verifyCloneJobResult({ kind, runDir }, { harnessDir: input.harnessDir, tier: input.tier ?? "stage2", validationConcurrency: options.validationConcurrency, viewportConcurrency: options.viewportConcurrency, log });
+      verify = done.verify;
+      verifyMs = done.verifyMs;
+    }
+
+    const files = collectFileMap(runDir);
+
+    return {
+      url: input.url,
+      kind,
+      options: requestOptions,
+      status: "succeeded",
+      compilerVersion: COMPILER_VERSION,
+      timings: { captureMs, generateMs: 0, ...(verifyMs !== undefined ? { verifyMs } : {}) },
+      routes,
+      files,
+      capture: sanity,
+      captureReused,
+      verify,
+      runDir,
+    };
+  } finally {
+    if (ownsTemp && !input.keepTemp) {
+      rmSync(runsDir, { recursive: true, force: true });
+    }
+  }
+}
+
+export async function verifyCloneJobResult(
+  result: Pick<CloneJobResult, "kind" | "runDir">,
+  opts?: {
+    harnessDir?: string;
+    tier?: string;
+    validationConcurrency?: number;
+    viewportConcurrency?: number;
+    log?: (e: Record<string, unknown>) => void;
+  },
+): Promise<{ verify: unknown; verifyMs: number }> {
+  const t0 = Date.now();
+  const tier = opts?.tier ?? "stage2";
+  const verify = result.kind === "clone_site"
+    ? await validateSite(result.runDir, { harnessDir: opts?.harnessDir, tier, routeConcurrency: opts?.validationConcurrency, viewportConcurrency: opts?.viewportConcurrency, log: opts?.log })
+    : await validateRun(result.runDir, { harnessDir: opts?.harnessDir, tier, log: opts?.log });
+  return { verify, verifyMs: Date.now() - t0 };
+}

packages/core/src/types.ts

@@ -0,0 +1,123 @@
+/**
+ * Service-layer types. These are the public wire shapes for clone options,
+ * clone results, and the eager output file map, plus the core's internal
+ * collected-file shape.
+ *
+ * The core layer is the ONLY place that imports the compiler. Everything above
+ * it (api, worker, mcp) speaks these types.
+ */
+
+export type CloneMode = "single" | "multi";
+export type CloneStyling = "tailwind" | "css";
+export type CloneFramework = "next" | "vite";
+
+/** Clone options accepted by the service/core boundary.
+ *
+ * Normal callers should use only the product choices:
+ * - `mode`: single page or multi-page site clone.
+ * - `styling`: Tailwind v4 or plain CSS output.
+ * - `framework`: Next.js App Router or Vite React output.
+ *
+ * The remaining fields are operational/dev controls or deprecated compatibility
+ * aliases. Internal pipeline features such as probing, recipes, component
+ * extraction, interactions, and motion stay automatic by default.
+ */
+export type CloneOptions = {
+  mode?: CloneMode;
+  styling?: CloneStyling;
+  framework?: CloneFramework;
+  verify?: boolean;
+  /** DB/worker mode: persist the clone first, then attach verify in the background. */
+  asyncVerify?: boolean;
+  maxRoutes?: number;
+  maxCollection?: number;
+  captureConcurrency?: number;
+  validationConcurrency?: number;
+  viewportConcurrency?: number;
+  /** Service-level: bypass the cache on read AND write (does not affect output). */
+  noCache?: boolean;
+
+  /** @deprecated Use `mode: "multi"` instead. */
+  multiPage?: boolean;
+  /** @deprecated Use `styling` instead. */
+  humanizeMode?: CloneStyling;
+
+  /** Dev-only/output-affecting escape hatches. Not part of the normal product surface. */
+  viewports?: number[];
+  interactions?: boolean;
+  components?: boolean;
+  motion?: boolean;
+};
+
+/** A single file in a clone's output, as collected from `generated/app/`.
+ *  Text files carry their content inline; binaries carry only a local path +
+ *  hash (the storage layer turns `absPath` into a presigned URL after upload). */
+export type CollectedFile = {
+  /** app-relative POSIX path, e.g. "src/app/page.tsx" or "public/assets/cloned/images/ab.png" */
+  path: string;
+  kind: "text" | "binary";
+  bytes: number;
+  /** sha256 of the raw file bytes (content-addressing for storage + cache integrity). */
+  sha256: string;
+  /** present for text files only */
+  content?: string;
+  /** local filesystem path (ephemeral — used by the storage layer to upload/stream). */
+  absPath: string;
+};
+
+export type FileMap = Record<string, CollectedFile>;
+
+/** Capture-sanity audit — surfaced so a structurally-perfect clone of the WRONG
+ *  page (bot wall, empty shell) is flagged rather than sold as success. */
+export type CaptureSanity = {
+  nodeCount: number;
+  pollution: boolean; // the pollution gate failed (degenerate capture)
+  blocked: boolean; // bot/egress wall text detected
+};
+
+export type CloneTimings = { captureMs: number; generateMs: number; verifyMs?: number };
+
+export type RouteInfo = { route: string; representativeOf?: string };
+
+/** What `runCloneJob` returns — the full result for one clone, BEFORE storage
+ *  upload assigns URLs to binaries. The worker/api persists + uploads from this. */
+export type CloneJobResult = {
+  url: string;
+  kind: "clone" | "clone_site";
+  options: CloneOptions;
+  status: "succeeded";
+  compilerVersion: string;
+  timings: CloneTimings;
+  routes?: RouteInfo[];
+  files: FileMap;
+  capture: CaptureSanity;
+  /** true when a multi-page job reused a prior single-page entry capture (no re-capture
+   *  of the entry route) — the "single page first, then expand" speed path. */
+  captureReused?: boolean;
+  /** present only when options.verify === true */
+  verify?: unknown; // compiler Report — kept opaque here to avoid leaking gate internals upward
+  /** the ephemeral local run dir (already collected into `files`); caller cleans up
+   *  unless `keepTemp` was set. */
+  runDir: string;
+};
+
+export type RunCloneJobInput = {
+  url: string;
+  options?: CloneOptions;
+  /** override the temp base dir (tests). When set, the dir is NOT auto-removed. */
+  runsDir?: string;
+  /** keep the run dir after collecting the file map (debug/tests). */
+  keepTemp?: boolean;
+  /** Persistent entry-capture cache base dir, keyed by URL. A single-page job copies its
+   *  capture here; a later multi-page job for the same URL REUSES it as the entry route
+   *  (no re-capture) — the "single page first, then expand to the full site" speed path. */
+  captureCacheDir?: string;
+  /** Max age (ms) of a cached entry capture that may be reused; older ⇒ re-capture.
+   *  Undefined/0 ⇒ no expiry (reuse whenever present). Bounds staleness for a service. */
+  captureCacheTtlMs?: number;
+  /** isolated build harness dir for verify (per-worker; defaults to the compiler's). */
+  harnessDir?: string;
+  /** tier threshold for the perceptual gate when verifying (default "stage2"). */
+  tier?: string;
+  log?: (e: Record<string, unknown>) => void;
+};

packages/core/test/cacheKey.test.ts

@@ -0,0 +1,41 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { cacheKey, normalizeUrl, canonicalOptions } from "../src/cacheKey.js";
+
+test("normalizeUrl: canonicalizes scheme/host/port/trailing-slash/fragment/query", () => {
+  assert.equal(normalizeUrl("HTTPS://Example.com:443/foo/#frag"), "https://example.com/foo");
+  assert.equal(normalizeUrl("http://example.com:80/"), "http://example.com/");
+  assert.equal(normalizeUrl("https://x.com"), "https://x.com/");
+  assert.equal(normalizeUrl("https://example.com/a?b=2&a=1"), "https://example.com/a?a=1&b=2");
+  assert.equal(normalizeUrl("not a url"), "not a url");
+});
+
+test("cacheKey: stable for equivalent input, varies by options + compilerVersion", () => {
+  const k1 = cacheKey("https://x.com/", { mode: "single", styling: "tailwind" }, "0.1.0");
+  const k2 = cacheKey("https://x.com", { mode: "single", styling: "tailwind" }, "0.1.0");
+  assert.equal(k1, k2, "trailing slash is normalized away");
+
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "multi", styling: "tailwind" }, "0.1.0"), "mode changes the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "css" }, "0.1.0"), "styling changes the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", framework: "vite" }, "0.1.0"), "framework changes the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind" }, "0.2.0"), "version bump invalidates");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", verify: true }, "0.1.0"), "verify is part of the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", asyncVerify: true }, "0.1.0"), "asyncVerify is part of the key");
+
+  // noCache is a request-time switch, not an output determinant — must not change the key.
+  assert.equal(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", noCache: true }, "0.1.0"));
+});
+
+test("canonicalOptions: deprecated aliases normalize to product options", () => {
+  assert.equal(
+    canonicalOptions({ mode: "multi", styling: "css" }),
+    canonicalOptions({ multiPage: true, humanizeMode: "css" }),
+  );
+});
+
+test("canonicalOptions: sorts viewports and is stable", () => {
+  assert.equal(
+    canonicalOptions({ viewports: [1920, 375, 768] }),
+    canonicalOptions({ viewports: [375, 768, 1920] }),
+  );
+});

packages/core/test/collectFileMap.test.ts

@@ -0,0 +1,54 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createHash } from "node:crypto";
+import { collectFileMap, fileMapStats } from "../src/collectFileMap.js";
+
+test("collectFileMap: inlines text, references binaries, sorts keys, hashes bytes", () => {
+  const root = mkdtempSync(join(tmpdir(), "cfm-"));
+  try {
+    const app = join(root, "generated", "app");
+    mkdirSync(join(app, "src", "app", "_clone"), { recursive: true });
+    mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+    writeFileSync(join(app, "package.json"), '{"name":"x"}\n');
+    writeFileSync(join(app, ".gitignore"), "node_modules\n");
+    writeFileSync(join(app, "src", "app", "page.tsx"), "export default function Page(){return null}\n");
+    writeFileSync(join(app, "src", "app", "globals.css"), "body{margin:0}\n");
+    const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]);
+    writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), png);
+
+    const map = collectFileMap(root);
+    const keys = Object.keys(map);
+    assert.deepEqual(keys, [...keys].sort(), "keys are sorted (deterministic)");
+
+    assert.equal(map["package.json"]!.kind, "text");
+    assert.equal(map[".gitignore"]!.kind, "text", ".gitignore is treated as text");
+    const page = map["src/app/page.tsx"]!;
+    assert.equal(page.kind, "text");
+    assert.ok(page.content!.includes("export default"));
+    assert.equal(page.sha256, createHash("sha256").update("export default function Page(){return null}\n").digest("hex"));
+
+    const bin = map["public/assets/cloned/images/a.png"]!;
+    assert.equal(bin.kind, "binary");
+    assert.equal(bin.content, undefined, "binaries are by reference, not inlined");
+    assert.equal(bin.sha256, createHash("sha256").update(png).digest("hex"));
+    assert.equal(bin.bytes, png.length);
+
+    const stats = fileMapStats(map);
+    assert.equal(stats.fileCount, 5);
+    assert.ok(stats.totalBytes > 0);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("collectFileMap: throws when there is no generated app", () => {
+  const root = mkdtempSync(join(tmpdir(), "cfm-empty-"));
+  try {
+    assert.throws(() => collectFileMap(root), /no generated app/);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});

packages/core/test/incremental.fixture.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runCloneJob } from "../src/runCloneJob.js";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+
+// The "single page first, then expand" speed path: a single-page clone stashes its
+// entry capture in captureCacheDir; a later multi-page clone of the SAME url reuses it
+// as the entry route (no re-capture) and regenerates the whole site on top of it.
+describe("runCloneJob incremental (single → multi reuse)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  let cacheDir: string;
+  before(async () => {
+    server = await serveDir(join(FIXTURES_DIR, "site"));
+    cacheDir = mkdtempSync(join(tmpdir(), "capture-cache-"));
+  });
+  after(async () => {
+    await server.close();
+    rmSync(cacheDir, { recursive: true, force: true });
+  });
+
+  it("reuses the single-page entry capture when expanding to the full site", async () => {
+    const url = server.url + "/";
+
+    // 1. Single page first — fast, returns one app; seeds the capture cache.
+    const single = await runCloneJob({ url, options: {}, captureCacheDir: cacheDir });
+    assert.equal(single.status, "succeeded");
+    assert.equal(single.kind, "clone");
+    assert.equal(single.captureReused, false, "first (single) job captures fresh");
+    assert.ok(single.files["src/app/page.tsx"], "single-page app emitted");
+
+    // 2. Expand to the full multi-route site — reuses page 1's capture (no re-capture),
+    //    captures the rest, regenerates ALL routes together.
+    const multi = await runCloneJob({ url, options: { mode: "multi" }, captureCacheDir: cacheDir });
+    assert.equal(multi.status, "succeeded");
+    assert.equal(multi.kind, "clone_site");
+    assert.equal(multi.captureReused, true, "multi-page job reused the cached entry capture");
+    assert.ok((multi.routes?.length ?? 0) >= 2, `expected >=2 routes, got ${multi.routes?.length}`);
+    assert.ok(multi.files["src/app/layout.tsx"], "shared layout emitted");
+    const subRoutePages = Object.keys(multi.files).filter((p) => /^src\/app\/.+\/page\.tsx$/.test(p));
+    assert.ok(subRoutePages.length >= 1, "at least one sub-route page beyond the entry");
+
+    // Tailwind is the default styling output end-to-end (service path included).
+    assert.ok(multi.files["postcss.config.mjs"], "Tailwind toolchain present");
+    assert.ok((multi.files["src/app/globals.css"]!.content ?? "").includes("tailwindcss"), "Tailwind globals");
+  });
+});

packages/core/test/multipage.fixture.test.ts

@@ -0,0 +1,34 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { join } from "node:path";
+import { runCloneJob } from "../src/runCloneJob.js";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+
+// Multi-page clone job: crawl a served fixture site (index → blog → faq) and clone
+// all routes into one Next app. Skipped without Chromium.
+describe("runCloneJob multi-page (served fixture site)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  before(async () => {
+    server = await serveDir(join(FIXTURES_DIR, "site"));
+  });
+  after(async () => {
+    await server.close();
+  });
+
+  it("crawls + clones multiple routes into one app (clone_site)", async () => {
+    const res = await runCloneJob({ url: server.url + "/", options: { mode: "multi", interactions: false, components: false } });
+    assert.equal(res.kind, "clone_site");
+    assert.ok(res.routes && res.routes.length >= 2, `expected >=2 routes, got ${res.routes?.length}`);
+    assert.ok(res.files["src/app/layout.tsx"], "shared layout emitted once");
+    assert.ok(res.files["src/app/page.tsx"], "home route page");
+    assert.ok(res.files["AGENTS.md"], "generated AGENTS.md emitted");
+    assert.ok(res.files["ARCHITECTURE.md"], "generated ARCHITECTURE.md emitted");
+    assert.ok(res.files["src/app/robots.ts"], "robots route emitted");
+    assert.ok(res.files["src/app/sitemap.ts"], "sitemap route emitted");
+    assert.ok(res.files["src/app/llms.txt/route.ts"], "llms route emitted");
+    assert.ok((res.files["src/app/llms.txt/route.ts"]!.content ?? "").includes("Build faster with Acme"), "generated llms includes captured route content");
+    const subRoutePages = Object.keys(res.files).filter((p) => /^src\/app\/.+\/page\.tsx$/.test(p));
+    assert.ok(subRoutePages.length >= 1, "at least one sub-route page");
+    assert.ok(res.capture.nodeCount > 0);
+  });
+});

packages/core/test/runCloneJob.fixture.test.ts

@@ -0,0 +1,170 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runClone } from "clone-static";
+import { runCloneJob } from "../src/runCloneJob.js";
+import { collectFileMap } from "../src/collectFileMap.js";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+
+const CHROMIUM = hasChromium();
+
+// End-to-end clone of a served fixture (zero external network). Skipped when no
+// Playwright Chromium is installed (run `npx playwright install chromium` first).
+describe("runCloneJob (served fixture)", { skip: CHROMIUM ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  before(async () => {
+    server = await serveDir(FIXTURES_DIR);
+  });
+  after(async () => {
+    await server.close();
+  });
+
+  it("produces the file-map contract for a single-page clone", async () => {
+    const url = server.url + "/components.html";
+    const res = await runCloneJob({
+      url,
+      options: { interactions: false, components: true, motion: false },
+    });
+
+    assert.equal(res.status, "succeeded");
+    assert.equal(res.kind, "clone");
+    assert.ok(res.compilerVersion);
+
+    // The essential scaffold is present and typed correctly.
+    for (const k of [
+      "package.json",
+      "tsconfig.json",
+      "next.config.mjs",
+      "src/app/layout.tsx",
+      "src/app/page.tsx",
+      "src/app/ditto.css",
+      "src/app/globals.css",
+    ]) {
+      assert.ok(res.files[k], `expected file ${k}`);
+      assert.equal(res.files[k]!.kind, "text");
+    }
+    assert.ok(res.files["src/app/page.tsx"]!.content!.includes("export default function Page"));
+
+    // Extracted components are split into their own files (compiler change); the
+    // service collects them as text automatically.
+    const componentFiles = Object.keys(res.files).filter((k) => k.startsWith("src/app/components/"));
+    assert.ok(componentFiles.length > 0, "extracted components emitted as separate files");
+    for (const k of componentFiles) {
+      assert.equal(res.files[k]!.kind, "text");
+      assert.ok((res.files[k]!.content ?? "").length > 0, `${k} has content`);
+    }
+
+    // Capture sanity: a real fixture is not degenerate / bot-walled.
+    assert.ok(res.capture.nodeCount > 0, "nodeCount > 0");
+    assert.equal(res.capture.blocked, false);
+
+    // temp dir is cleaned up by default.
+    const { existsSync } = await import("node:fs");
+    assert.equal(existsSync(res.runDir), false, "temp run dir removed");
+  });
+
+  it("can generate a Vite React app instead of Next", async () => {
+    const url = server.url + "/components.html";
+    const res = await runCloneJob({
+      url,
+      options: { framework: "vite", interactions: false, components: false, motion: false },
+    });
+
+    assert.equal(res.status, "succeeded");
+    assert.equal(res.options.framework, "vite");
+    for (const k of [
+      "index.html",
+      "vite.config.ts",
+      "src/main.tsx",
+      "src/page.tsx",
+      "src/ditto.css",
+      "src/globals.css",
+      "public/robots.txt",
+      "public/sitemap.xml",
+      "public/llms.txt",
+    ]) {
+      assert.ok(res.files[k], `expected file ${k}`);
+      assert.equal(res.files[k]!.kind, "text");
+    }
+    assert.ok(!res.files["next.config.mjs"], "Vite output should not include next.config.mjs");
+    assert.ok(!res.files["src/app/layout.tsx"], "Vite output should not include App Router layout");
+    assert.ok(res.files["package.json"]!.content!.includes('"dev": "vite"'));
+    assert.ok(res.files["src/main.tsx"]!.content!.includes('from "react-dom/client"'));
+    assert.ok((res.files["src/globals.css"]!.content ?? "").includes("#root { display: contents; }"));
+  });
+
+  it("names sections with valid JS identifiers even when a heading starts with a number", async () => {
+    // Repro: a section whose heading reads "0019 Iterate Faster" previously became the
+    // identifier `0019IterateSection` → `import 0019IterateSection` is a syntax error.
+    const url = server.url + "/numeric-section.html";
+    const res = await runCloneJob({ url, options: { interactions: false, components: false, motion: false } });
+    assert.equal(res.status, "succeeded");
+
+    const page = res.files["src/app/page.tsx"]!.content ?? "";
+    const imports = [...page.matchAll(/^import\s+([A-Za-z0-9_$]+)\s+from/gm)].map((m) => m[1]!);
+    assert.ok(imports.length > 0, "page imports section modules");
+    for (const id of imports) {
+      assert.match(id, /^[A-Za-z_$]/, `import identifier "${id}" must not start with a digit`);
+    }
+    // The numeric "0019" layer noise is dropped → a clean, valid name.
+    assert.ok(imports.includes("IterateFasterSection"), `expected IterateFasterSection, got ${imports.join(", ")}`);
+  });
+
+  it("preserves generated SEO metadata, icons, JSON-LD, llms, and docs", async () => {
+    const url = server.url + "/seo-rich.html";
+    const res = await runCloneJob({
+      url,
+      options: { interactions: false, components: false, motion: false },
+    });
+    assert.equal(res.status, "succeeded");
+
+    const layout = res.files["src/app/layout.tsx"]!.content ?? "";
+    assert.ok(layout.includes("SEO Rich Fixture"));
+    assert.ok(layout.includes("Open Graph description from the source page."));
+    assert.ok(layout.includes("summary_large_image"));
+    assert.ok(layout.includes("themeColor"));
+    assert.ok(layout.includes("colorScheme"));
+    assert.ok(layout.includes("application/ld+json"));
+
+    assert.equal(res.files["src/app/favicon.ico"]?.kind, "binary");
+    assert.equal(res.files["src/app/icon.png"]?.kind, "binary");
+    assert.equal(res.files["src/app/apple-icon.png"]?.kind, "binary");
+    assert.ok(Object.keys(res.files).some((k) => k.startsWith("public/assets/cloned/manifest/")), "web manifest materialized");
+    assert.ok(Object.keys(res.files).some((k) => k.startsWith("public/assets/cloned/images/")), "manifest/icon image assets materialized");
+
+    const llms = res.files["src/app/llms.txt/route.ts"]!.content ?? "";
+    assert.ok(llms.includes("Source LLMS"), "source llms.txt preserved");
+    const llmsFull = res.files["src/app/llms-full.txt/route.ts"]!.content ?? "";
+    assert.ok(llmsFull.includes("Source LLMS Full"), "source llms-full.txt preserved");
+
+    assert.ok(res.files["AGENTS.md"]!.content!.includes("generated ditto.site clone app"));
+    assert.ok(res.files["ARCHITECTURE.md"]!.content!.includes("data-ditto-id"));
+  });
+
+  it("generates byte-identical output from one frozen capture (golden / Gate 6)", async () => {
+    const url = server.url + "/components.html";
+    const a = mkdtempSync(join(tmpdir(), "golden-a-"));
+    const b = mkdtempSync(join(tmpdir(), "golden-b-"));
+    try {
+      const opts = { interactions: false, components: true, motion: false } as const;
+      const r1 = await runClone({ url, runsDir: a, ...opts });
+      // Regenerate from the SAME capture (no re-capture) → must be byte-identical.
+      const r2 = await runClone({ url, runsDir: b, reuseSource: r1.sourceDir, ...opts });
+
+      const m1 = collectFileMap(r1.runDir);
+      const m2 = collectFileMap(r2.runDir);
+      // Same file set + byte-identical contents across regenerations — covers the
+      // scaffold AND the split components/*.tsx, all from one frozen capture.
+      assert.deepEqual(Object.keys(m1).sort(), Object.keys(m2).sort(), "identical file set across regenerations");
+      for (const f of Object.keys(m1)) {
+        assert.equal(m1[f]!.sha256, m2[f]!.sha256, `${f} is byte-identical across regenerations`);
+      }
+      assert.ok(Object.keys(m1).some((k) => k.startsWith("src/app/components/")), "components are split into files");
+    } finally {
+      rmSync(a, { recursive: true, force: true });
+      rmSync(b, { recursive: true, force: true });
+    }
+  });
+});

packages/core/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"]
+}

packages/db/drizzle.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "drizzle-kit";
+
+export default defineConfig({
+  schema: "./src/schema.ts",
+  out: "./migrations",
+  dialect: "postgresql",
+  dbCredentials: {
+    url: process.env.DATABASE_URL ?? "postgresql://postgres@127.0.0.1:5432/postgres",
+  },
+});

packages/db/migrations/0000_plain_wither.sql

@@ -0,0 +1,49 @@
+CREATE TABLE "api_keys" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"key_hash" text NOT NULL,
+	"label" text,
+	"rate_limit" integer,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"revoked_at" timestamp with time zone,
+	CONSTRAINT "api_keys_key_hash_unique" UNIQUE("key_hash")
+);
+--> statement-breakpoint
+CREATE TABLE "cache" (
+	"cache_key" text PRIMARY KEY NOT NULL,
+	"job_id" uuid,
+	"url" text NOT NULL,
+	"options_hash" text NOT NULL,
+	"compiler_version" text NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"expires_at" timestamp with time zone NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "clones" (
+	"job_id" uuid PRIMARY KEY NOT NULL,
+	"url" text NOT NULL,
+	"route_count" integer DEFAULT 1 NOT NULL,
+	"file_manifest" jsonb NOT NULL,
+	"bundle_s3_key" text,
+	"verify" jsonb,
+	"capture_meta" jsonb NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "jobs" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"kind" text NOT NULL,
+	"url" text NOT NULL,
+	"options" jsonb DEFAULT '{}'::jsonb NOT NULL,
+	"status" text DEFAULT 'queued' NOT NULL,
+	"cache_key" text NOT NULL,
+	"attempts" integer DEFAULT 0 NOT NULL,
+	"error" text,
+	"compiler_version" text,
+	"timings" jsonb,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"started_at" timestamp with time zone,
+	"finished_at" timestamp with time zone
+);
+--> statement-breakpoint
+ALTER TABLE "cache" ADD CONSTRAINT "cache_job_id_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."jobs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "clones" ADD CONSTRAINT "clones_job_id_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."jobs"("id") ON DELETE cascade ON UPDATE no action;

packages/db/migrations/meta/0000_snapshot.json

@@ -0,0 +1,321 @@
+{
+  "id": "9000d72e-0529-4eaf-b53d-43d3b93b5620",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.api_keys": {
+      "name": "api_keys",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "key_hash": {
+          "name": "key_hash",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "label": {
+          "name": "label",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "rate_limit": {
+          "name": "rate_limit",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "revoked_at": {
+          "name": "revoked_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "api_keys_key_hash_unique": {
+          "name": "api_keys_key_hash_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "key_hash"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.cache": {
+      "name": "cache",
+      "schema": "",
+      "columns": {
+        "cache_key": {
+          "name": "cache_key",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "job_id": {
+          "name": "job_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "url": {
+          "name": "url",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "options_hash": {
+          "name": "options_hash",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "compiler_version": {
+          "name": "compiler_version",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "expires_at": {
+          "name": "expires_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "cache_job_id_jobs_id_fk": {
+          "name": "cache_job_id_jobs_id_fk",
+          "tableFrom": "cache",
+          "tableTo": "jobs",
+          "columnsFrom": [
+            "job_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.clones": {
+      "name": "clones",
+      "schema": "",
+      "columns": {
+        "job_id": {
+          "name": "job_id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "url": {
+          "name": "url",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "route_count": {
+          "name": "route_count",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "default": 1
+        },
+        "file_manifest": {
+          "name": "file_manifest",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "bundle_s3_key": {
+          "name": "bundle_s3_key",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "verify": {
+          "name": "verify",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "capture_meta": {
+          "name": "capture_meta",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "clones_job_id_jobs_id_fk": {
+          "name": "clones_job_id_jobs_id_fk",
+          "tableFrom": "clones",
+          "tableTo": "jobs",
+          "columnsFrom": [
+            "job_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.jobs": {
+      "name": "jobs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "kind": {
+          "name": "kind",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "url": {
+          "name": "url",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "options": {
+          "name": "options",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'{}'::jsonb"
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'queued'"
+        },
+        "cache_key": {
+          "name": "cache_key",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attempts": {
+          "name": "attempts",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "default": 0
+        },
+        "error": {
+          "name": "error",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "compiler_version": {
+          "name": "compiler_version",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "timings": {
+          "name": "timings",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "started_at": {
+          "name": "started_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "finished_at": {
+          "name": "finished_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {},
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

packages/db/migrations/meta/_journal.json

@@ -0,0 +1,13 @@
+{
+  "version": "7",
+  "dialect": "postgresql",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "7",
+      "when": 1782170309013,
+      "tag": "0000_plain_wither",
+      "breakpoints": true
+    }
+  ]
+}

packages/db/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@cloner/db",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "generate": "drizzle-kit generate",
+    "migrate": "tsx src/migrate.ts",
+    "test": "node --import tsx --test test/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "drizzle-orm": "^0.38.3",
+    "pg": "^8.13.1",
+    "pg-boss": "^10.1.5"
+  },
+  "devDependencies": {
+    "@cloner/test-utils": "*",
+    "@types/pg": "^8.11.10",
+    "drizzle-kit": "^0.30.1",
+    "tsx": "4.22.4",
+    "typescript": "5.7.3",
+    "@types/node": "22.10.5"
+  }
+}

packages/db/src/client.ts

@@ -0,0 +1,14 @@
+import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres";
+import pg from "pg";
+import * as schema from "./schema.js";
+
+export type Db = NodePgDatabase<typeof schema>;
+export type DbHandle = { db: Db; pool: pg.Pool };
+
+/** Create a Drizzle client + underlying pg Pool from a connection URL. Caller owns
+ *  the pool lifecycle (`handle.pool.end()` on shutdown). */
+export function createDb(connectionString: string): DbHandle {
+  const pool = new pg.Pool({ connectionString });
+  const db = drizzle(pool, { schema });
+  return { db, pool };
+}

packages/db/src/index.ts

@@ -0,0 +1,10 @@
+export * as schema from "./schema.js";
+export {
+  jobs, clones, cache, apiKeys,
+  type Job, type NewJob, type Clone, type NewClone, type CacheRow, type ApiKey,
+} from "./schema.js";
+export { createDb, type Db, type DbHandle } from "./client.js";
+export * as repo from "./repo.js";
+export { createBoss, enqueueClone, workClone, CLONE_QUEUE, type ClonePayload } from "./queue.js";
+export type { default as PgBoss } from "pg-boss";
+export { runMigrations, MIGRATIONS_DIR } from "./migrate.js";

packages/db/src/migrate.ts

@@ -0,0 +1,34 @@
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { migrate } from "drizzle-orm/node-postgres/migrator";
+import { createDb } from "./client.js";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+export const MIGRATIONS_DIR = join(HERE, "..", "migrations");
+
+/** Apply all pending Drizzle migrations to the target database. */
+export async function runMigrations(connectionString: string, migrationsFolder = MIGRATIONS_DIR): Promise<void> {
+  const { db, pool } = createDb(connectionString);
+  try {
+    await migrate(db, { migrationsFolder });
+  } finally {
+    await pool.end();
+  }
+}
+
+async function main(): Promise<void> {
+  const url = process.env.DATABASE_URL;
+  if (!url) {
+    console.error("DATABASE_URL is required");
+    process.exit(1);
+  }
+  await runMigrations(url);
+  console.log(JSON.stringify({ event: "migrations_applied" }));
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });
+}

packages/db/src/queue.ts

@@ -0,0 +1,43 @@
+import PgBoss from "pg-boss";
+
+export const CLONE_QUEUE = "clone-jobs";
+export type ClonePayload = { jobId: string };
+
+/** Start a pg-boss instance on the same Postgres (no Redis). pg-boss manages its
+ *  own schema/tables and gives us retries, heartbeats, and visibility timeouts. */
+export async function createBoss(connectionString: string): Promise<PgBoss> {
+  const boss = new PgBoss({ connectionString });
+  boss.on("error", (e) => console.error(JSON.stringify({ event: "pgboss_error", error: String(e).slice(0, 300) })));
+  await boss.start();
+  // pg-boss v10 requires queues to be created before send/work (idempotent).
+  const anyBoss = boss as unknown as { createQueue?: (name: string) => Promise<void> };
+  if (typeof anyBoss.createQueue === "function") {
+    try {
+      await anyBoss.createQueue(CLONE_QUEUE);
+    } catch {
+      /* already exists */
+    }
+  }
+  return boss;
+}
+
+/** Enqueue a clone job. Returns the pg-boss job id (or null if deduped). */
+export async function enqueueClone(boss: PgBoss, jobId: string): Promise<string | null> {
+  return boss.send(CLONE_QUEUE, { jobId } satisfies ClonePayload, {
+    retryLimit: 2,
+    retryBackoff: true,
+    expireInSeconds: 30 * 60,
+  });
+}
+
+/** Register the worker handler. Normalizes pg-boss v9 (single job) vs v10 (array)
+ *  callback shapes so the consumer just gets a jobId. */
+export async function workClone(boss: PgBoss, handler: (jobId: string) => Promise<void>): Promise<string> {
+  return boss.work(CLONE_QUEUE, async (job: unknown) => {
+    const arr = Array.isArray(job) ? job : [job];
+    for (const j of arr) {
+      const data = (j as { data?: ClonePayload }).data;
+      if (data?.jobId) await handler(data.jobId);
+    }
+  });
+}

packages/db/src/repo.ts

@@ -0,0 +1,95 @@
+import { and, desc, eq, gt, sql } from "drizzle-orm";
+import type { Db } from "./client.js";
+import { jobs, clones, cache, apiKeys, type Job, type NewJob, type Clone, type NewClone, type CacheRow, type ApiKey } from "./schema.js";
+
+// ---- jobs ----
+
+export async function createJob(db: Db, input: Omit<NewJob, "id" | "createdAt">): Promise<Job> {
+  const [row] = await db.insert(jobs).values(input).returning();
+  return row!;
+}
+
+export async function getJob(db: Db, id: string): Promise<Job | undefined> {
+  const [row] = await db.select().from(jobs).where(eq(jobs.id, id)).limit(1);
+  return row;
+}
+
+export async function markRunning(db: Db, id: string): Promise<void> {
+  await db
+    .update(jobs)
+    .set({ status: "running", startedAt: new Date(), attempts: sql`${jobs.attempts} + 1` })
+    .where(eq(jobs.id, id));
+}
+
+export async function markSucceeded(db: Db, id: string, fields: { compilerVersion: string; timings: unknown }): Promise<void> {
+  await db
+    .update(jobs)
+    .set({ status: "succeeded", finishedAt: new Date(), compilerVersion: fields.compilerVersion, timings: fields.timings })
+    .where(eq(jobs.id, id));
+}
+
+export async function updateJobTimings(db: Db, id: string, timings: unknown): Promise<void> {
+  await db.update(jobs).set({ timings }).where(eq(jobs.id, id));
+}
+
+export async function markFailed(db: Db, id: string, error: string): Promise<void> {
+  await db.update(jobs).set({ status: "failed", finishedAt: new Date(), error: error.slice(0, 2000) }).where(eq(jobs.id, id));
+}
+
+export async function listJobs(db: Db, limit = 50): Promise<Job[]> {
+  return db.select().from(jobs).orderBy(desc(jobs.createdAt)).limit(limit);
+}
+
+export async function deleteJob(db: Db, id: string): Promise<boolean> {
+  const rows = await db.delete(jobs).where(eq(jobs.id, id)).returning({ id: jobs.id });
+  return rows.length > 0;
+}
+
+// ---- clones ----
+
+export async function upsertClone(db: Db, input: NewClone): Promise<void> {
+  await db
+    .insert(clones)
+    .values(input)
+    .onConflictDoUpdate({ target: clones.jobId, set: { fileManifest: input.fileManifest, verify: input.verify, captureMeta: input.captureMeta, bundleS3Key: input.bundleS3Key, routeCount: input.routeCount } });
+}
+
+export async function getClone(db: Db, jobId: string): Promise<Clone | undefined> {
+  const [row] = await db.select().from(clones).where(eq(clones.jobId, jobId)).limit(1);
+  return row;
+}
+
+export async function updateCloneVerify(db: Db, jobId: string, verify: unknown): Promise<void> {
+  await db.update(clones).set({ verify: verify as Record<string, unknown> | null }).where(eq(clones.jobId, jobId));
+}
+
+// ---- cache ----
+
+/** A *fresh* cache hit: same compilerVersion and not yet expired. */
+export async function cacheGetFresh(db: Db, cacheKey: string, compilerVersion: string): Promise<CacheRow | undefined> {
+  const [row] = await db
+    .select()
+    .from(cache)
+    .where(and(eq(cache.cacheKey, cacheKey), eq(cache.compilerVersion, compilerVersion), gt(cache.expiresAt, new Date())))
+    .limit(1);
+  return row;
+}
+
+export async function cachePut(db: Db, input: { cacheKey: string; jobId: string; url: string; optionsHash: string; compilerVersion: string; expiresAt: Date }): Promise<void> {
+  await db
+    .insert(cache)
+    .values(input)
+    .onConflictDoUpdate({ target: cache.cacheKey, set: { jobId: input.jobId, expiresAt: input.expiresAt, compilerVersion: input.compilerVersion, createdAt: new Date() } });
+}
+
+// ---- api keys (M6) ----
+
+export async function getApiKeyByHash(db: Db, keyHash: string): Promise<ApiKey | undefined> {
+  const [row] = await db.select().from(apiKeys).where(eq(apiKeys.keyHash, keyHash)).limit(1);
+  return row;
+}
+
+export async function createApiKey(db: Db, input: { keyHash: string; label?: string; rateLimit?: number }): Promise<ApiKey> {
+  const [row] = await db.insert(apiKeys).values(input).returning();
+  return row!;
+}

packages/db/src/schema.ts

@@ -0,0 +1,64 @@
+import { pgTable, text, jsonb, integer, timestamp, uuid } from "drizzle-orm/pg-core";
+
+/** Job lifecycle row. Status mirrors the queue; `cacheKey` ties it to the cache row.
+ *  `options`/`timings` are jsonb (the service's typed shapes). */
+export const jobs = pgTable("jobs", {
+  id: uuid("id").defaultRandom().primaryKey(),
+  kind: text("kind").notNull(), // "clone" | "clone_site"
+  url: text("url").notNull(),
+  options: jsonb("options").notNull().default({}),
+  status: text("status").notNull().default("queued"), // queued|running|succeeded|failed|cached
+  cacheKey: text("cache_key").notNull(),
+  attempts: integer("attempts").notNull().default(0),
+  error: text("error"),
+  compilerVersion: text("compiler_version"),
+  timings: jsonb("timings"),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  startedAt: timestamp("started_at", { withTimezone: true }),
+  finishedAt: timestamp("finished_at", { withTimezone: true }),
+});
+
+/** The persisted result, written once on success. `fileManifest` holds the text
+ *  files inline + binary metadata (the eager file map sans bytes); binaries live in
+ *  blob storage referenced by `bundleS3Key` / per-file keys in the manifest. */
+export const clones = pgTable("clones", {
+  jobId: uuid("job_id")
+    .primaryKey()
+    .references(() => jobs.id, { onDelete: "cascade" }),
+  url: text("url").notNull(),
+  routeCount: integer("route_count").notNull().default(1),
+  fileManifest: jsonb("file_manifest").notNull(),
+  bundleS3Key: text("bundle_s3_key"),
+  verify: jsonb("verify"),
+  captureMeta: jsonb("capture_meta").notNull(),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+});
+
+/** Freshness-bounded "recently cloned this URL+options" cache. A hit requires
+ *  `now < expiresAt` and a matching compilerVersion. */
+export const cache = pgTable("cache", {
+  cacheKey: text("cache_key").primaryKey(),
+  jobId: uuid("job_id").references(() => jobs.id, { onDelete: "cascade" }),
+  url: text("url").notNull(),
+  optionsHash: text("options_hash").notNull(),
+  compilerVersion: text("compiler_version").notNull(),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
+});
+
+/** API keys (M6): only a hash is stored. `rateLimit` is requests/min (nullable = default). */
+export const apiKeys = pgTable("api_keys", {
+  id: uuid("id").defaultRandom().primaryKey(),
+  keyHash: text("key_hash").notNull().unique(),
+  label: text("label"),
+  rateLimit: integer("rate_limit"),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  revokedAt: timestamp("revoked_at", { withTimezone: true }),
+});
+
+export type Job = typeof jobs.$inferSelect;
+export type NewJob = typeof jobs.$inferInsert;
+export type Clone = typeof clones.$inferSelect;
+export type NewClone = typeof clones.$inferInsert;
+export type CacheRow = typeof cache.$inferSelect;
+export type ApiKey = typeof apiKeys.$inferSelect;

packages/db/test/cache.test.ts

@@ -0,0 +1,35 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { createDb, runMigrations, repo, type Db } from "../src/index.js";
+import { acquireTestPostgres, hasTestPostgres, type EphemeralPg } from "@cloner/test-utils";
+
+describe("cache freshness (TTL + compilerVersion)", { skip: hasTestPostgres() ? false : "no test Postgres" }, () => {
+  let pg: EphemeralPg;
+  let db: Db;
+  let pool: { end: () => Promise<void> };
+
+  before(async () => {
+    pg = (await acquireTestPostgres())!;
+    await runMigrations(pg.url);
+    const h = createDb(pg.url);
+    db = h.db;
+    pool = h.pool;
+  });
+  after(async () => {
+    await pool?.end().catch(() => {});
+    await pg?.stop().catch(() => {});
+  });
+
+  it("returns a hit only when not expired AND compilerVersion matches", async () => {
+    const job = await repo.createJob(db, { kind: "clone", url: "https://x/", options: {}, status: "succeeded", cacheKey: "k1", compilerVersion: "0.1.0" });
+
+    // Fresh row.
+    await repo.cachePut(db, { cacheKey: "k1", jobId: job.id, url: "https://x/", optionsHash: "{}", compilerVersion: "0.1.0", expiresAt: new Date(Date.now() + 60_000) });
+    assert.ok(await repo.cacheGetFresh(db, "k1", "0.1.0"), "fresh + matching version → hit");
+    assert.equal(await repo.cacheGetFresh(db, "k1", "0.2.0"), undefined, "version bump → miss");
+
+    // Expire it (upsert past expiry).
+    await repo.cachePut(db, { cacheKey: "k1", jobId: job.id, url: "https://x/", optionsHash: "{}", compilerVersion: "0.1.0", expiresAt: new Date(Date.now() - 1000) });
+    assert.equal(await repo.cacheGetFresh(db, "k1", "0.1.0"), undefined, "stale → miss");
+  });
+});

packages/db/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"]
+}

packages/storage/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@cloner/storage",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "node --import tsx --test test/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@cloner/core": "*",
+    "@aws-sdk/client-s3": "^3.726.0",
+    "@aws-sdk/s3-request-presigner": "^3.726.0"
+  },
+  "devDependencies": {
+    "tsx": "4.22.4",
+    "typescript": "5.7.3",
+    "@types/node": "22.10.5"
+  }
+}

packages/storage/src/blob.ts

@@ -0,0 +1,37 @@
+/** A minimal object-store client. The ArtifactStore is built on this so its logic
+ *  is testable with an in-memory client and runs on S3/R2 in production. */
+export interface BlobClient {
+  put(key: string, bytes: Buffer, contentType: string): Promise<void>;
+  get(key: string): Promise<Buffer | null>;
+  /** A URL a client can use to fetch the object directly (presigned for S3). */
+  presign(key: string, opts?: { expiresSeconds?: number; downloadName?: string; contentType?: string }): Promise<string>;
+  delete(key: string): Promise<void>;
+  /** Delete everything under a prefix (best effort). */
+  deletePrefix(prefix: string): Promise<void>;
+}
+
+/** In-memory BlobClient for tests. presign() returns a fake URL (no real fetch). */
+export class InMemoryBlobClient implements BlobClient {
+  private objs = new Map<string, { bytes: Buffer; contentType: string }>();
+  constructor(private publicBase = "memory://blobs") {}
+
+  async put(key: string, bytes: Buffer, contentType: string): Promise<void> {
+    this.objs.set(key, { bytes, contentType });
+  }
+  async get(key: string): Promise<Buffer | null> {
+    return this.objs.get(key)?.bytes ?? null;
+  }
+  async presign(key: string): Promise<string> {
+    return `${this.publicBase}/${key}?sig=test`;
+  }
+  async delete(key: string): Promise<void> {
+    this.objs.delete(key);
+  }
+  async deletePrefix(prefix: string): Promise<void> {
+    for (const k of [...this.objs.keys()]) if (k.startsWith(prefix)) this.objs.delete(k);
+  }
+  /** test helper */
+  has(key: string): boolean {
+    return this.objs.has(key);
+  }
+}

packages/storage/src/bundle.ts

@@ -0,0 +1,135 @@
+import { gzipSync, deflateRawSync } from "node:zlib";
+import { createHash } from "node:crypto";
+
+/** Build a single ustar tar header block (512 bytes) for a regular file. mtime is
+ *  fixed at 0 so the archive is deterministic for the same inputs. */
+function tarHeader(name: string, size: number): Buffer {
+  let prefix = "";
+  let nm = name;
+  if (Buffer.byteLength(name) > 100) {
+    let split = -1;
+    for (let p = 0; p < name.length; p++) {
+      if (name[p] === "/" && Buffer.byteLength(name.slice(p + 1)) <= 100 && Buffer.byteLength(name.slice(0, p)) <= 155) {
+        split = p;
+        break;
+      }
+    }
+    if (split < 0) throw new Error("path too long for tar: " + name);
+    prefix = name.slice(0, split);
+    nm = name.slice(split + 1);
+  }
+  const h = Buffer.alloc(512);
+  h.write(nm, 0, 100, "utf8");
+  h.write("0000644\0", 100, 8, "ascii"); // mode
+  h.write("0000000\0", 108, 8, "ascii"); // uid
+  h.write("0000000\0", 116, 8, "ascii"); // gid
+  h.write(size.toString(8).padStart(11, "0") + "\0", 124, 12, "ascii"); // size (octal)
+  h.write("00000000000\0", 136, 12, "ascii"); // mtime = 0
+  h.write("        ", 148, 8, "ascii"); // checksum placeholder (8 spaces)
+  h.write("0", 156, 1, "ascii"); // typeflag: regular file
+  h.write("ustar\0", 257, 6, "ascii");
+  h.write("00", 263, 2, "ascii");
+  if (prefix) h.write(prefix, 345, 155, "utf8");
+  let sum = 0;
+  for (let i = 0; i < 512; i++) sum += h[i]!;
+  h.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, 8, "ascii");
+  return h;
+}
+
+/** Pack files into a deterministic .tar.gz (sorted by path, zero mtime). */
+export function makeTarGz(files: Array<{ path: string; bytes: Buffer }>): Buffer {
+  const sorted = [...files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const parts: Buffer[] = [];
+  for (const f of sorted) {
+    parts.push(tarHeader(f.path, f.bytes.length));
+    parts.push(f.bytes);
+    const pad = (512 - (f.bytes.length % 512)) % 512;
+    if (pad) parts.push(Buffer.alloc(pad));
+  }
+  parts.push(Buffer.alloc(1024)); // two zero blocks = end of archive
+  return gzipSync(Buffer.concat(parts), { level: 9 });
+}
+
+export function sha256hex(buf: Buffer): string {
+  return createHash("sha256").update(buf).digest("hex");
+}
+
+// ---- ZIP (deflate) ----
+
+const CRC_TABLE = (() => {
+  const t = new Uint32Array(256);
+  for (let n = 0; n < 256; n++) {
+    let c = n;
+    for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+    t[n] = c >>> 0;
+  }
+  return t;
+})();
+
+function crc32(buf: Buffer): number {
+  let c = 0xffffffff;
+  for (let i = 0; i < buf.length; i++) c = CRC_TABLE[(c ^ buf[i]!) & 0xff]! ^ (c >>> 8);
+  return (c ^ 0xffffffff) >>> 0;
+}
+
+/** Pack files into a deterministic .zip (deflate; fixed DOS timestamp). */
+export function makeZip(files: Array<{ path: string; bytes: Buffer }>): Buffer {
+  const sorted = [...files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const DOS_TIME = 0; // fixed → deterministic
+  const DOS_DATE = 0x21; // 1980-01-01
+  const local: Buffer[] = [];
+  const central: Buffer[] = [];
+  let offset = 0;
+  for (const f of sorted) {
+    const name = Buffer.from(f.path, "utf8");
+    const crc = crc32(f.bytes);
+    const comp = deflateRawSync(f.bytes, { level: 9 });
+    const lh = Buffer.alloc(30);
+    lh.writeUInt32LE(0x04034b50, 0);
+    lh.writeUInt16LE(20, 4); // version needed
+    lh.writeUInt16LE(0, 6); // flags
+    lh.writeUInt16LE(8, 8); // method: deflate
+    lh.writeUInt16LE(DOS_TIME, 10);
+    lh.writeUInt16LE(DOS_DATE, 12);
+    lh.writeUInt32LE(crc, 14);
+    lh.writeUInt32LE(comp.length, 18);
+    lh.writeUInt32LE(f.bytes.length, 22);
+    lh.writeUInt16LE(name.length, 26);
+    lh.writeUInt16LE(0, 28);
+    local.push(lh, name, comp);
+
+    const ch = Buffer.alloc(46);
+    ch.writeUInt32LE(0x02014b50, 0);
+    ch.writeUInt16LE(20, 4); // version made by
+    ch.writeUInt16LE(20, 6); // version needed
+    ch.writeUInt16LE(0, 8); // flags
+    ch.writeUInt16LE(8, 10); // method
+    ch.writeUInt16LE(DOS_TIME, 12);
+    ch.writeUInt16LE(DOS_DATE, 14);
+    ch.writeUInt32LE(crc, 16);
+    ch.writeUInt32LE(comp.length, 20);
+    ch.writeUInt32LE(f.bytes.length, 24);
+    ch.writeUInt16LE(name.length, 28);
+    ch.writeUInt16LE(0, 30); // extra len
+    ch.writeUInt16LE(0, 32); // comment len
+    ch.writeUInt16LE(0, 34); // disk number
+    ch.writeUInt16LE(0, 36); // internal attrs
+    ch.writeUInt32LE(0, 38); // external attrs
+    ch.writeUInt32LE(offset, 42);
+    central.push(ch, name);
+
+    offset += lh.length + name.length + comp.length;
+  }
+  const centralBuf = Buffer.concat(central);
+  const localBuf = Buffer.concat(local);
+  const eocd = Buffer.alloc(22);
+  eocd.writeUInt32LE(0x06054b50, 0);
+  eocd.writeUInt16LE(0, 4);
+  eocd.writeUInt16LE(0, 6);
+  eocd.writeUInt16LE(sorted.length, 8);
+  eocd.writeUInt16LE(sorted.length, 10);
+  eocd.writeUInt32LE(centralBuf.length, 12);
+  eocd.writeUInt32LE(localBuf.length, 16);
+  eocd.writeUInt16LE(0, 20);
+  return Buffer.concat([localBuf, centralBuf, eocd]);
+}

packages/storage/src/factory.ts

@@ -0,0 +1,14 @@
+import { join } from "node:path";
+import type { ArtifactStore } from "./types.js";
+import { LocalArtifactStore } from "./local.js";
+import { ObjectArtifactStore } from "./objectStore.js";
+import { S3BlobClient, s3ConfigFromEnv } from "./s3.js";
+
+/** Pick the artifact store from the environment: S3/R2 when S3_BUCKET is set,
+ *  otherwise a local disk store (ARTIFACTS_DIR or ./local-data/artifacts). */
+export function artifactStoreFromEnv(): ArtifactStore {
+  const s3 = s3ConfigFromEnv();
+  if (s3) return new ObjectArtifactStore(new S3BlobClient(s3));
+  const dir = process.env.ARTIFACTS_DIR ?? join(process.cwd(), "local-data", "artifacts");
+  return new LocalArtifactStore(dir);
+}

packages/storage/src/index.ts

@@ -0,0 +1,7 @@
+export type { ArtifactStore, StoredFile, StoredManifest } from "./types.js";
+export { LocalArtifactStore } from "./local.js";
+export { ObjectArtifactStore } from "./objectStore.js";
+export { type BlobClient, InMemoryBlobClient } from "./blob.js";
+export { S3BlobClient, s3ConfigFromEnv, type S3Config } from "./s3.js";
+export { artifactStoreFromEnv } from "./factory.js";
+export { makeTarGz, makeZip, sha256hex } from "./bundle.js";

packages/storage/src/local.ts

@@ -0,0 +1,60 @@
+import { mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs";
+import { join, dirname, normalize } from "node:path";
+import type { FileMap } from "@cloner/core";
+import type { ArtifactStore, StoredFile, StoredManifest } from "./types.js";
+
+/** Reject path traversal — only allow relative paths that stay within the job dir. */
+function safeRel(path: string): string {
+  const n = normalize(path);
+  if (n.includes("\0") || n.startsWith("/") || /^[A-Za-z]:/.test(n) || n.split(/[/\\]/).includes("..")) {
+    throw new Error("unsafe path: " + path);
+  }
+  return n;
+}
+
+/**
+ * Disk-backed artifact store (M2, local dev). Writes every file under
+ * `<baseDir>/<jobId>/`, returns a manifest with text inline + binary keys, and
+ * serves bytes back for the API's /files/* route. In production this is swapped for
+ * the S3 store (M4); the API/worker depend only on the ArtifactStore interface.
+ */
+export class LocalArtifactStore implements ArtifactStore {
+  constructor(private baseDir: string) {}
+
+  private jobDir(jobId: string): string {
+    return join(this.baseDir, jobId);
+  }
+
+  async putClone(jobId: string, files: FileMap): Promise<StoredManifest> {
+    const root = this.jobDir(jobId);
+    const out: StoredFile[] = [];
+    for (const [path, f] of Object.entries(files)) {
+      const rel = safeRel(path);
+      const dest = join(root, rel);
+      mkdirSync(dirname(dest), { recursive: true });
+      writeFileSync(dest, readFileSync(f.absPath));
+      if (f.kind === "text") {
+        out.push({ path, kind: "text", bytes: f.bytes, sha256: f.sha256, content: f.content ?? "" });
+      } else {
+        out.push({ path, kind: "binary", bytes: f.bytes, sha256: f.sha256, key: `${jobId}/${rel}` });
+      }
+    }
+    out.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+    return { files: out };
+  }
+
+  async getFile(jobId: string, path: string): Promise<{ bytes: Buffer } | null> {
+    const dest = join(this.jobDir(jobId), safeRel(path));
+    if (!existsSync(dest)) return null;
+    return { bytes: readFileSync(dest) };
+  }
+
+  async binaryUrl(jobId: string, path: string): Promise<string> {
+    // Local: served by the API itself.
+    return `/v1/clones/${jobId}/files/${path}`;
+  }
+
+  async remove(jobId: string): Promise<void> {
+    rmSync(this.jobDir(jobId), { recursive: true, force: true });
+  }
+}

packages/storage/src/objectStore.ts

@@ -0,0 +1,62 @@
+import { extname } from "node:path";
+import type { FileMap } from "@cloner/core";
+import type { ArtifactStore, StoredFile, StoredManifest } from "./types.js";
+import type { BlobClient } from "./blob.js";
+
+const CT: Record<string, string> = {
+  ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp",
+  ".avif": "image/avif", ".gif": "image/gif", ".svg": "image/svg+xml", ".ico": "image/x-icon",
+  ".woff2": "font/woff2", ".woff": "font/woff", ".ttf": "font/ttf", ".otf": "font/otf",
+  ".mp4": "video/mp4", ".webm": "video/webm",
+};
+const ctFor = (p: string) => CT[extname(p).toLowerCase()] ?? "application/octet-stream";
+
+/**
+ * Object-store (S3 / R2 / MinIO) ArtifactStore. Binaries are uploaded
+ * content-addressably under `clones/<jobId>/<path>` and served via presigned URLs;
+ * text files stay inline in the DB manifest (small) and are not uploaded. The
+ * whole-app bundle is uploaded + presigned on demand.
+ */
+export class ObjectArtifactStore implements ArtifactStore {
+  constructor(private blob: BlobClient, private opts?: { presignExpiresSeconds?: number }) {}
+
+  private key(jobId: string, path: string): string {
+    return `clones/${jobId}/${path}`;
+  }
+
+  async putClone(jobId: string, files: FileMap): Promise<StoredManifest> {
+    const out: StoredFile[] = [];
+    for (const [path, f] of Object.entries(files)) {
+      if (f.kind === "text") {
+        out.push({ path, kind: "text", bytes: f.bytes, sha256: f.sha256, content: f.content ?? "" });
+      } else {
+        const { readFileSync } = await import("node:fs");
+        await this.blob.put(this.key(jobId, path), readFileSync(f.absPath), ctFor(path));
+        out.push({ path, kind: "binary", bytes: f.bytes, sha256: f.sha256, key: this.key(jobId, path) });
+      }
+    }
+    out.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+    return { files: out };
+  }
+
+  async getFile(jobId: string, path: string): Promise<{ bytes: Buffer } | null> {
+    const bytes = await this.blob.get(this.key(jobId, path));
+    return bytes ? { bytes } : null;
+  }
+
+  async binaryUrl(jobId: string, path: string): Promise<string> {
+    return this.blob.presign(this.key(jobId, path), { expiresSeconds: this.opts?.presignExpiresSeconds, contentType: ctFor(path) });
+  }
+
+  /** Upload the prebuilt archive and return a presigned download URL. */
+  async uploadBundle(jobId: string, format: "tgz" | "zip", bytes: Buffer): Promise<string> {
+    const key = `clones/${jobId}/bundle/clone.${format}`;
+    const ct = format === "zip" ? "application/zip" : "application/gzip";
+    await this.blob.put(key, bytes, ct);
+    return this.blob.presign(key, { expiresSeconds: this.opts?.presignExpiresSeconds, downloadName: `clone-${jobId}.${format}`, contentType: ct });
+  }
+
+  async remove(jobId: string): Promise<void> {
+    await this.blob.deletePrefix(`clones/${jobId}/`);
+  }
+}

packages/storage/src/s3.ts

@@ -0,0 +1,95 @@
+import {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand,
+  DeleteObjectCommand,
+  DeleteObjectsCommand,
+  ListObjectsV2Command,
+} from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import type { BlobClient } from "./blob.js";
+
+export type S3Config = {
+  bucket: string;
+  region?: string;
+  endpoint?: string; // e.g. http://localhost:9000 for MinIO, or R2 endpoint
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  forcePathStyle?: boolean; // required for MinIO
+  /** if set, objects are public at `${publicUrl}/${key}` (CDN) and not presigned. */
+  publicUrl?: string;
+  presignExpiresSeconds?: number;
+};
+
+/** BlobClient backed by S3 / R2 / MinIO. */
+export class S3BlobClient implements BlobClient {
+  private client: S3Client;
+  constructor(private cfg: S3Config) {
+    this.client = new S3Client({
+      region: cfg.region ?? "auto",
+      endpoint: cfg.endpoint,
+      forcePathStyle: cfg.forcePathStyle ?? !!cfg.endpoint,
+      credentials:
+        cfg.accessKeyId && cfg.secretAccessKey
+          ? { accessKeyId: cfg.accessKeyId, secretAccessKey: cfg.secretAccessKey }
+          : undefined,
+    });
+  }
+
+  async put(key: string, bytes: Buffer, contentType: string): Promise<void> {
+    await this.client.send(new PutObjectCommand({ Bucket: this.cfg.bucket, Key: key, Body: bytes, ContentType: contentType }));
+  }
+
+  async get(key: string): Promise<Buffer | null> {
+    try {
+      const r = await this.client.send(new GetObjectCommand({ Bucket: this.cfg.bucket, Key: key }));
+      if (!r.Body) return null;
+      const arr = await (r.Body as { transformToByteArray: () => Promise<Uint8Array> }).transformToByteArray();
+      return Buffer.from(arr);
+    } catch (e) {
+      if ((e as { name?: string }).name === "NoSuchKey" || (e as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode === 404) return null;
+      throw e;
+    }
+  }
+
+  async presign(key: string, opts?: { expiresSeconds?: number; downloadName?: string; contentType?: string }): Promise<string> {
+    if (this.cfg.publicUrl) return `${this.cfg.publicUrl.replace(/\/$/, "")}/${key}`;
+    const cmd = new GetObjectCommand({
+      Bucket: this.cfg.bucket,
+      Key: key,
+      ResponseContentDisposition: opts?.downloadName ? `attachment; filename="${opts.downloadName}"` : undefined,
+      ResponseContentType: opts?.contentType,
+    });
+    return getSignedUrl(this.client, cmd, { expiresIn: opts?.expiresSeconds ?? this.cfg.presignExpiresSeconds ?? 3600 });
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.client.send(new DeleteObjectCommand({ Bucket: this.cfg.bucket, Key: key }));
+  }
+
+  async deletePrefix(prefix: string): Promise<void> {
+    let token: string | undefined;
+    do {
+      const listed = await this.client.send(new ListObjectsV2Command({ Bucket: this.cfg.bucket, Prefix: prefix, ContinuationToken: token }));
+      const keys = (listed.Contents ?? []).map((o) => ({ Key: o.Key! })).filter((o) => o.Key);
+      if (keys.length) await this.client.send(new DeleteObjectsCommand({ Bucket: this.cfg.bucket, Delete: { Objects: keys } }));
+      token = listed.IsTruncated ? listed.NextContinuationToken : undefined;
+    } while (token);
+  }
+}
+
+/** Build an S3 config from env (S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_ACCESS_KEY_ID,
+ *  S3_SECRET_ACCESS_KEY, S3_FORCE_PATH_STYLE, S3_PUBLIC_URL). Returns null if no bucket. */
+export function s3ConfigFromEnv(): S3Config | null {
+  const bucket = process.env.S3_BUCKET;
+  if (!bucket) return null;
+  return {
+    bucket,
+    region: process.env.S3_REGION,
+    endpoint: process.env.S3_ENDPOINT,
+    accessKeyId: process.env.S3_ACCESS_KEY_ID,
+    secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
+    forcePathStyle: process.env.S3_FORCE_PATH_STYLE === "true",
+    publicUrl: process.env.S3_PUBLIC_URL,
+  };
+}

packages/storage/src/types.ts

@@ -0,0 +1,30 @@
+import type { FileMap } from "@cloner/core";
+
+/** A file as persisted: text inline (goes into the DB manifest), binaries by
+ *  reference to a storage key (local relative path or S3 key). */
+export type StoredFile =
+  | { path: string; kind: "text"; bytes: number; sha256: string; content: string }
+  | { path: string; kind: "binary"; bytes: number; sha256: string; key: string };
+
+export type StoredManifest = {
+  files: StoredFile[];
+  /** the whole app as one compressed archive (M4); absent until built. */
+  bundleKey?: string;
+};
+
+/** Blob backend for clone artifacts. LocalArtifactStore (M2) writes to disk;
+ *  S3ArtifactStore (M4) uploads to S3/R2 and presigns URLs. */
+export interface ArtifactStore {
+  /** Persist a clone's files; returns the manifest (text inline, binaries by key). */
+  putClone(jobId: string, files: FileMap): Promise<StoredManifest>;
+  /** Read one file's bytes (for the API's /files/* streaming). null if absent. */
+  getFile(jobId: string, path: string): Promise<{ bytes: Buffer } | null>;
+  /** A URL a client uses to fetch a binary out-of-band (local: API route;
+   *  S3: presigned URL). */
+  binaryUrl(jobId: string, path: string): Promise<string>;
+  /** Persist a prebuilt archive and return a download URL (presigned for S3). When
+   *  absent (local store), the API serves the bundle bytes itself. */
+  uploadBundle?(jobId: string, format: "tgz" | "zip", bytes: Buffer): Promise<string>;
+  /** Delete all artifacts for a job. */
+  remove(jobId: string): Promise<void>;
+}

packages/storage/test/bundle.test.ts

@@ -0,0 +1,35 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { gunzipSync, inflateRawSync } from "node:zlib";
+import { makeTarGz, makeZip, sha256hex } from "../src/bundle.js";
+
+const files = [
+  { path: "src/app/page.tsx", bytes: Buffer.from("hello world hello world") },
+  { path: "b.txt", bytes: Buffer.from("x") },
+];
+
+test("makeTarGz: deterministic (order-independent), valid gzip, contains files", () => {
+  const a = makeTarGz(files);
+  const b = makeTarGz([...files].reverse());
+  assert.equal(sha256hex(a), sha256hex(b), "sorted entries → deterministic regardless of input order");
+
+  const tar = gunzipSync(a).toString("latin1");
+  assert.ok(tar.includes("src/app/page.tsx"), "path present in tar header");
+  assert.ok(tar.includes("ustar"), "ustar magic present");
+  assert.ok(tar.includes("hello world hello world"), "content present");
+});
+
+test("makeZip: deterministic, valid signatures, deflate round-trips", () => {
+  const z = makeZip(files);
+  assert.equal(sha256hex(z), sha256hex(makeZip(files)), "deterministic");
+  assert.equal(z.readUInt32LE(0), 0x04034b50, "local file header signature");
+  assert.equal(z.readUInt32LE(z.length - 22), 0x06054b50, "end-of-central-directory signature");
+
+  // Extract the first entry and verify the deflate stream round-trips.
+  const nameLen = z.readUInt16LE(26);
+  const compSize = z.readUInt32LE(18);
+  const dataStart = 30 + nameLen;
+  const out = inflateRawSync(z.subarray(dataStart, dataStart + compSize));
+  // entries are sorted: "b.txt" sorts before "src/app/page.tsx"
+  assert.equal(out.toString(), "x");
+});

packages/storage/test/local.test.ts

@@ -0,0 +1,55 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { collectFileMap } from "@cloner/core";
+import { LocalArtifactStore } from "../src/local.js";
+
+test("LocalArtifactStore: persists, inlines text, references binaries, round-trips bytes", async () => {
+  const work = mkdtempSync(join(tmpdir(), "store-src-"));
+  const blobs = mkdtempSync(join(tmpdir(), "store-blobs-"));
+  try {
+    // Synthesize a generated app and collect it.
+    const app = join(work, "generated", "app");
+    mkdirSync(join(app, "src", "app"), { recursive: true });
+    mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+    writeFileSync(join(app, "src", "app", "page.tsx"), "export default () => null\n");
+    const png = Buffer.from([1, 2, 3, 4, 5]);
+    writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), png);
+    const files = collectFileMap(work);
+
+    const store = new LocalArtifactStore(blobs);
+    const manifest = await store.putClone("job-1", files);
+
+    const page = manifest.files.find((f) => f.path === "src/app/page.tsx")!;
+    assert.equal(page.kind, "text");
+    assert.ok(page.kind === "text" && page.content.includes("export default"));
+
+    const bin = manifest.files.find((f) => f.path.endsWith("a.png"))!;
+    assert.equal(bin.kind, "binary");
+    assert.ok(bin.kind === "binary" && bin.key === "job-1/public/assets/cloned/images/a.png");
+
+    const got = await store.getFile("job-1", "public/assets/cloned/images/a.png");
+    assert.ok(got);
+    assert.deepEqual([...got!.bytes], [1, 2, 3, 4, 5]);
+
+    assert.equal(await store.binaryUrl("job-1", "public/assets/cloned/images/a.png"), "/v1/clones/job-1/files/public/assets/cloned/images/a.png");
+
+    await store.remove("job-1");
+    assert.equal(await store.getFile("job-1", "public/assets/cloned/images/a.png"), null);
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+    rmSync(blobs, { recursive: true, force: true });
+  }
+});
+
+test("LocalArtifactStore: rejects path traversal", async () => {
+  const blobs = mkdtempSync(join(tmpdir(), "store-blobs2-"));
+  try {
+    const store = new LocalArtifactStore(blobs);
+    await assert.rejects(() => store.getFile("job-1", "../../etc/passwd"));
+  } finally {
+    rmSync(blobs, { recursive: true, force: true });
+  }
+});

packages/storage/test/objectStore.test.ts

@@ -0,0 +1,46 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { collectFileMap } from "@cloner/core";
+import { ObjectArtifactStore, InMemoryBlobClient } from "../src/index.js";
+
+test("ObjectArtifactStore: binaries → blob (presigned URL), text inline, getFile, bundle, remove", async () => {
+  const work = mkdtempSync(join(tmpdir(), "obj-src-"));
+  try {
+    const app = join(work, "generated", "app");
+    mkdirSync(join(app, "src", "app"), { recursive: true });
+    mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+    writeFileSync(join(app, "src", "app", "page.tsx"), "export default () => null\n");
+    writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), Buffer.from([9, 8, 7]));
+    const files = collectFileMap(work);
+
+    const blob = new InMemoryBlobClient("https://cdn.test");
+    const store = new ObjectArtifactStore(blob);
+    const manifest = await store.putClone("job-1", files);
+
+    const page = manifest.files.find((f) => f.path === "src/app/page.tsx")!;
+    assert.equal(page.kind, "text");
+
+    const bin = manifest.files.find((f) => f.path.endsWith("a.png"))!;
+    assert.equal(bin.kind, "binary");
+    assert.ok(bin.kind === "binary" && bin.key === "clones/job-1/public/assets/cloned/images/a.png");
+    assert.ok(blob.has("clones/job-1/public/assets/cloned/images/a.png"), "binary uploaded to blob");
+    assert.ok(!blob.has("clones/job-1/src/app/page.tsx"), "text NOT uploaded (stays in manifest)");
+
+    const got = await store.getFile("job-1", "public/assets/cloned/images/a.png");
+    assert.deepEqual([...got!.bytes], [9, 8, 7]);
+
+    const url = await store.binaryUrl("job-1", "public/assets/cloned/images/a.png");
+    assert.ok(url.startsWith("https://cdn.test/clones/job-1/"), "presigned/public URL");
+
+    const bundleUrl = await store.uploadBundle("job-1", "tgz", Buffer.from("archive"));
+    assert.ok(bundleUrl.includes("clones/job-1/bundle/clone.tgz"));
+
+    await store.remove("job-1");
+    assert.equal(await store.getFile("job-1", "public/assets/cloned/images/a.png"), null);
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});

packages/storage/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"]
+}

packages/test-utils/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@cloner/test-utils",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "tsx": "4.22.4",
+    "typescript": "5.7.3",
+    "@types/node": "22.10.5"
+  }
+}

packages/test-utils/src/index.ts

@@ -0,0 +1,87 @@
+import { createServer, type Server } from "node:http";
+import { readFileSync, existsSync, statSync, readdirSync } from "node:fs";
+import { join, extname, normalize, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { homedir } from "node:os";
+
+export { startEphemeralPostgres, canRunEphemeralPostgres, type EphemeralPg } from "./postgres.js";
+import { startEphemeralPostgres, canRunEphemeralPostgres, type EphemeralPg } from "./postgres.js";
+
+/** A Postgres available to tests: TEST_DATABASE_URL if set (e.g. a CI service),
+ *  else a throwaway local instance when we can run one. Sync check for skip. */
+export function hasTestPostgres(): boolean {
+  return !!process.env.TEST_DATABASE_URL || canRunEphemeralPostgres();
+}
+
+/** Acquire a test Postgres (env URL or ephemeral). Returns null if none available. */
+export async function acquireTestPostgres(): Promise<EphemeralPg | null> {
+  if (process.env.TEST_DATABASE_URL) return { url: process.env.TEST_DATABASE_URL, stop: async () => {} };
+  if (canRunEphemeralPostgres()) return startEphemeralPostgres();
+  return null;
+}
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+/** repo root = packages/test-utils/src → ../../.. */
+export const REPO_ROOT = join(HERE, "..", "..", "..");
+export const FIXTURES_DIR = join(REPO_ROOT, "compiler", "fixtures");
+
+const TYPES: Record<string, string> = {
+  ".html": "text/html; charset=utf-8",
+  ".css": "text/css",
+  ".js": "text/javascript",
+  ".mjs": "text/javascript",
+  ".json": "application/json",
+  ".webmanifest": "application/manifest+json",
+  ".xml": "application/xml",
+  ".png": "image/png",
+  ".ico": "image/x-icon",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".svg": "image/svg+xml",
+  ".webp": "image/webp",
+  ".gif": "image/gif",
+  ".woff2": "font/woff2",
+  ".woff": "font/woff",
+};
+
+/** Serve a directory of fixtures over loopback (zero external network). */
+export function serveDir(rootDir: string): Promise<{ url: string; close: () => Promise<void> }> {
+  const server: Server = createServer((req, res) => {
+    try {
+      let p = decodeURIComponent((req.url || "/").split("?")[0]!);
+      if (p === "/") p = "/index.html";
+      const file = join(rootDir, normalize(p).replace(/^(\.\.[/\\])+/, ""));
+      if (!existsSync(file) || statSync(file).isDirectory()) {
+        res.writeHead(404);
+        res.end("not found");
+        return;
+      }
+      res.writeHead(200, { "content-type": TYPES[extname(file).toLowerCase()] ?? "application/octet-stream" });
+      res.end(readFileSync(file));
+    } catch {
+      res.writeHead(500);
+      res.end("error");
+    }
+  });
+  return new Promise((resolve) => {
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address();
+      const port = typeof addr === "object" && addr ? addr.port : 0;
+      resolve({
+        url: `http://127.0.0.1:${port}`,
+        close: () => new Promise<void>((r) => server.close(() => r())),
+      });
+    });
+  });
+}
+
+/** Whether a Playwright Chromium browser appears installed (browser tests skip if not). */
+export function hasChromium(): boolean {
+  try {
+    const base = process.env.PLAYWRIGHT_BROWSERS_PATH || join(homedir(), ".cache", "ms-playwright");
+    if (!existsSync(base)) return false;
+    return readdirSync(base).some((d) => d.startsWith("chromium"));
+  } catch {
+    return false;
+  }
+}

packages/test-utils/src/postgres.ts

@@ -0,0 +1,97 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync, existsSync, readdirSync, chmodSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createServer } from "node:net";
+
+export type EphemeralPg = { url: string; stop: () => Promise<void> };
+
+/** Locate the newest installed PostgreSQL bin dir (e.g. /usr/lib/postgresql/16/bin). */
+function findPgBin(): string | null {
+  if (process.env.PG_BIN && existsSync(join(process.env.PG_BIN, "initdb"))) return process.env.PG_BIN;
+  const root = "/usr/lib/postgresql";
+  if (!existsSync(root)) return null;
+  const versions = readdirSync(root)
+    .filter((d) => /^\d+$/.test(d))
+    .sort((a, b) => Number(b) - Number(a));
+  for (const v of versions) {
+    const bin = join(root, v, "bin");
+    if (existsSync(join(bin, "initdb"))) return bin;
+  }
+  return null;
+}
+
+/** Whether we can stand up a throwaway Postgres here: root (to su to a non-root
+ *  user, since postgres refuses to run as root) + the binaries present. */
+export function canRunEphemeralPostgres(): boolean {
+  const isRoot = typeof process.getuid === "function" && process.getuid() === 0;
+  return isRoot && findPgBin() !== null;
+}
+
+function freePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const srv = createServer();
+    srv.listen(0, "127.0.0.1", () => {
+      const addr = srv.address();
+      const port = typeof addr === "object" && addr ? addr.port : 0;
+      srv.close(() => resolve(port));
+    });
+    srv.on("error", reject);
+  });
+}
+
+function sh(cmd: string): { ok: boolean; out: string } {
+  const r = spawnSync("bash", ["-lc", cmd], { encoding: "utf8" });
+  return { ok: r.status === 0, out: (r.stdout || "") + (r.stderr || "") };
+}
+
+/**
+ * Start a throwaway PostgreSQL instance for tests (no Docker required). initdb's a
+ * temp data dir, runs the server as a non-root user (`pgtest`) on a random loopback
+ * port with trust auth, and returns a connection URL + a stop()/cleanup. Use only
+ * after `canRunEphemeralPostgres()` returns true.
+ */
+export async function startEphemeralPostgres(): Promise<EphemeralPg> {
+  const pgbin = findPgBin();
+  if (!pgbin) throw new Error("no postgres binaries found");
+  const user = "pgtest";
+
+  // Ensure the non-root runner exists.
+  if (!sh(`id ${user}`).ok) {
+    const r = sh(`useradd -m -s /bin/bash ${user}`);
+    if (!sh(`id ${user}`).ok) throw new Error("could not create pgtest user: " + r.out);
+  }
+
+  const data = mkdtempSync(join(tmpdir(), "pgdata-"));
+  const sock = mkdtempSync(join(tmpdir(), "pgsock-"));
+  // The runner user must own the data + socket dirs.
+  chmodSync(data, 0o777);
+  chmodSync(sock, 0o777);
+  sh(`chown -R ${user} ${data} ${sock}`);
+
+  const init = sh(`su ${user} -c "${pgbin}/initdb -D ${data} -A trust -U postgres --no-sync"`);
+  if (!init.ok) {
+    rmSync(data, { recursive: true, force: true });
+    rmSync(sock, { recursive: true, force: true });
+    throw new Error("initdb failed: " + init.out.slice(-500));
+  }
+
+  const port = await freePort();
+  const start = sh(
+    `su ${user} -c "${pgbin}/pg_ctl -D ${data} -o '-p ${port} -k ${sock} -c listen_addresses=127.0.0.1' -w -l ${data}/server.log start"`,
+  );
+  if (!start.ok) {
+    const log = sh(`cat ${data}/server.log`).out;
+    rmSync(data, { recursive: true, force: true });
+    rmSync(sock, { recursive: true, force: true });
+    throw new Error("pg_ctl start failed: " + start.out + "\n" + log.slice(-500));
+  }
+
+  const url = `postgresql://postgres@127.0.0.1:${port}/postgres`;
+  const stop = async (): Promise<void> => {
+    sh(`su ${user} -c "${pgbin}/pg_ctl -D ${data} -w -m immediate stop"`);
+    rmSync(data, { recursive: true, force: true });
+    rmSync(sock, { recursive: true, force: true });
+  };
+  return { url, stop };
+}

packages/test-utils/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts"]
+}

packages/worker/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@cloner/worker",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "dev": "tsx watch src/worker.ts",
+    "start": "tsx src/worker.ts",
+    "test": "node --import tsx --test test/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@cloner/core": "*",
+    "@cloner/db": "*",
+    "@cloner/storage": "*"
+  },
+  "devDependencies": {
+    "@cloner/api": "*",
+    "@cloner/test-utils": "*",
+    "tsx": "4.22.4",
+    "typescript": "5.7.3",
+    "@types/node": "22.10.5"
+  }
+}

packages/worker/src/duration.ts

@@ -0,0 +1,13 @@
+/** Parse a duration like "24h", "30m", "500ms", or "0" (= disabled). A bare number
+ *  is milliseconds. Falls back to `defMs` on empty/invalid input. */
+export function parseDuration(s: string | undefined, defMs: number): number {
+  if (s === undefined || s.trim() === "") return defMs;
+  const t = s.trim();
+  if (t === "0") return 0;
+  const m = /^(\d+)(ms|s|m|h|d)?$/.exec(t);
+  if (!m) return defMs;
+  const n = parseInt(m[1]!, 10);
+  const unit = m[2] ?? "ms";
+  const mult: Record<string, number> = { ms: 1, s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 };
+  return n * (mult[unit] ?? 1);
+}

packages/worker/src/env.ts

@@ -0,0 +1,27 @@
+import { join } from "node:path";
+import { parseDuration } from "./duration.js";
+
+export type WorkerEnv = {
+  databaseUrl: string;
+  artifactsDir: string;
+  /** cache time-to-stale (CACHE_STALE_AFTER, default 24h; "0" disables caching). */
+  cacheStaleAfterMs: number;
+  /** this worker's isolated build harness dir for verify jobs (M5). */
+  harnessDir: string;
+  /** persistent entry-capture cache dir (the single→multi speed path). "" disables it. */
+  captureCacheDir: string;
+  tier: string;
+};
+
+export function loadWorkerEnv(): WorkerEnv {
+  const databaseUrl = process.env.DATABASE_URL;
+  if (!databaseUrl) throw new Error("DATABASE_URL is required for the worker");
+  return {
+    databaseUrl,
+    artifactsDir: process.env.ARTIFACTS_DIR ?? join(process.cwd(), "local-data", "artifacts"),
+    cacheStaleAfterMs: parseDuration(process.env.CACHE_STALE_AFTER, 24 * 60 * 60 * 1000),
+    harnessDir: process.env.HARNESS_DIR ?? join(process.cwd(), "local-data", "harness"),
+    captureCacheDir: process.env.CAPTURE_CACHE_DIR ?? join(process.cwd(), "local-data", "capture-cache"),
+    tier: process.env.VERIFY_TIER ?? "stage2",
+  };
+}

packages/worker/src/harness.ts

@@ -0,0 +1,51 @@
+import { cpSync, existsSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+
+/** The canonical build harness shipped with the compiler (has package.json + lock). */
+export function baseHarnessDir(): string {
+  // packages/worker/src → ../../../compiler/.harness
+  return join(HERE, "..", "..", "..", "compiler", ".harness");
+}
+
+function hasBuildDeps(dir: string): boolean {
+  return existsSync(join(dir, "node_modules", ".bin", "next"))
+    && existsSync(join(dir, "node_modules", ".bin", "vite"));
+}
+
+/**
+ * Ensure `targetDir` is a ready-to-build app harness with deps installed — the
+ * isolation seam for `verify` (each worker uses its OWN harness so concurrent
+ * framework builds never collide). Reuses the base harness's deps when present
+ * (fast copy), otherwise runs `npm install` from its package.json. Idempotent.
+ */
+export function provisionHarness(targetDir: string, baseDir = baseHarnessDir()): string {
+  if (hasBuildDeps(targetDir)) return targetDir;
+  mkdirSync(targetDir, { recursive: true });
+  if (baseDir !== targetDir) {
+    for (const f of ["package.json", "package-lock.json"]) {
+      if (existsSync(join(baseDir, f))) cpSync(join(baseDir, f), join(targetDir, f));
+    }
+  }
+  if (hasBuildDeps(baseDir) && baseDir !== targetDir) {
+    cpSync(join(baseDir, "node_modules"), join(targetDir, "node_modules"), { recursive: true });
+  } else if (!hasBuildDeps(targetDir)) {
+    const r = spawnSync("npm", ["install", "--no-audit", "--no-fund"], { cwd: targetDir, encoding: "utf8", stdio: "inherit" });
+    if (r.status !== 0) throw new Error("harness npm install failed in " + targetDir);
+  }
+  return targetDir;
+}
+
+/** A memoized provisioner: provisions once, then returns the same dir. */
+export function makeHarnessProvider(targetDir: string): () => Promise<string> {
+  let ready: string | null = null;
+  let inflight: Promise<string> | null = null;
+  return async () => {
+    if (ready) return ready;
+    if (!inflight) inflight = Promise.resolve().then(() => provisionHarness(targetDir)).then((d) => (ready = d));
+    return inflight;
+  };
+}

packages/worker/src/index.ts

@@ -0,0 +1,4 @@
+export { processCloneJob, type ProcessDeps, type RunJob } from "./processJob.js";
+export { parseDuration } from "./duration.js";
+export { provisionHarness, makeHarnessProvider, baseHarnessDir } from "./harness.js";
+export { loadWorkerEnv, type WorkerEnv } from "./env.js";

packages/worker/src/processJob.ts

@@ -0,0 +1,86 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { canonicalOptions, verifyCloneJobResult, type CloneJobResult, type CloneOptions, type RunCloneJobInput } from "@cloner/core";
+import { repo, type Db } from "@cloner/db";
+import type { ArtifactStore } from "@cloner/storage";
+
+export type RunJob = (input: RunCloneJobInput) => Promise<CloneJobResult>;
+
+export type ProcessDeps = {
+  db: Db;
+  store: ArtifactStore;
+  runJob: RunJob;
+  /** cache TTL in ms (0 = caching disabled — no cache row written). */
+  cacheTtlMs: number;
+  /** lazily provisions this worker's isolated build harness, used only for verify jobs (M5). */
+  harnessProvider?: () => Promise<string>;
+  /** persistent entry-capture cache dir for the single→multi reuse path ("" / undefined off). */
+  captureCacheDir?: string;
+  tier?: string;
+};
+
+/**
+ * Process one queued clone job: mark running → run the clone into a temp dir →
+ * persist artifacts (ArtifactStore) + the clone row → mark succeeded → write a
+ * freshness-bounded cache row. On failure, mark the job failed and rethrow so the
+ * queue can retry (capped). The temp dir is always cleaned up.
+ */
+export async function processCloneJob(deps: ProcessDeps, jobId: string): Promise<void> {
+  const { db, store } = deps;
+  const job = await repo.getJob(db, jobId);
+  if (!job) return; // job deleted before it ran
+  await repo.markRunning(db, jobId);
+
+  const base = mkdtempSync(join(tmpdir(), "worker-clone-"));
+  try {
+    const options = (job.options ?? {}) as CloneOptions;
+    // Provision the isolated harness only when this job actually verifies (build).
+    const harnessDir = (options.verify || options.asyncVerify) && deps.harnessProvider ? await deps.harnessProvider() : undefined;
+    const result = await deps.runJob({ url: job.url, options, runsDir: base, harnessDir, tier: deps.tier, captureCacheDir: deps.captureCacheDir || undefined, captureCacheTtlMs: deps.cacheTtlMs });
+
+    const manifest = await store.putClone(jobId, result.files);
+    const envelope = { files: manifest.files, routes: result.routes, bundleKey: manifest.bundleKey };
+    await repo.upsertClone(db, {
+      jobId,
+      url: result.url,
+      routeCount: result.routes?.length ?? 1,
+      fileManifest: envelope,
+      captureMeta: result.capture,
+      verify: (result.verify ?? null) as unknown as Record<string, unknown> | null,
+    });
+    await repo.markSucceeded(db, jobId, { compilerVersion: result.compilerVersion, timings: result.timings });
+
+    if (deps.cacheTtlMs > 0 && !options.noCache) {
+      await repo.cachePut(db, {
+        cacheKey: job.cacheKey,
+        jobId,
+        url: job.url,
+        optionsHash: canonicalOptions(options),
+        compilerVersion: result.compilerVersion,
+        expiresAt: new Date(Date.now() + deps.cacheTtlMs),
+      });
+    }
+
+    if (options.asyncVerify) {
+      try {
+        const done = await verifyCloneJobResult(result, {
+          harnessDir,
+          tier: deps.tier,
+          validationConcurrency: options.validationConcurrency,
+          viewportConcurrency: options.viewportConcurrency,
+        });
+        const timings = { ...result.timings, verifyMs: done.verifyMs };
+        await repo.updateCloneVerify(db, jobId, done.verify);
+        await repo.updateJobTimings(db, jobId, timings);
+      } catch (e) {
+        await repo.updateCloneVerify(db, jobId, { error: String(e).slice(0, 500), async: true });
+      }
+    }
+  } catch (e) {
+    await repo.markFailed(db, jobId, String(e));
+    throw e;
+  } finally {
+    rmSync(base, { recursive: true, force: true });
+  }
+}

packages/worker/src/worker.ts

@@ -0,0 +1,31 @@
+import { runCloneJob } from "@cloner/core";
+import { createDb, createBoss, workClone } from "@cloner/db";
+import { artifactStoreFromEnv } from "@cloner/storage";
+import { processCloneJob } from "./processJob.js";
+import { makeHarnessProvider } from "./harness.js";
+import { loadWorkerEnv } from "./env.js";
+
+async function main(): Promise<void> {
+  const env = loadWorkerEnv();
+  const { db } = createDb(env.databaseUrl);
+  const boss = await createBoss(env.databaseUrl);
+  const store = artifactStoreFromEnv();
+
+  const deps = {
+    db,
+    store,
+    runJob: runCloneJob,
+    cacheTtlMs: env.cacheStaleAfterMs,
+    harnessProvider: makeHarnessProvider(env.harnessDir),
+    captureCacheDir: env.captureCacheDir,
+    tier: env.tier,
+  };
+
+  await workClone(boss, (jobId) => processCloneJob(deps, jobId));
+  console.log(JSON.stringify({ event: "worker_started", artifactsDir: env.artifactsDir, harnessDir: env.harnessDir, cacheStaleAfterMs: env.cacheStaleAfterMs }));
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

packages/worker/test/integration.test.ts

@@ -0,0 +1,123 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type PgBoss from "pg-boss";
+import { collectFileMap, COMPILER_VERSION, type CloneJobResult } from "@cloner/core";
+import { createDb, createBoss, workClone, runMigrations, type Db } from "@cloner/db";
+import { LocalArtifactStore } from "@cloner/storage";
+import { createApp, DbBackend } from "@cloner/api";
+import { acquireTestPostgres, hasTestPostgres, type EphemeralPg } from "@cloner/test-utils";
+import { processCloneJob } from "../src/processJob.js";
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+async function waitFor<T>(fn: () => Promise<T | undefined | null>, ms = 30_000, step = 300): Promise<T> {
+  const t0 = Date.now();
+  for (;;) {
+    const v = await fn();
+    if (v) return v;
+    if (Date.now() - t0 > ms) throw new Error("waitFor timeout");
+    await sleep(step);
+  }
+}
+
+/** A browser-free fake clone: writes a tiny generated app under the worker's temp
+ *  base, then returns a real CloneJobResult via collectFileMap. Keeps the queue/DB
+ *  lifecycle test fast + hermetic (no Chromium). */
+const fakeRunJob = async (input: { url: string; runsDir?: string; options?: unknown }): Promise<CloneJobResult> => {
+  const base = input.runsDir!;
+  const app = join(base, "generated", "app");
+  mkdirSync(join(app, "src", "app"), { recursive: true });
+  mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+  writeFileSync(join(app, "package.json"), '{"name":"cloned-app"}\n');
+  writeFileSync(join(app, "src", "app", "page.tsx"), "export default function Page(){return <div/>}\n");
+  writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), Buffer.from([1, 2, 3, 4]));
+  return {
+    url: input.url,
+    kind: "clone",
+    options: (input.options as CloneJobResult["options"]) ?? {},
+    status: "succeeded",
+    compilerVersion: COMPILER_VERSION,
+    timings: { captureMs: 1, generateMs: 0 },
+    files: collectFileMap(base),
+    capture: { nodeCount: 7, pollution: false, blocked: false },
+    runDir: base,
+  };
+};
+
+describe("M2: async job lifecycle (Postgres queue + DB + worker)", { skip: hasTestPostgres() ? false : "no test Postgres" }, () => {
+  let pg: EphemeralPg;
+  let db: Db;
+  let pool: { end: () => Promise<void> };
+  let boss: PgBoss;
+  let store: LocalArtifactStore;
+  let blobs: string;
+
+  before(async () => {
+    pg = (await acquireTestPostgres())!;
+    await runMigrations(pg.url);
+    const h = createDb(pg.url);
+    db = h.db;
+    pool = h.pool;
+    boss = await createBoss(pg.url);
+    blobs = mkdtempSync(join(tmpdir(), "it-blobs-"));
+    store = new LocalArtifactStore(blobs);
+    // Start the worker consuming the queue.
+    await workClone(boss, (jobId) => processCloneJob({ db, store, runJob: fakeRunJob, cacheTtlMs: 60_000 }, jobId));
+  });
+
+  after(async () => {
+    await boss?.stop({ graceful: false }).catch(() => {});
+    await pool?.end().catch(() => {});
+    await pg?.stop().catch(() => {});
+    if (blobs) rmSync(blobs, { recursive: true, force: true });
+  });
+
+  it("enqueues, processes, persists, and serves the result; second submit is cached", async () => {
+    const app = createApp({ backend: new DbBackend({ db, boss, store }) });
+    const url = "https://example.com/?t=" + Date.now(); // unique → no stale cache on a reused DB
+    const reqBody = JSON.stringify({ url, options: { interactions: false } });
+
+    // Submit → 202 queued.
+    const submit = await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: reqBody });
+    assert.equal(submit.status, 202);
+    const { jobId, status } = await submit.json();
+    assert.ok(jobId);
+    assert.equal(status, "queued");
+
+    // Poll until the worker finishes.
+    const done = await waitFor(async () => {
+      const v = await (await app.request(`/v1/clones/${jobId}`)).json();
+      return v.status === "succeeded" ? v : undefined;
+    });
+    assert.equal(done.status, "succeeded");
+    assert.equal(done.capture.nodeCount, 7);
+    assert.equal(done.fileCount, 3);
+
+    // Full result + per-file streaming.
+    const result = await (await app.request(`/v1/clones/${jobId}/result`)).json();
+    assert.equal(result.files["src/app/page.tsx"].type, "text");
+    const bin = result.files["public/assets/cloned/images/a.png"];
+    assert.equal(bin.type, "binary");
+    const fileRes = await app.request(bin.url);
+    assert.equal(fileRes.status, 200);
+    assert.equal(Buffer.from(await fileRes.arrayBuffer()).length, 4);
+
+    // Second identical submit → cache hit (200, status cached), no new job.
+    const submit2 = await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: reqBody });
+    assert.equal(submit2.status, 200);
+    const cached = await submit2.json();
+    assert.equal(cached.status, "cached");
+    assert.equal(cached.jobId, jobId);
+
+    // noCache bypasses the cache → a fresh queued job.
+    const submit3 = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url, options: { interactions: false, noCache: true } }),
+    });
+    assert.equal(submit3.status, 202);
+    assert.notEqual((await submit3.json()).jobId, jobId);
+  });
+});

packages/worker/test/verify.fixture.test.ts

@@ -0,0 +1,44 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { runCloneJob } from "@cloner/core";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+import { provisionHarness, baseHarnessDir } from "../src/harness.js";
+
+// verify = build the generated app (next build) + serve + re-render + grade. Uses an
+// isolated, dep-installed harness. Skipped without Chromium; also skips gracefully if
+// the harness can't be provisioned (e.g. no network for npm install).
+describe("runCloneJob verify (build + gates via provisioned harness)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  let harnessDir: string | null = null;
+
+  before(async () => {
+    server = await serveDir(FIXTURES_DIR);
+    try {
+      harnessDir = provisionHarness(baseHarnessDir());
+    } catch (e) {
+      console.error("harness provisioning failed:", String(e).slice(0, 200));
+      harnessDir = null;
+    }
+  });
+  after(async () => {
+    await server.close();
+  });
+
+  it("builds the clone and attaches a verify report", { timeout: 300_000 }, async (t) => {
+    if (!harnessDir) {
+      t.skip("harness unavailable (npm install failed)");
+      return;
+    }
+    const res = await runCloneJob({
+      url: server.url + "/components.html",
+      options: { verify: true, interactions: false, components: true, motion: false },
+      harnessDir,
+      tier: "easy",
+    });
+    assert.ok(res.verify, "verify report attached");
+    const v = res.verify as { gates0to6Pass: boolean; scorecard: { total: number }; gates: Record<string, unknown> };
+    assert.equal(typeof v.gates0to6Pass, "boolean");
+    assert.ok(v.scorecard && typeof v.scorecard.total === "number", "scorecard present");
+    assert.ok(res.timings.verifyMs !== undefined && res.timings.verifyMs > 0, "verifyMs recorded");
+  });
+});

packages/worker/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"]
+}