Service: job event stream, async in-memory backend, preview routes

Ported from PR #15 (packages-side surface only; the fork's dev UI page is
intentionally dropped - the API surface is the product):
- job_events table + repository, worker emits stage transitions, additive
  GET /v1/clones/:id/events polling route. The fork's hand-written
  migration was never registered in Drizzle's journal (it silently never
  applied); regenerated properly via drizzle-kit from schema.ts with the
  (job_id, seq) index declared, and verified against a live Postgres
- In-memory backend moves to the documented enqueue-202 + poll contract
  with single-flight BUSY guard; POST honors backend httpStatus
- app-preview static serving routes; the preview BUILD half is deferred
  until the compiler exports buildApp/DEFAULT_HARNESS_DIR - routes 404
  cleanly until then
- preview option threaded through core types; cache key intentionally
  unchanged while the flag is inert

All workspaces typecheck; api 18/18, core 13/13, worker 1/1 pass;
migration verified applying (table + index + FK) on postgres:16.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

packages/api/src/app.ts

@@ -15,6 +15,7 @@ const OptionsSchema = z
     mode: z.enum(["single", "multi"]).optional(),
     styling: z.enum(["tailwind", "css"]).optional(),
     framework: z.enum(["next", "vite"]).optional(),
+    preview: z.boolean().optional(),
     verify: z.boolean().optional(),
     asyncVerify: z.boolean().optional(),
     maxRoutes: z.number().int().positive().optional(),
@@ -238,10 +239,12 @@ export function createApp(deps: AppDeps): Hono {
 
     try {
       const out = await backend.submit(url, opts);
-      if (out.status === "queued") return c.json({ jobId: out.jobId, status: "queued" }, 202);
+      if (out.status === "queued") return c.json({ jobId: out.jobId, status: "queued" }, out.httpStatus);
       return c.json(out.result, 200);
     } catch (e) {
-      return c.json({ status: "failed", error: String(e).slice(0, 500) }, 500);
+      const msg = String(e);
+      if (msg.startsWith("BUSY:")) return c.json({ error: msg.slice(5).trim() }, 429);
+      return c.json({ status: "failed", error: msg.slice(0, 500) }, 500);
     }
   });
 
@@ -282,6 +285,40 @@ export function createApp(deps: AppDeps): Hono {
     return c.body(file.bytes);
   });
 
+  // Pipeline progress events (poll every ~300ms while a clone runs).
+  app.get("/v1/clones/:id/events", async (c) => {
+    const after = Math.max(0, Number(c.req.query("after") ?? "0") || 0);
+    const events = backend.events ? await backend.events(c.req.param("id"), after) : null;
+    if (!events) return c.json({ error: "not found" }, 404);
+    return c.json({ jobId: c.req.param("id"), events });
+  });
+
+  // Browsable preview of the built clone (static export published by the preview
+  // build under public/app-preview/). References are relative, so the export works
+  // from this mount — the only requirement is a trailing slash on the root.
+  const previewFile = async (c: Context, sub: string) => {
+    const id = c.req.param("id") ?? "";
+    const tryPaths = sub.includes(".")
+      ? [`public/app-preview/${sub}`]
+      : [
+          `public/app-preview/${sub}`.replace(/\/$/, "") + "/index.html",
+          `public/app-preview/${sub}`,
+          `public/app-preview/${sub}.html`,
+        ];
+    for (const p of tryPaths) {
+      const file = await backend.file(id, p);
+      if (file) {
+        c.header("content-type", file.contentType);
+        c.header("content-length", String(file.bytes.length));
+        return c.body(file.bytes);
+      }
+    }
+    return c.json({ error: "no app preview for this clone (preview builds are on by default for single-page clones; pass options.preview=true otherwise)" }, 404);
+  };
+  app.get("/v1/clones/:id/app-preview", (c) => c.redirect(`/v1/clones/${c.req.param("id")}/app-preview/`, 302));
+  app.get("/v1/clones/:id/app-preview/", (c) => previewFile(c, "index.html"));
+  app.get("/v1/clones/:id/app-preview/:path{.+}", (c) => previewFile(c, c.req.param("path") ?? ""));
+
   app.delete("/v1/clones/:id", async (c) => {
     const ok = await backend.remove(c.req.param("id"));
     return c.json({ deleted: ok }, ok ? 200 : 404);

packages/api/src/backend.ts

@@ -1,5 +1,5 @@
 import type { CloneOptions, RouteInfo } from "@cloner/core";
-import type { RestCloneResult } from "./rest.js";
+import type { RestCloneResult, RestCloneSummary } from "./rest.js";
 
 export type JobStatus = "queued" | "running" | "succeeded" | "failed" | "cached";
 
@@ -22,7 +22,7 @@ export type JobView = {
 };
 
 export type SubmitOutcome =
-  | { jobId: string; status: "succeeded" | "cached"; httpStatus: 200; result: RestCloneResult }
+  | { jobId: string; status: "succeeded" | "cached"; httpStatus: 200; result: RestCloneResult | RestCloneSummary }
   | { jobId: string; status: "queued"; httpStatus: 202 };
 
 export type ResultOutcome =
@@ -56,4 +56,6 @@ export interface Backend {
   facets(jobId: string): Promise<FileFacet[] | null>;
   /** The whole app as one compressed archive (null if not ready/found). */
   bundle(jobId: string, format?: BundleFormat): Promise<CloneBundle | null>;
+  /** Pipeline progress events for polling UIs (null if unsupported or job unknown). */
+  events?(jobId: string, after?: number): Promise<Array<Record<string, unknown>> | null>;
 }

packages/api/src/backends/db.ts

@@ -142,4 +142,10 @@ export class DbBackend implements Backend {
     const url = this.deps.store.uploadBundle ? await this.deps.store.uploadBundle(jobId, format, bytes) : undefined;
     return { bytes, sha256: sha256hex(bytes), format, url };
   }
+
+  async events(jobId: string, after = 0): Promise<Array<Record<string, unknown>> | null> {
+    const job = await repo.getJob(this.deps.db, jobId);
+    if (!job) return null;
+    return repo.listJobEvents(this.deps.db, jobId, after);
+  }
 }

packages/api/src/backends/inMemory.ts

@@ -10,24 +10,51 @@ import type { Backend, BundleFormat, CloneBundle, FileFacet, JobView, ResultOutc
 
 export type RunJob = (input: RunCloneJobInput) => Promise<CloneJobResult>;
 
-/** Sync, in-memory backend (M1): runs the clone INLINE on submit and holds results
- *  in memory. POST returns the file map immediately (200). */
+/** In-memory backend: enqueues a clone (202) and runs it in the background so long
+ *  captures (multi-viewport + verify) never hold a single HTTP connection open for
+ *  minutes — which browsers surface as `TypeError: Failed to fetch` when the link
+ *  drops (server restart, hot-reload, idle timeout). Poll /v1/clones/:id + /events. */
 export class InMemoryBackend implements Backend {
   constructor(private deps: { store: InMemoryStore; runJob: RunJob; makeTempBase?: () => string; captureCacheDir?: string }) {}
 
+  private activeClones = 0;
+
   private makeBase(): string {
     return (this.deps.makeTempBase ?? (() => mkdtempSync(join(tmpdir(), "api-clone-"))))();
   }
 
   async submit(url: string, options: CloneOptions | undefined): Promise<SubmitOutcome> {
+    if (this.activeClones > 0 || this.deps.store.list().some((j) => j.status === "running")) {
+      throw new Error("BUSY: another clone is already running — wait for it to finish (use npm run dev:api:stable to avoid hot-reload killing Playwright)");
+    }
     const id = randomUUID();
     const base = this.makeBase();
+    const events: Array<Record<string, unknown>> = [];
+    const kind: "clone" | "clone_site" = resolveCloneMode(options) === "multi" ? "clone_site" : "clone";
+    const rec = { id, status: "running" as const, url, kind, options: options ?? {}, createdAt: Date.now(), base, events };
+    this.deps.store.put(rec);
+    void this.runInBackground(id, url, options, rec, base, events, kind);
+    return { jobId: id, status: "queued", httpStatus: 202 };
+  }
+
+  private async runInBackground(
+    id: string,
+    url: string,
+    options: CloneOptions | undefined,
+    rec: { id: string; status: "running"; url: string; kind: "clone" | "clone_site"; options: CloneOptions; createdAt: number; base: string; events: Array<Record<string, unknown>> },
+    base: string,
+    events: Array<Record<string, unknown>>,
+    kind: "clone" | "clone_site",
+  ): Promise<void> {
+    this.activeClones++;
+    const log = (e: Record<string, unknown>) => {
+      events.push({ t: Date.now(), ...e });
+      this.deps.store.put({ ...rec, events: [...events] });
+    };
     try {
-      // captureCacheDir (persistent, shared across submits) powers the single→multi
-      // reuse: a single-page submit stashes its capture; a later multi-page submit for
-      // the same URL reuses it as the entry route (no re-capture) and expands the site.
-      const result = await this.deps.runJob({ url, options, runsDir: base, captureCacheDir: this.deps.captureCacheDir });
-      this.deps.store.put({ id, status: "succeeded", url, kind: result.kind, options: result.options, createdAt: Date.now(), result, base });
+      const result = await this.deps.runJob({ url, options, runsDir: base, captureCacheDir: this.deps.captureCacheDir, log });
+      log({ event: "clone_done" });
+      this.deps.store.put({ id, status: "succeeded", url, kind: result.kind, options: result.options, createdAt: rec.createdAt, result, base, events });
       if (result.options.asyncVerify) {
         void verifyCloneJobResult(result, {
           validationConcurrency: result.options.validationConcurrency,
@@ -39,18 +66,25 @@ export class InMemoryBackend implements Backend {
           result.verify = { error: String(e).slice(0, 500), async: true };
         });
       }
-      return { jobId: id, status: "succeeded", httpStatus: 200, result: buildRestResult(id, result, `/v1/clones/${id}/files`) };
     } catch (e) {
       try {
         rmSync(base, { recursive: true, force: true });
       } catch {
         /* best effort */
       }
-      this.deps.store.put({ id, status: "failed", url, kind: resolveCloneMode(options) === "multi" ? "clone_site" : "clone", options: options ?? {}, createdAt: Date.now(), error: String(e) });
-      throw e;
+      log({ event: "clone_error", error: String(e).slice(0, 300) });
+      this.deps.store.put({ id, status: "failed", url, kind, options: options ?? {}, createdAt: rec.createdAt, error: String(e), events });
+    } finally {
+      this.activeClones = Math.max(0, this.activeClones - 1);
     }
   }
 
+  async events(jobId: string): Promise<Array<Record<string, unknown>> | null> {
+    const rec = this.deps.store.get(jobId);
+    if (!rec) return null;
+    return rec.events ?? [];
+  }
+
   async status(jobId: string): Promise<JobView | null> {
     const rec = this.deps.store.get(jobId);
     if (!rec) return null;

packages/api/src/store.ts

@@ -6,7 +6,7 @@ import type { CloneJobResult, CloneOptions } from "@cloner/core";
  *  it is removed on eviction. M2/M4 replace this with DB rows + S3 objects. */
 export type JobRecord = {
   id: string;
-  status: "succeeded" | "failed";
+  status: "running" | "succeeded" | "failed";
   url: string;
   kind: "clone" | "clone_site";
   options: CloneOptions;
@@ -14,6 +14,8 @@ export type JobRecord = {
   result?: CloneJobResult;
   base?: string;
   error?: string;
+  /** pipeline progress events (compiler log stream + service phases), poll via /events */
+  events?: Array<Record<string, unknown>>;
 };
 
 /** In-memory result store with TTL eviction. Swapped for a DB-backed store in M2. */

packages/api/test/app.fixture.test.ts

@@ -26,13 +26,25 @@ describe("POST /v1/clones (real clone, served fixture)", { skip: hasChromium() ?
       headers: { "content-type": "application/json" },
       body: JSON.stringify({ url: server.url + "/components.html", options: { interactions: false, components: true, motion: false } }),
     });
-    assert.equal(res.status, 200);
-    const body = await res.json();
-    assert.equal(body.status, "succeeded");
-    assert.ok(body.files["src/app/page.tsx"], "has page.tsx");
-    assert.equal(body.files["src/app/page.tsx"].type, "text");
-    assert.ok(body.files["package.json"], "has package.json");
-    assert.ok(body.capture.nodeCount > 0);
-    assert.equal(body.capture.blocked, false);
+    assert.equal(res.status, 202);
+    const queued = await res.json();
+    let body: Record<string, unknown> | undefined;
+    for (let i = 0; i < 600; i++) {
+      const view = await (await app.request(`/v1/clones/${queued.jobId}`)).json();
+      if (view.status === "succeeded") {
+        body = await (await app.request(`/v1/clones/${queued.jobId}/result`)).json();
+        break;
+      }
+      if (view.status === "failed") assert.fail(view.error);
+      await new Promise((r) => setTimeout(r, 50));
+    }
+    assert.ok(body);
+    const files = body!.files as Record<string, { type: string }>;
+    assert.equal(body!.status, "succeeded");
+    assert.ok(files["src/app/page.tsx"], "has page.tsx");
+    assert.equal(files["src/app/page.tsx"].type, "text");
+    assert.ok(files["package.json"], "has package.json");
+    assert.ok((body!.capture as { nodeCount: number }).nodeCount > 0);
+    assert.equal((body!.capture as { blocked: boolean }).blocked, false);
   });
 });

packages/api/test/app.test.ts

@@ -34,7 +34,17 @@ const fakeRunJob: RunJob = async (input) => {
   } satisfies CloneJobResult;
 };
 
-test("POST /v1/clones returns the eager file map; binaries by reference; streaming + lifecycle", async () => {
+async function waitForResult(app: ReturnType<typeof createApp>, jobId: string) {
+  for (let i = 0; i < 200; i++) {
+    const view = await (await app.request(`/v1/clones/${jobId}`)).json();
+    if (view.status === "succeeded") return await (await app.request(`/v1/clones/${jobId}/result`)).json();
+    if (view.status === "failed") throw new Error(String(view.error));
+    await new Promise((r) => setTimeout(r, 5));
+  }
+  throw new Error("timeout waiting for clone");
+}
+
+test("POST /v1/clones enqueues then completes; binaries by reference; streaming + lifecycle", async () => {
   const store = new InMemoryStore(60_000);
   const app = createApp({ backend: new InMemoryBackend({ store, runJob: fakeRunJob }) });
   try {
@@ -43,39 +53,47 @@ test("POST /v1/clones returns the eager file map; binaries by reference; streami
       headers: { "content-type": "application/json" },
       body: JSON.stringify({ url: "https://example.com/", options: {} }),
     });
-    assert.equal(res.status, 200);
-    const body = await res.json();
-    assert.ok(body.jobId);
-    assert.equal(body.status, "succeeded");
-    assert.equal(body.kind, "clone");
+    assert.equal(res.status, 202);
+    const queued = await res.json();
+    assert.ok(queued.jobId);
+    assert.equal(queued.status, "queued");
+
+    let body: Record<string, unknown> | undefined;
+    body = await waitForResult(app, queued.jobId);
+    assert.ok(body);
+    assert.equal(body!.status, "succeeded");
+    assert.equal(body!.kind, "clone");
 
     // Text inline.
-    const page = body.files["src/app/page.tsx"];
-    assert.equal(page.type, "text");
-    assert.ok(page.content.includes("Page"));
-    assert.ok(page.sha256);
+    const files = body!.files as Record<string, { type: string; content?: string; sha256?: string; url?: string }>;
+    const page = files["src/app/page.tsx"];
+    assert.ok(page);
+    assert.equal(page!.type, "text");
+    assert.ok(page!.content!.includes("Page"));
+    assert.ok(page!.sha256);
 
     // Binary by reference (URL, not bytes).
-    const bin = body.files["public/assets/cloned/images/a.png"];
-    assert.equal(bin.type, "binary");
-    assert.equal(bin.content, undefined);
-    assert.equal(bin.url, `/v1/clones/${body.jobId}/files/public/assets/cloned/images/a.png`);
+    const bin = files["public/assets/cloned/images/a.png"];
+    assert.ok(bin);
+    assert.equal(bin!.type, "binary");
+    assert.equal(bin!.content, undefined);
+    assert.equal(bin!.url, `/v1/clones/${queued.jobId}/files/public/assets/cloned/images/a.png`);
 
     // Per-file streaming returns the actual bytes.
-    const fileRes = await app.request(bin.url);
+    const fileRes = await app.request(bin!.url!);
     assert.equal(fileRes.status, 200);
     assert.equal(fileRes.headers.get("content-type"), "image/png");
     const bytes = Buffer.from(await fileRes.arrayBuffer());
     assert.equal(bytes.length, 7);
 
     // Cheap metadata overview (no file contents).
-    const meta = await (await app.request(`/v1/clones/${body.jobId}`)).json();
+    const meta = await (await app.request(`/v1/clones/${queued.jobId}`)).json();
     assert.equal(meta.fileCount, 3);
     assert.equal(meta.capture.nodeCount, 42);
     assert.equal(meta.totalBytes > 0, true);
 
     // Full result fetch.
-    const result = await app.request(`/v1/clones/${body.jobId}/result`);
+    const result = await app.request(`/v1/clones/${queued.jobId}/result`);
     assert.equal(result.status, 200);
 
     // List.
@@ -83,9 +101,9 @@ test("POST /v1/clones returns the eager file map; binaries by reference; streami
     assert.equal(list.clones.length, 1);
 
     // Delete purges; subsequent fetch 404s.
-    const del = await app.request(`/v1/clones/${body.jobId}`, { method: "DELETE" });
+    const del = await app.request(`/v1/clones/${queued.jobId}`, { method: "DELETE" });
     assert.equal((await del.json()).deleted, true);
-    assert.equal((await app.request(`/v1/clones/${body.jobId}`)).status, 404);
+    assert.equal((await app.request(`/v1/clones/${queued.jobId}`)).status, 404);
   } finally {
     store.clear();
   }
@@ -118,8 +136,9 @@ test("POST /v1/clones normalizes product options and legacy aliases", async () =
       headers: { "content-type": "application/json" },
       body: JSON.stringify({ url: "https://example.com/", options: { mode: "multi", styling: "css", framework: "vite" } }),
     });
-    assert.equal(product.status, 200);
-    const productBody = await product.json();
+    assert.equal(product.status, 202);
+    const productQueued = await product.json();
+    const productBody = await waitForResult(app, productQueued.jobId);
     assert.deepEqual(productBody.options, { mode: "multi", styling: "css", framework: "vite" });
 
     const legacy = await app.request("/v1/clones", {
@@ -127,8 +146,9 @@ test("POST /v1/clones normalizes product options and legacy aliases", async () =
       headers: { "content-type": "application/json" },
       body: JSON.stringify({ url: "https://example.com/", options: { multiPage: true, humanizeMode: "css" } }),
     });
-    assert.equal(legacy.status, 200);
-    const legacyBody = await legacy.json();
+    assert.equal(legacy.status, 202);
+    const legacyQueued = await legacy.json();
+    const legacyBody = await waitForResult(app, legacyQueued.jobId);
     assert.deepEqual(legacyBody.options, { mode: "multi", styling: "css", framework: "next" });
   } finally {
     store.clear();

packages/api/test/mcp.test.ts

@@ -52,12 +52,18 @@ test("MCP: list-then-read + bundle contract (never floods context)", async () =>
     // clone_website → jobId + status only (no files).
     const cw = parse(await client.callTool({ name: "clone_website", arguments: { url: "https://example.com/", options: {} } }));
     assert.ok(cw.jobId);
-    assert.equal(cw.status, "succeeded");
-    assert.ok(!("files" in cw));
+    assert.equal(cw.status, "queued");
     const jobId = cw.jobId;
 
-    // status
-    const status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
+    let status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
+    for (let i = 0; i < 200 && status.status !== "succeeded"; i++) {
+      await new Promise((r) => setTimeout(r, 5));
+      status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
+    }
+    assert.equal(status.status, "succeeded");
+
+    // status (re-fetch after completion)
+    status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
     assert.equal(status.status, "succeeded");
 
     // get_clone_result → metadata only, NO file contents.