diff --git a/packages/api/src/app.ts b/packages/api/src/app.ts index 8efda70..2c4de82 100644 --- a/packages/api/src/app.ts +++ b/packages/api/src/app.ts @@ -15,6 +15,7 @@ const OptionsSchema = z mode: z.enum(["single", "multi"]).optional(), styling: z.enum(["tailwind", "css"]).optional(), framework: z.enum(["next", "vite"]).optional(), + preview: z.boolean().optional(), verify: z.boolean().optional(), asyncVerify: z.boolean().optional(), maxRoutes: z.number().int().positive().optional(), @@ -238,10 +239,12 @@ export function createApp(deps: AppDeps): Hono { try { const out = await backend.submit(url, opts); - if (out.status === "queued") return c.json({ jobId: out.jobId, status: "queued" }, 202); + if (out.status === "queued") return c.json({ jobId: out.jobId, status: "queued" }, out.httpStatus); return c.json(out.result, 200); } catch (e) { - return c.json({ status: "failed", error: String(e).slice(0, 500) }, 500); + const msg = String(e); + if (msg.startsWith("BUSY:")) return c.json({ error: msg.slice(5).trim() }, 429); + return c.json({ status: "failed", error: msg.slice(0, 500) }, 500); } }); @@ -282,6 +285,40 @@ export function createApp(deps: AppDeps): Hono { return c.body(file.bytes); }); + // Pipeline progress events (poll every ~300ms while a clone runs). + app.get("/v1/clones/:id/events", async (c) => { + const after = Math.max(0, Number(c.req.query("after") ?? "0") || 0); + const events = backend.events ? await backend.events(c.req.param("id"), after) : null; + if (!events) return c.json({ error: "not found" }, 404); + return c.json({ jobId: c.req.param("id"), events }); + }); + + // Browsable preview of the built clone (static export published by the preview + // build under public/app-preview/). References are relative, so the export works + // from this mount — the only requirement is a trailing slash on the root. + const previewFile = async (c: Context, sub: string) => { + const id = c.req.param("id") ?? ""; + const tryPaths = sub.includes(".") + ? [`public/app-preview/${sub}`] + : [ + `public/app-preview/${sub}`.replace(/\/$/, "") + "/index.html", + `public/app-preview/${sub}`, + `public/app-preview/${sub}.html`, + ]; + for (const p of tryPaths) { + const file = await backend.file(id, p); + if (file) { + c.header("content-type", file.contentType); + c.header("content-length", String(file.bytes.length)); + return c.body(file.bytes); + } + } + return c.json({ error: "no app preview for this clone (preview builds are on by default for single-page clones; pass options.preview=true otherwise)" }, 404); + }; + app.get("/v1/clones/:id/app-preview", (c) => c.redirect(`/v1/clones/${c.req.param("id")}/app-preview/`, 302)); + app.get("/v1/clones/:id/app-preview/", (c) => previewFile(c, "index.html")); + app.get("/v1/clones/:id/app-preview/:path{.+}", (c) => previewFile(c, c.req.param("path") ?? "")); + app.delete("/v1/clones/:id", async (c) => { const ok = await backend.remove(c.req.param("id")); return c.json({ deleted: ok }, ok ? 200 : 404); diff --git a/packages/api/src/backend.ts b/packages/api/src/backend.ts index dbf8245..ae679de 100644 --- a/packages/api/src/backend.ts +++ b/packages/api/src/backend.ts @@ -1,5 +1,5 @@ import type { CloneOptions, RouteInfo } from "@cloner/core"; -import type { RestCloneResult } from "./rest.js"; +import type { RestCloneResult, RestCloneSummary } from "./rest.js"; export type JobStatus = "queued" | "running" | "succeeded" | "failed" | "cached"; @@ -22,7 +22,7 @@ export type JobView = { }; export type SubmitOutcome = - | { jobId: string; status: "succeeded" | "cached"; httpStatus: 200; result: RestCloneResult } + | { jobId: string; status: "succeeded" | "cached"; httpStatus: 200; result: RestCloneResult | RestCloneSummary } | { jobId: string; status: "queued"; httpStatus: 202 }; export type ResultOutcome = @@ -56,4 +56,6 @@ export interface Backend { facets(jobId: string): Promise; /** The whole app as one compressed archive (null if not ready/found). */ bundle(jobId: string, format?: BundleFormat): Promise; + /** Pipeline progress events for polling UIs (null if unsupported or job unknown). */ + events?(jobId: string, after?: number): Promise> | null>; } diff --git a/packages/api/src/backends/db.ts b/packages/api/src/backends/db.ts index c715c23..cee0def 100644 --- a/packages/api/src/backends/db.ts +++ b/packages/api/src/backends/db.ts @@ -142,4 +142,10 @@ export class DbBackend implements Backend { const url = this.deps.store.uploadBundle ? await this.deps.store.uploadBundle(jobId, format, bytes) : undefined; return { bytes, sha256: sha256hex(bytes), format, url }; } + + async events(jobId: string, after = 0): Promise> | null> { + const job = await repo.getJob(this.deps.db, jobId); + if (!job) return null; + return repo.listJobEvents(this.deps.db, jobId, after); + } } diff --git a/packages/api/src/backends/inMemory.ts b/packages/api/src/backends/inMemory.ts index cede034..2a48672 100644 --- a/packages/api/src/backends/inMemory.ts +++ b/packages/api/src/backends/inMemory.ts @@ -10,24 +10,51 @@ import type { Backend, BundleFormat, CloneBundle, FileFacet, JobView, ResultOutc export type RunJob = (input: RunCloneJobInput) => Promise; -/** Sync, in-memory backend (M1): runs the clone INLINE on submit and holds results - * in memory. POST returns the file map immediately (200). */ +/** In-memory backend: enqueues a clone (202) and runs it in the background so long + * captures (multi-viewport + verify) never hold a single HTTP connection open for + * minutes — which browsers surface as `TypeError: Failed to fetch` when the link + * drops (server restart, hot-reload, idle timeout). Poll /v1/clones/:id + /events. */ export class InMemoryBackend implements Backend { constructor(private deps: { store: InMemoryStore; runJob: RunJob; makeTempBase?: () => string; captureCacheDir?: string }) {} + private activeClones = 0; + private makeBase(): string { return (this.deps.makeTempBase ?? (() => mkdtempSync(join(tmpdir(), "api-clone-"))))(); } async submit(url: string, options: CloneOptions | undefined): Promise { + if (this.activeClones > 0 || this.deps.store.list().some((j) => j.status === "running")) { + throw new Error("BUSY: another clone is already running — wait for it to finish (use npm run dev:api:stable to avoid hot-reload killing Playwright)"); + } const id = randomUUID(); const base = this.makeBase(); + const events: Array> = []; + const kind: "clone" | "clone_site" = resolveCloneMode(options) === "multi" ? "clone_site" : "clone"; + const rec = { id, status: "running" as const, url, kind, options: options ?? {}, createdAt: Date.now(), base, events }; + this.deps.store.put(rec); + void this.runInBackground(id, url, options, rec, base, events, kind); + return { jobId: id, status: "queued", httpStatus: 202 }; + } + + private async runInBackground( + id: string, + url: string, + options: CloneOptions | undefined, + rec: { id: string; status: "running"; url: string; kind: "clone" | "clone_site"; options: CloneOptions; createdAt: number; base: string; events: Array> }, + base: string, + events: Array>, + kind: "clone" | "clone_site", + ): Promise { + this.activeClones++; + const log = (e: Record) => { + events.push({ t: Date.now(), ...e }); + this.deps.store.put({ ...rec, events: [...events] }); + }; try { - // captureCacheDir (persistent, shared across submits) powers the single→multi - // reuse: a single-page submit stashes its capture; a later multi-page submit for - // the same URL reuses it as the entry route (no re-capture) and expands the site. - const result = await this.deps.runJob({ url, options, runsDir: base, captureCacheDir: this.deps.captureCacheDir }); - this.deps.store.put({ id, status: "succeeded", url, kind: result.kind, options: result.options, createdAt: Date.now(), result, base }); + const result = await this.deps.runJob({ url, options, runsDir: base, captureCacheDir: this.deps.captureCacheDir, log }); + log({ event: "clone_done" }); + this.deps.store.put({ id, status: "succeeded", url, kind: result.kind, options: result.options, createdAt: rec.createdAt, result, base, events }); if (result.options.asyncVerify) { void verifyCloneJobResult(result, { validationConcurrency: result.options.validationConcurrency, @@ -39,18 +66,25 @@ export class InMemoryBackend implements Backend { result.verify = { error: String(e).slice(0, 500), async: true }; }); } - return { jobId: id, status: "succeeded", httpStatus: 200, result: buildRestResult(id, result, `/v1/clones/${id}/files`) }; } catch (e) { try { rmSync(base, { recursive: true, force: true }); } catch { /* best effort */ } - this.deps.store.put({ id, status: "failed", url, kind: resolveCloneMode(options) === "multi" ? "clone_site" : "clone", options: options ?? {}, createdAt: Date.now(), error: String(e) }); - throw e; + log({ event: "clone_error", error: String(e).slice(0, 300) }); + this.deps.store.put({ id, status: "failed", url, kind, options: options ?? {}, createdAt: rec.createdAt, error: String(e), events }); + } finally { + this.activeClones = Math.max(0, this.activeClones - 1); } } + async events(jobId: string): Promise> | null> { + const rec = this.deps.store.get(jobId); + if (!rec) return null; + return rec.events ?? []; + } + async status(jobId: string): Promise { const rec = this.deps.store.get(jobId); if (!rec) return null; diff --git a/packages/api/src/store.ts b/packages/api/src/store.ts index 2b9502b..073b314 100644 --- a/packages/api/src/store.ts +++ b/packages/api/src/store.ts @@ -6,7 +6,7 @@ import type { CloneJobResult, CloneOptions } from "@cloner/core"; * it is removed on eviction. M2/M4 replace this with DB rows + S3 objects. */ export type JobRecord = { id: string; - status: "succeeded" | "failed"; + status: "running" | "succeeded" | "failed"; url: string; kind: "clone" | "clone_site"; options: CloneOptions; @@ -14,6 +14,8 @@ export type JobRecord = { result?: CloneJobResult; base?: string; error?: string; + /** pipeline progress events (compiler log stream + service phases), poll via /events */ + events?: Array>; }; /** In-memory result store with TTL eviction. Swapped for a DB-backed store in M2. */ diff --git a/packages/api/test/app.fixture.test.ts b/packages/api/test/app.fixture.test.ts index 9038bbd..f126f12 100644 --- a/packages/api/test/app.fixture.test.ts +++ b/packages/api/test/app.fixture.test.ts @@ -26,13 +26,25 @@ describe("POST /v1/clones (real clone, served fixture)", { skip: hasChromium() ? headers: { "content-type": "application/json" }, body: JSON.stringify({ url: server.url + "/components.html", options: { interactions: false, components: true, motion: false } }), }); - assert.equal(res.status, 200); - const body = await res.json(); - assert.equal(body.status, "succeeded"); - assert.ok(body.files["src/app/page.tsx"], "has page.tsx"); - assert.equal(body.files["src/app/page.tsx"].type, "text"); - assert.ok(body.files["package.json"], "has package.json"); - assert.ok(body.capture.nodeCount > 0); - assert.equal(body.capture.blocked, false); + assert.equal(res.status, 202); + const queued = await res.json(); + let body: Record | undefined; + for (let i = 0; i < 600; i++) { + const view = await (await app.request(`/v1/clones/${queued.jobId}`)).json(); + if (view.status === "succeeded") { + body = await (await app.request(`/v1/clones/${queued.jobId}/result`)).json(); + break; + } + if (view.status === "failed") assert.fail(view.error); + await new Promise((r) => setTimeout(r, 50)); + } + assert.ok(body); + const files = body!.files as Record; + assert.equal(body!.status, "succeeded"); + assert.ok(files["src/app/page.tsx"], "has page.tsx"); + assert.equal(files["src/app/page.tsx"].type, "text"); + assert.ok(files["package.json"], "has package.json"); + assert.ok((body!.capture as { nodeCount: number }).nodeCount > 0); + assert.equal((body!.capture as { blocked: boolean }).blocked, false); }); }); diff --git a/packages/api/test/app.test.ts b/packages/api/test/app.test.ts index d28b537..1ef236b 100644 --- a/packages/api/test/app.test.ts +++ b/packages/api/test/app.test.ts @@ -34,7 +34,17 @@ const fakeRunJob: RunJob = async (input) => { } satisfies CloneJobResult; }; -test("POST /v1/clones returns the eager file map; binaries by reference; streaming + lifecycle", async () => { +async function waitForResult(app: ReturnType, jobId: string) { + for (let i = 0; i < 200; i++) { + const view = await (await app.request(`/v1/clones/${jobId}`)).json(); + if (view.status === "succeeded") return await (await app.request(`/v1/clones/${jobId}/result`)).json(); + if (view.status === "failed") throw new Error(String(view.error)); + await new Promise((r) => setTimeout(r, 5)); + } + throw new Error("timeout waiting for clone"); +} + +test("POST /v1/clones enqueues then completes; binaries by reference; streaming + lifecycle", async () => { const store = new InMemoryStore(60_000); const app = createApp({ backend: new InMemoryBackend({ store, runJob: fakeRunJob }) }); try { @@ -43,39 +53,47 @@ test("POST /v1/clones returns the eager file map; binaries by reference; streami headers: { "content-type": "application/json" }, body: JSON.stringify({ url: "https://example.com/", options: {} }), }); - assert.equal(res.status, 200); - const body = await res.json(); - assert.ok(body.jobId); - assert.equal(body.status, "succeeded"); - assert.equal(body.kind, "clone"); + assert.equal(res.status, 202); + const queued = await res.json(); + assert.ok(queued.jobId); + assert.equal(queued.status, "queued"); + + let body: Record | undefined; + body = await waitForResult(app, queued.jobId); + assert.ok(body); + assert.equal(body!.status, "succeeded"); + assert.equal(body!.kind, "clone"); // Text inline. - const page = body.files["src/app/page.tsx"]; - assert.equal(page.type, "text"); - assert.ok(page.content.includes("Page")); - assert.ok(page.sha256); + const files = body!.files as Record; + const page = files["src/app/page.tsx"]; + assert.ok(page); + assert.equal(page!.type, "text"); + assert.ok(page!.content!.includes("Page")); + assert.ok(page!.sha256); // Binary by reference (URL, not bytes). - const bin = body.files["public/assets/cloned/images/a.png"]; - assert.equal(bin.type, "binary"); - assert.equal(bin.content, undefined); - assert.equal(bin.url, `/v1/clones/${body.jobId}/files/public/assets/cloned/images/a.png`); + const bin = files["public/assets/cloned/images/a.png"]; + assert.ok(bin); + assert.equal(bin!.type, "binary"); + assert.equal(bin!.content, undefined); + assert.equal(bin!.url, `/v1/clones/${queued.jobId}/files/public/assets/cloned/images/a.png`); // Per-file streaming returns the actual bytes. - const fileRes = await app.request(bin.url); + const fileRes = await app.request(bin!.url!); assert.equal(fileRes.status, 200); assert.equal(fileRes.headers.get("content-type"), "image/png"); const bytes = Buffer.from(await fileRes.arrayBuffer()); assert.equal(bytes.length, 7); // Cheap metadata overview (no file contents). - const meta = await (await app.request(`/v1/clones/${body.jobId}`)).json(); + const meta = await (await app.request(`/v1/clones/${queued.jobId}`)).json(); assert.equal(meta.fileCount, 3); assert.equal(meta.capture.nodeCount, 42); assert.equal(meta.totalBytes > 0, true); // Full result fetch. - const result = await app.request(`/v1/clones/${body.jobId}/result`); + const result = await app.request(`/v1/clones/${queued.jobId}/result`); assert.equal(result.status, 200); // List. @@ -83,9 +101,9 @@ test("POST /v1/clones returns the eager file map; binaries by reference; streami assert.equal(list.clones.length, 1); // Delete purges; subsequent fetch 404s. - const del = await app.request(`/v1/clones/${body.jobId}`, { method: "DELETE" }); + const del = await app.request(`/v1/clones/${queued.jobId}`, { method: "DELETE" }); assert.equal((await del.json()).deleted, true); - assert.equal((await app.request(`/v1/clones/${body.jobId}`)).status, 404); + assert.equal((await app.request(`/v1/clones/${queued.jobId}`)).status, 404); } finally { store.clear(); } @@ -118,8 +136,9 @@ test("POST /v1/clones normalizes product options and legacy aliases", async () = headers: { "content-type": "application/json" }, body: JSON.stringify({ url: "https://example.com/", options: { mode: "multi", styling: "css", framework: "vite" } }), }); - assert.equal(product.status, 200); - const productBody = await product.json(); + assert.equal(product.status, 202); + const productQueued = await product.json(); + const productBody = await waitForResult(app, productQueued.jobId); assert.deepEqual(productBody.options, { mode: "multi", styling: "css", framework: "vite" }); const legacy = await app.request("/v1/clones", { @@ -127,8 +146,9 @@ test("POST /v1/clones normalizes product options and legacy aliases", async () = headers: { "content-type": "application/json" }, body: JSON.stringify({ url: "https://example.com/", options: { multiPage: true, humanizeMode: "css" } }), }); - assert.equal(legacy.status, 200); - const legacyBody = await legacy.json(); + assert.equal(legacy.status, 202); + const legacyQueued = await legacy.json(); + const legacyBody = await waitForResult(app, legacyQueued.jobId); assert.deepEqual(legacyBody.options, { mode: "multi", styling: "css", framework: "next" }); } finally { store.clear(); diff --git a/packages/api/test/mcp.test.ts b/packages/api/test/mcp.test.ts index 4e4d890..0f4395b 100644 --- a/packages/api/test/mcp.test.ts +++ b/packages/api/test/mcp.test.ts @@ -52,12 +52,18 @@ test("MCP: list-then-read + bundle contract (never floods context)", async () => // clone_website → jobId + status only (no files). const cw = parse(await client.callTool({ name: "clone_website", arguments: { url: "https://example.com/", options: {} } })); assert.ok(cw.jobId); - assert.equal(cw.status, "succeeded"); - assert.ok(!("files" in cw)); + assert.equal(cw.status, "queued"); const jobId = cw.jobId; - // status - const status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } })); + let status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } })); + for (let i = 0; i < 200 && status.status !== "succeeded"; i++) { + await new Promise((r) => setTimeout(r, 5)); + status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } })); + } + assert.equal(status.status, "succeeded"); + + // status (re-fetch after completion) + status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } })); assert.equal(status.status, "succeeded"); // get_clone_result → metadata only, NO file contents. diff --git a/packages/core/src/options.ts b/packages/core/src/options.ts index 138f98b..56b124b 100644 --- a/packages/core/src/options.ts +++ b/packages/core/src/options.ts @@ -9,6 +9,7 @@ export type ResolvedCloneOptions = CloneOptions & { interactions: boolean; components: boolean; motion: boolean; + preview: boolean; }; export function resolveCloneMode(options: CloneOptions = {}): CloneMode { @@ -53,5 +54,8 @@ export function resolveCloneOptions(options: CloneOptions = {}): ResolvedCloneOp interactions: options.interactions ?? true, components: options.components ?? true, motion: options.motion ?? true, + // Preview builds are the single-page product surface; multi-page exports can be + // large, so previewing a site clone stays an explicit opt-in. + preview: options.preview ?? mode === "single", }; } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index ec6de84..5f5174e 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -26,6 +26,9 @@ export type CloneOptions = { mode?: CloneMode; styling?: CloneStyling; framework?: CloneFramework; + /** Build a browsable static export of the clone into `public/app-preview/` + * (served by the API's app-preview route). Defaults ON for single-page clones. */ + preview?: boolean; verify?: boolean; /** DB/worker mode: persist the clone first, then attach verify in the background. */ asyncVerify?: boolean; @@ -75,7 +78,7 @@ export type CaptureSanity = { blocked: boolean; // bot/egress wall text detected }; -export type CloneTimings = { captureMs: number; generateMs: number; verifyMs?: number }; +export type CloneTimings = { captureMs: number; generateMs: number; verifyMs?: number; previewMs?: number }; export type RouteInfo = { route: string; representativeOf?: string }; diff --git a/packages/db/migrations/0002_productive_wallflower.sql b/packages/db/migrations/0002_productive_wallflower.sql new file mode 100644 index 0000000..df1554c --- /dev/null +++ b/packages/db/migrations/0002_productive_wallflower.sql @@ -0,0 +1,10 @@ +CREATE TABLE "job_events" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "job_id" uuid NOT NULL, + "seq" integer NOT NULL, + "payload" jsonb NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "job_events" ADD CONSTRAINT "job_events_job_id_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."jobs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "job_events_job_id_seq_idx" ON "job_events" USING btree ("job_id","seq"); \ No newline at end of file diff --git a/packages/db/migrations/meta/0002_snapshot.json b/packages/db/migrations/meta/0002_snapshot.json new file mode 100644 index 0000000..fb3109a --- /dev/null +++ b/packages/db/migrations/meta/0002_snapshot.json @@ -0,0 +1,461 @@ +{ + "id": "de9b94b8-cefb-4d2a-a848-79a552976943", + "prevId": "e076bd1e-a7ce-40ab-abd1-9eb11d00ddb9", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rate_limit": { + "name": "rate_limit", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "key_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cache": { + "name": "cache", + "schema": "", + "columns": { + "cache_key": { + "name": "cache_key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "options_hash": { + "name": "options_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "compiler_version": { + "name": "compiler_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "cache_job_id_jobs_id_fk": { + "name": "cache_job_id_jobs_id_fk", + "tableFrom": "cache", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.clones": { + "name": "clones", + "schema": "", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "route_count": { + "name": "route_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "file_manifest": { + "name": "file_manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "bundle_s3_key": { + "name": "bundle_s3_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "verify": { + "name": "verify", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "capture_meta": { + "name": "capture_meta", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "clones_job_id_jobs_id_fk": { + "name": "clones_job_id_jobs_id_fk", + "tableFrom": "clones", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.job_events": { + "name": "job_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "job_events_job_id_seq_idx": { + "name": "job_events_job_id_seq_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "job_events_job_id_jobs_id_fk": { + "name": "job_events_job_id_jobs_id_fk", + "tableFrom": "job_events", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jobs": { + "name": "jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "cache_key": { + "name": "cache_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "compiler_version": { + "name": "compiler_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "timings": { + "name": "timings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.signup_tokens": { + "name": "signup_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "signup_tokens_token_hash_unique": { + "name": "signup_tokens_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index 06f02ca..ac09d1d 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1782790200000, "tag": "0001_signup_tokens", "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1783202414928, + "tag": "0002_productive_wallflower", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/packages/db/src/repo.ts b/packages/db/src/repo.ts index 75e392a..0f7f089 100644 --- a/packages/db/src/repo.ts +++ b/packages/db/src/repo.ts @@ -1,6 +1,6 @@ import { and, desc, eq, gt, isNull, sql } from "drizzle-orm"; import type { Db } from "./client.js"; -import { jobs, clones, cache, apiKeys, signupTokens, type Job, type NewJob, type Clone, type NewClone, type CacheRow, type ApiKey, type SignupToken } from "./schema.js"; +import { jobs, clones, cache, apiKeys, signupTokens, jobEvents, type Job, type NewJob, type Clone, type NewClone, type CacheRow, type ApiKey, type SignupToken } from "./schema.js"; // ---- jobs ---- @@ -111,3 +111,24 @@ export async function consumeSignupToken(db: Db, tokenHash: string): Promise): Promise { + const [row] = await db + .select({ n: sql`coalesce(max(${jobEvents.seq}), 0)` }) + .from(jobEvents) + .where(eq(jobEvents.jobId, jobId)); + const seq = (row?.n ?? 0) + 1; + await db.insert(jobEvents).values({ jobId, seq, payload: { t: Date.now(), ...payload } }); + return seq; +} + +export async function listJobEvents(db: Db, jobId: string, after = 0): Promise>> { + const rows = await db + .select() + .from(jobEvents) + .where(and(eq(jobEvents.jobId, jobId), sql`${jobEvents.seq} > ${after}`)) + .orderBy(jobEvents.seq); + return rows.map((r) => r.payload as Record); +} diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 30f28a5..bbbeca8 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -1,4 +1,4 @@ -import { pgTable, text, jsonb, integer, timestamp, uuid } from "drizzle-orm/pg-core"; +import { pgTable, text, jsonb, integer, timestamp, uuid, index } from "drizzle-orm/pg-core"; /** Job lifecycle row. Status mirrors the queue; `cacheKey` ties it to the cache row. * `options`/`timings` are jsonb (the service's typed shapes). */ @@ -66,6 +66,17 @@ export const signupTokens = pgTable("signup_tokens", { consumedAt: timestamp("consumed_at", { withTimezone: true }), }); +/** Structured clone progress events (mirrors in-memory backend event stream). */ +export const jobEvents = pgTable("job_events", { + id: uuid("id").defaultRandom().primaryKey(), + jobId: uuid("job_id") + .notNull() + .references(() => jobs.id, { onDelete: "cascade" }), + seq: integer("seq").notNull(), + payload: jsonb("payload").notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), +}, (t) => [index("job_events_job_id_seq_idx").on(t.jobId, t.seq)]); + export type Job = typeof jobs.$inferSelect; export type NewJob = typeof jobs.$inferInsert; export type Clone = typeof clones.$inferSelect; @@ -74,3 +85,4 @@ export type CacheRow = typeof cache.$inferSelect; export type ApiKey = typeof apiKeys.$inferSelect; export type SignupToken = typeof signupTokens.$inferSelect; export type NewSignupToken = typeof signupTokens.$inferInsert; +export type JobEvent = typeof jobEvents.$inferSelect; diff --git a/packages/worker/src/processJob.ts b/packages/worker/src/processJob.ts index 21550c5..809ffce 100644 --- a/packages/worker/src/processJob.ts +++ b/packages/worker/src/processJob.ts @@ -37,7 +37,16 @@ export async function processCloneJob(deps: ProcessDeps, jobId: string): Promise const options = (job.options ?? {}) as CloneOptions; // Provision the isolated harness only when this job actually verifies (build). const harnessDir = (options.verify || options.asyncVerify) && deps.harnessProvider ? await deps.harnessProvider() : undefined; - const result = await deps.runJob({ url: job.url, options, runsDir: base, harnessDir, tier: deps.tier, captureCacheDir: deps.captureCacheDir || undefined, captureCacheTtlMs: deps.cacheTtlMs }); + const result = await deps.runJob({ + url: job.url, + options, + runsDir: base, + harnessDir, + tier: deps.tier, + captureCacheDir: deps.captureCacheDir || undefined, + captureCacheTtlMs: deps.cacheTtlMs, + log: (e) => { void repo.appendJobEvent(db, jobId, e); }, + }); const manifest = await store.putClone(jobId, result.files); const envelope = { files: manifest.files, routes: result.routes, bundleKey: manifest.bundleKey };