bzuccaro/ditto.site
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Service: job event stream, async in-memory backend, preview routes

Browse Source 
Ported from PR #15 (packages-side surface only; the fork's dev UI page is
intentionally dropped - the API surface is the product):
- job_events table + repository, worker emits stage transitions, additive
  GET /v1/clones/:id/events polling route. The fork's hand-written
  migration was never registered in Drizzle's journal (it silently never
  applied); regenerated properly via drizzle-kit from schema.ts with the
  (job_id, seq) index declared, and verified against a live Postgres
- In-memory backend moves to the documented enqueue-202 + poll contract
  with single-flight BUSY guard; POST honors backend httpStatus
- app-preview static serving routes; the preview BUILD half is deferred
  until the compiler exports buildApp/DEFAULT_HARNESS_DIR - routes 404
  cleanly until then
- preview option threaded through core types; cache key intentionally
  unchanged while the flag is inert

All workspaces typecheck; api 18/18, core 13/13, worker 1/1 pass;
migration verified applying (table + index + FK) on postgres:16.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Samraaj Bath
2026-07-04 15:00:55 -07:00
co-authored by Claude Fable 5
parent f991f469cf
commit bbe4a5643b
16 changed files with 701 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/api/test/app.fixture.test.ts
+20 -8
View File
@@ -26,13 +26,25 @@ describe("POST /v1/clones (real clone, served fixture)", { skip: hasChromium() ?
headers: { "content-type": "application/json" },
body: JSON.stringify({ url: server.url + "/components.html", options: { interactions: false, components: true, motion: false } }),
});
assert.equal(res.status, 200);
const body = await res.json();
assert.equal(body.status, "succeeded");
assert.ok(body.files["src/app/page.tsx"], "has page.tsx");
assert.equal(body.files["src/app/page.tsx"].type, "text");
assert.ok(body.files["package.json"], "has package.json");
assert.ok(body.capture.nodeCount > 0);
assert.equal(body.capture.blocked, false);
assert.equal(res.status, 202);
const queued = await res.json();
let body: Record<string, unknown> | undefined;
for (let i = 0; i < 600; i++) {
const view = await (await app.request(`/v1/clones/${queued.jobId}`)).json();
if (view.status === "succeeded") {
body = await (await app.request(`/v1/clones/${queued.jobId}/result`)).json();
break;
}
if (view.status === "failed") assert.fail(view.error);
await new Promise((r) => setTimeout(r, 50));
}
assert.ok(body);
const files = body!.files as Record<string, { type: string }>;
assert.equal(body!.status, "succeeded");
assert.ok(files["src/app/page.tsx"], "has page.tsx");
assert.equal(files["src/app/page.tsx"].type, "text");
assert.ok(files["package.json"], "has package.json");
assert.ok((body!.capture as { nodeCount: number }).nodeCount > 0);
assert.equal((body!.capture as { blocked: boolean }).blocked, false);
});
});
packages/api/test/app.test.ts
+43 -23
View File
@@ -34,7 +34,17 @@ const fakeRunJob: RunJob = async (input) => {
} satisfies CloneJobResult;
};
test("POST /v1/clones returns the eager file map; binaries by reference; streaming + lifecycle", async () => {
async function waitForResult(app: ReturnType<typeof createApp>, jobId: string) {
for (let i = 0; i < 200; i++) {
const view = await (await app.request(`/v1/clones/${jobId}`)).json();
if (view.status === "succeeded") return await (await app.request(`/v1/clones/${jobId}/result`)).json();
if (view.status === "failed") throw new Error(String(view.error));
await new Promise((r) => setTimeout(r, 5));
}
throw new Error("timeout waiting for clone");
}
test("POST /v1/clones enqueues then completes; binaries by reference; streaming + lifecycle", async () => {
const store = new InMemoryStore(60_000);
const app = createApp({ backend: new InMemoryBackend({ store, runJob: fakeRunJob }) });
try {
@@ -43,39 +53,47 @@ test("POST /v1/clones returns the eager file map; binaries by reference; streami
headers: { "content-type": "application/json" },
body: JSON.stringify({ url: "https://example.com/", options: {} }),
});
assert.equal(res.status, 200);
const body = await res.json();
assert.ok(body.jobId);
assert.equal(body.status, "succeeded");
assert.equal(body.kind, "clone");
assert.equal(res.status, 202);
const queued = await res.json();
assert.ok(queued.jobId);
assert.equal(queued.status, "queued");
let body: Record<string, unknown> | undefined;
body = await waitForResult(app, queued.jobId);
assert.ok(body);
assert.equal(body!.status, "succeeded");
assert.equal(body!.kind, "clone");
// Text inline.
const page = body.files["src/app/page.tsx"];
assert.equal(page.type, "text");
assert.ok(page.content.includes("Page"));
assert.ok(page.sha256);
const files = body!.files as Record<string, { type: string; content?: string; sha256?: string; url?: string }>;
const page = files["src/app/page.tsx"];
assert.ok(page);
assert.equal(page!.type, "text");
assert.ok(page!.content!.includes("Page"));
assert.ok(page!.sha256);
// Binary by reference (URL, not bytes).
const bin = body.files["public/assets/cloned/images/a.png"];
assert.equal(bin.type, "binary");
assert.equal(bin.content, undefined);
assert.equal(bin.url, `/v1/clones/${body.jobId}/files/public/assets/cloned/images/a.png`);
const bin = files["public/assets/cloned/images/a.png"];
assert.ok(bin);
assert.equal(bin!.type, "binary");
assert.equal(bin!.content, undefined);
assert.equal(bin!.url, `/v1/clones/${queued.jobId}/files/public/assets/cloned/images/a.png`);
// Per-file streaming returns the actual bytes.
const fileRes = await app.request(bin.url);
const fileRes = await app.request(bin!.url!);
assert.equal(fileRes.status, 200);
assert.equal(fileRes.headers.get("content-type"), "image/png");
const bytes = Buffer.from(await fileRes.arrayBuffer());
assert.equal(bytes.length, 7);
// Cheap metadata overview (no file contents).
const meta = await (await app.request(`/v1/clones/${body.jobId}`)).json();
const meta = await (await app.request(`/v1/clones/${queued.jobId}`)).json();
assert.equal(meta.fileCount, 3);
assert.equal(meta.capture.nodeCount, 42);
assert.equal(meta.totalBytes > 0, true);
// Full result fetch.
const result = await app.request(`/v1/clones/${body.jobId}/result`);
const result = await app.request(`/v1/clones/${queued.jobId}/result`);
assert.equal(result.status, 200);
// List.
@@ -83,9 +101,9 @@ test("POST /v1/clones returns the eager file map; binaries by reference; streami
assert.equal(list.clones.length, 1);
// Delete purges; subsequent fetch 404s.
const del = await app.request(`/v1/clones/${body.jobId}`, { method: "DELETE" });
const del = await app.request(`/v1/clones/${queued.jobId}`, { method: "DELETE" });
assert.equal((await del.json()).deleted, true);
assert.equal((await app.request(`/v1/clones/${body.jobId}`)).status, 404);
assert.equal((await app.request(`/v1/clones/${queued.jobId}`)).status, 404);
} finally {
store.clear();
}
@@ -118,8 +136,9 @@ test("POST /v1/clones normalizes product options and legacy aliases", async () =
headers: { "content-type": "application/json" },
body: JSON.stringify({ url: "https://example.com/", options: { mode: "multi", styling: "css", framework: "vite" } }),
});
assert.equal(product.status, 200);
const productBody = await product.json();
assert.equal(product.status, 202);
const productQueued = await product.json();
const productBody = await waitForResult(app, productQueued.jobId);
assert.deepEqual(productBody.options, { mode: "multi", styling: "css", framework: "vite" });
const legacy = await app.request("/v1/clones", {
@@ -127,8 +146,9 @@ test("POST /v1/clones normalizes product options and legacy aliases", async () =
headers: { "content-type": "application/json" },
body: JSON.stringify({ url: "https://example.com/", options: { multiPage: true, humanizeMode: "css" } }),
});
assert.equal(legacy.status, 200);
const legacyBody = await legacy.json();
assert.equal(legacy.status, 202);
const legacyQueued = await legacy.json();
const legacyBody = await waitForResult(app, legacyQueued.jobId);
assert.deepEqual(legacyBody.options, { mode: "multi", styling: "css", framework: "next" });
} finally {
store.clear();
packages/api/test/mcp.test.ts
+10 -4
View File
@@ -52,12 +52,18 @@ test("MCP: list-then-read + bundle contract (never floods context)", async () =>
// clone_website → jobId + status only (no files).
const cw = parse(await client.callTool({ name: "clone_website", arguments: { url: "https://example.com/", options: {} } }));
assert.ok(cw.jobId);
assert.equal(cw.status, "succeeded");
assert.ok(!("files" in cw));
assert.equal(cw.status, "queued");
const jobId = cw.jobId;
// status
const status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
let status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
for (let i = 0; i < 200 && status.status !== "succeeded"; i++) {
await new Promise((r) => setTimeout(r, 5));
status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
}
assert.equal(status.status, "succeeded");
// status (re-fetch after completion)
status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
assert.equal(status.status, "succeeded");
// get_clone_result → metadata only, NO file contents.