Initial commit

packages/api/test/app.fixture.test.ts

@@ -0,0 +1,38 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { runCloneJob } from "@cloner/core";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+import { createApp } from "../src/app.js";
+import { InMemoryStore } from "../src/store.js";
+import { InMemoryBackend } from "../src/backends/inMemory.js";
+
+// End-to-end: real clone of a served fixture through the HTTP layer. Skipped when
+// no Chromium is installed.
+describe("POST /v1/clones (real clone, served fixture)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  const store = new InMemoryStore(60_000);
+  before(async () => {
+    server = await serveDir(FIXTURES_DIR);
+  });
+  after(async () => {
+    store.clear();
+    await server.close();
+  });
+
+  it("clones a fixture and returns the real generated app file map", async () => {
+    const app = createApp({ backend: new InMemoryBackend({ store, runJob: runCloneJob }) });
+    const res = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url: server.url + "/components.html", options: { interactions: false, components: true, motion: false } }),
+    });
+    assert.equal(res.status, 200);
+    const body = await res.json();
+    assert.equal(body.status, "succeeded");
+    assert.ok(body.files["src/app/page.tsx"], "has page.tsx");
+    assert.equal(body.files["src/app/page.tsx"].type, "text");
+    assert.ok(body.files["package.json"], "has package.json");
+    assert.ok(body.capture.nodeCount > 0);
+    assert.equal(body.capture.blocked, false);
+  });
+});

packages/api/test/app.test.ts

@@ -0,0 +1,143 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { collectFileMap, type CloneJobResult } from "@cloner/core";
+import { createApp } from "../src/app.js";
+import { InMemoryStore } from "../src/store.js";
+import { InMemoryBackend, type RunJob } from "../src/backends/inMemory.js";
+
+/** A browser-free fake clone: writes a tiny generated app under the provided temp
+ *  base, then returns a real CloneJobResult via collectFileMap. Proves the REST
+ *  file-map contract (text inline + binary by reference + per-file streaming)
+ *  without launching Chromium. */
+const fakeRunJob: RunJob = async (input) => {
+  const base = input.runsDir!;
+  const app = join(base, "generated", "app");
+  mkdirSync(join(app, "src", "app"), { recursive: true });
+  mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+  writeFileSync(join(app, "package.json"), '{"name":"cloned-app"}\n');
+  writeFileSync(join(app, "src", "app", "page.tsx"), "export default function Page(){return <div/>}\n");
+  const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]);
+  writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), png);
+  const files = collectFileMap(base);
+  return {
+    url: input.url,
+    kind: "clone",
+    options: input.options ?? {},
+    status: "succeeded",
+    compilerVersion: "test-0",
+    timings: { captureMs: 5, generateMs: 0 },
+    files,
+    capture: { nodeCount: 42, pollution: false, blocked: false },
+    runDir: base,
+  } satisfies CloneJobResult;
+};
+
+test("POST /v1/clones returns the eager file map; binaries by reference; streaming + lifecycle", async () => {
+  const store = new InMemoryStore(60_000);
+  const app = createApp({ backend: new InMemoryBackend({ store, runJob: fakeRunJob }) });
+  try {
+    const res = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url: "https://example.com/", options: {} }),
+    });
+    assert.equal(res.status, 200);
+    const body = await res.json();
+    assert.ok(body.jobId);
+    assert.equal(body.status, "succeeded");
+    assert.equal(body.kind, "clone");
+
+    // Text inline.
+    const page = body.files["src/app/page.tsx"];
+    assert.equal(page.type, "text");
+    assert.ok(page.content.includes("Page"));
+    assert.ok(page.sha256);
+
+    // Binary by reference (URL, not bytes).
+    const bin = body.files["public/assets/cloned/images/a.png"];
+    assert.equal(bin.type, "binary");
+    assert.equal(bin.content, undefined);
+    assert.equal(bin.url, `/v1/clones/${body.jobId}/files/public/assets/cloned/images/a.png`);
+
+    // Per-file streaming returns the actual bytes.
+    const fileRes = await app.request(bin.url);
+    assert.equal(fileRes.status, 200);
+    assert.equal(fileRes.headers.get("content-type"), "image/png");
+    const bytes = Buffer.from(await fileRes.arrayBuffer());
+    assert.equal(bytes.length, 7);
+
+    // Cheap metadata overview (no file contents).
+    const meta = await (await app.request(`/v1/clones/${body.jobId}`)).json();
+    assert.equal(meta.fileCount, 3);
+    assert.equal(meta.capture.nodeCount, 42);
+    assert.equal(meta.totalBytes > 0, true);
+
+    // Full result fetch.
+    const result = await app.request(`/v1/clones/${body.jobId}/result`);
+    assert.equal(result.status, 200);
+
+    // List.
+    const list = await (await app.request("/v1/clones")).json();
+    assert.equal(list.clones.length, 1);
+
+    // Delete purges; subsequent fetch 404s.
+    const del = await app.request(`/v1/clones/${body.jobId}`, { method: "DELETE" });
+    assert.equal((await del.json()).deleted, true);
+    assert.equal((await app.request(`/v1/clones/${body.jobId}`)).status, 404);
+  } finally {
+    store.clear();
+  }
+});
+
+test("POST /v1/clones validates the body", async () => {
+  const store = new InMemoryStore(60_000);
+  const app = createApp({ backend: new InMemoryBackend({ store, runJob: fakeRunJob }) });
+  try {
+    assert.equal((await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status, 400);
+    assert.equal(
+      (await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ url: "ftp://x" }) })).status,
+      400,
+    );
+    assert.equal(
+      (await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ url: "https://x.com", options: { bogus: 1 } }) })).status,
+      400,
+    );
+  } finally {
+    store.clear();
+  }
+});
+
+test("POST /v1/clones normalizes product options and legacy aliases", async () => {
+  const store = new InMemoryStore(60_000);
+  const app = createApp({ backend: new InMemoryBackend({ store, runJob: fakeRunJob }) });
+  try {
+    const product = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url: "https://example.com/", options: { mode: "multi", styling: "css", framework: "vite" } }),
+    });
+    assert.equal(product.status, 200);
+    const productBody = await product.json();
+    assert.deepEqual(productBody.options, { mode: "multi", styling: "css", framework: "vite" });
+
+    const legacy = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url: "https://example.com/", options: { multiPage: true, humanizeMode: "css" } }),
+    });
+    assert.equal(legacy.status, 200);
+    const legacyBody = await legacy.json();
+    assert.deepEqual(legacyBody.options, { mode: "multi", styling: "css", framework: "next" });
+  } finally {
+    store.clear();
+  }
+});
+
+test("GET /healthz", async () => {
+  const app = createApp({ backend: new InMemoryBackend({ store: new InMemoryStore(1000), runJob: fakeRunJob }) });
+  const res = await app.request("/healthz");
+  assert.equal(res.status, 200);
+  assert.deepEqual(await res.json(), { ok: true });
+});

packages/api/test/auth.test.ts

@@ -0,0 +1,48 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { createApp } from "../src/app.js";
+import { InMemoryStore } from "../src/store.js";
+import { InMemoryBackend, type RunJob } from "../src/backends/inMemory.js";
+import { hashApiKey } from "../src/auth.js";
+
+const noopRun: RunJob = async () => {
+  throw new Error("clone should not run in these tests");
+};
+
+function appWith(opts: Partial<Parameters<typeof createApp>[0]>) {
+  return createApp({ backend: new InMemoryBackend({ store: new InMemoryStore(1000), runJob: noopRun }), ...opts });
+}
+
+test("auth: 401 without key / with wrong key, 200 with a valid key; /healthz stays open", async () => {
+  const app = appWith({ auth: { keyHashes: new Set([hashApiKey("s3cret")]) } });
+  assert.equal((await app.request("/v1/clones")).status, 401);
+  assert.equal((await app.request("/v1/clones", { headers: { authorization: "Bearer wrong" } })).status, 401);
+  assert.equal((await app.request("/v1/clones", { headers: { "x-api-key": "s3cret" } })).status, 200);
+  assert.equal((await app.request("/v1/clones", { headers: { authorization: "Bearer s3cret" } })).status, 200);
+  assert.equal((await app.request("/healthz")).status, 200, "health check is unauthenticated");
+});
+
+test("rate limit: 429 once the per-minute cap is exceeded", async () => {
+  const app = appWith({ rateLimitPerMinute: 2 });
+  const headers = { "x-forwarded-for": "9.9.9.9" };
+  assert.equal((await app.request("/v1/clones", { headers })).status, 200);
+  assert.equal((await app.request("/v1/clones", { headers })).status, 200);
+  const limited = await app.request("/v1/clones", { headers });
+  assert.equal(limited.status, 429);
+  assert.ok(limited.headers.get("retry-after"));
+});
+
+test("ssrf: a submit is rejected (400) when the URL guard throws", async () => {
+  const app = appWith({
+    assertUrl: async (u) => {
+      if (u.includes("169.254")) throw new Error("blocked address");
+    },
+  });
+  const res = await app.request("/v1/clones", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ url: "http://169.254.169.254/" }),
+  });
+  assert.equal(res.status, 400);
+  assert.equal((await res.json()).error, "url not allowed");
+});

packages/api/test/mcp.test.ts

@@ -0,0 +1,107 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { collectFileMap, type CloneJobResult } from "@cloner/core";
+import { createMcpServer } from "../src/mcp.js";
+import { InMemoryStore } from "../src/store.js";
+import { InMemoryBackend, type RunJob } from "../src/backends/inMemory.js";
+
+const fakeRunJob: RunJob = async (input) => {
+  const base = input.runsDir!;
+  const app = join(base, "generated", "app");
+  mkdirSync(join(app, "src", "app"), { recursive: true });
+  mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+  writeFileSync(join(app, "package.json"), '{"name":"cloned-app"}\n');
+  writeFileSync(join(app, "src", "app", "page.tsx"), "export default function Page(){return <div/>}\n");
+  writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), Buffer.from([1, 2, 3, 4]));
+  return {
+    url: input.url,
+    kind: "clone",
+    options: input.options ?? {},
+    status: "succeeded",
+    compilerVersion: "test-0",
+    timings: { captureMs: 1, generateMs: 0 },
+    files: collectFileMap(base),
+    capture: { nodeCount: 9, pollution: false, blocked: false },
+    runDir: base,
+  } satisfies CloneJobResult;
+};
+
+async function connect(backend: InMemoryBackend) {
+  const server = createMcpServer(backend, { baseUrl: "http://test" });
+  const [clientT, serverT] = InMemoryTransport.createLinkedPair();
+  const client = new Client({ name: "test-client", version: "0.0.0" });
+  await Promise.all([server.connect(serverT), client.connect(clientT)]);
+  return client;
+}
+const parse = (res: unknown): any => JSON.parse((res as { content: { text: string }[] }).content[0]!.text);
+
+test("MCP: list-then-read + bundle contract (never floods context)", async () => {
+  const store = new InMemoryStore(60_000);
+  const backend = new InMemoryBackend({ store, runJob: fakeRunJob });
+  const client = await connect(backend);
+  try {
+    const tools = (await client.listTools()).tools.map((t) => t.name);
+    for (const n of ["clone_website", "get_clone_status", "get_clone_result", "list_clone_files", "read_clone_files", "get_clone_bundle", "cancel_clone"]) {
+      assert.ok(tools.includes(n), `tool ${n} registered`);
+    }
+
+    // clone_website → jobId + status only (no files).
+    const cw = parse(await client.callTool({ name: "clone_website", arguments: { url: "https://example.com/", options: {} } }));
+    assert.ok(cw.jobId);
+    assert.equal(cw.status, "succeeded");
+    assert.ok(!("files" in cw));
+    const jobId = cw.jobId;
+
+    // status
+    const status = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
+    assert.equal(status.status, "succeeded");
+
+    // get_clone_result → metadata only, NO file contents.
+    const meta = parse(await client.callTool({ name: "get_clone_result", arguments: { jobId } }));
+    assert.equal(meta.fileCount, 3);
+    assert.equal(meta.capture.nodeCount, 9);
+    assert.ok(!("files" in meta), "result metadata must not include file contents");
+    assert.ok(meta.bundleUrl.includes("/bundle"));
+
+    // list_clone_files (glob) → manifest only, no content.
+    const list = parse(await client.callTool({ name: "list_clone_files", arguments: { jobId, glob: "**/*.tsx" } }));
+    assert.ok(list.files.length >= 1);
+    assert.ok(list.files.every((f: any) => f.path.endsWith(".tsx")));
+    assert.ok(list.files.some((f: any) => f.path === "src/app/page.tsx"));
+    assert.ok(list.files.every((f: any) => !("content" in f)), "list must not include content");
+
+    // read_clone_files → text inline, binary by URL.
+    const read = parse(await client.callTool({ name: "read_clone_files", arguments: { jobId, paths: ["src/app/page.tsx", "public/assets/cloned/images/a.png"] } }));
+    const page = read.files.find((f: any) => f.path === "src/app/page.tsx");
+    assert.equal(page.type, "text");
+    assert.ok(page.content.includes("Page"));
+    const bin = read.files.find((f: any) => f.path.endsWith("a.png"));
+    assert.equal(bin.type, "binary");
+    assert.ok(bin.url.startsWith("http://test/"), "binary returned as absolute URL");
+    assert.ok(!("content" in bin), "binary must not inline bytes");
+
+    // read size budget → oversized text flagged skipped, not dumped.
+    const tiny = parse(await client.callTool({ name: "read_clone_files", arguments: { jobId, paths: ["src/app/page.tsx"], maxBytes: 1 } }));
+    assert.equal(tiny.files[0].skipped, true);
+    assert.equal(tiny.truncated, true);
+
+    // get_clone_bundle → a download reference, not bytes.
+    const bundle = parse(await client.callTool({ name: "get_clone_bundle", arguments: { jobId } }));
+    assert.equal(bundle.format, "tgz");
+    assert.ok(bundle.bytes > 0);
+    assert.ok(/^[0-9a-f]{64}$/.test(bundle.sha256));
+    assert.ok(bundle.url.includes(`/v1/clones/${jobId}/bundle`));
+
+    // cancel_clone → purges.
+    const cancel = parse(await client.callTool({ name: "cancel_clone", arguments: { jobId } }));
+    assert.equal(cancel.cancelled, true);
+    const after = parse(await client.callTool({ name: "get_clone_status", arguments: { jobId } }));
+    assert.ok(after.error, "status after cancel reports not found");
+  } finally {
+    store.clear();
+  }
+});

packages/api/test/ssrf.test.ts

@@ -0,0 +1,35 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { assertPublicUrl, isBlockedIp, SsrfError } from "../src/ssrf.js";
+
+test("isBlockedIp: private / loopback / link-local / metadata / ULA are blocked", () => {
+  for (const ip of ["10.0.0.1", "127.0.0.1", "169.254.169.254", "192.168.1.1", "172.16.0.1", "100.64.0.1", "::1", "fe80::1", "fc00::1", "::ffff:127.0.0.1"]) {
+    assert.equal(isBlockedIp(ip), true, `${ip} should be blocked`);
+  }
+});
+
+test("isBlockedIp: public addresses are allowed", () => {
+  for (const ip of ["8.8.8.8", "1.1.1.1", "93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"]) {
+    assert.equal(isBlockedIp(ip), false, `${ip} should be allowed`);
+  }
+  assert.equal(isBlockedIp("127.0.0.1", true), false, "loopback allowed when opted in");
+});
+
+test("assertPublicUrl: rejects non-http, IP literals in blocked ranges, localhost", async () => {
+  await assert.rejects(() => assertPublicUrl("ftp://example.com/"), SsrfError);
+  await assert.rejects(() => assertPublicUrl("http://169.254.169.254/latest/meta-data/"), SsrfError);
+  await assert.rejects(() => assertPublicUrl("http://10.1.2.3/"), SsrfError);
+  await assert.rejects(() => assertPublicUrl("http://localhost:3000/"), SsrfError);
+});
+
+test("assertPublicUrl: blocks hostnames that resolve to private IPs (DNS rebinding)", async () => {
+  await assert.rejects(() => assertPublicUrl("http://rebind.evil.test/", { resolver: async () => ["10.0.0.5"] }), SsrfError);
+  await assert.rejects(() => assertPublicUrl("http://nope.test/", { resolver: async () => [] }), SsrfError);
+});
+
+test("assertPublicUrl: allows public hostnames + opt-in loopback", async () => {
+  const u = await assertPublicUrl("https://example.test/path", { resolver: async () => ["93.184.216.34"] });
+  assert.equal(u.hostname, "example.test");
+  const u2 = await assertPublicUrl("http://127.0.0.1:8080/", { allowLoopback: true });
+  assert.equal(u2.port, "8080");
+});