Initial commit

packages/core/test/cacheKey.test.ts

@@ -0,0 +1,41 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { cacheKey, normalizeUrl, canonicalOptions } from "../src/cacheKey.js";
+
+test("normalizeUrl: canonicalizes scheme/host/port/trailing-slash/fragment/query", () => {
+  assert.equal(normalizeUrl("HTTPS://Example.com:443/foo/#frag"), "https://example.com/foo");
+  assert.equal(normalizeUrl("http://example.com:80/"), "http://example.com/");
+  assert.equal(normalizeUrl("https://x.com"), "https://x.com/");
+  assert.equal(normalizeUrl("https://example.com/a?b=2&a=1"), "https://example.com/a?a=1&b=2");
+  assert.equal(normalizeUrl("not a url"), "not a url");
+});
+
+test("cacheKey: stable for equivalent input, varies by options + compilerVersion", () => {
+  const k1 = cacheKey("https://x.com/", { mode: "single", styling: "tailwind" }, "0.1.0");
+  const k2 = cacheKey("https://x.com", { mode: "single", styling: "tailwind" }, "0.1.0");
+  assert.equal(k1, k2, "trailing slash is normalized away");
+
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "multi", styling: "tailwind" }, "0.1.0"), "mode changes the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "css" }, "0.1.0"), "styling changes the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", framework: "vite" }, "0.1.0"), "framework changes the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind" }, "0.2.0"), "version bump invalidates");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", verify: true }, "0.1.0"), "verify is part of the key");
+  assert.notEqual(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", asyncVerify: true }, "0.1.0"), "asyncVerify is part of the key");
+
+  // noCache is a request-time switch, not an output determinant — must not change the key.
+  assert.equal(k1, cacheKey("https://x.com/", { mode: "single", styling: "tailwind", noCache: true }, "0.1.0"));
+});
+
+test("canonicalOptions: deprecated aliases normalize to product options", () => {
+  assert.equal(
+    canonicalOptions({ mode: "multi", styling: "css" }),
+    canonicalOptions({ multiPage: true, humanizeMode: "css" }),
+  );
+});
+
+test("canonicalOptions: sorts viewports and is stable", () => {
+  assert.equal(
+    canonicalOptions({ viewports: [1920, 375, 768] }),
+    canonicalOptions({ viewports: [375, 768, 1920] }),
+  );
+});

packages/core/test/collectFileMap.test.ts

@@ -0,0 +1,54 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createHash } from "node:crypto";
+import { collectFileMap, fileMapStats } from "../src/collectFileMap.js";
+
+test("collectFileMap: inlines text, references binaries, sorts keys, hashes bytes", () => {
+  const root = mkdtempSync(join(tmpdir(), "cfm-"));
+  try {
+    const app = join(root, "generated", "app");
+    mkdirSync(join(app, "src", "app", "_clone"), { recursive: true });
+    mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+    writeFileSync(join(app, "package.json"), '{"name":"x"}\n');
+    writeFileSync(join(app, ".gitignore"), "node_modules\n");
+    writeFileSync(join(app, "src", "app", "page.tsx"), "export default function Page(){return null}\n");
+    writeFileSync(join(app, "src", "app", "globals.css"), "body{margin:0}\n");
+    const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]);
+    writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), png);
+
+    const map = collectFileMap(root);
+    const keys = Object.keys(map);
+    assert.deepEqual(keys, [...keys].sort(), "keys are sorted (deterministic)");
+
+    assert.equal(map["package.json"]!.kind, "text");
+    assert.equal(map[".gitignore"]!.kind, "text", ".gitignore is treated as text");
+    const page = map["src/app/page.tsx"]!;
+    assert.equal(page.kind, "text");
+    assert.ok(page.content!.includes("export default"));
+    assert.equal(page.sha256, createHash("sha256").update("export default function Page(){return null}\n").digest("hex"));
+
+    const bin = map["public/assets/cloned/images/a.png"]!;
+    assert.equal(bin.kind, "binary");
+    assert.equal(bin.content, undefined, "binaries are by reference, not inlined");
+    assert.equal(bin.sha256, createHash("sha256").update(png).digest("hex"));
+    assert.equal(bin.bytes, png.length);
+
+    const stats = fileMapStats(map);
+    assert.equal(stats.fileCount, 5);
+    assert.ok(stats.totalBytes > 0);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("collectFileMap: throws when there is no generated app", () => {
+  const root = mkdtempSync(join(tmpdir(), "cfm-empty-"));
+  try {
+    assert.throws(() => collectFileMap(root), /no generated app/);
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+});

packages/core/test/incremental.fixture.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runCloneJob } from "../src/runCloneJob.js";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+
+// The "single page first, then expand" speed path: a single-page clone stashes its
+// entry capture in captureCacheDir; a later multi-page clone of the SAME url reuses it
+// as the entry route (no re-capture) and regenerates the whole site on top of it.
+describe("runCloneJob incremental (single → multi reuse)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  let cacheDir: string;
+  before(async () => {
+    server = await serveDir(join(FIXTURES_DIR, "site"));
+    cacheDir = mkdtempSync(join(tmpdir(), "capture-cache-"));
+  });
+  after(async () => {
+    await server.close();
+    rmSync(cacheDir, { recursive: true, force: true });
+  });
+
+  it("reuses the single-page entry capture when expanding to the full site", async () => {
+    const url = server.url + "/";
+
+    // 1. Single page first — fast, returns one app; seeds the capture cache.
+    const single = await runCloneJob({ url, options: {}, captureCacheDir: cacheDir });
+    assert.equal(single.status, "succeeded");
+    assert.equal(single.kind, "clone");
+    assert.equal(single.captureReused, false, "first (single) job captures fresh");
+    assert.ok(single.files["src/app/page.tsx"], "single-page app emitted");
+
+    // 2. Expand to the full multi-route site — reuses page 1's capture (no re-capture),
+    //    captures the rest, regenerates ALL routes together.
+    const multi = await runCloneJob({ url, options: { mode: "multi" }, captureCacheDir: cacheDir });
+    assert.equal(multi.status, "succeeded");
+    assert.equal(multi.kind, "clone_site");
+    assert.equal(multi.captureReused, true, "multi-page job reused the cached entry capture");
+    assert.ok((multi.routes?.length ?? 0) >= 2, `expected >=2 routes, got ${multi.routes?.length}`);
+    assert.ok(multi.files["src/app/layout.tsx"], "shared layout emitted");
+    const subRoutePages = Object.keys(multi.files).filter((p) => /^src\/app\/.+\/page\.tsx$/.test(p));
+    assert.ok(subRoutePages.length >= 1, "at least one sub-route page beyond the entry");
+
+    // Tailwind is the default styling output end-to-end (service path included).
+    assert.ok(multi.files["postcss.config.mjs"], "Tailwind toolchain present");
+    assert.ok((multi.files["src/app/globals.css"]!.content ?? "").includes("tailwindcss"), "Tailwind globals");
+  });
+});

packages/core/test/multipage.fixture.test.ts

@@ -0,0 +1,34 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { join } from "node:path";
+import { runCloneJob } from "../src/runCloneJob.js";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+
+// Multi-page clone job: crawl a served fixture site (index → blog → faq) and clone
+// all routes into one Next app. Skipped without Chromium.
+describe("runCloneJob multi-page (served fixture site)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  before(async () => {
+    server = await serveDir(join(FIXTURES_DIR, "site"));
+  });
+  after(async () => {
+    await server.close();
+  });
+
+  it("crawls + clones multiple routes into one app (clone_site)", async () => {
+    const res = await runCloneJob({ url: server.url + "/", options: { mode: "multi", interactions: false, components: false } });
+    assert.equal(res.kind, "clone_site");
+    assert.ok(res.routes && res.routes.length >= 2, `expected >=2 routes, got ${res.routes?.length}`);
+    assert.ok(res.files["src/app/layout.tsx"], "shared layout emitted once");
+    assert.ok(res.files["src/app/page.tsx"], "home route page");
+    assert.ok(res.files["AGENTS.md"], "generated AGENTS.md emitted");
+    assert.ok(res.files["ARCHITECTURE.md"], "generated ARCHITECTURE.md emitted");
+    assert.ok(res.files["src/app/robots.ts"], "robots route emitted");
+    assert.ok(res.files["src/app/sitemap.ts"], "sitemap route emitted");
+    assert.ok(res.files["src/app/llms.txt/route.ts"], "llms route emitted");
+    assert.ok((res.files["src/app/llms.txt/route.ts"]!.content ?? "").includes("Build faster with Acme"), "generated llms includes captured route content");
+    const subRoutePages = Object.keys(res.files).filter((p) => /^src\/app\/.+\/page\.tsx$/.test(p));
+    assert.ok(subRoutePages.length >= 1, "at least one sub-route page");
+    assert.ok(res.capture.nodeCount > 0);
+  });
+});

packages/core/test/runCloneJob.fixture.test.ts

@@ -0,0 +1,170 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runClone } from "clone-static";
+import { runCloneJob } from "../src/runCloneJob.js";
+import { collectFileMap } from "../src/collectFileMap.js";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+
+const CHROMIUM = hasChromium();
+
+// End-to-end clone of a served fixture (zero external network). Skipped when no
+// Playwright Chromium is installed (run `npx playwright install chromium` first).
+describe("runCloneJob (served fixture)", { skip: CHROMIUM ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  before(async () => {
+    server = await serveDir(FIXTURES_DIR);
+  });
+  after(async () => {
+    await server.close();
+  });
+
+  it("produces the file-map contract for a single-page clone", async () => {
+    const url = server.url + "/components.html";
+    const res = await runCloneJob({
+      url,
+      options: { interactions: false, components: true, motion: false },
+    });
+
+    assert.equal(res.status, "succeeded");
+    assert.equal(res.kind, "clone");
+    assert.ok(res.compilerVersion);
+
+    // The essential scaffold is present and typed correctly.
+    for (const k of [
+      "package.json",
+      "tsconfig.json",
+      "next.config.mjs",
+      "src/app/layout.tsx",
+      "src/app/page.tsx",
+      "src/app/ditto.css",
+      "src/app/globals.css",
+    ]) {
+      assert.ok(res.files[k], `expected file ${k}`);
+      assert.equal(res.files[k]!.kind, "text");
+    }
+    assert.ok(res.files["src/app/page.tsx"]!.content!.includes("export default function Page"));
+
+    // Extracted components are split into their own files (compiler change); the
+    // service collects them as text automatically.
+    const componentFiles = Object.keys(res.files).filter((k) => k.startsWith("src/app/components/"));
+    assert.ok(componentFiles.length > 0, "extracted components emitted as separate files");
+    for (const k of componentFiles) {
+      assert.equal(res.files[k]!.kind, "text");
+      assert.ok((res.files[k]!.content ?? "").length > 0, `${k} has content`);
+    }
+
+    // Capture sanity: a real fixture is not degenerate / bot-walled.
+    assert.ok(res.capture.nodeCount > 0, "nodeCount > 0");
+    assert.equal(res.capture.blocked, false);
+
+    // temp dir is cleaned up by default.
+    const { existsSync } = await import("node:fs");
+    assert.equal(existsSync(res.runDir), false, "temp run dir removed");
+  });
+
+  it("can generate a Vite React app instead of Next", async () => {
+    const url = server.url + "/components.html";
+    const res = await runCloneJob({
+      url,
+      options: { framework: "vite", interactions: false, components: false, motion: false },
+    });
+
+    assert.equal(res.status, "succeeded");
+    assert.equal(res.options.framework, "vite");
+    for (const k of [
+      "index.html",
+      "vite.config.ts",
+      "src/main.tsx",
+      "src/page.tsx",
+      "src/ditto.css",
+      "src/globals.css",
+      "public/robots.txt",
+      "public/sitemap.xml",
+      "public/llms.txt",
+    ]) {
+      assert.ok(res.files[k], `expected file ${k}`);
+      assert.equal(res.files[k]!.kind, "text");
+    }
+    assert.ok(!res.files["next.config.mjs"], "Vite output should not include next.config.mjs");
+    assert.ok(!res.files["src/app/layout.tsx"], "Vite output should not include App Router layout");
+    assert.ok(res.files["package.json"]!.content!.includes('"dev": "vite"'));
+    assert.ok(res.files["src/main.tsx"]!.content!.includes('from "react-dom/client"'));
+    assert.ok((res.files["src/globals.css"]!.content ?? "").includes("#root { display: contents; }"));
+  });
+
+  it("names sections with valid JS identifiers even when a heading starts with a number", async () => {
+    // Repro: a section whose heading reads "0019 Iterate Faster" previously became the
+    // identifier `0019IterateSection` → `import 0019IterateSection` is a syntax error.
+    const url = server.url + "/numeric-section.html";
+    const res = await runCloneJob({ url, options: { interactions: false, components: false, motion: false } });
+    assert.equal(res.status, "succeeded");
+
+    const page = res.files["src/app/page.tsx"]!.content ?? "";
+    const imports = [...page.matchAll(/^import\s+([A-Za-z0-9_$]+)\s+from/gm)].map((m) => m[1]!);
+    assert.ok(imports.length > 0, "page imports section modules");
+    for (const id of imports) {
+      assert.match(id, /^[A-Za-z_$]/, `import identifier "${id}" must not start with a digit`);
+    }
+    // The numeric "0019" layer noise is dropped → a clean, valid name.
+    assert.ok(imports.includes("IterateFasterSection"), `expected IterateFasterSection, got ${imports.join(", ")}`);
+  });
+
+  it("preserves generated SEO metadata, icons, JSON-LD, llms, and docs", async () => {
+    const url = server.url + "/seo-rich.html";
+    const res = await runCloneJob({
+      url,
+      options: { interactions: false, components: false, motion: false },
+    });
+    assert.equal(res.status, "succeeded");
+
+    const layout = res.files["src/app/layout.tsx"]!.content ?? "";
+    assert.ok(layout.includes("SEO Rich Fixture"));
+    assert.ok(layout.includes("Open Graph description from the source page."));
+    assert.ok(layout.includes("summary_large_image"));
+    assert.ok(layout.includes("themeColor"));
+    assert.ok(layout.includes("colorScheme"));
+    assert.ok(layout.includes("application/ld+json"));
+
+    assert.equal(res.files["src/app/favicon.ico"]?.kind, "binary");
+    assert.equal(res.files["src/app/icon.png"]?.kind, "binary");
+    assert.equal(res.files["src/app/apple-icon.png"]?.kind, "binary");
+    assert.ok(Object.keys(res.files).some((k) => k.startsWith("public/assets/cloned/manifest/")), "web manifest materialized");
+    assert.ok(Object.keys(res.files).some((k) => k.startsWith("public/assets/cloned/images/")), "manifest/icon image assets materialized");
+
+    const llms = res.files["src/app/llms.txt/route.ts"]!.content ?? "";
+    assert.ok(llms.includes("Source LLMS"), "source llms.txt preserved");
+    const llmsFull = res.files["src/app/llms-full.txt/route.ts"]!.content ?? "";
+    assert.ok(llmsFull.includes("Source LLMS Full"), "source llms-full.txt preserved");
+
+    assert.ok(res.files["AGENTS.md"]!.content!.includes("generated ditto.site clone app"));
+    assert.ok(res.files["ARCHITECTURE.md"]!.content!.includes("data-ditto-id"));
+  });
+
+  it("generates byte-identical output from one frozen capture (golden / Gate 6)", async () => {
+    const url = server.url + "/components.html";
+    const a = mkdtempSync(join(tmpdir(), "golden-a-"));
+    const b = mkdtempSync(join(tmpdir(), "golden-b-"));
+    try {
+      const opts = { interactions: false, components: true, motion: false } as const;
+      const r1 = await runClone({ url, runsDir: a, ...opts });
+      // Regenerate from the SAME capture (no re-capture) → must be byte-identical.
+      const r2 = await runClone({ url, runsDir: b, reuseSource: r1.sourceDir, ...opts });
+
+      const m1 = collectFileMap(r1.runDir);
+      const m2 = collectFileMap(r2.runDir);
+      // Same file set + byte-identical contents across regenerations — covers the
+      // scaffold AND the split components/*.tsx, all from one frozen capture.
+      assert.deepEqual(Object.keys(m1).sort(), Object.keys(m2).sort(), "identical file set across regenerations");
+      for (const f of Object.keys(m1)) {
+        assert.equal(m1[f]!.sha256, m2[f]!.sha256, `${f} is byte-identical across regenerations`);
+      }
+      assert.ok(Object.keys(m1).some((k) => k.startsWith("src/app/components/")), "components are split into files");
+    } finally {
+      rmSync(a, { recursive: true, force: true });
+      rmSync(b, { recursive: true, force: true });
+    }
+  });
+});