Initial commit

packages/worker/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@cloner/worker",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "dev": "tsx watch src/worker.ts",
+    "start": "tsx src/worker.ts",
+    "test": "node --import tsx --test test/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@cloner/core": "*",
+    "@cloner/db": "*",
+    "@cloner/storage": "*"
+  },
+  "devDependencies": {
+    "@cloner/api": "*",
+    "@cloner/test-utils": "*",
+    "tsx": "4.22.4",
+    "typescript": "5.7.3",
+    "@types/node": "22.10.5"
+  }
+}

packages/worker/src/duration.ts

@@ -0,0 +1,13 @@
+/** Parse a duration like "24h", "30m", "500ms", or "0" (= disabled). A bare number
+ *  is milliseconds. Falls back to `defMs` on empty/invalid input. */
+export function parseDuration(s: string | undefined, defMs: number): number {
+  if (s === undefined || s.trim() === "") return defMs;
+  const t = s.trim();
+  if (t === "0") return 0;
+  const m = /^(\d+)(ms|s|m|h|d)?$/.exec(t);
+  if (!m) return defMs;
+  const n = parseInt(m[1]!, 10);
+  const unit = m[2] ?? "ms";
+  const mult: Record<string, number> = { ms: 1, s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 };
+  return n * (mult[unit] ?? 1);
+}

packages/worker/src/env.ts

@@ -0,0 +1,27 @@
+import { join } from "node:path";
+import { parseDuration } from "./duration.js";
+
+export type WorkerEnv = {
+  databaseUrl: string;
+  artifactsDir: string;
+  /** cache time-to-stale (CACHE_STALE_AFTER, default 24h; "0" disables caching). */
+  cacheStaleAfterMs: number;
+  /** this worker's isolated build harness dir for verify jobs (M5). */
+  harnessDir: string;
+  /** persistent entry-capture cache dir (the single→multi speed path). "" disables it. */
+  captureCacheDir: string;
+  tier: string;
+};
+
+export function loadWorkerEnv(): WorkerEnv {
+  const databaseUrl = process.env.DATABASE_URL;
+  if (!databaseUrl) throw new Error("DATABASE_URL is required for the worker");
+  return {
+    databaseUrl,
+    artifactsDir: process.env.ARTIFACTS_DIR ?? join(process.cwd(), "local-data", "artifacts"),
+    cacheStaleAfterMs: parseDuration(process.env.CACHE_STALE_AFTER, 24 * 60 * 60 * 1000),
+    harnessDir: process.env.HARNESS_DIR ?? join(process.cwd(), "local-data", "harness"),
+    captureCacheDir: process.env.CAPTURE_CACHE_DIR ?? join(process.cwd(), "local-data", "capture-cache"),
+    tier: process.env.VERIFY_TIER ?? "stage2",
+  };
+}

packages/worker/src/harness.ts

@@ -0,0 +1,51 @@
+import { cpSync, existsSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+
+/** The canonical build harness shipped with the compiler (has package.json + lock). */
+export function baseHarnessDir(): string {
+  // packages/worker/src → ../../../compiler/.harness
+  return join(HERE, "..", "..", "..", "compiler", ".harness");
+}
+
+function hasBuildDeps(dir: string): boolean {
+  return existsSync(join(dir, "node_modules", ".bin", "next"))
+    && existsSync(join(dir, "node_modules", ".bin", "vite"));
+}
+
+/**
+ * Ensure `targetDir` is a ready-to-build app harness with deps installed — the
+ * isolation seam for `verify` (each worker uses its OWN harness so concurrent
+ * framework builds never collide). Reuses the base harness's deps when present
+ * (fast copy), otherwise runs `npm install` from its package.json. Idempotent.
+ */
+export function provisionHarness(targetDir: string, baseDir = baseHarnessDir()): string {
+  if (hasBuildDeps(targetDir)) return targetDir;
+  mkdirSync(targetDir, { recursive: true });
+  if (baseDir !== targetDir) {
+    for (const f of ["package.json", "package-lock.json"]) {
+      if (existsSync(join(baseDir, f))) cpSync(join(baseDir, f), join(targetDir, f));
+    }
+  }
+  if (hasBuildDeps(baseDir) && baseDir !== targetDir) {
+    cpSync(join(baseDir, "node_modules"), join(targetDir, "node_modules"), { recursive: true });
+  } else if (!hasBuildDeps(targetDir)) {
+    const r = spawnSync("npm", ["install", "--no-audit", "--no-fund"], { cwd: targetDir, encoding: "utf8", stdio: "inherit" });
+    if (r.status !== 0) throw new Error("harness npm install failed in " + targetDir);
+  }
+  return targetDir;
+}
+
+/** A memoized provisioner: provisions once, then returns the same dir. */
+export function makeHarnessProvider(targetDir: string): () => Promise<string> {
+  let ready: string | null = null;
+  let inflight: Promise<string> | null = null;
+  return async () => {
+    if (ready) return ready;
+    if (!inflight) inflight = Promise.resolve().then(() => provisionHarness(targetDir)).then((d) => (ready = d));
+    return inflight;
+  };
+}

packages/worker/src/index.ts

@@ -0,0 +1,4 @@
+export { processCloneJob, type ProcessDeps, type RunJob } from "./processJob.js";
+export { parseDuration } from "./duration.js";
+export { provisionHarness, makeHarnessProvider, baseHarnessDir } from "./harness.js";
+export { loadWorkerEnv, type WorkerEnv } from "./env.js";

packages/worker/src/processJob.ts

@@ -0,0 +1,86 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { canonicalOptions, verifyCloneJobResult, type CloneJobResult, type CloneOptions, type RunCloneJobInput } from "@cloner/core";
+import { repo, type Db } from "@cloner/db";
+import type { ArtifactStore } from "@cloner/storage";
+
+export type RunJob = (input: RunCloneJobInput) => Promise<CloneJobResult>;
+
+export type ProcessDeps = {
+  db: Db;
+  store: ArtifactStore;
+  runJob: RunJob;
+  /** cache TTL in ms (0 = caching disabled — no cache row written). */
+  cacheTtlMs: number;
+  /** lazily provisions this worker's isolated build harness, used only for verify jobs (M5). */
+  harnessProvider?: () => Promise<string>;
+  /** persistent entry-capture cache dir for the single→multi reuse path ("" / undefined off). */
+  captureCacheDir?: string;
+  tier?: string;
+};
+
+/**
+ * Process one queued clone job: mark running → run the clone into a temp dir →
+ * persist artifacts (ArtifactStore) + the clone row → mark succeeded → write a
+ * freshness-bounded cache row. On failure, mark the job failed and rethrow so the
+ * queue can retry (capped). The temp dir is always cleaned up.
+ */
+export async function processCloneJob(deps: ProcessDeps, jobId: string): Promise<void> {
+  const { db, store } = deps;
+  const job = await repo.getJob(db, jobId);
+  if (!job) return; // job deleted before it ran
+  await repo.markRunning(db, jobId);
+
+  const base = mkdtempSync(join(tmpdir(), "worker-clone-"));
+  try {
+    const options = (job.options ?? {}) as CloneOptions;
+    // Provision the isolated harness only when this job actually verifies (build).
+    const harnessDir = (options.verify || options.asyncVerify) && deps.harnessProvider ? await deps.harnessProvider() : undefined;
+    const result = await deps.runJob({ url: job.url, options, runsDir: base, harnessDir, tier: deps.tier, captureCacheDir: deps.captureCacheDir || undefined, captureCacheTtlMs: deps.cacheTtlMs });
+
+    const manifest = await store.putClone(jobId, result.files);
+    const envelope = { files: manifest.files, routes: result.routes, bundleKey: manifest.bundleKey };
+    await repo.upsertClone(db, {
+      jobId,
+      url: result.url,
+      routeCount: result.routes?.length ?? 1,
+      fileManifest: envelope,
+      captureMeta: result.capture,
+      verify: (result.verify ?? null) as unknown as Record<string, unknown> | null,
+    });
+    await repo.markSucceeded(db, jobId, { compilerVersion: result.compilerVersion, timings: result.timings });
+
+    if (deps.cacheTtlMs > 0 && !options.noCache) {
+      await repo.cachePut(db, {
+        cacheKey: job.cacheKey,
+        jobId,
+        url: job.url,
+        optionsHash: canonicalOptions(options),
+        compilerVersion: result.compilerVersion,
+        expiresAt: new Date(Date.now() + deps.cacheTtlMs),
+      });
+    }
+
+    if (options.asyncVerify) {
+      try {
+        const done = await verifyCloneJobResult(result, {
+          harnessDir,
+          tier: deps.tier,
+          validationConcurrency: options.validationConcurrency,
+          viewportConcurrency: options.viewportConcurrency,
+        });
+        const timings = { ...result.timings, verifyMs: done.verifyMs };
+        await repo.updateCloneVerify(db, jobId, done.verify);
+        await repo.updateJobTimings(db, jobId, timings);
+      } catch (e) {
+        await repo.updateCloneVerify(db, jobId, { error: String(e).slice(0, 500), async: true });
+      }
+    }
+  } catch (e) {
+    await repo.markFailed(db, jobId, String(e));
+    throw e;
+  } finally {
+    rmSync(base, { recursive: true, force: true });
+  }
+}

packages/worker/src/worker.ts

@@ -0,0 +1,31 @@
+import { runCloneJob } from "@cloner/core";
+import { createDb, createBoss, workClone } from "@cloner/db";
+import { artifactStoreFromEnv } from "@cloner/storage";
+import { processCloneJob } from "./processJob.js";
+import { makeHarnessProvider } from "./harness.js";
+import { loadWorkerEnv } from "./env.js";
+
+async function main(): Promise<void> {
+  const env = loadWorkerEnv();
+  const { db } = createDb(env.databaseUrl);
+  const boss = await createBoss(env.databaseUrl);
+  const store = artifactStoreFromEnv();
+
+  const deps = {
+    db,
+    store,
+    runJob: runCloneJob,
+    cacheTtlMs: env.cacheStaleAfterMs,
+    harnessProvider: makeHarnessProvider(env.harnessDir),
+    captureCacheDir: env.captureCacheDir,
+    tier: env.tier,
+  };
+
+  await workClone(boss, (jobId) => processCloneJob(deps, jobId));
+  console.log(JSON.stringify({ event: "worker_started", artifactsDir: env.artifactsDir, harnessDir: env.harnessDir, cacheStaleAfterMs: env.cacheStaleAfterMs }));
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

packages/worker/test/integration.test.ts

@@ -0,0 +1,123 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type PgBoss from "pg-boss";
+import { collectFileMap, COMPILER_VERSION, type CloneJobResult } from "@cloner/core";
+import { createDb, createBoss, workClone, runMigrations, type Db } from "@cloner/db";
+import { LocalArtifactStore } from "@cloner/storage";
+import { createApp, DbBackend } from "@cloner/api";
+import { acquireTestPostgres, hasTestPostgres, type EphemeralPg } from "@cloner/test-utils";
+import { processCloneJob } from "../src/processJob.js";
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+async function waitFor<T>(fn: () => Promise<T | undefined | null>, ms = 30_000, step = 300): Promise<T> {
+  const t0 = Date.now();
+  for (;;) {
+    const v = await fn();
+    if (v) return v;
+    if (Date.now() - t0 > ms) throw new Error("waitFor timeout");
+    await sleep(step);
+  }
+}
+
+/** A browser-free fake clone: writes a tiny generated app under the worker's temp
+ *  base, then returns a real CloneJobResult via collectFileMap. Keeps the queue/DB
+ *  lifecycle test fast + hermetic (no Chromium). */
+const fakeRunJob = async (input: { url: string; runsDir?: string; options?: unknown }): Promise<CloneJobResult> => {
+  const base = input.runsDir!;
+  const app = join(base, "generated", "app");
+  mkdirSync(join(app, "src", "app"), { recursive: true });
+  mkdirSync(join(app, "public", "assets", "cloned", "images"), { recursive: true });
+  writeFileSync(join(app, "package.json"), '{"name":"cloned-app"}\n');
+  writeFileSync(join(app, "src", "app", "page.tsx"), "export default function Page(){return <div/>}\n");
+  writeFileSync(join(app, "public", "assets", "cloned", "images", "a.png"), Buffer.from([1, 2, 3, 4]));
+  return {
+    url: input.url,
+    kind: "clone",
+    options: (input.options as CloneJobResult["options"]) ?? {},
+    status: "succeeded",
+    compilerVersion: COMPILER_VERSION,
+    timings: { captureMs: 1, generateMs: 0 },
+    files: collectFileMap(base),
+    capture: { nodeCount: 7, pollution: false, blocked: false },
+    runDir: base,
+  };
+};
+
+describe("M2: async job lifecycle (Postgres queue + DB + worker)", { skip: hasTestPostgres() ? false : "no test Postgres" }, () => {
+  let pg: EphemeralPg;
+  let db: Db;
+  let pool: { end: () => Promise<void> };
+  let boss: PgBoss;
+  let store: LocalArtifactStore;
+  let blobs: string;
+
+  before(async () => {
+    pg = (await acquireTestPostgres())!;
+    await runMigrations(pg.url);
+    const h = createDb(pg.url);
+    db = h.db;
+    pool = h.pool;
+    boss = await createBoss(pg.url);
+    blobs = mkdtempSync(join(tmpdir(), "it-blobs-"));
+    store = new LocalArtifactStore(blobs);
+    // Start the worker consuming the queue.
+    await workClone(boss, (jobId) => processCloneJob({ db, store, runJob: fakeRunJob, cacheTtlMs: 60_000 }, jobId));
+  });
+
+  after(async () => {
+    await boss?.stop({ graceful: false }).catch(() => {});
+    await pool?.end().catch(() => {});
+    await pg?.stop().catch(() => {});
+    if (blobs) rmSync(blobs, { recursive: true, force: true });
+  });
+
+  it("enqueues, processes, persists, and serves the result; second submit is cached", async () => {
+    const app = createApp({ backend: new DbBackend({ db, boss, store }) });
+    const url = "https://example.com/?t=" + Date.now(); // unique → no stale cache on a reused DB
+    const reqBody = JSON.stringify({ url, options: { interactions: false } });
+
+    // Submit → 202 queued.
+    const submit = await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: reqBody });
+    assert.equal(submit.status, 202);
+    const { jobId, status } = await submit.json();
+    assert.ok(jobId);
+    assert.equal(status, "queued");
+
+    // Poll until the worker finishes.
+    const done = await waitFor(async () => {
+      const v = await (await app.request(`/v1/clones/${jobId}`)).json();
+      return v.status === "succeeded" ? v : undefined;
+    });
+    assert.equal(done.status, "succeeded");
+    assert.equal(done.capture.nodeCount, 7);
+    assert.equal(done.fileCount, 3);
+
+    // Full result + per-file streaming.
+    const result = await (await app.request(`/v1/clones/${jobId}/result`)).json();
+    assert.equal(result.files["src/app/page.tsx"].type, "text");
+    const bin = result.files["public/assets/cloned/images/a.png"];
+    assert.equal(bin.type, "binary");
+    const fileRes = await app.request(bin.url);
+    assert.equal(fileRes.status, 200);
+    assert.equal(Buffer.from(await fileRes.arrayBuffer()).length, 4);
+
+    // Second identical submit → cache hit (200, status cached), no new job.
+    const submit2 = await app.request("/v1/clones", { method: "POST", headers: { "content-type": "application/json" }, body: reqBody });
+    assert.equal(submit2.status, 200);
+    const cached = await submit2.json();
+    assert.equal(cached.status, "cached");
+    assert.equal(cached.jobId, jobId);
+
+    // noCache bypasses the cache → a fresh queued job.
+    const submit3 = await app.request("/v1/clones", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ url, options: { interactions: false, noCache: true } }),
+    });
+    assert.equal(submit3.status, 202);
+    assert.notEqual((await submit3.json()).jobId, jobId);
+  });
+});

packages/worker/test/verify.fixture.test.ts

@@ -0,0 +1,44 @@
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { runCloneJob } from "@cloner/core";
+import { serveDir, FIXTURES_DIR, hasChromium } from "@cloner/test-utils";
+import { provisionHarness, baseHarnessDir } from "../src/harness.js";
+
+// verify = build the generated app (next build) + serve + re-render + grade. Uses an
+// isolated, dep-installed harness. Skipped without Chromium; also skips gracefully if
+// the harness can't be provisioned (e.g. no network for npm install).
+describe("runCloneJob verify (build + gates via provisioned harness)", { skip: hasChromium() ? false : "no Chromium installed" }, () => {
+  let server: { url: string; close: () => Promise<void> };
+  let harnessDir: string | null = null;
+
+  before(async () => {
+    server = await serveDir(FIXTURES_DIR);
+    try {
+      harnessDir = provisionHarness(baseHarnessDir());
+    } catch (e) {
+      console.error("harness provisioning failed:", String(e).slice(0, 200));
+      harnessDir = null;
+    }
+  });
+  after(async () => {
+    await server.close();
+  });
+
+  it("builds the clone and attaches a verify report", { timeout: 300_000 }, async (t) => {
+    if (!harnessDir) {
+      t.skip("harness unavailable (npm install failed)");
+      return;
+    }
+    const res = await runCloneJob({
+      url: server.url + "/components.html",
+      options: { verify: true, interactions: false, components: true, motion: false },
+      harnessDir,
+      tier: "easy",
+    });
+    assert.ok(res.verify, "verify report attached");
+    const v = res.verify as { gates0to6Pass: boolean; scorecard: { total: number }; gates: Record<string, unknown> };
+    assert.equal(typeof v.gates0to6Pass, "boolean");
+    assert.ok(v.scorecard && typeof v.scorecard.total === "number", "scorecard present");
+    assert.ok(res.timings.verifyMs !== undefined && res.timings.verifyMs > 0, "verifyMs recorded");
+  });
+});

packages/worker/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"]
+}