Initial commit

packages/worker/src/duration.ts

@@ -0,0 +1,13 @@
+/** Parse a duration like "24h", "30m", "500ms", or "0" (= disabled). A bare number
+ *  is milliseconds. Falls back to `defMs` on empty/invalid input. */
+export function parseDuration(s: string | undefined, defMs: number): number {
+  if (s === undefined || s.trim() === "") return defMs;
+  const t = s.trim();
+  if (t === "0") return 0;
+  const m = /^(\d+)(ms|s|m|h|d)?$/.exec(t);
+  if (!m) return defMs;
+  const n = parseInt(m[1]!, 10);
+  const unit = m[2] ?? "ms";
+  const mult: Record<string, number> = { ms: 1, s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 };
+  return n * (mult[unit] ?? 1);
+}

packages/worker/src/env.ts

@@ -0,0 +1,27 @@
+import { join } from "node:path";
+import { parseDuration } from "./duration.js";
+
+export type WorkerEnv = {
+  databaseUrl: string;
+  artifactsDir: string;
+  /** cache time-to-stale (CACHE_STALE_AFTER, default 24h; "0" disables caching). */
+  cacheStaleAfterMs: number;
+  /** this worker's isolated build harness dir for verify jobs (M5). */
+  harnessDir: string;
+  /** persistent entry-capture cache dir (the single→multi speed path). "" disables it. */
+  captureCacheDir: string;
+  tier: string;
+};
+
+export function loadWorkerEnv(): WorkerEnv {
+  const databaseUrl = process.env.DATABASE_URL;
+  if (!databaseUrl) throw new Error("DATABASE_URL is required for the worker");
+  return {
+    databaseUrl,
+    artifactsDir: process.env.ARTIFACTS_DIR ?? join(process.cwd(), "local-data", "artifacts"),
+    cacheStaleAfterMs: parseDuration(process.env.CACHE_STALE_AFTER, 24 * 60 * 60 * 1000),
+    harnessDir: process.env.HARNESS_DIR ?? join(process.cwd(), "local-data", "harness"),
+    captureCacheDir: process.env.CAPTURE_CACHE_DIR ?? join(process.cwd(), "local-data", "capture-cache"),
+    tier: process.env.VERIFY_TIER ?? "stage2",
+  };
+}

packages/worker/src/harness.ts

@@ -0,0 +1,51 @@
+import { cpSync, existsSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+
+/** The canonical build harness shipped with the compiler (has package.json + lock). */
+export function baseHarnessDir(): string {
+  // packages/worker/src → ../../../compiler/.harness
+  return join(HERE, "..", "..", "..", "compiler", ".harness");
+}
+
+function hasBuildDeps(dir: string): boolean {
+  return existsSync(join(dir, "node_modules", ".bin", "next"))
+    && existsSync(join(dir, "node_modules", ".bin", "vite"));
+}
+
+/**
+ * Ensure `targetDir` is a ready-to-build app harness with deps installed — the
+ * isolation seam for `verify` (each worker uses its OWN harness so concurrent
+ * framework builds never collide). Reuses the base harness's deps when present
+ * (fast copy), otherwise runs `npm install` from its package.json. Idempotent.
+ */
+export function provisionHarness(targetDir: string, baseDir = baseHarnessDir()): string {
+  if (hasBuildDeps(targetDir)) return targetDir;
+  mkdirSync(targetDir, { recursive: true });
+  if (baseDir !== targetDir) {
+    for (const f of ["package.json", "package-lock.json"]) {
+      if (existsSync(join(baseDir, f))) cpSync(join(baseDir, f), join(targetDir, f));
+    }
+  }
+  if (hasBuildDeps(baseDir) && baseDir !== targetDir) {
+    cpSync(join(baseDir, "node_modules"), join(targetDir, "node_modules"), { recursive: true });
+  } else if (!hasBuildDeps(targetDir)) {
+    const r = spawnSync("npm", ["install", "--no-audit", "--no-fund"], { cwd: targetDir, encoding: "utf8", stdio: "inherit" });
+    if (r.status !== 0) throw new Error("harness npm install failed in " + targetDir);
+  }
+  return targetDir;
+}
+
+/** A memoized provisioner: provisions once, then returns the same dir. */
+export function makeHarnessProvider(targetDir: string): () => Promise<string> {
+  let ready: string | null = null;
+  let inflight: Promise<string> | null = null;
+  return async () => {
+    if (ready) return ready;
+    if (!inflight) inflight = Promise.resolve().then(() => provisionHarness(targetDir)).then((d) => (ready = d));
+    return inflight;
+  };
+}

packages/worker/src/index.ts

@@ -0,0 +1,4 @@
+export { processCloneJob, type ProcessDeps, type RunJob } from "./processJob.js";
+export { parseDuration } from "./duration.js";
+export { provisionHarness, makeHarnessProvider, baseHarnessDir } from "./harness.js";
+export { loadWorkerEnv, type WorkerEnv } from "./env.js";

packages/worker/src/processJob.ts

@@ -0,0 +1,86 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { canonicalOptions, verifyCloneJobResult, type CloneJobResult, type CloneOptions, type RunCloneJobInput } from "@cloner/core";
+import { repo, type Db } from "@cloner/db";
+import type { ArtifactStore } from "@cloner/storage";
+
+export type RunJob = (input: RunCloneJobInput) => Promise<CloneJobResult>;
+
+export type ProcessDeps = {
+  db: Db;
+  store: ArtifactStore;
+  runJob: RunJob;
+  /** cache TTL in ms (0 = caching disabled — no cache row written). */
+  cacheTtlMs: number;
+  /** lazily provisions this worker's isolated build harness, used only for verify jobs (M5). */
+  harnessProvider?: () => Promise<string>;
+  /** persistent entry-capture cache dir for the single→multi reuse path ("" / undefined off). */
+  captureCacheDir?: string;
+  tier?: string;
+};
+
+/**
+ * Process one queued clone job: mark running → run the clone into a temp dir →
+ * persist artifacts (ArtifactStore) + the clone row → mark succeeded → write a
+ * freshness-bounded cache row. On failure, mark the job failed and rethrow so the
+ * queue can retry (capped). The temp dir is always cleaned up.
+ */
+export async function processCloneJob(deps: ProcessDeps, jobId: string): Promise<void> {
+  const { db, store } = deps;
+  const job = await repo.getJob(db, jobId);
+  if (!job) return; // job deleted before it ran
+  await repo.markRunning(db, jobId);
+
+  const base = mkdtempSync(join(tmpdir(), "worker-clone-"));
+  try {
+    const options = (job.options ?? {}) as CloneOptions;
+    // Provision the isolated harness only when this job actually verifies (build).
+    const harnessDir = (options.verify || options.asyncVerify) && deps.harnessProvider ? await deps.harnessProvider() : undefined;
+    const result = await deps.runJob({ url: job.url, options, runsDir: base, harnessDir, tier: deps.tier, captureCacheDir: deps.captureCacheDir || undefined, captureCacheTtlMs: deps.cacheTtlMs });
+
+    const manifest = await store.putClone(jobId, result.files);
+    const envelope = { files: manifest.files, routes: result.routes, bundleKey: manifest.bundleKey };
+    await repo.upsertClone(db, {
+      jobId,
+      url: result.url,
+      routeCount: result.routes?.length ?? 1,
+      fileManifest: envelope,
+      captureMeta: result.capture,
+      verify: (result.verify ?? null) as unknown as Record<string, unknown> | null,
+    });
+    await repo.markSucceeded(db, jobId, { compilerVersion: result.compilerVersion, timings: result.timings });
+
+    if (deps.cacheTtlMs > 0 && !options.noCache) {
+      await repo.cachePut(db, {
+        cacheKey: job.cacheKey,
+        jobId,
+        url: job.url,
+        optionsHash: canonicalOptions(options),
+        compilerVersion: result.compilerVersion,
+        expiresAt: new Date(Date.now() + deps.cacheTtlMs),
+      });
+    }
+
+    if (options.asyncVerify) {
+      try {
+        const done = await verifyCloneJobResult(result, {
+          harnessDir,
+          tier: deps.tier,
+          validationConcurrency: options.validationConcurrency,
+          viewportConcurrency: options.viewportConcurrency,
+        });
+        const timings = { ...result.timings, verifyMs: done.verifyMs };
+        await repo.updateCloneVerify(db, jobId, done.verify);
+        await repo.updateJobTimings(db, jobId, timings);
+      } catch (e) {
+        await repo.updateCloneVerify(db, jobId, { error: String(e).slice(0, 500), async: true });
+      }
+    }
+  } catch (e) {
+    await repo.markFailed(db, jobId, String(e));
+    throw e;
+  } finally {
+    rmSync(base, { recursive: true, force: true });
+  }
+}

packages/worker/src/worker.ts

@@ -0,0 +1,31 @@
+import { runCloneJob } from "@cloner/core";
+import { createDb, createBoss, workClone } from "@cloner/db";
+import { artifactStoreFromEnv } from "@cloner/storage";
+import { processCloneJob } from "./processJob.js";
+import { makeHarnessProvider } from "./harness.js";
+import { loadWorkerEnv } from "./env.js";
+
+async function main(): Promise<void> {
+  const env = loadWorkerEnv();
+  const { db } = createDb(env.databaseUrl);
+  const boss = await createBoss(env.databaseUrl);
+  const store = artifactStoreFromEnv();
+
+  const deps = {
+    db,
+    store,
+    runJob: runCloneJob,
+    cacheTtlMs: env.cacheStaleAfterMs,
+    harnessProvider: makeHarnessProvider(env.harnessDir),
+    captureCacheDir: env.captureCacheDir,
+    tier: env.tier,
+  };
+
+  await workClone(boss, (jobId) => processCloneJob(deps, jobId));
+  console.log(JSON.stringify({ event: "worker_started", artifactsDir: env.artifactsDir, harnessDir: env.harnessDir, cacheStaleAfterMs: env.cacheStaleAfterMs }));
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});