Parallelize the clone queue: in-flight dedup + WORKER_CONCURRENCY

Two changes that multiply effective queue throughput:

- In-flight dedup: POST /v1/clones now attaches identical submits (same
  cacheKey) to the already queued/running job instead of enqueueing a
  duplicate capture. Retry-spam previously multiplied queue load (observed
  4x same-URL submissions in prod); now it's free. Best-effort — a
  same-instant race can still double-submit. Gated on !noCache.

- WORKER_CONCURRENCY (default 1): the worker registers N independent
  pg-boss pollers, running up to N captures concurrently per process with
  no head-of-line blocking. Each slot gets its own verify harness dir
  (harnessDir/slot-i) so concurrent framework builds never collide;
  concurrency 1 keeps the flat harnessDir layout.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

packages/worker/src/env.ts

@@ -11,11 +11,15 @@ export type WorkerEnv = {
   /** persistent entry-capture cache dir (the single→multi speed path). "" disables it. */
   captureCacheDir: string;
   tier: string;
+  /** concurrent clone jobs in this process (WORKER_CONCURRENCY, default 1). Each
+   *  slot runs a full Chromium capture — size the container accordingly. */
+  concurrency: number;
 };
 
 export function loadWorkerEnv(): WorkerEnv {
   const databaseUrl = process.env.DATABASE_URL;
   if (!databaseUrl) throw new Error("DATABASE_URL is required for the worker");
+  const concurrency = Math.max(1, parseInt(process.env.WORKER_CONCURRENCY ?? "1", 10) || 1);
   return {
     databaseUrl,
     artifactsDir: process.env.ARTIFACTS_DIR ?? join(process.cwd(), "local-data", "artifacts"),
@@ -23,5 +27,6 @@ export function loadWorkerEnv(): WorkerEnv {
     harnessDir: process.env.HARNESS_DIR ?? join(process.cwd(), "local-data", "harness"),
     captureCacheDir: process.env.CAPTURE_CACHE_DIR ?? join(process.cwd(), "local-data", "capture-cache"),
     tier: process.env.VERIFY_TIER ?? "stage2",
+    concurrency,
   };
 }

packages/worker/src/worker.ts

@@ -1,3 +1,4 @@
+import { join } from "node:path";
 import { runCloneJob } from "@cloner/core";
 import { createDb, createBoss, workClone } from "@cloner/db";
 import { artifactStoreFromEnv } from "@cloner/storage";
@@ -16,13 +17,32 @@ async function main(): Promise<void> {
     store,
     runJob: runCloneJob,
     cacheTtlMs: env.cacheStaleAfterMs,
-    harnessProvider: makeHarnessProvider(env.harnessDir),
     captureCacheDir: env.captureCacheDir,
     tier: env.tier,
   };
 
-  await workClone(boss, (jobId) => processCloneJob(deps, jobId));
-  console.log(JSON.stringify({ event: "worker_started", artifactsDir: env.artifactsDir, harnessDir: env.harnessDir, cacheStaleAfterMs: env.cacheStaleAfterMs }));
+  // One harness per concurrency slot: verify builds must not share a harness dir
+  // (concurrent framework builds collide). With concurrency 1 this is the plain
+  // harnessDir, preserving existing layouts. At most `concurrency` jobs run at
+  // once (one per poller), so a free slot always exists at checkout.
+  const harnesses = Array.from({ length: env.concurrency }, (_, i) =>
+    makeHarnessProvider(env.concurrency > 1 ? join(env.harnessDir, `slot-${i}`) : env.harnessDir),
+  );
+  const freeSlots = harnesses.map((_, i) => i);
+
+  await workClone(
+    boss,
+    async (jobId) => {
+      const slot = freeSlots.pop() ?? 0;
+      try {
+        await processCloneJob({ ...deps, harnessProvider: harnesses[slot]! }, jobId);
+      } finally {
+        freeSlots.push(slot);
+      }
+    },
+    env.concurrency,
+  );
+  console.log(JSON.stringify({ event: "worker_started", concurrency: env.concurrency, artifactsDir: env.artifactsDir, harnessDir: env.harnessDir, cacheStaleAfterMs: env.cacheStaleAfterMs }));
 }
 
 main().catch((e) => {