Files
ditto.site/packages/db/src/repo.ts
T
Samraaj BathandClaude Fable 5 4c1dea77e1 Parallelize the clone queue: in-flight dedup + WORKER_CONCURRENCY
Two changes that multiply effective queue throughput:

- In-flight dedup: POST /v1/clones now attaches identical submits (same
  cacheKey) to the already queued/running job instead of enqueueing a
  duplicate capture. Retry-spam previously multiplied queue load (observed
  4x same-URL submissions in prod); now it's free. Best-effort — a
  same-instant race can still double-submit. Gated on !noCache.

- WORKER_CONCURRENCY (default 1): the worker registers N independent
  pg-boss pollers, running up to N captures concurrently per process with
  no head-of-line blocking. Each slot gets its own verify harness dir
  (harnessDir/slot-i) so concurrent framework builds never collide;
  concurrency 1 keeps the flat harnessDir layout.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-06 22:44:42 -07:00

148 lines
5.8 KiB
TypeScript

import { and, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm";
import type { Db } from "./client.js";
import { jobs, clones, cache, apiKeys, signupTokens, jobEvents, type Job, type NewJob, type Clone, type NewClone, type CacheRow, type ApiKey, type SignupToken } from "./schema.js";
// ---- jobs ----
export async function createJob(db: Db, input: Omit<NewJob, "id" | "createdAt">): Promise<Job> {
const [row] = await db.insert(jobs).values(input).returning();
return row!;
}
export async function getJob(db: Db, id: string): Promise<Job | undefined> {
const [row] = await db.select().from(jobs).where(eq(jobs.id, id)).limit(1);
return row;
}
export async function markRunning(db: Db, id: string): Promise<void> {
await db
.update(jobs)
.set({ status: "running", startedAt: new Date(), attempts: sql`${jobs.attempts} + 1` })
.where(eq(jobs.id, id));
}
export async function markSucceeded(db: Db, id: string, fields: { compilerVersion: string; timings: unknown }): Promise<void> {
await db
.update(jobs)
.set({ status: "succeeded", finishedAt: new Date(), compilerVersion: fields.compilerVersion, timings: fields.timings })
.where(eq(jobs.id, id));
}
export async function updateJobTimings(db: Db, id: string, timings: unknown): Promise<void> {
await db.update(jobs).set({ timings }).where(eq(jobs.id, id));
}
export async function markFailed(db: Db, id: string, error: string): Promise<void> {
await db.update(jobs).set({ status: "failed", finishedAt: new Date(), error: error.slice(0, 2000) }).where(eq(jobs.id, id));
}
export async function listJobs(db: Db, limit = 50): Promise<Job[]> {
return db.select().from(jobs).orderBy(desc(jobs.createdAt)).limit(limit);
}
/** The newest still-active (queued/running) job for a cache key, if any — the
* in-flight dedup lookup: identical submits attach to it instead of enqueueing
* a duplicate capture. */
export async function findActiveJobByCacheKey(db: Db, key: string): Promise<Job | undefined> {
const [row] = await db
.select()
.from(jobs)
.where(and(eq(jobs.cacheKey, key), inArray(jobs.status, ["queued", "running"])))
.orderBy(desc(jobs.createdAt))
.limit(1);
return row;
}
export async function deleteJob(db: Db, id: string): Promise<boolean> {
const rows = await db.delete(jobs).where(eq(jobs.id, id)).returning({ id: jobs.id });
return rows.length > 0;
}
// ---- clones ----
export async function upsertClone(db: Db, input: NewClone): Promise<void> {
await db
.insert(clones)
.values(input)
.onConflictDoUpdate({ target: clones.jobId, set: { fileManifest: input.fileManifest, verify: input.verify, captureMeta: input.captureMeta, bundleS3Key: input.bundleS3Key, routeCount: input.routeCount } });
}
export async function getClone(db: Db, jobId: string): Promise<Clone | undefined> {
const [row] = await db.select().from(clones).where(eq(clones.jobId, jobId)).limit(1);
return row;
}
export async function updateCloneVerify(db: Db, jobId: string, verify: unknown): Promise<void> {
await db.update(clones).set({ verify: verify as Record<string, unknown> | null }).where(eq(clones.jobId, jobId));
}
// ---- cache ----
/** A *fresh* cache hit: same compilerVersion and not yet expired. */
export async function cacheGetFresh(db: Db, cacheKey: string, compilerVersion: string): Promise<CacheRow | undefined> {
const [row] = await db
.select()
.from(cache)
.where(and(eq(cache.cacheKey, cacheKey), eq(cache.compilerVersion, compilerVersion), gt(cache.expiresAt, new Date())))
.limit(1);
return row;
}
export async function cachePut(db: Db, input: { cacheKey: string; jobId: string; url: string; optionsHash: string; compilerVersion: string; expiresAt: Date }): Promise<void> {
await db
.insert(cache)
.values(input)
.onConflictDoUpdate({ target: cache.cacheKey, set: { jobId: input.jobId, expiresAt: input.expiresAt, compilerVersion: input.compilerVersion, createdAt: new Date() } });
}
// ---- api keys (M6) ----
export async function getApiKeyByHash(db: Db, keyHash: string): Promise<ApiKey | undefined> {
const [row] = await db.select().from(apiKeys).where(eq(apiKeys.keyHash, keyHash)).limit(1);
return row;
}
export async function createApiKey(db: Db, input: { keyHash: string; label?: string; rateLimit?: number }): Promise<ApiKey> {
const [row] = await db.insert(apiKeys).values(input).returning();
return row!;
}
// ---- signup tokens ----
export async function createSignupToken(db: Db, input: { email: string; tokenHash: string; expiresAt: Date }): Promise<SignupToken> {
const [row] = await db.insert(signupTokens).values(input).returning();
return row!;
}
/** Atomically consume a still-fresh signup token. Returns undefined for missing,
* expired, or already-used tokens. */
export async function consumeSignupToken(db: Db, tokenHash: string): Promise<SignupToken | undefined> {
const [row] = await db
.update(signupTokens)
.set({ consumedAt: new Date() })
.where(and(eq(signupTokens.tokenHash, tokenHash), gt(signupTokens.expiresAt, new Date()), isNull(signupTokens.consumedAt)))
.returning();
return row;
}
// ---- job events ----
export async function appendJobEvent(db: Db, jobId: string, payload: Record<string, unknown>): Promise<number> {
const [row] = await db
.select({ n: sql<number>`coalesce(max(${jobEvents.seq}), 0)` })
.from(jobEvents)
.where(eq(jobEvents.jobId, jobId));
const seq = (row?.n ?? 0) + 1;
await db.insert(jobEvents).values({ jobId, seq, payload: { t: Date.now(), ...payload } });
return seq;
}
export async function listJobEvents(db: Db, jobId: string, after = 0): Promise<Array<Record<string, unknown>>> {
const rows = await db
.select()
.from(jobEvents)
.where(and(eq(jobEvents.jobId, jobId), sql`${jobEvents.seq} > ${after}`))
.orderBy(jobEvents.seq);
return rows.map((r) => r.payload as Record<string, unknown>);
}