Initial commit

packages/db/src/client.ts

@@ -0,0 +1,14 @@
+import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres";
+import pg from "pg";
+import * as schema from "./schema.js";
+
+export type Db = NodePgDatabase<typeof schema>;
+export type DbHandle = { db: Db; pool: pg.Pool };
+
+/** Create a Drizzle client + underlying pg Pool from a connection URL. Caller owns
+ *  the pool lifecycle (`handle.pool.end()` on shutdown). */
+export function createDb(connectionString: string): DbHandle {
+  const pool = new pg.Pool({ connectionString });
+  const db = drizzle(pool, { schema });
+  return { db, pool };
+}

packages/db/src/index.ts

@@ -0,0 +1,10 @@
+export * as schema from "./schema.js";
+export {
+  jobs, clones, cache, apiKeys,
+  type Job, type NewJob, type Clone, type NewClone, type CacheRow, type ApiKey,
+} from "./schema.js";
+export { createDb, type Db, type DbHandle } from "./client.js";
+export * as repo from "./repo.js";
+export { createBoss, enqueueClone, workClone, CLONE_QUEUE, type ClonePayload } from "./queue.js";
+export type { default as PgBoss } from "pg-boss";
+export { runMigrations, MIGRATIONS_DIR } from "./migrate.js";

packages/db/src/migrate.ts

@@ -0,0 +1,34 @@
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { migrate } from "drizzle-orm/node-postgres/migrator";
+import { createDb } from "./client.js";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+export const MIGRATIONS_DIR = join(HERE, "..", "migrations");
+
+/** Apply all pending Drizzle migrations to the target database. */
+export async function runMigrations(connectionString: string, migrationsFolder = MIGRATIONS_DIR): Promise<void> {
+  const { db, pool } = createDb(connectionString);
+  try {
+    await migrate(db, { migrationsFolder });
+  } finally {
+    await pool.end();
+  }
+}
+
+async function main(): Promise<void> {
+  const url = process.env.DATABASE_URL;
+  if (!url) {
+    console.error("DATABASE_URL is required");
+    process.exit(1);
+  }
+  await runMigrations(url);
+  console.log(JSON.stringify({ event: "migrations_applied" }));
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });
+}

packages/db/src/queue.ts

@@ -0,0 +1,43 @@
+import PgBoss from "pg-boss";
+
+export const CLONE_QUEUE = "clone-jobs";
+export type ClonePayload = { jobId: string };
+
+/** Start a pg-boss instance on the same Postgres (no Redis). pg-boss manages its
+ *  own schema/tables and gives us retries, heartbeats, and visibility timeouts. */
+export async function createBoss(connectionString: string): Promise<PgBoss> {
+  const boss = new PgBoss({ connectionString });
+  boss.on("error", (e) => console.error(JSON.stringify({ event: "pgboss_error", error: String(e).slice(0, 300) })));
+  await boss.start();
+  // pg-boss v10 requires queues to be created before send/work (idempotent).
+  const anyBoss = boss as unknown as { createQueue?: (name: string) => Promise<void> };
+  if (typeof anyBoss.createQueue === "function") {
+    try {
+      await anyBoss.createQueue(CLONE_QUEUE);
+    } catch {
+      /* already exists */
+    }
+  }
+  return boss;
+}
+
+/** Enqueue a clone job. Returns the pg-boss job id (or null if deduped). */
+export async function enqueueClone(boss: PgBoss, jobId: string): Promise<string | null> {
+  return boss.send(CLONE_QUEUE, { jobId } satisfies ClonePayload, {
+    retryLimit: 2,
+    retryBackoff: true,
+    expireInSeconds: 30 * 60,
+  });
+}
+
+/** Register the worker handler. Normalizes pg-boss v9 (single job) vs v10 (array)
+ *  callback shapes so the consumer just gets a jobId. */
+export async function workClone(boss: PgBoss, handler: (jobId: string) => Promise<void>): Promise<string> {
+  return boss.work(CLONE_QUEUE, async (job: unknown) => {
+    const arr = Array.isArray(job) ? job : [job];
+    for (const j of arr) {
+      const data = (j as { data?: ClonePayload }).data;
+      if (data?.jobId) await handler(data.jobId);
+    }
+  });
+}

packages/db/src/repo.ts

@@ -0,0 +1,95 @@
+import { and, desc, eq, gt, sql } from "drizzle-orm";
+import type { Db } from "./client.js";
+import { jobs, clones, cache, apiKeys, type Job, type NewJob, type Clone, type NewClone, type CacheRow, type ApiKey } from "./schema.js";
+
+// ---- jobs ----
+
+export async function createJob(db: Db, input: Omit<NewJob, "id" | "createdAt">): Promise<Job> {
+  const [row] = await db.insert(jobs).values(input).returning();
+  return row!;
+}
+
+export async function getJob(db: Db, id: string): Promise<Job | undefined> {
+  const [row] = await db.select().from(jobs).where(eq(jobs.id, id)).limit(1);
+  return row;
+}
+
+export async function markRunning(db: Db, id: string): Promise<void> {
+  await db
+    .update(jobs)
+    .set({ status: "running", startedAt: new Date(), attempts: sql`${jobs.attempts} + 1` })
+    .where(eq(jobs.id, id));
+}
+
+export async function markSucceeded(db: Db, id: string, fields: { compilerVersion: string; timings: unknown }): Promise<void> {
+  await db
+    .update(jobs)
+    .set({ status: "succeeded", finishedAt: new Date(), compilerVersion: fields.compilerVersion, timings: fields.timings })
+    .where(eq(jobs.id, id));
+}
+
+export async function updateJobTimings(db: Db, id: string, timings: unknown): Promise<void> {
+  await db.update(jobs).set({ timings }).where(eq(jobs.id, id));
+}
+
+export async function markFailed(db: Db, id: string, error: string): Promise<void> {
+  await db.update(jobs).set({ status: "failed", finishedAt: new Date(), error: error.slice(0, 2000) }).where(eq(jobs.id, id));
+}
+
+export async function listJobs(db: Db, limit = 50): Promise<Job[]> {
+  return db.select().from(jobs).orderBy(desc(jobs.createdAt)).limit(limit);
+}
+
+export async function deleteJob(db: Db, id: string): Promise<boolean> {
+  const rows = await db.delete(jobs).where(eq(jobs.id, id)).returning({ id: jobs.id });
+  return rows.length > 0;
+}
+
+// ---- clones ----
+
+export async function upsertClone(db: Db, input: NewClone): Promise<void> {
+  await db
+    .insert(clones)
+    .values(input)
+    .onConflictDoUpdate({ target: clones.jobId, set: { fileManifest: input.fileManifest, verify: input.verify, captureMeta: input.captureMeta, bundleS3Key: input.bundleS3Key, routeCount: input.routeCount } });
+}
+
+export async function getClone(db: Db, jobId: string): Promise<Clone | undefined> {
+  const [row] = await db.select().from(clones).where(eq(clones.jobId, jobId)).limit(1);
+  return row;
+}
+
+export async function updateCloneVerify(db: Db, jobId: string, verify: unknown): Promise<void> {
+  await db.update(clones).set({ verify: verify as Record<string, unknown> | null }).where(eq(clones.jobId, jobId));
+}
+
+// ---- cache ----
+
+/** A *fresh* cache hit: same compilerVersion and not yet expired. */
+export async function cacheGetFresh(db: Db, cacheKey: string, compilerVersion: string): Promise<CacheRow | undefined> {
+  const [row] = await db
+    .select()
+    .from(cache)
+    .where(and(eq(cache.cacheKey, cacheKey), eq(cache.compilerVersion, compilerVersion), gt(cache.expiresAt, new Date())))
+    .limit(1);
+  return row;
+}
+
+export async function cachePut(db: Db, input: { cacheKey: string; jobId: string; url: string; optionsHash: string; compilerVersion: string; expiresAt: Date }): Promise<void> {
+  await db
+    .insert(cache)
+    .values(input)
+    .onConflictDoUpdate({ target: cache.cacheKey, set: { jobId: input.jobId, expiresAt: input.expiresAt, compilerVersion: input.compilerVersion, createdAt: new Date() } });
+}
+
+// ---- api keys (M6) ----
+
+export async function getApiKeyByHash(db: Db, keyHash: string): Promise<ApiKey | undefined> {
+  const [row] = await db.select().from(apiKeys).where(eq(apiKeys.keyHash, keyHash)).limit(1);
+  return row;
+}
+
+export async function createApiKey(db: Db, input: { keyHash: string; label?: string; rateLimit?: number }): Promise<ApiKey> {
+  const [row] = await db.insert(apiKeys).values(input).returning();
+  return row!;
+}

packages/db/src/schema.ts

@@ -0,0 +1,64 @@
+import { pgTable, text, jsonb, integer, timestamp, uuid } from "drizzle-orm/pg-core";
+
+/** Job lifecycle row. Status mirrors the queue; `cacheKey` ties it to the cache row.
+ *  `options`/`timings` are jsonb (the service's typed shapes). */
+export const jobs = pgTable("jobs", {
+  id: uuid("id").defaultRandom().primaryKey(),
+  kind: text("kind").notNull(), // "clone" | "clone_site"
+  url: text("url").notNull(),
+  options: jsonb("options").notNull().default({}),
+  status: text("status").notNull().default("queued"), // queued|running|succeeded|failed|cached
+  cacheKey: text("cache_key").notNull(),
+  attempts: integer("attempts").notNull().default(0),
+  error: text("error"),
+  compilerVersion: text("compiler_version"),
+  timings: jsonb("timings"),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  startedAt: timestamp("started_at", { withTimezone: true }),
+  finishedAt: timestamp("finished_at", { withTimezone: true }),
+});
+
+/** The persisted result, written once on success. `fileManifest` holds the text
+ *  files inline + binary metadata (the eager file map sans bytes); binaries live in
+ *  blob storage referenced by `bundleS3Key` / per-file keys in the manifest. */
+export const clones = pgTable("clones", {
+  jobId: uuid("job_id")
+    .primaryKey()
+    .references(() => jobs.id, { onDelete: "cascade" }),
+  url: text("url").notNull(),
+  routeCount: integer("route_count").notNull().default(1),
+  fileManifest: jsonb("file_manifest").notNull(),
+  bundleS3Key: text("bundle_s3_key"),
+  verify: jsonb("verify"),
+  captureMeta: jsonb("capture_meta").notNull(),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+});
+
+/** Freshness-bounded "recently cloned this URL+options" cache. A hit requires
+ *  `now < expiresAt` and a matching compilerVersion. */
+export const cache = pgTable("cache", {
+  cacheKey: text("cache_key").primaryKey(),
+  jobId: uuid("job_id").references(() => jobs.id, { onDelete: "cascade" }),
+  url: text("url").notNull(),
+  optionsHash: text("options_hash").notNull(),
+  compilerVersion: text("compiler_version").notNull(),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
+});
+
+/** API keys (M6): only a hash is stored. `rateLimit` is requests/min (nullable = default). */
+export const apiKeys = pgTable("api_keys", {
+  id: uuid("id").defaultRandom().primaryKey(),
+  keyHash: text("key_hash").notNull().unique(),
+  label: text("label"),
+  rateLimit: integer("rate_limit"),
+  createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  revokedAt: timestamp("revoked_at", { withTimezone: true }),
+});
+
+export type Job = typeof jobs.$inferSelect;
+export type NewJob = typeof jobs.$inferInsert;
+export type Clone = typeof clones.$inferSelect;
+export type NewClone = typeof clones.$inferInsert;
+export type CacheRow = typeof cache.$inferSelect;
+export type ApiKey = typeof apiKeys.$inferSelect;