Initial commit

This commit is contained in:
Samraaj Bath
2026-06-29 15:11:48 -07:00
commit 66dfdcc58d
404 changed files with 45970 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
import { defineConfig } from "drizzle-kit";
export default defineConfig({
schema: "./src/schema.ts",
out: "./migrations",
dialect: "postgresql",
dbCredentials: {
url: process.env.DATABASE_URL ?? "postgresql://postgres@127.0.0.1:5432/postgres",
},
});
@@ -0,0 +1,49 @@
CREATE TABLE "api_keys" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"key_hash" text NOT NULL,
"label" text,
"rate_limit" integer,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"revoked_at" timestamp with time zone,
CONSTRAINT "api_keys_key_hash_unique" UNIQUE("key_hash")
);
--> statement-breakpoint
CREATE TABLE "cache" (
"cache_key" text PRIMARY KEY NOT NULL,
"job_id" uuid,
"url" text NOT NULL,
"options_hash" text NOT NULL,
"compiler_version" text NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"expires_at" timestamp with time zone NOT NULL
);
--> statement-breakpoint
CREATE TABLE "clones" (
"job_id" uuid PRIMARY KEY NOT NULL,
"url" text NOT NULL,
"route_count" integer DEFAULT 1 NOT NULL,
"file_manifest" jsonb NOT NULL,
"bundle_s3_key" text,
"verify" jsonb,
"capture_meta" jsonb NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE "jobs" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"kind" text NOT NULL,
"url" text NOT NULL,
"options" jsonb DEFAULT '{}'::jsonb NOT NULL,
"status" text DEFAULT 'queued' NOT NULL,
"cache_key" text NOT NULL,
"attempts" integer DEFAULT 0 NOT NULL,
"error" text,
"compiler_version" text,
"timings" jsonb,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"started_at" timestamp with time zone,
"finished_at" timestamp with time zone
);
--> statement-breakpoint
ALTER TABLE "cache" ADD CONSTRAINT "cache_job_id_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."jobs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "clones" ADD CONSTRAINT "clones_job_id_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."jobs"("id") ON DELETE cascade ON UPDATE no action;
@@ -0,0 +1,321 @@
{
"id": "9000d72e-0529-4eaf-b53d-43d3b93b5620",
"prevId": "00000000-0000-0000-0000-000000000000",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.api_keys": {
"name": "api_keys",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"key_hash": {
"name": "key_hash",
"type": "text",
"primaryKey": false,
"notNull": true
},
"label": {
"name": "label",
"type": "text",
"primaryKey": false,
"notNull": false
},
"rate_limit": {
"name": "rate_limit",
"type": "integer",
"primaryKey": false,
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"revoked_at": {
"name": "revoked_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {
"api_keys_key_hash_unique": {
"name": "api_keys_key_hash_unique",
"nullsNotDistinct": false,
"columns": [
"key_hash"
]
}
},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.cache": {
"name": "cache",
"schema": "",
"columns": {
"cache_key": {
"name": "cache_key",
"type": "text",
"primaryKey": true,
"notNull": true
},
"job_id": {
"name": "job_id",
"type": "uuid",
"primaryKey": false,
"notNull": false
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true
},
"options_hash": {
"name": "options_hash",
"type": "text",
"primaryKey": false,
"notNull": true
},
"compiler_version": {
"name": "compiler_version",
"type": "text",
"primaryKey": false,
"notNull": true
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"expires_at": {
"name": "expires_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true
}
},
"indexes": {},
"foreignKeys": {
"cache_job_id_jobs_id_fk": {
"name": "cache_job_id_jobs_id_fk",
"tableFrom": "cache",
"tableTo": "jobs",
"columnsFrom": [
"job_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.clones": {
"name": "clones",
"schema": "",
"columns": {
"job_id": {
"name": "job_id",
"type": "uuid",
"primaryKey": true,
"notNull": true
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true
},
"route_count": {
"name": "route_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"default": 1
},
"file_manifest": {
"name": "file_manifest",
"type": "jsonb",
"primaryKey": false,
"notNull": true
},
"bundle_s3_key": {
"name": "bundle_s3_key",
"type": "text",
"primaryKey": false,
"notNull": false
},
"verify": {
"name": "verify",
"type": "jsonb",
"primaryKey": false,
"notNull": false
},
"capture_meta": {
"name": "capture_meta",
"type": "jsonb",
"primaryKey": false,
"notNull": true
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
}
},
"indexes": {},
"foreignKeys": {
"clones_job_id_jobs_id_fk": {
"name": "clones_job_id_jobs_id_fk",
"tableFrom": "clones",
"tableTo": "jobs",
"columnsFrom": [
"job_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.jobs": {
"name": "jobs",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"kind": {
"name": "kind",
"type": "text",
"primaryKey": false,
"notNull": true
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true
},
"options": {
"name": "options",
"type": "jsonb",
"primaryKey": false,
"notNull": true,
"default": "'{}'::jsonb"
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"default": "'queued'"
},
"cache_key": {
"name": "cache_key",
"type": "text",
"primaryKey": false,
"notNull": true
},
"attempts": {
"name": "attempts",
"type": "integer",
"primaryKey": false,
"notNull": true,
"default": 0
},
"error": {
"name": "error",
"type": "text",
"primaryKey": false,
"notNull": false
},
"compiler_version": {
"name": "compiler_version",
"type": "text",
"primaryKey": false,
"notNull": false
},
"timings": {
"name": "timings",
"type": "jsonb",
"primaryKey": false,
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"started_at": {
"name": "started_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": false
},
"finished_at": {
"name": "finished_at",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
}
},
"enums": {},
"schemas": {},
"sequences": {},
"roles": {},
"policies": {},
"views": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}
+13
View File
@@ -0,0 +1,13 @@
{
"version": "7",
"dialect": "postgresql",
"entries": [
{
"idx": 0,
"version": "7",
"when": 1782170309013,
"tag": "0000_plain_wither",
"breakpoints": true
}
]
}
+28
View File
@@ -0,0 +1,28 @@
{
"name": "@cloner/db",
"version": "0.1.0",
"private": true,
"type": "module",
"exports": {
".": "./src/index.ts"
},
"scripts": {
"generate": "drizzle-kit generate",
"migrate": "tsx src/migrate.ts",
"test": "node --import tsx --test test/*.test.ts",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"drizzle-orm": "^0.38.3",
"pg": "^8.13.1",
"pg-boss": "^10.1.5"
},
"devDependencies": {
"@cloner/test-utils": "*",
"@types/pg": "^8.11.10",
"drizzle-kit": "^0.30.1",
"tsx": "4.22.4",
"typescript": "5.7.3",
"@types/node": "22.10.5"
}
}
+14
View File
@@ -0,0 +1,14 @@
import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres";
import pg from "pg";
import * as schema from "./schema.js";
export type Db = NodePgDatabase<typeof schema>;
export type DbHandle = { db: Db; pool: pg.Pool };
/** Create a Drizzle client + underlying pg Pool from a connection URL. Caller owns
* the pool lifecycle (`handle.pool.end()` on shutdown). */
export function createDb(connectionString: string): DbHandle {
const pool = new pg.Pool({ connectionString });
const db = drizzle(pool, { schema });
return { db, pool };
}
+10
View File
@@ -0,0 +1,10 @@
export * as schema from "./schema.js";
export {
jobs, clones, cache, apiKeys,
type Job, type NewJob, type Clone, type NewClone, type CacheRow, type ApiKey,
} from "./schema.js";
export { createDb, type Db, type DbHandle } from "./client.js";
export * as repo from "./repo.js";
export { createBoss, enqueueClone, workClone, CLONE_QUEUE, type ClonePayload } from "./queue.js";
export type { default as PgBoss } from "pg-boss";
export { runMigrations, MIGRATIONS_DIR } from "./migrate.js";
+34
View File
@@ -0,0 +1,34 @@
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { migrate } from "drizzle-orm/node-postgres/migrator";
import { createDb } from "./client.js";
const HERE = dirname(fileURLToPath(import.meta.url));
export const MIGRATIONS_DIR = join(HERE, "..", "migrations");
/** Apply all pending Drizzle migrations to the target database. */
export async function runMigrations(connectionString: string, migrationsFolder = MIGRATIONS_DIR): Promise<void> {
const { db, pool } = createDb(connectionString);
try {
await migrate(db, { migrationsFolder });
} finally {
await pool.end();
}
}
async function main(): Promise<void> {
const url = process.env.DATABASE_URL;
if (!url) {
console.error("DATABASE_URL is required");
process.exit(1);
}
await runMigrations(url);
console.log(JSON.stringify({ event: "migrations_applied" }));
}
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((e) => {
console.error(e);
process.exit(1);
});
}
+43
View File
@@ -0,0 +1,43 @@
import PgBoss from "pg-boss";
export const CLONE_QUEUE = "clone-jobs";
export type ClonePayload = { jobId: string };
/** Start a pg-boss instance on the same Postgres (no Redis). pg-boss manages its
* own schema/tables and gives us retries, heartbeats, and visibility timeouts. */
export async function createBoss(connectionString: string): Promise<PgBoss> {
const boss = new PgBoss({ connectionString });
boss.on("error", (e) => console.error(JSON.stringify({ event: "pgboss_error", error: String(e).slice(0, 300) })));
await boss.start();
// pg-boss v10 requires queues to be created before send/work (idempotent).
const anyBoss = boss as unknown as { createQueue?: (name: string) => Promise<void> };
if (typeof anyBoss.createQueue === "function") {
try {
await anyBoss.createQueue(CLONE_QUEUE);
} catch {
/* already exists */
}
}
return boss;
}
/** Enqueue a clone job. Returns the pg-boss job id (or null if deduped). */
export async function enqueueClone(boss: PgBoss, jobId: string): Promise<string | null> {
return boss.send(CLONE_QUEUE, { jobId } satisfies ClonePayload, {
retryLimit: 2,
retryBackoff: true,
expireInSeconds: 30 * 60,
});
}
/** Register the worker handler. Normalizes pg-boss v9 (single job) vs v10 (array)
* callback shapes so the consumer just gets a jobId. */
export async function workClone(boss: PgBoss, handler: (jobId: string) => Promise<void>): Promise<string> {
return boss.work(CLONE_QUEUE, async (job: unknown) => {
const arr = Array.isArray(job) ? job : [job];
for (const j of arr) {
const data = (j as { data?: ClonePayload }).data;
if (data?.jobId) await handler(data.jobId);
}
});
}
+95
View File
@@ -0,0 +1,95 @@
import { and, desc, eq, gt, sql } from "drizzle-orm";
import type { Db } from "./client.js";
import { jobs, clones, cache, apiKeys, type Job, type NewJob, type Clone, type NewClone, type CacheRow, type ApiKey } from "./schema.js";
// ---- jobs ----
export async function createJob(db: Db, input: Omit<NewJob, "id" | "createdAt">): Promise<Job> {
const [row] = await db.insert(jobs).values(input).returning();
return row!;
}
export async function getJob(db: Db, id: string): Promise<Job | undefined> {
const [row] = await db.select().from(jobs).where(eq(jobs.id, id)).limit(1);
return row;
}
export async function markRunning(db: Db, id: string): Promise<void> {
await db
.update(jobs)
.set({ status: "running", startedAt: new Date(), attempts: sql`${jobs.attempts} + 1` })
.where(eq(jobs.id, id));
}
export async function markSucceeded(db: Db, id: string, fields: { compilerVersion: string; timings: unknown }): Promise<void> {
await db
.update(jobs)
.set({ status: "succeeded", finishedAt: new Date(), compilerVersion: fields.compilerVersion, timings: fields.timings })
.where(eq(jobs.id, id));
}
export async function updateJobTimings(db: Db, id: string, timings: unknown): Promise<void> {
await db.update(jobs).set({ timings }).where(eq(jobs.id, id));
}
export async function markFailed(db: Db, id: string, error: string): Promise<void> {
await db.update(jobs).set({ status: "failed", finishedAt: new Date(), error: error.slice(0, 2000) }).where(eq(jobs.id, id));
}
export async function listJobs(db: Db, limit = 50): Promise<Job[]> {
return db.select().from(jobs).orderBy(desc(jobs.createdAt)).limit(limit);
}
export async function deleteJob(db: Db, id: string): Promise<boolean> {
const rows = await db.delete(jobs).where(eq(jobs.id, id)).returning({ id: jobs.id });
return rows.length > 0;
}
// ---- clones ----
export async function upsertClone(db: Db, input: NewClone): Promise<void> {
await db
.insert(clones)
.values(input)
.onConflictDoUpdate({ target: clones.jobId, set: { fileManifest: input.fileManifest, verify: input.verify, captureMeta: input.captureMeta, bundleS3Key: input.bundleS3Key, routeCount: input.routeCount } });
}
export async function getClone(db: Db, jobId: string): Promise<Clone | undefined> {
const [row] = await db.select().from(clones).where(eq(clones.jobId, jobId)).limit(1);
return row;
}
export async function updateCloneVerify(db: Db, jobId: string, verify: unknown): Promise<void> {
await db.update(clones).set({ verify: verify as Record<string, unknown> | null }).where(eq(clones.jobId, jobId));
}
// ---- cache ----
/** A *fresh* cache hit: same compilerVersion and not yet expired. */
export async function cacheGetFresh(db: Db, cacheKey: string, compilerVersion: string): Promise<CacheRow | undefined> {
const [row] = await db
.select()
.from(cache)
.where(and(eq(cache.cacheKey, cacheKey), eq(cache.compilerVersion, compilerVersion), gt(cache.expiresAt, new Date())))
.limit(1);
return row;
}
export async function cachePut(db: Db, input: { cacheKey: string; jobId: string; url: string; optionsHash: string; compilerVersion: string; expiresAt: Date }): Promise<void> {
await db
.insert(cache)
.values(input)
.onConflictDoUpdate({ target: cache.cacheKey, set: { jobId: input.jobId, expiresAt: input.expiresAt, compilerVersion: input.compilerVersion, createdAt: new Date() } });
}
// ---- api keys (M6) ----
export async function getApiKeyByHash(db: Db, keyHash: string): Promise<ApiKey | undefined> {
const [row] = await db.select().from(apiKeys).where(eq(apiKeys.keyHash, keyHash)).limit(1);
return row;
}
export async function createApiKey(db: Db, input: { keyHash: string; label?: string; rateLimit?: number }): Promise<ApiKey> {
const [row] = await db.insert(apiKeys).values(input).returning();
return row!;
}
+64
View File
@@ -0,0 +1,64 @@
import { pgTable, text, jsonb, integer, timestamp, uuid } from "drizzle-orm/pg-core";
/** Job lifecycle row. Status mirrors the queue; `cacheKey` ties it to the cache row.
* `options`/`timings` are jsonb (the service's typed shapes). */
export const jobs = pgTable("jobs", {
id: uuid("id").defaultRandom().primaryKey(),
kind: text("kind").notNull(), // "clone" | "clone_site"
url: text("url").notNull(),
options: jsonb("options").notNull().default({}),
status: text("status").notNull().default("queued"), // queued|running|succeeded|failed|cached
cacheKey: text("cache_key").notNull(),
attempts: integer("attempts").notNull().default(0),
error: text("error"),
compilerVersion: text("compiler_version"),
timings: jsonb("timings"),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
startedAt: timestamp("started_at", { withTimezone: true }),
finishedAt: timestamp("finished_at", { withTimezone: true }),
});
/** The persisted result, written once on success. `fileManifest` holds the text
* files inline + binary metadata (the eager file map sans bytes); binaries live in
* blob storage referenced by `bundleS3Key` / per-file keys in the manifest. */
export const clones = pgTable("clones", {
jobId: uuid("job_id")
.primaryKey()
.references(() => jobs.id, { onDelete: "cascade" }),
url: text("url").notNull(),
routeCount: integer("route_count").notNull().default(1),
fileManifest: jsonb("file_manifest").notNull(),
bundleS3Key: text("bundle_s3_key"),
verify: jsonb("verify"),
captureMeta: jsonb("capture_meta").notNull(),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
});
/** Freshness-bounded "recently cloned this URL+options" cache. A hit requires
* `now < expiresAt` and a matching compilerVersion. */
export const cache = pgTable("cache", {
cacheKey: text("cache_key").primaryKey(),
jobId: uuid("job_id").references(() => jobs.id, { onDelete: "cascade" }),
url: text("url").notNull(),
optionsHash: text("options_hash").notNull(),
compilerVersion: text("compiler_version").notNull(),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
});
/** API keys (M6): only a hash is stored. `rateLimit` is requests/min (nullable = default). */
export const apiKeys = pgTable("api_keys", {
id: uuid("id").defaultRandom().primaryKey(),
keyHash: text("key_hash").notNull().unique(),
label: text("label"),
rateLimit: integer("rate_limit"),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
revokedAt: timestamp("revoked_at", { withTimezone: true }),
});
export type Job = typeof jobs.$inferSelect;
export type NewJob = typeof jobs.$inferInsert;
export type Clone = typeof clones.$inferSelect;
export type NewClone = typeof clones.$inferInsert;
export type CacheRow = typeof cache.$inferSelect;
export type ApiKey = typeof apiKeys.$inferSelect;
+35
View File
@@ -0,0 +1,35 @@
import { describe, it, before, after } from "node:test";
import assert from "node:assert/strict";
import { createDb, runMigrations, repo, type Db } from "../src/index.js";
import { acquireTestPostgres, hasTestPostgres, type EphemeralPg } from "@cloner/test-utils";
describe("cache freshness (TTL + compilerVersion)", { skip: hasTestPostgres() ? false : "no test Postgres" }, () => {
let pg: EphemeralPg;
let db: Db;
let pool: { end: () => Promise<void> };
before(async () => {
pg = (await acquireTestPostgres())!;
await runMigrations(pg.url);
const h = createDb(pg.url);
db = h.db;
pool = h.pool;
});
after(async () => {
await pool?.end().catch(() => {});
await pg?.stop().catch(() => {});
});
it("returns a hit only when not expired AND compilerVersion matches", async () => {
const job = await repo.createJob(db, { kind: "clone", url: "https://x/", options: {}, status: "succeeded", cacheKey: "k1", compilerVersion: "0.1.0" });
// Fresh row.
await repo.cachePut(db, { cacheKey: "k1", jobId: job.id, url: "https://x/", optionsHash: "{}", compilerVersion: "0.1.0", expiresAt: new Date(Date.now() + 60_000) });
assert.ok(await repo.cacheGetFresh(db, "k1", "0.1.0"), "fresh + matching version → hit");
assert.equal(await repo.cacheGetFresh(db, "k1", "0.2.0"), undefined, "version bump → miss");
// Expire it (upsert past expiry).
await repo.cachePut(db, { cacheKey: "k1", jobId: job.id, url: "https://x/", optionsHash: "{}", compilerVersion: "0.1.0", expiresAt: new Date(Date.now() - 1000) });
assert.equal(await repo.cacheGetFresh(db, "k1", "0.1.0"), undefined, "stale → miss");
});
});
+7
View File
@@ -0,0 +1,7 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"types": ["node"]
},
"include": ["src/**/*.ts", "test/**/*.ts"]
}