Initial commit

packages/storage/src/blob.ts

@@ -0,0 +1,37 @@
+/** A minimal object-store client. The ArtifactStore is built on this so its logic
+ *  is testable with an in-memory client and runs on S3/R2 in production. */
+export interface BlobClient {
+  put(key: string, bytes: Buffer, contentType: string): Promise<void>;
+  get(key: string): Promise<Buffer | null>;
+  /** A URL a client can use to fetch the object directly (presigned for S3). */
+  presign(key: string, opts?: { expiresSeconds?: number; downloadName?: string; contentType?: string }): Promise<string>;
+  delete(key: string): Promise<void>;
+  /** Delete everything under a prefix (best effort). */
+  deletePrefix(prefix: string): Promise<void>;
+}
+
+/** In-memory BlobClient for tests. presign() returns a fake URL (no real fetch). */
+export class InMemoryBlobClient implements BlobClient {
+  private objs = new Map<string, { bytes: Buffer; contentType: string }>();
+  constructor(private publicBase = "memory://blobs") {}
+
+  async put(key: string, bytes: Buffer, contentType: string): Promise<void> {
+    this.objs.set(key, { bytes, contentType });
+  }
+  async get(key: string): Promise<Buffer | null> {
+    return this.objs.get(key)?.bytes ?? null;
+  }
+  async presign(key: string): Promise<string> {
+    return `${this.publicBase}/${key}?sig=test`;
+  }
+  async delete(key: string): Promise<void> {
+    this.objs.delete(key);
+  }
+  async deletePrefix(prefix: string): Promise<void> {
+    for (const k of [...this.objs.keys()]) if (k.startsWith(prefix)) this.objs.delete(k);
+  }
+  /** test helper */
+  has(key: string): boolean {
+    return this.objs.has(key);
+  }
+}

packages/storage/src/bundle.ts

@@ -0,0 +1,135 @@
+import { gzipSync, deflateRawSync } from "node:zlib";
+import { createHash } from "node:crypto";
+
+/** Build a single ustar tar header block (512 bytes) for a regular file. mtime is
+ *  fixed at 0 so the archive is deterministic for the same inputs. */
+function tarHeader(name: string, size: number): Buffer {
+  let prefix = "";
+  let nm = name;
+  if (Buffer.byteLength(name) > 100) {
+    let split = -1;
+    for (let p = 0; p < name.length; p++) {
+      if (name[p] === "/" && Buffer.byteLength(name.slice(p + 1)) <= 100 && Buffer.byteLength(name.slice(0, p)) <= 155) {
+        split = p;
+        break;
+      }
+    }
+    if (split < 0) throw new Error("path too long for tar: " + name);
+    prefix = name.slice(0, split);
+    nm = name.slice(split + 1);
+  }
+  const h = Buffer.alloc(512);
+  h.write(nm, 0, 100, "utf8");
+  h.write("0000644\0", 100, 8, "ascii"); // mode
+  h.write("0000000\0", 108, 8, "ascii"); // uid
+  h.write("0000000\0", 116, 8, "ascii"); // gid
+  h.write(size.toString(8).padStart(11, "0") + "\0", 124, 12, "ascii"); // size (octal)
+  h.write("00000000000\0", 136, 12, "ascii"); // mtime = 0
+  h.write("        ", 148, 8, "ascii"); // checksum placeholder (8 spaces)
+  h.write("0", 156, 1, "ascii"); // typeflag: regular file
+  h.write("ustar\0", 257, 6, "ascii");
+  h.write("00", 263, 2, "ascii");
+  if (prefix) h.write(prefix, 345, 155, "utf8");
+  let sum = 0;
+  for (let i = 0; i < 512; i++) sum += h[i]!;
+  h.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, 8, "ascii");
+  return h;
+}
+
+/** Pack files into a deterministic .tar.gz (sorted by path, zero mtime). */
+export function makeTarGz(files: Array<{ path: string; bytes: Buffer }>): Buffer {
+  const sorted = [...files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const parts: Buffer[] = [];
+  for (const f of sorted) {
+    parts.push(tarHeader(f.path, f.bytes.length));
+    parts.push(f.bytes);
+    const pad = (512 - (f.bytes.length % 512)) % 512;
+    if (pad) parts.push(Buffer.alloc(pad));
+  }
+  parts.push(Buffer.alloc(1024)); // two zero blocks = end of archive
+  return gzipSync(Buffer.concat(parts), { level: 9 });
+}
+
+export function sha256hex(buf: Buffer): string {
+  return createHash("sha256").update(buf).digest("hex");
+}
+
+// ---- ZIP (deflate) ----
+
+const CRC_TABLE = (() => {
+  const t = new Uint32Array(256);
+  for (let n = 0; n < 256; n++) {
+    let c = n;
+    for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+    t[n] = c >>> 0;
+  }
+  return t;
+})();
+
+function crc32(buf: Buffer): number {
+  let c = 0xffffffff;
+  for (let i = 0; i < buf.length; i++) c = CRC_TABLE[(c ^ buf[i]!) & 0xff]! ^ (c >>> 8);
+  return (c ^ 0xffffffff) >>> 0;
+}
+
+/** Pack files into a deterministic .zip (deflate; fixed DOS timestamp). */
+export function makeZip(files: Array<{ path: string; bytes: Buffer }>): Buffer {
+  const sorted = [...files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const DOS_TIME = 0; // fixed → deterministic
+  const DOS_DATE = 0x21; // 1980-01-01
+  const local: Buffer[] = [];
+  const central: Buffer[] = [];
+  let offset = 0;
+  for (const f of sorted) {
+    const name = Buffer.from(f.path, "utf8");
+    const crc = crc32(f.bytes);
+    const comp = deflateRawSync(f.bytes, { level: 9 });
+    const lh = Buffer.alloc(30);
+    lh.writeUInt32LE(0x04034b50, 0);
+    lh.writeUInt16LE(20, 4); // version needed
+    lh.writeUInt16LE(0, 6); // flags
+    lh.writeUInt16LE(8, 8); // method: deflate
+    lh.writeUInt16LE(DOS_TIME, 10);
+    lh.writeUInt16LE(DOS_DATE, 12);
+    lh.writeUInt32LE(crc, 14);
+    lh.writeUInt32LE(comp.length, 18);
+    lh.writeUInt32LE(f.bytes.length, 22);
+    lh.writeUInt16LE(name.length, 26);
+    lh.writeUInt16LE(0, 28);
+    local.push(lh, name, comp);
+
+    const ch = Buffer.alloc(46);
+    ch.writeUInt32LE(0x02014b50, 0);
+    ch.writeUInt16LE(20, 4); // version made by
+    ch.writeUInt16LE(20, 6); // version needed
+    ch.writeUInt16LE(0, 8); // flags
+    ch.writeUInt16LE(8, 10); // method
+    ch.writeUInt16LE(DOS_TIME, 12);
+    ch.writeUInt16LE(DOS_DATE, 14);
+    ch.writeUInt32LE(crc, 16);
+    ch.writeUInt32LE(comp.length, 20);
+    ch.writeUInt32LE(f.bytes.length, 24);
+    ch.writeUInt16LE(name.length, 28);
+    ch.writeUInt16LE(0, 30); // extra len
+    ch.writeUInt16LE(0, 32); // comment len
+    ch.writeUInt16LE(0, 34); // disk number
+    ch.writeUInt16LE(0, 36); // internal attrs
+    ch.writeUInt32LE(0, 38); // external attrs
+    ch.writeUInt32LE(offset, 42);
+    central.push(ch, name);
+
+    offset += lh.length + name.length + comp.length;
+  }
+  const centralBuf = Buffer.concat(central);
+  const localBuf = Buffer.concat(local);
+  const eocd = Buffer.alloc(22);
+  eocd.writeUInt32LE(0x06054b50, 0);
+  eocd.writeUInt16LE(0, 4);
+  eocd.writeUInt16LE(0, 6);
+  eocd.writeUInt16LE(sorted.length, 8);
+  eocd.writeUInt16LE(sorted.length, 10);
+  eocd.writeUInt32LE(centralBuf.length, 12);
+  eocd.writeUInt32LE(localBuf.length, 16);
+  eocd.writeUInt16LE(0, 20);
+  return Buffer.concat([localBuf, centralBuf, eocd]);
+}

packages/storage/src/factory.ts

@@ -0,0 +1,14 @@
+import { join } from "node:path";
+import type { ArtifactStore } from "./types.js";
+import { LocalArtifactStore } from "./local.js";
+import { ObjectArtifactStore } from "./objectStore.js";
+import { S3BlobClient, s3ConfigFromEnv } from "./s3.js";
+
+/** Pick the artifact store from the environment: S3/R2 when S3_BUCKET is set,
+ *  otherwise a local disk store (ARTIFACTS_DIR or ./local-data/artifacts). */
+export function artifactStoreFromEnv(): ArtifactStore {
+  const s3 = s3ConfigFromEnv();
+  if (s3) return new ObjectArtifactStore(new S3BlobClient(s3));
+  const dir = process.env.ARTIFACTS_DIR ?? join(process.cwd(), "local-data", "artifacts");
+  return new LocalArtifactStore(dir);
+}

packages/storage/src/index.ts

@@ -0,0 +1,7 @@
+export type { ArtifactStore, StoredFile, StoredManifest } from "./types.js";
+export { LocalArtifactStore } from "./local.js";
+export { ObjectArtifactStore } from "./objectStore.js";
+export { type BlobClient, InMemoryBlobClient } from "./blob.js";
+export { S3BlobClient, s3ConfigFromEnv, type S3Config } from "./s3.js";
+export { artifactStoreFromEnv } from "./factory.js";
+export { makeTarGz, makeZip, sha256hex } from "./bundle.js";

packages/storage/src/local.ts

@@ -0,0 +1,60 @@
+import { mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs";
+import { join, dirname, normalize } from "node:path";
+import type { FileMap } from "@cloner/core";
+import type { ArtifactStore, StoredFile, StoredManifest } from "./types.js";
+
+/** Reject path traversal — only allow relative paths that stay within the job dir. */
+function safeRel(path: string): string {
+  const n = normalize(path);
+  if (n.includes("\0") || n.startsWith("/") || /^[A-Za-z]:/.test(n) || n.split(/[/\\]/).includes("..")) {
+    throw new Error("unsafe path: " + path);
+  }
+  return n;
+}
+
+/**
+ * Disk-backed artifact store (M2, local dev). Writes every file under
+ * `<baseDir>/<jobId>/`, returns a manifest with text inline + binary keys, and
+ * serves bytes back for the API's /files/* route. In production this is swapped for
+ * the S3 store (M4); the API/worker depend only on the ArtifactStore interface.
+ */
+export class LocalArtifactStore implements ArtifactStore {
+  constructor(private baseDir: string) {}
+
+  private jobDir(jobId: string): string {
+    return join(this.baseDir, jobId);
+  }
+
+  async putClone(jobId: string, files: FileMap): Promise<StoredManifest> {
+    const root = this.jobDir(jobId);
+    const out: StoredFile[] = [];
+    for (const [path, f] of Object.entries(files)) {
+      const rel = safeRel(path);
+      const dest = join(root, rel);
+      mkdirSync(dirname(dest), { recursive: true });
+      writeFileSync(dest, readFileSync(f.absPath));
+      if (f.kind === "text") {
+        out.push({ path, kind: "text", bytes: f.bytes, sha256: f.sha256, content: f.content ?? "" });
+      } else {
+        out.push({ path, kind: "binary", bytes: f.bytes, sha256: f.sha256, key: `${jobId}/${rel}` });
+      }
+    }
+    out.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+    return { files: out };
+  }
+
+  async getFile(jobId: string, path: string): Promise<{ bytes: Buffer } | null> {
+    const dest = join(this.jobDir(jobId), safeRel(path));
+    if (!existsSync(dest)) return null;
+    return { bytes: readFileSync(dest) };
+  }
+
+  async binaryUrl(jobId: string, path: string): Promise<string> {
+    // Local: served by the API itself.
+    return `/v1/clones/${jobId}/files/${path}`;
+  }
+
+  async remove(jobId: string): Promise<void> {
+    rmSync(this.jobDir(jobId), { recursive: true, force: true });
+  }
+}

packages/storage/src/objectStore.ts

@@ -0,0 +1,62 @@
+import { extname } from "node:path";
+import type { FileMap } from "@cloner/core";
+import type { ArtifactStore, StoredFile, StoredManifest } from "./types.js";
+import type { BlobClient } from "./blob.js";
+
+const CT: Record<string, string> = {
+  ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp",
+  ".avif": "image/avif", ".gif": "image/gif", ".svg": "image/svg+xml", ".ico": "image/x-icon",
+  ".woff2": "font/woff2", ".woff": "font/woff", ".ttf": "font/ttf", ".otf": "font/otf",
+  ".mp4": "video/mp4", ".webm": "video/webm",
+};
+const ctFor = (p: string) => CT[extname(p).toLowerCase()] ?? "application/octet-stream";
+
+/**
+ * Object-store (S3 / R2 / MinIO) ArtifactStore. Binaries are uploaded
+ * content-addressably under `clones/<jobId>/<path>` and served via presigned URLs;
+ * text files stay inline in the DB manifest (small) and are not uploaded. The
+ * whole-app bundle is uploaded + presigned on demand.
+ */
+export class ObjectArtifactStore implements ArtifactStore {
+  constructor(private blob: BlobClient, private opts?: { presignExpiresSeconds?: number }) {}
+
+  private key(jobId: string, path: string): string {
+    return `clones/${jobId}/${path}`;
+  }
+
+  async putClone(jobId: string, files: FileMap): Promise<StoredManifest> {
+    const out: StoredFile[] = [];
+    for (const [path, f] of Object.entries(files)) {
+      if (f.kind === "text") {
+        out.push({ path, kind: "text", bytes: f.bytes, sha256: f.sha256, content: f.content ?? "" });
+      } else {
+        const { readFileSync } = await import("node:fs");
+        await this.blob.put(this.key(jobId, path), readFileSync(f.absPath), ctFor(path));
+        out.push({ path, kind: "binary", bytes: f.bytes, sha256: f.sha256, key: this.key(jobId, path) });
+      }
+    }
+    out.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+    return { files: out };
+  }
+
+  async getFile(jobId: string, path: string): Promise<{ bytes: Buffer } | null> {
+    const bytes = await this.blob.get(this.key(jobId, path));
+    return bytes ? { bytes } : null;
+  }
+
+  async binaryUrl(jobId: string, path: string): Promise<string> {
+    return this.blob.presign(this.key(jobId, path), { expiresSeconds: this.opts?.presignExpiresSeconds, contentType: ctFor(path) });
+  }
+
+  /** Upload the prebuilt archive and return a presigned download URL. */
+  async uploadBundle(jobId: string, format: "tgz" | "zip", bytes: Buffer): Promise<string> {
+    const key = `clones/${jobId}/bundle/clone.${format}`;
+    const ct = format === "zip" ? "application/zip" : "application/gzip";
+    await this.blob.put(key, bytes, ct);
+    return this.blob.presign(key, { expiresSeconds: this.opts?.presignExpiresSeconds, downloadName: `clone-${jobId}.${format}`, contentType: ct });
+  }
+
+  async remove(jobId: string): Promise<void> {
+    await this.blob.deletePrefix(`clones/${jobId}/`);
+  }
+}

packages/storage/src/s3.ts

@@ -0,0 +1,95 @@
+import {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand,
+  DeleteObjectCommand,
+  DeleteObjectsCommand,
+  ListObjectsV2Command,
+} from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import type { BlobClient } from "./blob.js";
+
+export type S3Config = {
+  bucket: string;
+  region?: string;
+  endpoint?: string; // e.g. http://localhost:9000 for MinIO, or R2 endpoint
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  forcePathStyle?: boolean; // required for MinIO
+  /** if set, objects are public at `${publicUrl}/${key}` (CDN) and not presigned. */
+  publicUrl?: string;
+  presignExpiresSeconds?: number;
+};
+
+/** BlobClient backed by S3 / R2 / MinIO. */
+export class S3BlobClient implements BlobClient {
+  private client: S3Client;
+  constructor(private cfg: S3Config) {
+    this.client = new S3Client({
+      region: cfg.region ?? "auto",
+      endpoint: cfg.endpoint,
+      forcePathStyle: cfg.forcePathStyle ?? !!cfg.endpoint,
+      credentials:
+        cfg.accessKeyId && cfg.secretAccessKey
+          ? { accessKeyId: cfg.accessKeyId, secretAccessKey: cfg.secretAccessKey }
+          : undefined,
+    });
+  }
+
+  async put(key: string, bytes: Buffer, contentType: string): Promise<void> {
+    await this.client.send(new PutObjectCommand({ Bucket: this.cfg.bucket, Key: key, Body: bytes, ContentType: contentType }));
+  }
+
+  async get(key: string): Promise<Buffer | null> {
+    try {
+      const r = await this.client.send(new GetObjectCommand({ Bucket: this.cfg.bucket, Key: key }));
+      if (!r.Body) return null;
+      const arr = await (r.Body as { transformToByteArray: () => Promise<Uint8Array> }).transformToByteArray();
+      return Buffer.from(arr);
+    } catch (e) {
+      if ((e as { name?: string }).name === "NoSuchKey" || (e as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode === 404) return null;
+      throw e;
+    }
+  }
+
+  async presign(key: string, opts?: { expiresSeconds?: number; downloadName?: string; contentType?: string }): Promise<string> {
+    if (this.cfg.publicUrl) return `${this.cfg.publicUrl.replace(/\/$/, "")}/${key}`;
+    const cmd = new GetObjectCommand({
+      Bucket: this.cfg.bucket,
+      Key: key,
+      ResponseContentDisposition: opts?.downloadName ? `attachment; filename="${opts.downloadName}"` : undefined,
+      ResponseContentType: opts?.contentType,
+    });
+    return getSignedUrl(this.client, cmd, { expiresIn: opts?.expiresSeconds ?? this.cfg.presignExpiresSeconds ?? 3600 });
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.client.send(new DeleteObjectCommand({ Bucket: this.cfg.bucket, Key: key }));
+  }
+
+  async deletePrefix(prefix: string): Promise<void> {
+    let token: string | undefined;
+    do {
+      const listed = await this.client.send(new ListObjectsV2Command({ Bucket: this.cfg.bucket, Prefix: prefix, ContinuationToken: token }));
+      const keys = (listed.Contents ?? []).map((o) => ({ Key: o.Key! })).filter((o) => o.Key);
+      if (keys.length) await this.client.send(new DeleteObjectsCommand({ Bucket: this.cfg.bucket, Delete: { Objects: keys } }));
+      token = listed.IsTruncated ? listed.NextContinuationToken : undefined;
+    } while (token);
+  }
+}
+
+/** Build an S3 config from env (S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_ACCESS_KEY_ID,
+ *  S3_SECRET_ACCESS_KEY, S3_FORCE_PATH_STYLE, S3_PUBLIC_URL). Returns null if no bucket. */
+export function s3ConfigFromEnv(): S3Config | null {
+  const bucket = process.env.S3_BUCKET;
+  if (!bucket) return null;
+  return {
+    bucket,
+    region: process.env.S3_REGION,
+    endpoint: process.env.S3_ENDPOINT,
+    accessKeyId: process.env.S3_ACCESS_KEY_ID,
+    secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
+    forcePathStyle: process.env.S3_FORCE_PATH_STYLE === "true",
+    publicUrl: process.env.S3_PUBLIC_URL,
+  };
+}

packages/storage/src/types.ts

@@ -0,0 +1,30 @@
+import type { FileMap } from "@cloner/core";
+
+/** A file as persisted: text inline (goes into the DB manifest), binaries by
+ *  reference to a storage key (local relative path or S3 key). */
+export type StoredFile =
+  | { path: string; kind: "text"; bytes: number; sha256: string; content: string }
+  | { path: string; kind: "binary"; bytes: number; sha256: string; key: string };
+
+export type StoredManifest = {
+  files: StoredFile[];
+  /** the whole app as one compressed archive (M4); absent until built. */
+  bundleKey?: string;
+};
+
+/** Blob backend for clone artifacts. LocalArtifactStore (M2) writes to disk;
+ *  S3ArtifactStore (M4) uploads to S3/R2 and presigns URLs. */
+export interface ArtifactStore {
+  /** Persist a clone's files; returns the manifest (text inline, binaries by key). */
+  putClone(jobId: string, files: FileMap): Promise<StoredManifest>;
+  /** Read one file's bytes (for the API's /files/* streaming). null if absent. */
+  getFile(jobId: string, path: string): Promise<{ bytes: Buffer } | null>;
+  /** A URL a client uses to fetch a binary out-of-band (local: API route;
+   *  S3: presigned URL). */
+  binaryUrl(jobId: string, path: string): Promise<string>;
+  /** Persist a prebuilt archive and return a download URL (presigned for S3). When
+   *  absent (local store), the API serves the bundle bytes itself. */
+  uploadBundle?(jobId: string, format: "tgz" | "zip", bytes: Buffer): Promise<string>;
+  /** Delete all artifacts for a job. */
+  remove(jobId: string): Promise<void>;
+}