Add ditto unpack CLI to turn clone result JSON into a project tree

The REST API returns a clone as a giant `files` map JSON blob with no
first-party way to use it — users were left staring at kilobytes of
escaped JSON with no path from "you got JSON" to "here's a project".

Add packages/cli: a zero-dependency `ditto` CLI whose `unpack` command
walks the files{} map from POST /v1/clones (or GET .../result) and
writes a real project tree to disk:

- text files written from inline content;
- binary assets materialized from inline base64, else fetched from their
  reference URL using $DITTO_API_URL / $DITTO_API_KEY (--no-fetch writes
  just the text tree and lists skipped assets);
- path-traversal guards (clone results are untrusted) and sha256
  integrity checks;
- reads JSON from stdin ("-") so a curl response pipes straight in;
- clear errors for queued jobs with no files yet, and a tip pointing at
  the /bundle endpoint when assets can't be fetched.

Document the pipe-from-curl one-liner prominently next to the REST
examples in README.md and docs/SERVICE.md, add a package README, a
Repository Map row, a root `unpack` script, and a CHANGELOG entry.
Covered by 9 tests (text tree, stdin, inline base64, live URL fetch with
auth + relative-URL resolution, --no-fetch, traversal refusal, queued-job
detection, sha256 mismatch).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Michael Cole
2026-07-01 15:40:28 -04:00
co-authored by Claude Opus 4.8
parent 4de6e244a0
commit 638a0516c5
9 changed files with 589 additions and 5 deletions
+46
View File
@@ -0,0 +1,46 @@
# ditto (CLI)
The official ditto.site command-line helper. Turns a clone **result JSON**
the giant blob you get back from `POST /v1/clones` or
`GET /v1/clones/:id/result` — into an actual project tree on disk.
Zero dependencies. Needs Node >= 20.
## Unpack
```bash
# from a saved file
ditto unpack clone.json ./out
# straight from curl, no temp file
curl -sS -X POST "$DITTO_API_URL/v1/clones" \
-H "authorization: Bearer $DITTO_API_KEY" \
-H "content-type: application/json" \
-d '{"url":"https://example.com/","options":{"mode":"single"}}' \
| ditto unpack - ./out
```
`ditto unpack <clone.json|-> <out-dir>`:
- writes every text file from its inline `content`,
- materializes binary assets from inline base64 when present, otherwise fetches
them from their reference `url`,
- refuses paths that escape `<out-dir>` and verifies `sha256` when the result
carries it.
### Binary assets
Clone results return binaries by reference (`{ "type": "binary", "url": ... }`)
rather than inlining megabytes of base64. To fetch them, `ditto` needs to know
where the API lives:
| Source | Flag | Env |
| --- | --- | --- |
| Base URL for relative asset URLs | `--base-url <url>` | `DITTO_API_URL` |
| Bearer key for authenticated APIs | `--api-key <key>` | `DITTO_API_KEY` |
Use `--no-fetch` to write only the text tree and list the binaries as skipped.
To grab everything in one shot instead, download the archive directly:
`GET /v1/clones/<id>/bundle?format=tgz`.
Run `ditto --help` for the full option list.
+255
View File
@@ -0,0 +1,255 @@
#!/usr/bin/env node
// ditto — official command-line helper for ditto.site.
//
// `ditto unpack <clone.json> <out-dir>` turns the JSON returned by
// `POST /v1/clones` (or `GET /v1/clones/:id/result`) into a real project tree
// on disk: text files are written from their inline `content`, and binary
// assets are materialized from inline base64 or fetched from their reference
// URL. Zero dependencies — just Node >= 20.
import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, join, relative, resolve, sep } from "node:path";
const USAGE = `ditto — unpack a ditto.site clone result into a project tree
Usage:
ditto unpack <clone.json|-> <out-dir> [options]
Arguments:
<clone.json> Path to the saved JSON (POST /v1/clones or GET .../result),
or "-" to read the JSON from stdin (e.g. piped from curl).
<out-dir> Directory to write the project into (created if missing).
Options:
--base-url <url> Base URL used to resolve relative binary asset URLs.
Defaults to $DITTO_API_URL.
--api-key <key> Bearer key sent when fetching binary assets.
Defaults to $DITTO_API_KEY.
--no-fetch Do not fetch binary assets over the network; report them
as skipped instead. Text files are still written.
--quiet Only print the final summary line.
-h, --help Show this help.
Examples:
curl -sS -X POST "$DITTO_API_URL/v1/clones" \\
-H "authorization: Bearer $DITTO_API_KEY" \\
-H "content-type: application/json" \\
-d '{"url":"https://example.com/","options":{"mode":"single"}}' \\
| ditto unpack - ./out
ditto unpack clone.json ./out
`;
/** Print to stderr and exit non-zero. */
function fail(msg) {
process.stderr.write(`ditto: ${msg}\n`);
process.exit(1);
}
/** Minimal flag parser: pulls known --flags out, returns { positionals, flags }. */
function parseArgs(argv) {
const positionals = [];
const flags = { fetch: true, quiet: false };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
switch (a) {
case "-h":
case "--help":
flags.help = true;
break;
case "--no-fetch":
flags.fetch = false;
break;
case "--quiet":
flags.quiet = true;
break;
case "--base-url":
flags.baseUrl = argv[++i];
break;
case "--api-key":
flags.apiKey = argv[++i];
break;
default:
if (a.startsWith("--")) fail(`unknown option: ${a}`);
positionals.push(a);
}
}
return { positionals, flags };
}
/** Read the whole of stdin as a string. */
async function readStdin() {
const chunks = [];
for await (const chunk of process.stdin) chunks.push(chunk);
return Buffer.concat(chunks).toString("utf8");
}
/** Accept either the full result envelope ({ files: {...} }) or a bare file map. */
function extractFiles(doc) {
if (doc && typeof doc === "object" && doc.files && typeof doc.files === "object") {
return doc.files;
}
// A bare file map: values look like clone file entries.
if (doc && typeof doc === "object") {
const vals = Object.values(doc);
if (vals.length && vals.every((v) => v && typeof v === "object" && ("content" in v || "url" in v || "type" in v))) {
return doc;
}
}
return null;
}
/** Resolve `path` under `outDir`, refusing anything that escapes the tree
* (absolute paths, `..`, leading slashes) — clone results are untrusted input. */
function safeJoin(outDir, path) {
const cleaned = String(path).replace(/^[/\\]+/, "");
const dest = resolve(outDir, cleaned);
const rel = relative(outDir, dest);
if (rel === "" || rel.startsWith("..") || rel.split(sep).includes("..")) {
throw new Error(`refusing to write outside output dir: ${path}`);
}
return dest;
}
/** Decode a binary entry's inline bytes, if present (base64 or plain). */
function inlineBytes(entry) {
if (typeof entry.base64 === "string") return Buffer.from(entry.base64, "base64");
if (typeof entry.content === "string") {
const enc = entry.encoding || (entry.type === "binary" ? "base64" : "utf8");
return Buffer.from(entry.content, enc === "base64" ? "base64" : "utf8");
}
return null;
}
function sha256(buf) {
return createHash("sha256").update(buf).digest("hex");
}
async function fetchBinary(url, baseUrl, apiKey) {
const abs = /^https?:\/\//i.test(url) ? url : baseUrl ? new URL(url, baseUrl).toString() : null;
if (!abs) throw new Error("relative asset URL but no --base-url / $DITTO_API_URL set");
const headers = apiKey ? { authorization: `Bearer ${apiKey}` } : {};
const res = await fetch(abs, { headers });
if (!res.ok) throw new Error(`GET ${abs} -> ${res.status} ${res.statusText}`);
return Buffer.from(await res.arrayBuffer());
}
async function unpack(positionals, flags) {
const [input, outDir] = positionals;
if (!input || !outDir) fail("unpack needs <clone.json|-> and <out-dir>\n\n" + USAGE);
const raw = input === "-" ? await readStdin() : await readFile(input, "utf8").catch((e) => fail(`cannot read ${input}: ${e.message}`));
let doc;
try {
doc = JSON.parse(raw);
} catch (e) {
fail(`input is not valid JSON: ${e.message}`);
}
if (doc && (doc.status === "queued" || doc.jobId) && !doc.files) {
fail(
`this looks like a queued job (${doc.jobId ? `jobId ${doc.jobId}` : doc.status}), not a finished result.\n` +
`Poll GET /v1/clones/<id>/result until it has a "files" map, then unpack that.`,
);
}
const files = extractFiles(doc);
if (!files) fail(`no "files" map found in the input — is this a clone result JSON?`);
const outAbs = resolve(outDir);
const baseUrl = flags.baseUrl || process.env.DITTO_API_URL || "";
const apiKey = flags.apiKey || process.env.DITTO_API_KEY || "";
const log = (m) => {
if (!flags.quiet) process.stderr.write(m + "\n");
};
let written = 0;
let bytes = 0;
let hashMismatch = 0;
const skipped = [];
for (const [path, entry] of Object.entries(files)) {
if (!entry || typeof entry !== "object") {
skipped.push({ path, reason: "malformed entry" });
continue;
}
const isBinary = entry.type === "binary" || entry.kind === "binary";
let buf;
if (isBinary) {
buf = inlineBytes(entry);
if (!buf) {
if (!flags.fetch) {
skipped.push({ path, reason: "binary asset (fetch disabled)" });
continue;
}
if (typeof entry.url !== "string") {
skipped.push({ path, reason: "binary asset with no inline bytes or url" });
continue;
}
try {
buf = await fetchBinary(entry.url, baseUrl, apiKey);
} catch (e) {
skipped.push({ path, reason: `could not fetch asset: ${e.message}` });
continue;
}
}
} else {
buf = Buffer.from(typeof entry.content === "string" ? entry.content : "", "utf8");
}
const dest = safeJoin(outAbs, path);
await mkdir(dirname(dest), { recursive: true });
await writeFile(dest, buf);
written++;
bytes += buf.length;
if (typeof entry.sha256 === "string" && entry.sha256 && sha256(buf) !== entry.sha256) {
hashMismatch++;
log(` ! ${path}: sha256 mismatch`);
} else {
log(` + ${path}`);
}
}
const kb = (bytes / 1024).toFixed(1);
log("");
process.stdout.write(`Wrote ${written} file${written === 1 ? "" : "s"} (${kb} KB) to ${relative(process.cwd(), outAbs) || "."}\n`);
if (skipped.length) {
process.stderr.write(`\n${skipped.length} file${skipped.length === 1 ? "" : "s"} skipped:\n`);
for (const s of skipped) process.stderr.write(` - ${s.path}: ${s.reason}\n`);
const anyAsset = skipped.some((s) => /asset|binary/.test(s.reason));
if (anyAsset) {
process.stderr.write(
`\nTip: binary assets are referenced by URL. Set $DITTO_API_URL (and $DITTO_API_KEY if\n` +
`the API is authenticated) so ditto can fetch them, or download the whole app in one\n` +
`shot with GET /v1/clones/<id>/bundle?format=tgz.\n`,
);
}
}
if (hashMismatch) fail(`${hashMismatch} file(s) failed sha256 integrity check`);
}
async function main() {
const argv = process.argv.slice(2);
const { positionals, flags } = parseArgs(argv);
const command = positionals.shift();
if (flags.help || !command) {
process.stdout.write(USAGE);
// No command at all is a misuse (exit 1); an explicit --help is success.
process.exit(flags.help ? 0 : 1);
}
switch (command) {
case "unpack":
await unpack(positionals, flags);
break;
default:
fail(`unknown command: ${command}\n\n${USAGE}`);
}
}
main().catch((e) => fail(e?.stack || String(e)));
+30
View File
@@ -0,0 +1,30 @@
{
"name": "ditto",
"version": "0.1.0",
"private": true,
"description": "Official ditto.site command-line helper: unpack a clone JSON result into a project tree on disk.",
"license": "MIT",
"author": "ion-design",
"repository": {
"type": "git",
"url": "https://github.com/ion-design/ditto.site.git",
"directory": "packages/cli"
},
"bugs": {
"url": "https://github.com/ion-design/ditto.site/issues"
},
"homepage": "https://github.com/ion-design/ditto.site#readme",
"type": "module",
"bin": {
"ditto": "./bin/ditto.mjs"
},
"files": [
"bin"
],
"engines": {
"node": ">=20"
},
"scripts": {
"test": "node --test test/*.test.mjs"
}
}
+185
View File
@@ -0,0 +1,185 @@
import test from "node:test";
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
const BIN = fileURLToPath(new URL("../bin/ditto.mjs", import.meta.url));
function sha256(s) {
return createHash("sha256").update(Buffer.from(s)).digest("hex");
}
/** Run the CLI. `stdin` (optional) is written to the child's stdin. */
function run(args, { stdin, env } = {}) {
return new Promise((resolvePromise) => {
const child = spawn(process.execPath, [BIN, ...args], {
env: { ...process.env, ...env },
});
let out = "";
let err = "";
child.stdout.on("data", (d) => (out += d));
child.stderr.on("data", (d) => (err += d));
child.on("close", (code) => resolvePromise({ code, out, err }));
if (stdin !== undefined) child.stdin.end(stdin);
else child.stdin.end();
});
}
async function withTmp(fn) {
const dir = await mkdtemp(join(tmpdir(), "ditto-cli-"));
try {
return await fn(dir);
} finally {
await rm(dir, { recursive: true, force: true });
}
}
test("unpack: writes a text file tree from a result envelope", async () => {
await withTmp(async (dir) => {
const content = "export default function Page() { return null; }\n";
const doc = {
jobId: "job_1",
files: {
"package.json": { type: "text", content: '{"name":"x"}\n', bytes: 13, sha256: sha256('{"name":"x"}\n') },
"src/app/page.tsx": { type: "text", content, bytes: content.length, sha256: sha256(content) },
},
};
const jsonPath = join(dir, "clone.json");
const out = join(dir, "out");
await writeFile(jsonPath, JSON.stringify(doc));
const res = await run(["unpack", jsonPath, out]);
assert.equal(res.code, 0, res.err);
assert.equal(await readFile(join(out, "package.json"), "utf8"), '{"name":"x"}\n');
assert.equal(await readFile(join(out, "src/app/page.tsx"), "utf8"), content);
assert.match(res.out, /Wrote 2 files/);
});
});
test("unpack: reads JSON from stdin with '-'", async () => {
await withTmp(async (dir) => {
const doc = { files: { "a.txt": { type: "text", content: "hi" } } };
const out = join(dir, "out");
const res = await run(["unpack", "-", out], { stdin: JSON.stringify(doc) });
assert.equal(res.code, 0, res.err);
assert.equal(await readFile(join(out, "a.txt"), "utf8"), "hi");
});
});
test("unpack: materializes inline base64 binary assets", async () => {
await withTmp(async (dir) => {
const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]);
const doc = {
files: {
"public/logo.png": {
type: "binary",
content: bytes.toString("base64"),
encoding: "base64",
bytes: bytes.length,
sha256: createHash("sha256").update(bytes).digest("hex"),
},
},
};
const out = join(dir, "out");
const res = await run(["unpack", "-", out], { stdin: JSON.stringify(doc) });
assert.equal(res.code, 0, res.err);
const written = await readFile(join(out, "public/logo.png"));
assert.deepEqual([...written], [...bytes]);
});
});
test("unpack: fetches binary assets by URL and resolves relative URLs against base", async () => {
await withTmp(async (dir) => {
const asset = Buffer.from("PNGDATA");
const { createServer } = await import("node:http");
const server = createServer((req, res) => {
if (req.url === "/v1/clones/job_1/files/public/img.png" && req.headers.authorization === "Bearer k") {
res.writeHead(200);
res.end(asset);
} else {
res.writeHead(404);
res.end("no");
}
});
await new Promise((r) => server.listen(0, r));
const port = server.address().port;
try {
const doc = {
files: {
"public/img.png": {
type: "binary",
url: "/v1/clones/job_1/files/public/img.png",
bytes: asset.length,
sha256: createHash("sha256").update(asset).digest("hex"),
},
},
};
const out = join(dir, "out");
const res = await run(["unpack", "-", out], {
stdin: JSON.stringify(doc),
env: { DITTO_API_URL: `http://127.0.0.1:${port}`, DITTO_API_KEY: "k" },
});
assert.equal(res.code, 0, res.err);
assert.deepEqual([...(await readFile(join(out, "public/img.png")))], [...asset]);
} finally {
server.close();
}
});
});
test("unpack: --no-fetch skips remote binaries but still writes text", async () => {
await withTmp(async (dir) => {
const doc = {
files: {
"index.html": { type: "text", content: "<h1>hi</h1>" },
"public/img.png": { type: "binary", url: "/v1/clones/x/files/public/img.png" },
},
};
const out = join(dir, "out");
const res = await run(["unpack", "-", out, "--no-fetch"], { stdin: JSON.stringify(doc) });
assert.equal(res.code, 0, res.err);
assert.equal(await readFile(join(out, "index.html"), "utf8"), "<h1>hi</h1>");
await assert.rejects(stat(join(out, "public/img.png")));
assert.match(res.err, /skipped/);
});
});
test("unpack: refuses path traversal", async () => {
await withTmp(async (dir) => {
const doc = { files: { "../escape.txt": { type: "text", content: "x" } } };
const out = join(dir, "out");
const res = await run(["unpack", "-", out], { stdin: JSON.stringify(doc) });
assert.notEqual(res.code, 0);
assert.match(res.err, /outside output dir/);
});
});
test("unpack: fails clearly on a queued job with no files", async () => {
await withTmp(async (dir) => {
const doc = { jobId: "job_9", status: "queued" };
const out = join(dir, "out");
const res = await run(["unpack", "-", out], { stdin: JSON.stringify(doc) });
assert.notEqual(res.code, 0);
assert.match(res.err, /queued job/);
});
});
test("unpack: reports sha256 mismatch as a failure", async () => {
await withTmp(async (dir) => {
const doc = { files: { "a.txt": { type: "text", content: "hi", sha256: "deadbeef" } } };
const out = join(dir, "out");
const res = await run(["unpack", "-", out], { stdin: JSON.stringify(doc) });
assert.notEqual(res.code, 0);
assert.match(res.err, /integrity check/);
});
});
test("help: exits 0 and prints usage", async () => {
const res = await run(["--help"]);
assert.equal(res.code, 0);
assert.match(res.out, /ditto unpack/);
});