diff --git a/com.bryce.llamawatch.sdPlugin/ui/gpu.html b/com.bryce.llamawatch.sdPlugin/ui/gpu.html index 91e6410..993ccfc 100644 --- a/com.bryce.llamawatch.sdPlugin/ui/gpu.html +++ b/com.bryce.llamawatch.sdPlugin/ui/gpu.html @@ -11,6 +11,9 @@ + + + diff --git a/com.bryce.llamawatch.sdPlugin/ui/inflight.html b/com.bryce.llamawatch.sdPlugin/ui/inflight.html index 550f6cf..38c6a0e 100644 --- a/com.bryce.llamawatch.sdPlugin/ui/inflight.html +++ b/com.bryce.llamawatch.sdPlugin/ui/inflight.html @@ -11,6 +11,9 @@ + + + diff --git a/package-lock.json b/package-lock.json index 63c4ce2..eeaeb60 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,8 @@ "name": "llama-watch", "version": "1.0.0", "dependencies": { - "@elgato/streamdeck": "^2.1.1" + "@elgato/streamdeck": "^2.1.1", + "undici": "^6.28.0" }, "devDependencies": { "@elgato/cli": "^1.8.1", @@ -2477,6 +2478,15 @@ "node": ">=14.17" } }, + "node_modules/undici": { + "version": "6.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", + "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/undici-types": { "version": "6.20.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", diff --git a/package.json b/package.json index d78d09f..5eaa3d7 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "typescript": "^5.7.3" }, "dependencies": { - "@elgato/streamdeck": "^2.1.1" + "@elgato/streamdeck": "^2.1.1", + "undici": "^6.28.0" } } diff --git a/src/lib/event-feed.ts b/src/lib/event-feed.ts index f2644a4..b25ea8a 100644 --- a/src/lib/event-feed.ts +++ b/src/lib/event-feed.ts @@ -1,3 +1,4 @@ +import { fetchWith } from "./http"; import { type LlamaSwapConfig } from "./util"; import { parseSse, type SseMessage } from "./sse"; import { type FeedEvent, type InflightRequest, type ModelState } from "./inflight-tracker"; @@ -80,7 +81,7 @@ export class EventFeed { try { const headers: Record = { Accept: "text/event-stream" }; if (this.cfg.apiKey) headers["Authorization"] = `Bearer ${this.cfg.apiKey}`; - const res = await fetch(`${this.cfg.baseUrl}/api/events`, { + const res = await fetchWith(this.cfg, `${this.cfg.baseUrl}/api/events`, { headers, signal: this.controller.signal, }); diff --git a/src/lib/http.ts b/src/lib/http.ts new file mode 100644 index 0000000..a02cb24 --- /dev/null +++ b/src/lib/http.ts @@ -0,0 +1,9 @@ +import { Agent, fetch as undiciFetch } from "undici"; +import { type LlamaSwapConfig } from "./util"; + +const INSECURE_AGENT = new Agent({ connect: { rejectUnauthorized: false } }); + +export function fetchWith(cfg: LlamaSwapConfig, input: string, init?: RequestInit): Promise { + if (cfg.insecure !== true) return fetch(input, init); + return undiciFetch(input, { ...(init ?? {}), dispatcher: INSECURE_AGENT }) as unknown as Promise; +} \ No newline at end of file diff --git a/src/lib/llamaswap.ts b/src/lib/llamaswap.ts index 68fb9e6..80ae9a5 100644 --- a/src/lib/llamaswap.ts +++ b/src/lib/llamaswap.ts @@ -1,3 +1,4 @@ +import { fetchWith } from "./http"; import { type LlamaSwapConfig } from "./util"; const TIMEOUT_MS = 5000; @@ -14,7 +15,7 @@ function headersFor(cfg: LlamaSwapConfig): Record { } export async function fetchMetrics(cfg: LlamaSwapConfig): Promise { - const res = await fetch(`${cfg.baseUrl}/metrics`, { + const res = await fetchWith(cfg, `${cfg.baseUrl}/metrics`, { headers: headersFor(cfg), signal: AbortSignal.timeout(TIMEOUT_MS), }); @@ -23,7 +24,7 @@ export async function fetchMetrics(cfg: LlamaSwapConfig): Promise { } export async function fetchModels(cfg: LlamaSwapConfig): Promise { - const res = await fetch(`${cfg.baseUrl}/v1/models`, { + const res = await fetchWith(cfg, `${cfg.baseUrl}/v1/models`, { headers: headersFor(cfg), signal: AbortSignal.timeout(TIMEOUT_MS), }); diff --git a/src/lib/runtime.ts b/src/lib/runtime.ts index 6e4ecfb..feceff0 100644 --- a/src/lib/runtime.ts +++ b/src/lib/runtime.ts @@ -16,7 +16,11 @@ class Runtime { private listeners = new Set<() => void>(); ensureConnections(cfg: LlamaSwapConfig): void { - const changed = !this.cfg || this.cfg.baseUrl !== cfg.baseUrl || this.cfg.apiKey !== cfg.apiKey; + const changed = + !this.cfg || + this.cfg.baseUrl !== cfg.baseUrl || + this.cfg.apiKey !== cfg.apiKey || + this.cfg.insecure !== cfg.insecure; if (changed) { this.feed?.stop(); this.poller?.stop(); diff --git a/src/lib/stats.ts b/src/lib/stats.ts index 5793edf..727ce29 100644 --- a/src/lib/stats.ts +++ b/src/lib/stats.ts @@ -1,3 +1,4 @@ +import { fetchWith } from "./http"; import { type LlamaSwapConfig } from "./util"; export interface UsageStats { @@ -24,7 +25,7 @@ export async function fetchStats(cfg: LlamaSwapConfig, modelId: string): Promise const query = modelId === "all" ? "" : `?model=${encodeURIComponent(modelId)}`; const headers: Record = {}; if (cfg.apiKey) headers["Authorization"] = `Bearer ${cfg.apiKey}`; - const res = await fetch(`${cfg.baseUrl}/api/metrics/stats${query}`, { + const res = await fetchWith(cfg, `${cfg.baseUrl}/api/metrics/stats${query}`, { headers, signal: AbortSignal.timeout(5000), }); diff --git a/src/lib/util.ts b/src/lib/util.ts index ff3636a..4ef3b6e 100644 --- a/src/lib/util.ts +++ b/src/lib/util.ts @@ -5,17 +5,20 @@ export const DEFAULT_BASE_URL = "http://localhost:9292"; export type CfgSettings = { baseUrl?: string; apiKey?: string; + insecure?: boolean; } & JsonObject; export interface LlamaSwapConfig { baseUrl: string; apiKey?: string; + insecure?: boolean; } export function cfgFromSettings(settings: CfgSettings): LlamaSwapConfig { return { baseUrl: normalizeBaseUrl(settings.baseUrl ?? DEFAULT_BASE_URL), apiKey: settings.apiKey || undefined, + insecure: settings.insecure === true, }; } diff --git a/tests/fixtures/selfsigned-cert.pem b/tests/fixtures/selfsigned-cert.pem new file mode 100644 index 0000000..4dd1157 --- /dev/null +++ b/tests/fixtures/selfsigned-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCTCCAfGgAwIBAgIUXQ30Z78Fhhti2Ct6DXczY/jvDjcwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDgyNzIxMTAyNloXDTM2MDgy +NDIxMTAyNlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAtJNuhOYp4Qi7W17LRvph2EZH7WfQsbELROrT3ijJ2ys0 +iA+Bee0ATpc/1R+NgSGfQZsPlAYh2TlYK0UWBIA1BGKm5tSHpBvfxs+GXcgmFM0k +nWFE4Kof6d8zvcd3l7aU+8w8Tz/LnhLpaYQiixZ4ZIoAC1SEWKO0wm+AoMSSeLjH +CBaGmLuD1MQ+I6kZPwDK2YluZ28LHMH9ZCbgfVKXR0FVjgF0fEw6645IzgWGUznN +8pcEisVG2Veyr83Q7uqEL3CtvrdYFjgFJP0Qzez2zgCGXKB+tiBsWIzL2rwVnDcV +ss2bwAVz9JMUxaj8aoMq0Bx5hJkzK9JipGRB9C58CQIDAQABo1MwUTAdBgNVHQ4E +FgQUCtzsN0GW6A2iKrJF28+UVJOY7zkwHwYDVR0jBBgwFoAUCtzsN0GW6A2iKrJF +28+UVJOY7zkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEApZfd +QLXrtRfu6NG1BPfzaROjwTLs5O0tU8+/SSyi3WjoIZmYUBkP7nJm/sR2ZxpSUEfq +OFZwux8bDJTccS9Au0/OR9vYbHWmDoY1e14v2GehNRWXz8vvaD3AURluAYXcmgol +5WOnSB4W8rp6A5gEKX7n4hsHrkUx/Mt+uSg7KZv/fFnwQqyu9OthPKVNV1v79dgm +C0ZUNhpaDDX5Ae+khY88kGADwCKY9BTNzftQG0/4ZJgu7O34eHNgC+2MAgXDtLxm +TIk3XSEaFDoT+HOX38Lh3JYnzJCFO59tQfMMGrPuWfgOISxCe5tkZ22zTtPId+wB ++TptauhacNtwwpc1Jw== +-----END CERTIFICATE----- diff --git a/tests/fixtures/selfsigned-key.pem b/tests/fixtures/selfsigned-key.pem new file mode 100644 index 0000000..b884b21 --- /dev/null +++ b/tests/fixtures/selfsigned-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC0k26E5inhCLtb +XstG+mHYRkftZ9CxsQtE6tPeKMnbKzSID4F57QBOlz/VH42BIZ9Bmw+UBiHZOVgr +RRYEgDUEYqbm1IekG9/Gz4ZdyCYUzSSdYUTgqh/p3zO9x3eXtpT7zDxPP8ueEulp +hCKLFnhkigALVIRYo7TCb4CgxJJ4uMcIFoaYu4PUxD4jqRk/AMrZiW5nbwscwf1k +JuB9UpdHQVWOAXR8TDrrjkjOBYZTOc3ylwSKxUbZV7KvzdDu6oQvcK2+t1gWOAUk +/RDN7PbOAIZcoH62IGxYjMvavBWcNxWyzZvABXP0kxTFqPxqgyrQHHmEmTMr0mKk +ZEH0LnwJAgMBAAECggEADgjFVqvixlwc36GS7+3Gy/3OWkuuwxis9QrBM6t84L1P +ZGG8IONEGleT/PbqUwZvb7Ri9hCx8cWMrjQ83VWviSs3qIoNDrqh3jxDx6cmGojF +Fzw3k7R1LYKM7WuCxnZIxvcdGtWs+BilLm+4FZJGAh5dmYPUk2UJx/DNkPEmJx6n +4aSEwUnV1QGPR7IjchGEHImBj8T8L3HBQnrsRsCRMwgUs3z0XMgfZoFm9/dq08YA +NPzyxAgSsQQrtVOyJMOFj+wi2NrUFtQ0OfS617cUMQXOyceQPpUECn2zPU1UIoPy +VgXfkxSXVdraetgNACELJvBI1bvvZctmkJYkEkY5lwKBgQDXcBDCHn4Any+T4gV8 +Jp9XQf9lCWQnyegP7sSB9vLxzFKU49tuEDDG3m1wOPw17OV/Cs1sOzbiaFXSMM78 +rD2ljRIODaewr3/Js8B6XbYmmkJqth8ByYuKV6kyq1aFPb4zTmEtcWbnnx45mUHr +yZr+8H33yl1iWhR7rdgAtvNP4wKBgQDWkw1C6qR9T8v8sf/vNbwJUHC5xD6B5d18 +XIV8+Uax1tgmIupy/1hVllLBcaizBpE5eFPsWMMNw2pujnSKNER2hfADSFXYUlMO +Ror6e+Dfhr9MctiJjqhNW9kIPBvvnqzpOqwHun6P1GEC/qOi4GIdDDP95OqDaDsV +BvKNKVAwIwKBgQCS+iKEvNbLx85mvrFtRNA6cI0zuhd5Sbcnf4bS/845BmNkrpsa +WLNeSYsyH755b7gWVyFUcIV+Kx45uxDLsxqPolGqAsjfsqukyRxMnzhQ17buJHe8 ++WpYpHuLVPc/CaOETznfDdndtWGifBtMKIu02A+oiIfzPG9y/WQ7AJW4bwKBgFmq +FW6TEq1yvPEZiLNzJuJVhOV7xgsN/SHMn9N7bzk9aBF3obTwUv9g07AWSMKWyfTT +/W3UIZ4MvNr6GGTwNnO4wHT+szC0JhTfEZBeV7fQXPwbObUxsc6xxN2WEK5vBh5n +8B9CpUSBIRDZS5PyY81znf5IvF6xHY9J2e13CBU1AoGARfyoIH6gF92bI1DNZ4cT +mtgPYEGbWYr28/ADF25CrlZ3HvWDqwt2Oe1EvWWyALLjTyKdZXuc0iFiKUSsgUSm +qlBjoh1lWXFrtm+uu8rtGaWrp9q0xjNhcmdM0TqjXpvifDXLALGUiRGvUjZHGo8J +mgw1KgJviTZzZu2cy7iFCqg= +-----END PRIVATE KEY----- diff --git a/tests/http.test.ts b/tests/http.test.ts new file mode 100644 index 0000000..065cc1b --- /dev/null +++ b/tests/http.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { createServer, type Server } from "node:https"; +import { after, before, test } from "node:test"; +import { fetchWith } from "../src/lib/http"; + +const TLS_OPTIONS = { + cert: readFileSync(new URL("./fixtures/selfsigned-cert.pem", import.meta.url)), + key: readFileSync(new URL("./fixtures/selfsigned-key.pem", import.meta.url)), +}; + +let server: Server; +let base: string; + +before(async () => { + server = createServer(TLS_OPTIONS, (req, res) => { + res.writeHead(200, { "content-type": "text/plain" }); + res.end("ok"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + base = `https://127.0.0.1:${(server.address() as { port: number }).port}`; +}); + +after(async () => new Promise((resolve) => server.close(() => resolve()))); + +test("fetchWith verifies certificates by default", async () => { + await assert.rejects(fetchWith({ baseUrl: base }, `${base}/`)); +}); + +test("fetchWith accepts self-signed certificates when insecure", async () => { + const res = await fetchWith({ baseUrl: base, insecure: true }, `${base}/`); + assert.equal(res.status, 200); + assert.equal(await res.text(), "ok"); +}); + +test("fetchWith honors init headers and signal when insecure", async () => { + let seen: string | undefined; + const spy = createServer(TLS_OPTIONS, (req, res) => { + seen = req.headers.authorization; + res.writeHead(200); + res.end("ok"); + }); + await new Promise((resolve) => spy.listen(0, "127.0.0.1", resolve)); + const spyBase = `https://127.0.0.1:${(spy.address() as { port: number }).port}`; + try { + const res = await fetchWith({ baseUrl: spyBase, insecure: true }, `${spyBase}/`, { + headers: { Authorization: "Bearer tok" }, + signal: AbortSignal.timeout(3000), + }); + assert.equal(res.status, 200); + assert.equal(seen, "Bearer tok"); + } finally { + await new Promise((resolve) => spy.close(() => resolve())); + } +}); \ No newline at end of file diff --git a/tests/util.test.ts b/tests/util.test.ts index 61003e6..acdac14 100644 --- a/tests/util.test.ts +++ b/tests/util.test.ts @@ -16,10 +16,13 @@ test("cfgFromSettings defaults baseUrl and omits empty apiKey", () => { const cfg = cfgFromSettings({}); assert.equal(cfg.baseUrl, "http://localhost:9292"); assert.equal(cfg.apiKey, undefined); - assert.deepEqual(cfgFromSettings({ baseUrl: "http://x/", apiKey: "abc" }), { + assert.equal(cfg.insecure, false); + assert.deepEqual(cfgFromSettings({ baseUrl: "http://x/", apiKey: "abc", insecure: true }), { baseUrl: "http://x", apiKey: "abc", + insecure: true, }); + assert.equal(cfgFromSettings({ insecure: "yes" }).insecure, false); }); test("shorten keeps short names and truncates long ones", () => {